Merge pull request #63 from axiom-nz/feature/url_path_regex_to_ignore

Add UrlPathRegexToIgnore argument for ignoring paths by regex

Author: jsakamoto

BlazorWasmPreRendering.Build.Test/StaticlizeCrawlerTest.cs

@@ -20,6 +20,7 @@ public async Task SaveToStaticFileAsync_IndexHtmlInSubFolder_Style_Test()
         var crawler = new StaticlizeCrawler(
             baseUrl,
             urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: null,
             webRootPath: outDir,
             locales: new string[] { },
             OutputStyle.IndexHtmlInSubFolders,
@@ -40,6 +41,40 @@ public async Task SaveToStaticFileAsync_IndexHtmlInSubFolder_Style_Test()
                 $"Getting {baseUrl}/fetchdata/weather-forecast...");
     }
 
+    [Test]
+    public async Task SaveToStaticFileAsync_UrlPathRegexToIgnore_Test()
+    {
+        // Given
+        const string baseUrl = "http://127.0.0.1:5058";
+        await using var testSiteServer = await TestSites.StartTestSite1(baseUrl);
+        using var outDir = new WorkDirectory();
+        var logger = new TestLogger();
+
+        // When
+        var crawler = new StaticlizeCrawler(
+            baseUrl,
+            urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: "counter", // Skip "/counter"
+            webRootPath: outDir,
+            locales: new string[] { },
+            OutputStyle.IndexHtmlInSubFolders,
+            enableBrotliCompression: false,
+            enableGZipCompression: false,
+            logger: logger);
+        var result = await crawler.SaveToStaticFileAsync();
+
+        // Then
+        result.Is(StaticlizeCrawlingResult.Nothing);
+        File.Exists(Path.Combine(outDir, "index.html")).IsTrue();
+        File.Exists(Path.Combine(outDir, "counter", "index.html")).IsFalse();
+        File.Exists(Path.Combine(outDir, "fetchdata", "weather-forecast", "index.html")).IsTrue();
+
+        logger.LogLines.OrderBy(line => line)
+            .Is($"Getting {baseUrl}/...",
+                $"Getting {baseUrl}/fetchdata/weather-forecast...",
+                $"Skipping /counter (matched by UrlPathRegexToIgnore)");
+    }
+
     [Test]
     public async Task SaveToStaticFileAsync_AppendHtmlExtension_Style_Test()
     {
@@ -53,6 +88,7 @@ public async Task SaveToStaticFileAsync_AppendHtmlExtension_Style_Test()
         var crawler = new StaticlizeCrawler(
             baseUrl,
             urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: null,
             webRootPath: outDir,
             locales: new[] { "en" },
             OutputStyle.AppendHtmlExtension,
@@ -87,6 +123,7 @@ public async Task SaveToStaticFileAsync_ServiceNotRegisteredError_Test(int port,
         var crawler = new StaticlizeCrawler(
             baseUrl,
             urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: null,
             webRootPath: outDir,
             locales: new string[] { },
             OutputStyle.AppendHtmlExtension,
@@ -113,6 +150,7 @@ public async Task SaveToStaticFileAsync_FollowsAlternateLinks_Test()
         var crawler = new StaticlizeCrawler(
             baseUrl,
             urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: null,
             webRootPath: outDir,
             locales: new string[] { },
             OutputStyle.IndexHtmlInSubFolders,
@@ -148,6 +186,7 @@ public async Task SaveToStaticFileAsync_JsInvokeOnServerError_Test()
         var crawler = new StaticlizeCrawler(
             baseUrl,
             urlPathToExplicitFetch: null,
+            urlPathRegexToIgnore: null,
             webRootPath: outDir,
             locales: new string[] { },
             OutputStyle.AppendHtmlExtension,

BlazorWasmPreRendering.Build/CommandLineOptions.cs

@@ -48,6 +48,8 @@ public bool DeleteLoadingContents
 
     public string? UrlPathToExplicitFetch { get; set; }
 
+    public string? UrlPathRegexToIgnore { get; set; }
+
     public bool KeepRunning { get; init; }
 
     public static readonly string DefaultServerPort = "5050-5999";

BlazorWasmPreRendering.Build/Program.cs

@@ -41,6 +41,7 @@ private static async Task<StaticlizeCrawlingResult> PreRenderToStaticFilesAsync(
         var crawler = new StaticlizeCrawler(
             baseUrl,
             commandLineOptions.UrlPathToExplicitFetch,
+            commandLineOptions.UrlPathRegexToIgnore,
             prerenderingOptions.WebRootPath,
             prerenderingOptions.Locales,
             commandLineOptions.OutputStyle,

BlazorWasmPreRendering.Build/StaticlizeCrawler.cs

@@ -32,13 +32,16 @@ internal class StaticlizeCrawler
 
     private IEnumerable<string> UrlPathToExplicitFetch { get; }
 
+    private Regex? UrlPathRegexToIgnore { get; }
+
     private readonly List<string> _StaticalizedFiles = new List<string>();
 
     public IEnumerable<string> StaticalizedFiles => this._StaticalizedFiles;
 
     public StaticlizeCrawler(
         string baseUrl,
         string? urlPathToExplicitFetch,
+        string? urlPathRegexToIgnore,
         string webRootPath,
         IEnumerable<string> locales,
         OutputStyle outputStyle,
@@ -59,6 +62,21 @@ public StaticlizeCrawler(
             .Where(s => !string.IsNullOrEmpty(s))
             .ToArray();
 
+        if (!string.IsNullOrEmpty(urlPathRegexToIgnore))
+        {
+            try
+            {
+                this.UrlPathRegexToIgnore = new Regex(urlPathRegexToIgnore, RegexOptions.Compiled | RegexOptions.IgnoreCase);
+            }
+            catch (ArgumentException ex)
+            {
+                throw new ArgumentException(
+                    "Invalid regex pattern for --urlpathregextoignore / BlazorWasmPrerenderingUrlPathRegexToIgnore.",
+                    nameof(urlPathRegexToIgnore),
+                    ex);
+            }
+        }
+
         if (locales.Any())
         {
             this.HttpClient.DefaultRequestHeaders.AcceptLanguage.Clear();
@@ -90,6 +108,14 @@ private async Task SaveToStaticFileAsync((string Href, string Protocol, string P
     {
         var href = args.Href.Split('#').FirstOrDefault() ?? "";
         if (this.SavedPathSet.Contains(href)) return;
+
+        if (this.UrlPathRegexToIgnore != null && this.UrlPathRegexToIgnore.IsMatch(args.PathName))
+        {
+            this.SavedPathSet.Add(href);
+            this.Logger.LogInformation($"Skipping {args.PathName} (matched by UrlPathRegexToIgnore)");
+            return;
+        }
+
         this.SavedPathSet.Add(href);
 
         // DEBUG: Console.WriteLine($"Protocol:[{args.Protocol}], PathName:[{args.PathName}], Href:[{args.Href}]");

BlazorWasmPreRendering.Build/build/BlazorWasmPreRendering.Build.targets

@@ -31,6 +31,7 @@
       <BlazorWasmPrerenderingKeepServer Condition=" '$(BlazorWasmPrerenderingKeepServer)' == '' ">false</BlazorWasmPrerenderingKeepServer>
       <BlazorWasmPrerenderingDeleteLoadingContents Condition=" '$(BlazorWasmPrerenderingKeepServer)' == '' ">false</BlazorWasmPrerenderingDeleteLoadingContents>
       <BlazorWasmPrerenderingServerPort Condition=" '$(BlazorWasmPrerenderingServerPort)' == '' ">5050-5999</BlazorWasmPrerenderingServerPort>
+      <BlazorWasmPrerenderingUrlPathRegexToIgnore Condition=" '$(BlazorWasmPrerenderingUrlPathRegexToIgnore)' == '' "></BlazorWasmPrerenderingUrlPathRegexToIgnore>
       <BlazorWasmPrerenderingDotNetHost Condition=" '$(BlazorWasmPrerenderingDotNetHost)' == '' ">$(DOTNET_HOST_PATH)</BlazorWasmPrerenderingDotNetHost>
       <BlazorWasmPrerenderingDotNetHost Condition=" '$(BlazorWasmPrerenderingDotNetHost)' == '' ">$(_DotNetHostDirectory)$(_DotNetHostFileName)</BlazorWasmPrerenderingDotNetHost>
     </PropertyGroup>
@@ -63,6 +64,6 @@
       <BlazorWasmPrerenderingDeleteLoadingContentsSwitch Condition=" '$(BlazorWasmPrerenderingDeleteLoadingContents)' == 'true' "> -d</BlazorWasmPrerenderingDeleteLoadingContentsSwitch>
     </PropertyGroup>
 
-    <Exec Command="&quot;$(BlazorWasmPrerenderingDotNetHost)&quot; &quot;$(BlazorWasmPrerenderingServerDll)&quot; --assemblyname &quot;$(BlazorWasmPrerenderingAssembly)&quot; -t &quot;$(BlazorWasmPrerenderingRootComponentType)&quot; --selectorofrootcomponent &quot;$(BlazorWasmPrerenderingRootComponentSelector)&quot; --selectorofheadoutletcomponent &quot;$(BlazorWasmPrerenderingHeadOutletComponentSelector)&quot; -p &quot;$(BlazorWasmPrerenderingPublishDir)&quot; -i &quot;$(BlazorWasmPrerenderingIntermediateDir)&quot; --assemblydir &quot;$(BlazorWasmPrerenderingTargetDir)&quot; -m &quot;$(BlazorWasmPrerenderingMiddlewareArg)&quot; -f &quot;$(BlazorWasmPrerenderingTFM)&quot; --serviceworkerassetsmanifest &quot;$(ServiceWorkerAssetsManifest)&quot; --environment &quot;$(BlazorWasmPrerenderingEnvironment)&quot; --emulateauthme &quot;$(BlazorWasmPrerenderingEmulateAuthMe)&quot; --locale &quot;$(BlazorWasmPrerenderingLocale)&quot; -o &quot;$(BlazorWasmPrerenderingOutputStyle)&quot; $(BlazorWasmPrerenderingKeepServerSwitch)$(BlazorWasmPrerenderingDeleteLoadingContentsSwitch) -u &quot;$(BlazorWasmPrerenderingUrlPathToExplicitFetch)&quot; -r &quot;$(BlazorWasmPrerenderingMode)&quot; --serverport &quot;$(BlazorWasmPrerenderingServerPort)&quot; --bwapoptionsdllext &quot;$(BlazorWasmPrerenderingBWAPDllExt)&quot;" />
+    <Exec Command="&quot;$(BlazorWasmPrerenderingDotNetHost)&quot; &quot;$(BlazorWasmPrerenderingServerDll)&quot; --assemblyname &quot;$(BlazorWasmPrerenderingAssembly)&quot; -t &quot;$(BlazorWasmPrerenderingRootComponentType)&quot; --selectorofrootcomponent &quot;$(BlazorWasmPrerenderingRootComponentSelector)&quot; --selectorofheadoutletcomponent &quot;$(BlazorWasmPrerenderingHeadOutletComponentSelector)&quot; -p &quot;$(BlazorWasmPrerenderingPublishDir)&quot; -i &quot;$(BlazorWasmPrerenderingIntermediateDir)&quot; --assemblydir &quot;$(BlazorWasmPrerenderingTargetDir)&quot; -m &quot;$(BlazorWasmPrerenderingMiddlewareArg)&quot; -f &quot;$(BlazorWasmPrerenderingTFM)&quot; --serviceworkerassetsmanifest &quot;$(ServiceWorkerAssetsManifest)&quot; --environment &quot;$(BlazorWasmPrerenderingEnvironment)&quot; --emulateauthme &quot;$(BlazorWasmPrerenderingEmulateAuthMe)&quot; --locale &quot;$(BlazorWasmPrerenderingLocale)&quot; -o &quot;$(BlazorWasmPrerenderingOutputStyle)&quot; $(BlazorWasmPrerenderingKeepServerSwitch)$(BlazorWasmPrerenderingDeleteLoadingContentsSwitch) -u &quot;$(BlazorWasmPrerenderingUrlPathToExplicitFetch)&quot; --urlpathregextoignore &quot;$(BlazorWasmPrerenderingUrlPathRegexToIgnore)&quot; -r &quot;$(BlazorWasmPrerenderingMode)&quot; --serverport &quot;$(BlazorWasmPrerenderingServerPort)&quot; --bwapoptionsdllext &quot;$(BlazorWasmPrerenderingBWAPDllExt)&quot;" />
   </Target>
 </Project>

MSBUILD-PROPERTIES.md

@@ -11,6 +11,7 @@ BlazorWasmPrerenderingHeadOutletComponentSelector| `head::after` | Set the DOM e
 BlazorWasmPrerenderingOutputStyle           | `IndexHtmlInSubFolders` | When it is set to `AppendHtmlExtension`, the page of the URL path `foo/bar` will be saved as the `foo/bar.html` instead of the `foo/bar/index.html`.
 BlazorWasmPrerenderingDeleteLoadingContents | `false`        | When it is set to `true`, the "Loading..." contents will be deleted from prerendered output HTML files, and prerendered contents to be visible immediately even before the Blazor WebAssembly runtime has warmed up.
 BlazorWasmPrerenderingUrlPathToExplicitFetch|                | Set the semicolon-separated URL paths explicitly that are not linked from anywhere, such as easter-egg pages, to be prerendered.
+BlazorWasmPrerenderingUrlPathRegexToIgnore   |                | Set a regular expression to skip paths that match it during the crawling process.
 BlazorWasmPrerenderingEnvironment           | `Prerendering` | Set a name of a host environment that can retrieve via `IWebHostEnvironment.Environment`.
 BlazorWasmPrerenderingEmulateAuthMe         | `true`         | When it is set to `true`, prerendering server emulates Azure App Services Auth. That means the ULR endpoint **"/.auth/me"** will return the JSON content `{"clientPrincipal":null}`
 BlazorWasmPrerenderingLocale                | `en`           | Set a comma-separated locale list such as "en", "ja-JP,en-US", etc., those used when crawling. **⚠️Attention:** when you specify this MSBuild property via "dotnet" command line, you have to replace `,` (comma) with `%2c`.

README.md

@@ -203,6 +203,21 @@ To support that case, please **set the URL path list that you want to fetch expl
     <BlazorWasmPrerenderingUrlPathToExplicitFetch>/unkinked/page1;/unlinked/page2</BlazorWasmPrerenderingUrlPathToExplicitFetch>
     ...
 ```
+
+Similarly, if you want to skip some URL paths during the crawling process, you can **set a regular expression to the `BlazorWasmPrerenderingUrlPathRegexToIgnore` MSBuild property**.
+
+```xml
+<Project Sdk="Microsoft.NET.Sdk.BlazorWebAssembly">
+  ...
+  <PropertyGroup>
+    <!--
+    👇 If you set this, each URL path matched with the regex
+       will be skipped during the crawling process.
+    -->
+    <BlazorWasmPrerenderingUrlPathRegexToIgnore>/(admin|private)</BlazorWasmPrerenderingUrlPathRegexToIgnore>
+    ...
+```
+
 (* See also: [_MSBuild properties reference for the "BlazorWasmPreRendering.Build"_](https://github.com/jsakamoto/BlazorWasmPreRendering.Build/blob/master/MSBUILD-PROPERTIES.md))
 
 ### Render mode