Merge pull request #818 from EdiWang/feature/index-now

Add IndexNow Support

src/Moonglade.IndexNow.Client/IIndexNowClient.cs

@@ -0,0 +1,6 @@
+namespace Moonglade.IndexNow.Client;
+
+public interface IIndexNowClient
+{
+    Task SendRequestAsync(Uri url);
+}

src/Moonglade.IndexNow.Client/IndexNowClient.cs

@@ -0,0 +1,89 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using System.Net;
+using System.Text;
+
+namespace Moonglade.IndexNow.Client;
+
+public class IndexNowClient(ILogger<IndexNowClient> logger, IConfiguration configuration, IHttpClientFactory httpClientFactory) : IIndexNowClient
+{
+    private readonly string[] _pingTargets = configuration.GetSection("IndexNow:PingTargets").Get<string[]>();
+    private readonly string _apiKey = configuration["IndexNow:ApiKey"] ?? throw new InvalidOperationException("IndexNow:ApiKey is not configured.");
+
+    public async Task SendRequestAsync(Uri uri)
+    {
+        if (string.IsNullOrWhiteSpace(_apiKey))
+        {
+            logger.LogWarning("IndexNow:ApiKey is not configured.");
+            return;
+        }
+
+        if (_pingTargets == null || _pingTargets.Length == 0)
+        {
+            throw new InvalidOperationException("IndexNow:PingTargets is not configured.");
+        }
+
+        foreach (var pingTarget in _pingTargets)
+        {
+            var client = httpClientFactory.CreateClient(pingTarget);
+
+            var requestBody = CreateRequestBody(uri);
+            var content = new StringContent(System.Text.Json.JsonSerializer.Serialize(requestBody), Encoding.UTF8, "application/json");
+
+            try
+            {
+                var response = await client.PostAsync("/indexnow", content);
+                await HandleResponseAsync(pingTarget, response);
+            }
+            catch (Exception e)
+            {
+                logger.LogError(e, $"Failed to send index request to '{pingTarget}'");
+            }
+        }
+    }
+
+    private IndexNowRequest CreateRequestBody(Uri uri)
+    {
+        // https://www.indexnow.org/documentation
+        return new IndexNowRequest
+        {
+            Host = uri.Host,
+            Key = _apiKey,
+            KeyLocation = $"https://{uri.Host}/indexnowkey.txt",
+            UrlList = new[] { uri.ToString() }
+        };
+    }
+
+    private async Task HandleResponseAsync(string pingTarget, HttpResponseMessage response)
+    {
+        var responseBody = await response.Content.ReadAsStringAsync();
+
+        switch (response.StatusCode)
+        {
+            // Success cases
+            case HttpStatusCode.OK:
+                logger.LogInformation($"Index request sent to '{pingTarget}', {response.StatusCode}: {responseBody}. URL submitted successfully.");
+                break;
+            case HttpStatusCode.Accepted:
+                logger.LogWarning($"Index request sent to '{pingTarget}', {response.StatusCode}. URL received. IndexNow key validation pending.");
+                break;
+
+            // Error cases
+            case HttpStatusCode.BadRequest:
+                logger.LogError($"Index request sent to '{pingTarget}', {response.StatusCode}: {responseBody}. Invalid format.");
+                break;
+            case HttpStatusCode.Forbidden:
+                logger.LogError($"Index request sent to '{pingTarget}', {response.StatusCode}: {responseBody}. Key not valid (e.g., key not found, file found but key not in the file).");
+                break;
+            case HttpStatusCode.UnprocessableEntity:
+                logger.LogError($"Index request sent to '{pingTarget}', {response.StatusCode}: {responseBody}. URLs which don’t belong to the host or the key is not matching the schema in the protocol.");
+                break;
+            case HttpStatusCode.TooManyRequests:
+                logger.LogError($"Index request sent to '{pingTarget}', {response.StatusCode}: {responseBody}. Too many requests (potential spam).");
+                break;
+            default:
+                response.EnsureSuccessStatusCode();
+                break;
+        }
+    }
+}

src/Moonglade.IndexNow.Client/IndexNowMapHandler.cs

@@ -0,0 +1,28 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
+using System.Text;
+
+namespace Moonglade.IndexNow.Client;
+
+public class IndexNowMapHandler
+{
+    public static Delegate Handler => async (HttpContext httpContext, IConfiguration configuration) =>
+    {
+        await Handle(httpContext, configuration);
+    };
+
+    public static async Task Handle(HttpContext httpContext, IConfiguration configuration)
+    {
+        var apiKey = configuration["IndexNow:ApiKey"];
+        if (string.IsNullOrWhiteSpace(apiKey))
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status404NotFound;
+            await httpContext.Response.WriteAsync("No indexnowkey.txt is present.", httpContext.RequestAborted);
+        }
+        else
+        {
+            httpContext.Response.ContentType = "text/plain";
+            await httpContext.Response.WriteAsync(apiKey, Encoding.UTF8, httpContext.RequestAborted);
+        }
+    }
+}

src/Moonglade.IndexNow.Client/IndexNowRequest.cs

@@ -0,0 +1,9 @@
+namespace Moonglade.IndexNow.Client;
+
+public class IndexNowRequest
+{
+    public string Host { get; set; }
+    public string Key { get; set; }
+    public string KeyLocation { get; set; }
+    public string[] UrlList { get; set; }
+}

src/Moonglade.IndexNow.Client/Moonglade.IndexNow.Client.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+    </PropertyGroup>
+    <ItemGroup>
+        <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    </ItemGroup>
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.1" />
+    </ItemGroup>
+    <ItemGroup>
+      <ProjectReference Include="..\Moonglade.Utils\Moonglade.Utils.csproj" />
+    </ItemGroup>
+</Project>

src/Moonglade.IndexNow.Client/ServiceCollectionExtension.cs

@@ -0,0 +1,30 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Moonglade.Utils;
+using System.Net.Http.Headers;
+
+namespace Moonglade.IndexNow.Client;
+
+public static class ServiceCollectionExtension
+{
+    public static IServiceCollection AddIndexNowClient(this IServiceCollection services, IConfigurationSection configurationSection)
+    {
+        var pingTargets = configurationSection.GetSection("PingTargets").Get<string[]>();
+
+        foreach (var pingTarget in pingTargets)
+        {
+            services.AddHttpClient(pingTarget, o =>
+                    {
+                        o.BaseAddress = new Uri($"https://{pingTarget}");
+                        o.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+                        o.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Moonglade", Helper.AppVersionBasic));
+                        o.DefaultRequestHeaders.Host = pingTarget;
+                    })
+                    .AddStandardResilienceHandler();
+        }
+
+        services.AddScoped<IIndexNowClient, IndexNowClient>();
+
+        return services;
+    }
+}

src/Moonglade.Web/Controllers/PostController.cs

@@ -1,4 +1,5 @@
 using Moonglade.Core.PostFeature;
+using Moonglade.IndexNow.Client;
 using Moonglade.Pingback;
 using Moonglade.Web.Attributes;
 using Moonglade.Webmention;
@@ -59,6 +60,8 @@ await mediator.Send(new CreatePostCommand(model)) :
                 {
                     cannonService.FireAsync<IWebmentionSender>(async sender => await sender.SendWebmentionAsync(link.ToString(), postEntity.PostContent));
                 }
+
+                cannonService.FireAsync<IIndexNowClient>(async sender => await sender.SendRequestAsync(link));
             }
 
             return Ok(new { PostId = postEntity.Id });

src/Moonglade.Web/Moonglade.Web.csproj

@@ -52,6 +52,7 @@
     <ProjectReference Include="..\Moonglade.FriendLink\Moonglade.FriendLink.csproj" />
     <ProjectReference Include="..\Moonglade.ImageStorage\Moonglade.ImageStorage.csproj" />
     <ProjectReference Include="..\Moonglade.Email.Client\Moonglade.Email.Client.csproj" />
+    <ProjectReference Include="..\Moonglade.IndexNow.Client\Moonglade.IndexNow.Client.csproj" />
     <ProjectReference Include="..\Moonglade.Pingback\Moonglade.Pingback.csproj" />
     <ProjectReference Include="..\Moonglade.Setup\Moonglade.Setup.csproj" />
     <ProjectReference Include="..\Moonglade.Syndication\Moonglade.Syndication.csproj" />

src/Moonglade.Web/Program.cs

@@ -1,25 +1,21 @@
 using Edi.Captcha;
 using Edi.PasswordGenerator;
-
 using Microsoft.AspNetCore.Rewrite;
-
 using Moonglade.Comments.Moderator;
 using Moonglade.Data.MySql;
 using Moonglade.Data.PostgreSql;
 using Moonglade.Data.SqlServer;
 using Moonglade.Email.Client;
+using Moonglade.IndexNow.Client;
 using Moonglade.Mention.Common;
 using Moonglade.Pingback;
 using Moonglade.Setup;
 using Moonglade.Syndication;
 using Moonglade.Web.Handlers;
 using Moonglade.Webmention;
-
 using SixLabors.Fonts;
-
 using System.Globalization;
 using System.Text.Json.Serialization;
-
 using Encoder = Moonglade.Web.Configuration.Encoder;
 
 AppDomain.CurrentDomain.Load("Moonglade.Setup");
@@ -132,6 +128,7 @@
         .AddImageStorage(builder.Configuration, options => options.ContentRootPath = builder.Environment.ContentRootPath);
 
 services.AddEmailClient();
+services.AddIndexNowClient(builder.Configuration.GetSection("IndexNow"));
 services.AddContentModerator(builder.Configuration);
 
 services.AddSingleton<CannonService>();
@@ -223,6 +220,7 @@
 app.MapRazorPages();
 
 app.MapGet("/robots.txt", RobotsTxtMapHandler.Handler);
+app.MapGet("/indexnowkey.txt", IndexNowMapHandler.Handler);
 app.MapGet("/manifest.webmanifest", WebManifestMapHandler.Handler);
 
 var bc = app.Services.GetRequiredService<IBlogConfig>();

src/Moonglade.Web/appsettings.json

@@ -23,6 +23,15 @@
     "ApiKey": "",
     "ApiKeyHeader": "x-functions-key"
   },
+  "IndexNow": {
+    "ApiKey": "",
+    "PingTargets": [
+      "api.indexnow.org",
+      "www.bing.com",
+      "search.seznam.cz",
+      "yandex.com"
+    ]
+  },
   "ForwardedHeaders": {
     "Enabled": false,
     "HeaderName": "",

src/Moonglade.sln

@@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Mention", "Mention", "{1AE9
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Moonglade.Setup", "Moonglade.Setup\Moonglade.Setup.csproj", "{648FDD86-E047-49A4-9054-BF041FCD0447}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Moonglade.IndexNow.Client", "Moonglade.IndexNow.Client\Moonglade.IndexNow.Client.csproj", "{F06B2391-938E-44FB-8281-6764DC2BC024}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -148,6 +150,10 @@ Global
 		{648FDD86-E047-49A4-9054-BF041FCD0447}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{648FDD86-E047-49A4-9054-BF041FCD0447}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{648FDD86-E047-49A4-9054-BF041FCD0447}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F06B2391-938E-44FB-8281-6764DC2BC024}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F06B2391-938E-44FB-8281-6764DC2BC024}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F06B2391-938E-44FB-8281-6764DC2BC024}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F06B2391-938E-44FB-8281-6764DC2BC024}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -174,9 +180,10 @@ Global
 		{F8E8ECCB-0379-416A-8D34-3F3D8428F2E1} = {1AE9B003-34B0-4F5D-BF84-E1C714A7EE31}
 		{1AE9B003-34B0-4F5D-BF84-E1C714A7EE31} = {97439361-03B4-46C9-BC06-49F76BA57879}
 		{648FDD86-E047-49A4-9054-BF041FCD0447} = {97439361-03B4-46C9-BC06-49F76BA57879}
+		{F06B2391-938E-44FB-8281-6764DC2BC024} = {97439361-03B4-46C9-BC06-49F76BA57879}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
-		RESX_CultureCountyOverrides = zh=zh-Hant
 		SolutionGuid = {EBD14882-9899-4C4B-B9BC-22C0B93BD559}
+		RESX_CultureCountyOverrides = zh=zh-Hant
 	EndGlobalSection
 EndGlobal