WIP: compile for .NET 8.0 and 9.0

Signed-off-by: Yves Bastide <yves@botify.com>
Signed-off-by: Yves Bastide <stid@acm.org>

Author: ybastide

ConsoleApplication/ConsoleApplication.csproj

@@ -2,10 +2,11 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <RootNamespace>ConsoleApp</RootNamespace>
+        <TargetFrameworks>net7.0;net8.0;net9.0</TargetFrameworks>
+        <LangVersion>default</LangVersion>
     </PropertyGroup>
 
     <ItemGroup>

ConsoleApplication/Program.cs

@@ -1,5 +1,7 @@
 // See https://aka.ms/new-console-template for more information
 
+using System.Text;
+
 using RobotsTxt;
 
 var filename = args.Length > 0 ? args[0] : "";
@@ -18,7 +20,7 @@
 
 var robotsContent = File.ReadAllBytes(filename);
 var userAgent = args[1];
-var userAgents = new List<string> { userAgent };
+var userAgents = new List<byte[]> { new UTF8Encoding().GetBytes(userAgent) };
 
 var matcher = new RobotsMatcher();
 var url = args[2];

RobotsTxt/Extensions.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace RobotsTxt
 {
     public static class MyExtensions

RobotsTxt/IRobotsParseHandler.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace RobotsTxt
 {
     public interface IRobotsParseHandler

RobotsTxt/LongestMatchRobotsMatchStrategy.cs

@@ -1,4 +1,6 @@
-namespace RobotsTxt
+using System;
+
+namespace RobotsTxt
 {
 
     /// <summary>

RobotsTxt/ParsedRobotsKey.cs

@@ -1,5 +1,5 @@
+using System;
 using System.Diagnostics;
-using System.Text;
 
 namespace RobotsTxt
 {
@@ -10,7 +10,7 @@ class ParsedRobotsKey
 
         public enum KeyType
         {
-            // Generic highlevel fields.
+            // Generic high level fields.
             UserAgent,
             Sitemap,
 

RobotsTxt/RobotsMatcher.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Text;
 
@@ -163,29 +165,24 @@ public void HandleUnknownAction(int lineNum, ReadOnlySpan<byte> action, ReadOnly
         {
         }
 
-        private void InitUserAgentsAndPath(List<string> userAgents, byte[] path)
+        private void InitUserAgentsAndPath(List<byte[]> userAgents, byte[] path)
         {
-            _userAgents = new List<byte[]>(userAgents.Count);
-            foreach (var ua in userAgents)
-            {
-                _userAgents.Add(Encoding.UTF8.GetBytes(ua));
-            }
-
+            _userAgents = userAgents;
             Debug.Assert(path.Length > 0 && path[0] == '/');
             _path = path;
         }
 
         private bool SeenAnyAgent => _seenGlobalAgent || _seenSpecificAgent;
 
-        public bool AllowedByRobots(byte[] robotsBody, List<string> userAgents, string url)
+        public bool AllowedByRobots(byte[] robotsBody, List<byte[]> userAgents, string url)
         {
             // The url is not normalized (escaped, percent encoded) here because the user
             // is asked to provide it in escaped form already.
             var path = GetPathParamsQuery(url);
             return PathAllowedByRobots(robotsBody, userAgents, new UTF8Encoding().GetBytes(path));
         }
 
-        public bool PathAllowedByRobots(byte[] robotsBody, List<string> userAgents, byte[] path)
+        public bool PathAllowedByRobots(byte[] robotsBody, List<byte[]> userAgents, byte[] path)
         {
             InitUserAgentsAndPath(userAgents, path);
             ParseRobotsTxt(robotsBody, this);
@@ -313,9 +310,9 @@ public void Clear()
         byte[]? _path;
         private List<byte[]>? _userAgents; // Set by InitUserAgentsAndPath.
 
-        public bool OneAgentAllowedByRobots(byte[] robotsContent, string userAgent, string url)
+        public bool OneAgentAllowedByRobots(byte[] robotsContent, byte[] userAgent, string url)
         {
-            var userAgents = new List<string> { userAgent };
+            var userAgents = new List<byte[]> { userAgent, };
             return AllowedByRobots(robotsContent, userAgents, url);
         }
 

RobotsTxt/RobotsTxt.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFrameworks>net7.0;net8.0;net9.0</TargetFrameworks>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <Configurations>Debug;Release</Configurations>

RobotsTxt/RobotsTxtParser.cs

@@ -1,11 +1,11 @@
-using System.Text;
+using System;
 
 namespace RobotsTxt
 {
     public class RobotsTxtParser
     {
         static readonly byte[] UtfBom = { 0xEF, 0xBB, 0xBF };
-        static readonly byte[] HexDigits = Encoding.ASCII.GetBytes("0123456789ABCDEF");
+        static readonly byte[] HexDigits = "0123456789ABCDEF"u8.ToArray();
 
         public void Parse()
         {

TestRobotsTxt/GoogleTests.cs

@@ -10,7 +10,7 @@ public class GoogleTests
         bool IsUserAgentAllowed(string robotsTxt, string userAgent, string url)
         {
             RobotsMatcher matcher = new RobotsMatcher();
-            return matcher.OneAgentAllowedByRobots(Encoding.UTF8.GetBytes(robotsTxt), userAgent, url);
+            return matcher.OneAgentAllowedByRobots(Encoding.UTF8.GetBytes(robotsTxt), new UTF8Encoding().GetBytes(userAgent), url);
         }
 
         // Google-specific: system test.
@@ -940,7 +940,7 @@ public void ID_Utf8BOMSomewhereInMiddleOfFile()
             var part1 = Encoding.UTF8.GetBytes("User-Agent: foo\n");
             byte[] bom = { 0xEF, 0xBB, 0xBF };
             var part2 = Encoding.UTF8.GetBytes("Allow: /AnyValue\n");
-            
+
             var ms = new MemoryStream();
             ms.Write(part1, 0, part1.Length);
             ms.Write(bom, 0, bom.Length);