From e8204affb3f9a3ddb16cd1e07ad3eb873cb428c8 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Thu, 16 May 2024 15:43:52 +0900 Subject: [PATCH] Implemented chibias pre-checking with parser. --- README.ja.md | 22 ++-- README.md | 18 +++- chibias/chibias.core/Assembler.cs | 100 ++++++++++++++++-- .../chibild.core.Tests/LinkerTestRunner.cs | 5 +- .../chibild.core.Tests/LinkerTests_Common.cs | 4 +- chibild/chibild.core/CilLinker.cs | 12 +-- chibild/chibild.core/Cli/CliOptions.cs | 3 +- .../Internal/MultipleSymbolReaderProvider.cs | 5 +- .../chibild.core/Internal/NetCoreWriter.cs | 9 +- chibild/chibild.core/Internal/Utilities.cs | 8 -- toolchain.common/Internal/CommonUtilities.cs | 11 +- toolchain.common/Properties/AssemblyInfo.cs | 1 + 12 files changed, 150 insertions(+), 48 deletions(-) diff --git a/README.ja.md b/README.ja.md index af31bf0..633e768 100644 --- a/README.ja.md +++ b/README.ja.md @@ -52,21 +52,27 @@ chibicc-toolchainは、以下のツールチェインプログラムで構成さ * chibias: CILアセンブラ * chibild: CILオブジェクトリンカ +* chibiar: CILオブジェクトアーカイバ ![chibicc-toolchain overview](Images/toolchain.png) chibiasは、CILソースコード群('*.s')をほぼそのままオブジェクトファイル('*.o')として出力し、 CILソースコードの実質的なアセンブル処理は、chibildが行います。 -従って、chibiasは構文チェックを行いません。 -chibiasとchibildの奇妙な挙動は、実装の制約によるものですが、 -ツールチェイン使用者にとって見れば、POSIXで想定されるasやldなどのツールチェインと対比させて使用方法を理解できるという強みがあります。 +* chibiasは、パーサーによる構文チェックを行いますが、シンボルの妥当性など細かい検査を行いません。 +* chibiasが出力するオブジェクトファイルは、実際には入力となるCILソースファイルをgzipで圧縮しただけです。 + このことは、実際にオブジェクトファイルを `gzip -d` することで確かめることができます。 +* 同様に、chibiarが出力するアーカイブファイルは、実際にはシンボルテーブルファイルを含めたzipファイルです。 + 全く同様に、`unzip` コマンドで確かめることができます。 + +このような、chibias, chibild, chibiarの奇妙な挙動は実装の制約によるものですが、 +ツールチェイン使用者にとって見れば、POSIXで想定されるas,ld,arなどのツールチェインと対比させて使用方法を理解できるという強みがあります。 以降の説明では、CILアセンブリソースコードをchibildで直接使用して解説します。 ### CILアセンブル処理 -chibildは、複数のCILオブジェクトファイル（ソースコード）を入力として、アセンブルを行い、 +chibildは、複数のCILオブジェクトファイル（またはCILソースファイル）を入力として、アセンブルを行い、 結果を.NETアセンブリとして出力します。 この時、参照アセンブリ群を指定して、オブジェクトファイルから参照出来るようにします。 @@ -82,13 +88,15 @@ chibildはchibiccのバックエンドアセンブラとして開発しました CLIバージョンのツールチェインを、nugetからインストール出来ます * chibias: [chibias-cli](https://www.nuget.org/packages/chibias-cli) -* chibild: [chibild-cli](https://www.nuget.org/packages/chibild-cli) +* chibild: [chibild-cli](https://www.nuget.org/packages/chibild-cli). +* chibiar: [chibiar-cli](https://www.nuget.org/packages/chibiar-cli). -(紛らわしいのですが、 'chibias-cil' や 'chibild-cil' ではありません :) +(紛らわしいのですが、 'chibias-cil' ではありません :) ```bash $ dotnet tool install -g chibias-cli $ dotnet tool install -g chibild-cli +$ dotnet tool install -g chibiar-cli ``` 使用可能になったかどうかは、以下のように確認できます: @@ -132,7 +140,7 @@ usage: cil-ecma-chibild [options] [ ...] * chibildの実際のコマンド名は、`cil-ecma-chibild` です。chibiarやchibiasも同様です。 この命名規則は、GNU binutilsになぞらえたものです。 -* chibildは、コマンドラインで指摘された複数の入力ファイル（オブジェクト '*.o'、CILソース '*.s'） +* chibildは、コマンドラインで指示された複数の入力ファイル（オブジェクト '*.o'、CILソース '*.s'） をアセンブルして、1つの.NETアセンブリにまとめます。 * 参照ライブラリ名 `-l` は、アーカイブファイルパス ('*.a') を含めて、先頭から順に評価されます。 この機能は、重複するシンボル(関数/グローバル変数)にも適用されます。 diff --git a/README.md b/README.md index 2797b5e..ce39751 100644 --- a/README.md +++ b/README.md @@ -55,22 +55,28 @@ chibicc-toolchain consists of the following toolchain programs: * chibias: A CIL assembler. * chibild: A CIL object linker. +* chibild: A CIL object archiver. ![chibicc-toolchain overview](Images/toolchain.png) chibias outputs the CIL source code ('*.s') almost as is as an object file ('*.o'), and chibild performs the actual assembly process of the CIL source code. -Therefore, chibias does not perform syntax checking. -The strange behavior of chibias and chibild is due to implementation limitations. +* chibias performs syntax checking by the parser, but does not perform detailed checking such as symbol validity. +* The '*.o' file output by chibias is actually just a compression with gzip format of the input CIL source file. + This can be verified by actually `gzip -d` the object file. +* Similarly, the '*.a' file output by chibiar is actually a zip file format including the symbol table file. + Exactly the same can be verified with the `unzip` command. + +The strange internal behavior of chibias, chibild and chibiar is due to implementation limitations. For toolchain users, the advantage is that they can contrast their usage with -that of the "as" and "ld" toolchains expected on POSIX. +that of the "as", "ld" and "ar" toolchains expected on POSIX. In the following sections, the CIL assembly source code is used directly in chibild. ### CIL assembling -chibild takes multiple CIL object file (source codes) as input, performs assembly, +chibild takes multiple CIL object file (or CIL source file) as input, performs assembly, and outputs the result as .NET assemblies. At this time, reference assemblies can be specified so that they can be referenced from the CIL object file. @@ -87,12 +93,14 @@ Install CLI via nuget: * chibias: [chibias-cli](https://www.nuget.org/packages/chibias-cli) * chibild: [chibild-cli](https://www.nuget.org/packages/chibild-cli). +* chibiar: [chibiar-cli](https://www.nuget.org/packages/chibiar-cli). -* (It is not `chibias-cil` and `chibild-cil` :) +* (It is NOT `chibias-cil` :) ```bash $ dotnet tool install -g chibias-cli $ dotnet tool install -g chibild-cli +$ dotnet tool install -g chibiar-cli ``` Then: diff --git a/chibias/chibias.core/Assembler.cs b/chibias/chibias.core/Assembler.cs index c305b6e..06629ab 100644 --- a/chibias/chibias.core/Assembler.cs +++ b/chibias/chibias.core/Assembler.cs @@ -7,8 +7,17 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using System; using chibicc.toolchain.IO; using chibicc.toolchain.Logging; +using chibicc.toolchain.Parsing; +using chibicc.toolchain.Tokenizing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using chibicc.toolchain.Internal; namespace chibias; @@ -24,18 +33,91 @@ public bool Assemble( string sourceFilePath, bool isDryrun) { - using var outputStream = isDryrun ? - null : ObjectStreamUtilities.OpenObjectStream(outputObjectFilePath, true); + var outputTemporaryFilePath = + Path.Combine( + CommonUtilities.GetDirectoryPath(outputObjectFilePath), + Guid.NewGuid().ToString("N")); + var count = 0; + + var tasks = new[] + { + () => + { + using (var inputStream = StreamUtilities.OpenStream(sourceFilePath, false)) + { + var tr = new StreamReader(inputStream, Encoding.UTF8, true); - using var inputStream = StreamUtilities.OpenStream(sourceFilePath, false); + var parser = new CilParser(this.logger); + var _ = parser.Parse( + CilTokenizer.TokenizeAll("", sourceFilePath, tr), + false). + ToArray(); - if (outputStream != null) - { - inputStream.CopyTo(outputStream); + if (!parser.CaughtError) + { + Interlocked.Increment(ref count); + } + } + }, + () => + { + using var outputStream = isDryrun ? + null : ObjectStreamUtilities.OpenObjectStream(outputTemporaryFilePath, true); + + if (outputStream != null) + { + using (var inputStream = StreamUtilities.OpenStream(sourceFilePath, false)) + { + inputStream.CopyTo(outputStream); + outputStream.Flush(); + } + } - outputStream.Flush(); + Interlocked.Increment(ref count); + }, + }; + + try + { + Parallel.Invoke(tasks); + } + catch + { + try + { + File.Delete(outputTemporaryFilePath); + } + catch + { + } + throw; } - return false; + if (!isDryrun) + { + if (count == 2) + { + try + { + File.Delete(outputObjectFilePath); + } + catch + { + } + File.Move(outputTemporaryFilePath, outputObjectFilePath); + } + else + { + try + { + File.Delete(outputTemporaryFilePath); + } + catch + { + } + } + } + + return count == 2; } -} \ No newline at end of file +} diff --git a/chibild/chibild.core.Tests/LinkerTestRunner.cs b/chibild/chibild.core.Tests/LinkerTestRunner.cs index 8175fff..7fd64cc 100644 --- a/chibild/chibild.core.Tests/LinkerTestRunner.cs +++ b/chibild/chibild.core.Tests/LinkerTestRunner.cs @@ -7,6 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Internal; using chibicc.toolchain.Logging; using chibild.Internal; using System; @@ -72,7 +73,7 @@ public static string RunCore( coreLibPath, tmp2Path, }. Concat(additionalReferencePaths ?? Array.Empty()). - Select(Utilities.GetDirectoryPath). + Select(CommonUtilities.GetDirectoryPath). Distinct(). ToArray(); var libraryReferences = new[] @@ -129,7 +130,7 @@ public static string RunCore( var psi = new ProcessStartInfo() { FileName = Path.Combine(ArtifactsBasePath, - Utilities.IsInWindows ? "ildasm.exe" : "ildasm.linux-x64"), + CommonUtilities.IsInWindows ? "ildasm.exe" : "ildasm.linux-x64"), Arguments = $"-utf8 -out={disassembledPath} {outputAssemblyPath}" }; diff --git a/chibild/chibild.core.Tests/LinkerTests_Common.cs b/chibild/chibild.core.Tests/LinkerTests_Common.cs index e136811..6029886 100644 --- a/chibild/chibild.core.Tests/LinkerTests_Common.cs +++ b/chibild/chibild.core.Tests/LinkerTests_Common.cs @@ -7,7 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// -using chibild.Internal; +using chibicc.toolchain.Internal; using System; using System.IO; using System.Runtime.CompilerServices; @@ -33,7 +33,7 @@ private string Run( var appHostTemplatePath = Path.GetFullPath( Path.Combine( LinkerTestRunner.ArtifactsBasePath, - Utilities.IsInWindows ? "apphost.exe" : "apphost.linux-x64")); + CommonUtilities.IsInWindows ? "apphost.exe" : "apphost.linux-x64")); var tf = TargetFramework.TryParse(targetFrameworkMoniker, out var tf1) ? tf1 : throw new InvalidOperationException(); return new() diff --git a/chibild/chibild.core/CilLinker.cs b/chibild/chibild.core/CilLinker.cs index c3036c2..19ed72b 100644 --- a/chibild/chibild.core/CilLinker.cs +++ b/chibild/chibild.core/CilLinker.cs @@ -313,7 +313,7 @@ injectToAssemblyPath is { } injectPath ? Concat(inputReferences. OfType(). Where(ir => Path.GetExtension(ir.RelativePath) == ".dll"). - Select(ir => Utilities.GetDirectoryPath(Path.Combine(baseInputPath, ir.RelativePath)))). + Select(ir => CommonUtilities.GetDirectoryPath(Path.Combine(baseInputPath, ir.RelativePath)))). Distinct(). ToArray(); @@ -346,7 +346,7 @@ injectToAssemblyPath is { } injectPath ? var outputAssemblyFullPath = Path.GetFullPath(outputAssemblyPath); var outputAssemblyCandidateFullPath = requireAppHost? Path.Combine( - Utilities.GetDirectoryPath(outputAssemblyFullPath), + CommonUtilities.GetDirectoryPath(outputAssemblyFullPath), Path.GetFileNameWithoutExtension(outputAssemblyFullPath) + ".dll") : outputAssemblyFullPath; @@ -412,7 +412,7 @@ injectToAssemblyPath is { } injectPath ? } var outputAssemblyBasePath = - Utilities.GetDirectoryPath(outputAssemblyCandidateFullPath); + CommonUtilities.GetDirectoryPath(outputAssemblyCandidateFullPath); try { if (!Directory.Exists(outputAssemblyBasePath)) @@ -490,7 +490,7 @@ injectToAssemblyPath is { } injectPath ? if (cachedAssemblies.FirstOrDefault( assembly => assembly.Name.Name == corlibName) is { } corlibAssembly) { - var corlibBasePath = Utilities.GetDirectoryPath( + var corlibBasePath = CommonUtilities.GetDirectoryPath( corlibAssembly.MainModule.FileName); requiredAssemblyBasePaths = requiredAssemblyBasePaths. @@ -501,7 +501,7 @@ injectToAssemblyPath is { } injectPath ? Parallel.ForEach(cachedAssemblies. SelectMany(assembly => assembly.Modules). Where(module => requiredAssemblyBasePaths. - Contains(Utilities.GetDirectoryPath(module.FileName))), + Contains(CommonUtilities.GetDirectoryPath(module.FileName))), module => { var tp = Path.Combine( @@ -519,7 +519,7 @@ injectToAssemblyPath is { } injectPath ? foreach (var ext in new[] { ".pdb", ".mdb" }) { var fp = Path.Combine( - Utilities.GetDirectoryPath(module.FileName), + CommonUtilities.GetDirectoryPath(module.FileName), Path.GetFileNameWithoutExtension(module.FileName) + ext); tp = Path.Combine( outputAssemblyBasePath, diff --git a/chibild/chibild.core/Cli/CliOptions.cs b/chibild/chibild.core/Cli/CliOptions.cs index abc5c38..1d6f4ae 100644 --- a/chibild/chibild.core/Cli/CliOptions.cs +++ b/chibild/chibild.core/Cli/CliOptions.cs @@ -9,7 +9,6 @@ using chibicc.toolchain.Internal; using chibicc.toolchain.Logging; -using chibild.Internal; using System; using System.Collections.Generic; using System.IO; @@ -376,7 +375,7 @@ static bool TryGetOptionArgument( break; case ObjectFilePathReference(var path): options.OutputAssemblyPath = Path.Combine( - Utilities.GetDirectoryPath(path), + CommonUtilities.GetDirectoryPath(path), Path.GetFileNameWithoutExtension(path) + ".dll"); break; } diff --git a/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs b/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs index 7897e4f..096e03a 100644 --- a/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs +++ b/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs @@ -7,6 +7,8 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Internal; +using chibicc.toolchain.Logging; using Mono.Cecil; using Mono.Cecil.Cil; using Mono.Cecil.Mdb; @@ -15,7 +17,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using chibicc.toolchain.Logging; namespace chibild.Internal; @@ -44,7 +45,7 @@ public MultipleSymbolReaderProvider(ILogger logger) => where TSymbolReaderProvider : ISymbolReaderProvider { var path = Path.Combine( - Utilities.GetDirectoryPath(fullPath), + CommonUtilities.GetDirectoryPath(fullPath), Path.GetFileNameWithoutExtension(fullPath) + extension); try diff --git a/chibild/chibild.core/Internal/NetCoreWriter.cs b/chibild/chibild.core/Internal/NetCoreWriter.cs index 87d33cd..eb7c87d 100644 --- a/chibild/chibild.core/Internal/NetCoreWriter.cs +++ b/chibild/chibild.core/Internal/NetCoreWriter.cs @@ -7,12 +7,13 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Internal; +using chibicc.toolchain.Logging; using System.Diagnostics; using System; using System.IO; using System.Runtime.InteropServices; using System.Text; -using chibicc.toolchain.Logging; namespace chibild.Internal; @@ -52,7 +53,7 @@ public static void WriteRuntimeConfiguration( Debug.Assert(co != null); var runtimeConfigJsonPath = Path.Combine( - Utilities.GetDirectoryPath(outputAssemblyCandidateFullPath), + CommonUtilities.GetDirectoryPath(outputAssemblyCandidateFullPath), Path.GetFileNameWithoutExtension(outputAssemblyCandidateFullPath) + ".runtimeconfig.json"); logger.Information( @@ -111,7 +112,7 @@ public static void WriteAppHost( var isPEImage = PEUtils.IsPEImage(ms); var outputFullPath = Path.Combine( - Utilities.GetDirectoryPath(outputAssemblyFullPath), + CommonUtilities.GetDirectoryPath(outputAssemblyFullPath), Path.GetFileNameWithoutExtension(outputAssemblyFullPath) + (isPEImage ? ".exe" : "")); logger.Information( @@ -138,7 +139,7 @@ public static void WriteAppHost( fs.Flush(); } - if (!Utilities.IsInWindows) + if (!CommonUtilities.IsInWindows) { while (true) { diff --git a/chibild/chibild.core/Internal/Utilities.cs b/chibild/chibild.core/Internal/Utilities.cs index fdcd282..f457e51 100644 --- a/chibild/chibild.core/Internal/Utilities.cs +++ b/chibild/chibild.core/Internal/Utilities.cs @@ -33,19 +33,11 @@ internal enum chmodFlags internal static class Utilities { - public static readonly bool IsInWindows = - Environment.OSVersion.Platform == PlatformID.Win32NT; - public const int EINTR = 4; [DllImport("libc", SetLastError = true)] public static extern int chmod(string path, chmodFlags mode); - public static string GetDirectoryPath(string path) => - Path.GetDirectoryName(path) is { } d ? - Path.GetFullPath(string.IsNullOrWhiteSpace(d) ? "." : d) : - Path.DirectorySeparatorChar.ToString(); - #if NETFRAMEWORK || NETSTANDARD2_0 public static bool TryAdd( this Dictionary dict, diff --git a/toolchain.common/Internal/CommonUtilities.cs b/toolchain.common/Internal/CommonUtilities.cs index 14b08ec..0381461 100644 --- a/toolchain.common/Internal/CommonUtilities.cs +++ b/toolchain.common/Internal/CommonUtilities.cs @@ -9,6 +9,7 @@ using System; using System.Globalization; +using System.IO; #if NET45 || NET461 using System.Collections.Generic; @@ -50,7 +51,15 @@ namespace chibicc.toolchain.Internal internal static class CommonUtilities { private static readonly IFormatProvider invariantCulture = CultureInfo.InvariantCulture; - + + public static readonly bool IsInWindows = + Environment.OSVersion.Platform == PlatformID.Win32NT; + + public static string GetDirectoryPath(string path) => + Path.GetDirectoryName(path) is { } d ? + Path.GetFullPath(string.IsNullOrWhiteSpace(d) ? "." : d) : + Path.DirectorySeparatorChar.ToString(); + #if NET40 || NET45 private static class ArrayEmpty { diff --git a/toolchain.common/Properties/AssemblyInfo.cs b/toolchain.common/Properties/AssemblyInfo.cs index 857253a..1d44976 100644 --- a/toolchain.common/Properties/AssemblyInfo.cs +++ b/toolchain.common/Properties/AssemblyInfo.cs @@ -12,3 +12,4 @@ [assembly: InternalsVisibleTo("chibiar.core")] [assembly: InternalsVisibleTo("chibias.core")] [assembly: InternalsVisibleTo("chibild.core")] +[assembly: InternalsVisibleTo("chibild.core.Tests")]