Additional VerificationEngine improvements/features

eng/dependabot/Packages.props

@@ -26,7 +26,7 @@
     <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.3.2" />
     <PackageReference Update="xunit.abstractions" Version="2.0.3" />
     <PackageReference Update="Newtonsoft.Json.Schema" Version="3.0.14" />
-    <PackageReference Update="Verify.XUnit" Version="18.0.0-beta.18" />
+    <PackageReference Update="Verify.XUnit" Version="18.0.0-beta.31" />
     <PackageReference Update="Verify.DiffPlex" Version="1.3.0" />
   </ItemGroup>
 </Project>

src/Microsoft.TemplateEngine.Authoring.CLI/Commands/Verify/VerifyCommand.cs

@@ -50,6 +50,11 @@ internal class VerifyCommand : ExecutableCommand<VerifyCommandArgs>
             Description = LocalizableStrings.command_verify_help_expectationsDirPath_description,
         };
 
+        private readonly Option<string> _scenarioNameOption = new(new[] { "--scenario-name" })
+        {
+            Description = LocalizableStrings.command_verify_help_scenarioName_description,
+        };
+
         private readonly Option<bool> _disableDiffToolOption = new("--disable-diff-tool")
         {
             Description = LocalizableStrings.command_verify_help_disableDiffTool_description,
@@ -66,6 +71,12 @@ internal class VerifyCommand : ExecutableCommand<VerifyCommandArgs>
             Arity = new ArgumentArity(0, 999)
         };
 
+        private readonly Option<IEnumerable<string>> _includePatternOption = new("--include-pattern")
+        {
+            Description = LocalizableStrings.command_verify_help_customIncludes_description,
+            Arity = new ArgumentArity(0, 999)
+        };
+
         private readonly Option<bool> _verifyCommandOutputOption = new("--verify-std")
         {
             Description = LocalizableStrings.command_verify_help_verifyOutputs_description,
@@ -92,14 +103,16 @@ public VerifyCommand(ILoggerFactory loggerFactory)
             AddOption(_newCommandPathOption);
             AddOption(_templateOutputPathOption);
             AddOption(_expectationsDirectoryOption);
+            AddOption(_scenarioNameOption);
             AddOption(_disableDiffToolOption);
             AddOption(_disableDefaultExcludePatternsOption);
             AddOption(_excludePatternOption);
+            AddOption(_includePatternOption);
             AddOption(_verifyCommandOutputOption);
             AddOption(_isCommandExpectedToFailOption);
             FromAmongCaseInsensitive(
                 _uniqueForOption,
-                System.Enum.GetNames(typeof(UniqueForOption))
+                Enum.GetNames(typeof(UniqueForOption))
                     .Where(v => !v.Equals(UniqueForOption.None.ToString(), StringComparison.OrdinalIgnoreCase))
                     .ToArray());
             AddOption(_uniqueForOption);
@@ -113,10 +126,12 @@ internal static VerifyCommandArgs ExtractArguments(VerifyCommand verifyCommand,
                 templatePath: parseResult.GetValueForOption(verifyCommand._templatePathOption),
                 dotnetNewCommandAssemblyPath: parseResult.GetValueForOption(verifyCommand._newCommandPathOption),
                 expectationsDirectory: parseResult.GetValueForOption(verifyCommand._expectationsDirectoryOption),
+                scenarioDistinguisher: parseResult.GetValueForOption(verifyCommand._scenarioNameOption),
                 outputDirectory: parseResult.GetValueForOption(verifyCommand._templateOutputPathOption),
                 disableDiffTool: parseResult.GetValueForOption(verifyCommand._disableDiffToolOption),
                 disableDefaultVerificationExcludePatterns: parseResult.GetValueForOption(verifyCommand._disableDefaultExcludePatternsOption),
                 verificationExcludePatterns: parseResult.GetValueForOption(verifyCommand._excludePatternOption),
+                verificationIncludePatterns: parseResult.GetValueForOption(verifyCommand._includePatternOption),
                 verifyCommandOutput: parseResult.GetValueForOption(verifyCommand._verifyCommandOutputOption),
                 isCommandExpectedToFail: parseResult.GetValueForOption(verifyCommand._isCommandExpectedToFailOption),
                 uniqueForOptions: parseResult.GetValueForOption(verifyCommand._uniqueForOption));
@@ -136,12 +151,15 @@ protected override async Task<int> ExecuteAsync(VerifyCommandArgs args, Cancella
                     DisableDiffTool = args.DisableDiffTool,
                     DisableDefaultVerificationExcludePatterns = args.DisableDefaultVerificationExcludePatterns,
                     VerificationExcludePatterns = args.VerificationExcludePatterns,
+                    VerificationIncludePatterns = args.VerificationIncludePatterns,
                     DotnetNewCommandAssemblyPath = args.DotnetNewCommandAssemblyPath,
                     ExpectationsDirectory = args.ExpectationsDirectory,
+                    ScenarioName = args.ScenarioDistinguisher,
                     OutputDirectory = args.OutputDirectory,
                     VerifyCommandOutput = args.VerifyCommandOutput,
                     IsCommandExpectedToFail = args.IsCommandExpectedToFail,
                     UniqueFor = args.UniqueFor,
+                    DoNotPrependCallerMethodNameToScenarioName = true
                 };
                 await engine.Execute(
                     options,

src/Microsoft.TemplateEngine.Authoring.CLI/Commands/Verify/VerifyCommandArgs.cs

@@ -23,22 +23,26 @@ public VerifyCommandArgs(
             string? templateSpecificArgs,
             string? dotnetNewCommandAssemblyPath,
             string? expectationsDirectory,
+            string? scenarioDistinguisher,
             string? outputDirectory,
-            bool? disableDiffTool,
-            bool? disableDefaultVerificationExcludePatterns,
+            bool disableDiffTool,
+            bool disableDefaultVerificationExcludePatterns,
             IEnumerable<string>? verificationExcludePatterns,
-            bool? verifyCommandOutput,
+            IEnumerable<string>? verificationIncludePatterns,
+            bool verifyCommandOutput,
             bool isCommandExpectedToFail,
             IEnumerable<UniqueForOption>? uniqueForOptions)
         : this(templateName, templateSpecificArgs)
         {
             TemplatePath = templatePath;
             DotnetNewCommandAssemblyPath = dotnetNewCommandAssemblyPath;
             ExpectationsDirectory = expectationsDirectory;
+            ScenarioDistinguisher = scenarioDistinguisher;
             OutputDirectory = outputDirectory;
             DisableDiffTool = disableDiffTool;
             DisableDefaultVerificationExcludePatterns = disableDefaultVerificationExcludePatterns;
             VerificationExcludePatterns = verificationExcludePatterns;
+            VerificationIncludePatterns = verificationIncludePatterns;
             VerifyCommandOutput = verifyCommandOutput;
             IsCommandExpectedToFail = isCommandExpectedToFail;
             UniqueFor = ToUniqueForOptionFlags(uniqueForOptions);
@@ -69,6 +73,11 @@ public VerifyCommandArgs(
         /// </summary>
         public string? ExpectationsDirectory { get; init; }
 
+        /// <summary>
+        /// Gets a custom prefix prepended in front of generated scenario name - result used for naming verification subdirectories.
+        /// </summary>
+        public string? ScenarioDistinguisher { get; init; }
+
         /// <summary>
         /// Gets the target directory to output the generated template.
         /// </summary>
@@ -77,23 +86,29 @@ public VerifyCommandArgs(
         /// <summary>
         /// If set to true - the diff tool won't be automatically started by the Verifier on verification failures.
         /// </summary>
-        public bool? DisableDiffTool { get; init; }
+        public bool DisableDiffTool { get; init; }
 
         /// <summary>
         /// If set to true - all template output files will be verified, unless <see cref="VerificationExcludePatterns"/> are specified.
         /// Otherwise a default exclusions (to be documented - mostly binaries etc.).
         /// </summary>
-        public bool? DisableDefaultVerificationExcludePatterns { get; init; }
+        public bool DisableDefaultVerificationExcludePatterns { get; init; }
 
         /// <summary>
         /// Set of patterns defining files to be excluded from verification.
         /// </summary>
         public IEnumerable<string>? VerificationExcludePatterns { get; init; }
 
+        /// <summary>
+        /// Set of patterns defining files to be included into verification (unless excluded by <see cref="VerificationExcludePatterns"/>).
+        /// By default all files are included (unless excluded).
+        /// </summary>
+        public IEnumerable<string>? VerificationIncludePatterns { get; init; }
+
         /// <summary>
         /// If set to true - 'dotnet new' command standard output and error contents will be verified along with the produced template files.
         /// </summary>
-        public bool? VerifyCommandOutput { get; init; }
+        public bool VerifyCommandOutput { get; init; }
 
         /// <summary>
         /// If set to true - 'dotnet new' command is expected to return nonzero return code.

src/Microsoft.TemplateEngine.Authoring.CLI/LocalizableStrings.resx

@@ -171,6 +171,9 @@ Do not localize: "template.json"</comment>
   <data name="command_verify_help_customExcludes_description" xml:space="preserve">
     <value>Specifies pattern(s) defining files to be excluded from verification.</value>
   </data>
+  <data name="command_verify_help_customIncludes_description" xml:space="preserve">
+    <value>Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</value>
+  </data>
   <data name="command_verify_help_description" xml:space="preserve">
     <value>Runs the template with specified arguments and compares the result with expectations files.</value>
   </data>
@@ -196,6 +199,9 @@ Do not localize: "template.json"</comment>
   <data name="command_verify_help_outputPath_description" xml:space="preserve">
     <value>Specifies the path to target directory to output the generated template to.</value>
   </data>
+  <data name="command_verify_help_scenarioName_description" xml:space="preserve">
+    <value>Specifies optional scenario name to be used in the snapshot folder name.</value>
+  </data>
   <data name="command_verify_help_templateName_description" xml:space="preserve">
     <value>Name of the template to be verified. Can be already installed template or a template within local path specified with -p/--template-path option.</value>
     <comment>Do not translate the '-p/--template-path'</comment>
@@ -217,4 +223,4 @@ Do not localize: "template.json"</comment>
   <data name="generic_log_commandExecutionFailedWithErrorMessage" xml:space="preserve">
     <value>There was an error while running the "--{0}" command. Error message: "{1}". See the logs for more details.</value>
   </data>
-</root>
+</root>

src/Microsoft.TemplateEngine.Authoring.CLI/xlf/LocalizableStrings.cs.xlf

@@ -78,6 +78,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Určuje vzory definující soubory, které se mají vyloučit z ověřování.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_customIncludes_description">
+        <source>Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</source>
+        <target state="new">Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_description">
         <source>Runs the template with specified arguments and compares the result with expectations files.</source>
         <target state="translated">Spustí šablonu se zadanými argumenty a porovná výsledek s očekávanými soubory.</target>
@@ -113,6 +118,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Určuje cestu k cílovému adresáři pro výstup vygenerované šablony.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_scenarioName_description">
+        <source>Specifies optional scenario name to be used in the snapshot folder name.</source>
+        <target state="new">Specifies optional scenario name to be used in the snapshot folder name.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_templateName_description">
         <source>Name of the template to be verified. Can be already installed template or a template within local path specified with -p/--template-path option.</source>
         <target state="translated">Název šablony, která se má ověřit. Může jít o již nainstalovanou šablonu, nebo o šablonu v místní cestě zadané s parametrem -p/--template-path.</target>

src/Microsoft.TemplateEngine.Authoring.CLI/xlf/LocalizableStrings.de.xlf

@@ -78,6 +78,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Gibt Muster an, die Dateien definieren, die von der Überprüfung ausgeschlossen werden sollen.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_customIncludes_description">
+        <source>Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</source>
+        <target state="new">Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_description">
         <source>Runs the template with specified arguments and compares the result with expectations files.</source>
         <target state="translated">Führt die Vorlage mit angegebenen Argumenten aus und vergleicht das Ergebnis mit den Erwartungsdateien.</target>
@@ -113,6 +118,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Gibt den Pfad zum Zielverzeichnis an, in das die generierte Vorlage ausgegeben werden soll.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_scenarioName_description">
+        <source>Specifies optional scenario name to be used in the snapshot folder name.</source>
+        <target state="new">Specifies optional scenario name to be used in the snapshot folder name.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_templateName_description">
         <source>Name of the template to be verified. Can be already installed template or a template within local path specified with -p/--template-path option.</source>
         <target state="translated">Name der zu überprüfenden Vorlage. Kann eine bereits installierte Vorlage oder eine Vorlage innerhalb des lokalen Pfads sein, der mit der Option „-p/--template-path“ angegeben ist.</target>

src/Microsoft.TemplateEngine.Authoring.CLI/xlf/LocalizableStrings.es.xlf

@@ -78,6 +78,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Especifica los patrones que definen los archivos que se excluirán de la comprobación.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_customIncludes_description">
+        <source>Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</source>
+        <target state="new">Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_description">
         <source>Runs the template with specified arguments and compares the result with expectations files.</source>
         <target state="translated">Ejecuta la plantilla con los argumentos especificados y compara el resultado con los archivos de expectativas.</target>
@@ -113,6 +118,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Especifica la ruta de acceso al directorio de destino en el que se generará la plantilla generada.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_scenarioName_description">
+        <source>Specifies optional scenario name to be used in the snapshot folder name.</source>
+        <target state="new">Specifies optional scenario name to be used in the snapshot folder name.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_templateName_description">
         <source>Name of the template to be verified. Can be already installed template or a template within local path specified with -p/--template-path option.</source>
         <target state="translated">Nombre de la plantilla por verificar. Puede ser una plantilla ya instalada o una plantilla dentro de la ruta local especificada con la opción -p/--template-path.</target>

src/Microsoft.TemplateEngine.Authoring.CLI/xlf/LocalizableStrings.fr.xlf

@@ -78,6 +78,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Spécifie le ou les modèles définissant les fichiers à exclure de la vérification.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_customIncludes_description">
+        <source>Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</source>
+        <target state="new">Specifies pattern(s) defining files to be included to verification (all files are included if not specified).</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_description">
         <source>Runs the template with specified arguments and compares the result with expectations files.</source>
         <target state="translated">Exécute le modèle avec les arguments spécifiés et compare le résultat aux fichiers attendus.</target>
@@ -113,6 +118,11 @@ Do not localize: "template.json"</note>
         <target state="translated">Spécifie le chemin d’accès au répertoire cible vers lequel générer la sortie du modèle généré.</target>
         <note />
       </trans-unit>
+      <trans-unit id="command_verify_help_scenarioName_description">
+        <source>Specifies optional scenario name to be used in the snapshot folder name.</source>
+        <target state="new">Specifies optional scenario name to be used in the snapshot folder name.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="command_verify_help_templateName_description">
         <source>Name of the template to be verified. Can be already installed template or a template within local path specified with -p/--template-path option.</source>
         <target state="translated">Nom du modèle à vérifier. Peut déjà un modèle être installé ou un modèle dans le chemin d’accès local spécifié avec l’option -p/--template-path.</target>