Merge pull request #7 from wild-devops/develop

v1.5.0

.gitignore

@@ -1,3 +1,4 @@
 *.log
+*.log.json
 .*/
 *.xml

README.md

@@ -70,12 +70,12 @@ attributes:
 scripts_directories:
   - .
 
-# Declaration of tasks that compose and determine order of executing for scripts
+# Declaration of tasks that compose and determine order of executing for scripts, inline commands and other tasks
 tasks:
   hello:
   - powershell: Write-Host "Hello PWSHAKE!"
 
-# Tasks to current execute
+# List of tasks to current execute
 invoke_tasks:
 - hello
 
@@ -92,42 +92,44 @@ This installs all required components from the `PSGallery` and invokes scripts a
 
 
 The result looks like the followed output including information about:
+*  `PWSHAKE arguments:` - arguments passed to `Invoke-pwshake` command (useful in logs inspection)
 *  `PWSHAKE config:` - loaded config file content (may be rearranged due to overriding, interpolation, and merging metadata)
-*  `Arranged tasks:` - tasks to be executed in actual order
 *  `Invoke task:` - invoked tasks name
 *  `Execute step:` - invoked steps caption
 *  Everything else - invoked scripts output
 
 ```
+PWSHAKE arguments:
+Verbosity: Default
+DryRun: false
+MetaData:
+ConfigPath: /absolute/path/to/your/working/directory/my_pwshake.yaml
+Tasks: []
+WorkDir: /absolute/path/to/process/working/directory
+
 PWSHAKE config:
 tasks:
   hello:
   - powershell: Write-Host "Hello PWSHAKE!"
+filters: {}
+resources: []
 includes: []
 templates: {}
 attributes_overrides: []
 scripts_directories:
 - .
-resources: []
-invoke_tasks: hello
+invoke_tasks:
+- hello
 attributes:
   pwshake_module_path: /path/to/pwshake/module/source
   pwshake_path: /absolute/path/to/your/working/directory
-  pwshake_version: 1.4.0
+  pwshake_version: 1.5.0
   work_dir: /absolute/path/to/process/working/directory
   pwshake_log_path: /absolute/path/to/your/working/directory/my_pwshake.log
   some_attribute: this is an attribute value
 
-Arranged tasks:
-- when: $true
-  steps:
-    powershell: Write-Host "Hello PWSHAKE!"
-  depends_on: []
-  work_dir: ""
-  name: hello
-
 Invoke task: hello
-Execute step: step_66875131
+Execute step: powershell_1
 Hello PWSHAKE!
 ```
 ## Test **PWSHAKE** by itself
@@ -152,6 +154,7 @@ Since some examples use third party cli tools, make sure that you have installed
 * [`pwshake.yaml` configuration file structure](/doc/config.md)
   * [`attributes:` element](/doc/attributes.md)
   * [`attributes_overrides:` element](/doc/attributes_overrides.md)
+  * [`filters:` element](/doc/filters.md)
   * [`includes:` element](/doc/includes.md)
   * [`invoke_tasks:` element](/doc/invoke_tasks.md)
   * [`tasks:` element](/doc/tasks.md)

doc/bootstrapper.md

@@ -11,7 +11,7 @@ All parameters are optional since they all have conventional default values:
 
 * ### **`-ConfigPath`**
 
-  Alias: **`-Path`**
+  Aliases: **`-Path`, `-File`, `-ConfigFile`**
 
   Default is: `[string]"$PSScriptRoot\pwshake.yaml"` - i.e. `pwshake.yaml` from the same directory as the bootstrapper.
 
@@ -53,6 +53,8 @@ All parameters are optional since they all have conventional default values:
   ```
 
 * ### **`-MetaData`**
+  Alias: **`-Attributes`**
+
   Default is: `$null` - i.e. nothing to merge into the `yaml` config's `attributes:` element.
 
   Gives **PWSHAKE** engine an ability to populate the `yaml` config's `attributes:` element with an external data (**metadata** as a term) passed from the outside world (CI server, cloud provider agent, canny developer, etc).
@@ -109,6 +111,24 @@ All parameters are optional since they all have conventional default values:
     ...
     ```
 
+* ### **`-Verbosity`**
+
+  Alias: **`-LogLevel`**
+
+  Default is: `Verbose` - for backward compatibility.
+
+  Available values:
+  * `Quiet` - logs nothing, except `Powershell` host process exceptions, the olny way to check if run was successful - inspect if process (`powershell.exe` or `pwsh`) exit code was not 0;
+  * `Error` - logs only script exceptions and third party cli tools failures (exit code is not 0);
+  * `Warning` - reserved for future;
+  * `Minimal` - logs only raw scripts and cli tools output passed to **stdout**, **stderr** and any **Powershell** streams;
+  * `Information` - additionally logs task and step captions before each task\step invocation;
+  * `Verbose` - additionally logs `Invoke-pwshake` call arguments and **PWSHAKE** config file content after all initialization stages performed by the **PWSHAKE** engine (merging metadata, attributes interpolation, etc)
+  * `Debug` - logs tons of tracing information about each action performed during the **PWSHAKE** config file processing;
+  * `Silent = Quiet` - alias;
+  * `Normal = Information` - alias;
+  * `Default = Verbose` - alias for backward compatibility.
+
 * ### **`-DryRun`**
 
   Aliases: **`-WhatIf`**, **`-Noop`**

doc/config.md

@@ -153,3 +153,21 @@
   Tells to **PWSHAKE** engine about the source(-s) of outer resources that should be loaded before the current execution starts.
   
   [See more about `resources:`](/doc/resources.md)
+
+* ### **`filters:` element**
+
+    Optional
+
+    ```
+    filters:
+      f-$capsed: |-
+        { "$_".ToUpper() }
+      f-braced: |-
+        { "[$_]" }
+    ```
+    Tells to **PWSHAKE** engine to add small functions into the script execution context.
+
+    These functions will be used into the  **PWSHAKE** processing as attributes interpolators or helpers.
+
+    [See more about `filters:`](/doc/filters.md)
+

doc/filters.md

@@ -0,0 +1,39 @@
+## `filters:` **element**
+
+**Optional**
+
+Contains definitions of small functions that will be loaded into the **PWSHAKE** execution context.
+
+These functions will be used into the  **PWSHAKE** processing as attributes interpolators or helpers.
+
+Added functions are actually of the **Powershell** `filter` type, so they have some peculiarities:
+
+* have to take the first argument from  **Powershell** pipeline
+* have the only `process` stage (block) in  **Powershell** execution
+* have the first argument taken from pipeline is referred as `$_` variable in the `process` stage (block)
+
+Example:
+```
+PS>cat pwshake.yaml
+attributes:
+    my_attr: '{{$capsed:my_attr_value}}'
+    sq_braced: '{{$("{{my_attr}}" | sq-braced)}}'
+filters:
+  f-$capsed: |-
+    { "$_".ToUpper() }
+  sq-braced: |-
+    { "[$_]" }
+...
+```
+```
+PS>Invoke-pwshake
+PWSHAKE config:
+attributes:
+    my_attr: MY_ATTR_VALUE
+    sq_braced: '[MY_ATTR_VALUE]'
+...
+```
+
+Note that if the filter name starts with `f-$` chars it could be used to interpolate attributes.
+
+In this case the text following of `:` is passed to the filter as the first argument and result of the filter execution is substituted as the attribute value.

doc/step.md

@@ -86,15 +86,15 @@ In this case the `-` sign means that subsequent items in `yaml` hierarchy are ke
   ```
 
 * ### - `powershell:` element implicit shortenings
-  Since the `powershell:` element contains inline code that can be too long and\or complex to use it as the meaningful name, so the `name:` property for these shortenings is generated from the `.GetHashCode()` method of the supplied inline string to distinct each other inline step  in the **PWSHAKE** execution log.
+  Since the `powershell:` element contains inline code that can be too long and\or complex to use it as the meaningful name, so the `name:` property for these shortenings is generated from the step type and incremented number to distinct each other inline step in the **PWSHAKE** execution log.
 
   Example:
   ```
-  - powershell: rm ./ -recurse -force
+  - pwsh: rm ./ -recurse -force
   ```
   This is the same as:
   ```
-  - name: step_2122574676
+  - name: pwsh_1
     powershell: rm ./ -recurse -force
   ```
 
@@ -125,7 +125,7 @@ In this case the `-` sign means that subsequent items in `yaml` hierarchy are ke
   ```
   This is the same as:
   ```
-  - name: step_2122574676
+  - name: step_1
     powershell: rm ./ -recurse -force
     when: $true
   ```
@@ -139,7 +139,7 @@ In this case the `-` sign means that subsequent items in `yaml` hierarchy are ke
   ```
   This is the same as:
   ```
-  - name: step_2122574676
+  - name: powershell_2
     powershell: rm ./ -recurse -force
     when: -not ($env:SOME_VALUE -eq '42')
   ```

doc/templates.md

@@ -56,18 +56,18 @@ This tells to **PWSHAKE** engine how to substitute any structured `yaml` input i
     PS>Invoke-pwshake ./python_pwshake.yaml
     ...
     Invoke task: test_python_template
-    Execute step: step_25540982
+    Execute step: python_1
     Python 3.6.8
     Execute step: Run python
     bash: python --version
     Python 3.6.8
-    Execute step: step_46664441
+    Execute step: python_2
     Hello pwshake!
-    Execute step: step_6213368
+    Execute step: python_3
     bash: python /workdir/examples/5.templates/v1.2/hello.py again
 
     Hello again!
-    Execute step: step_32712664
+    Execute step: python_4
     bash: python /workdir/examples/5.templates/v1.2/hello.py twice
     Hello twice!
     ```

examples/1.hello/v1.3/my_pwshake.yaml

@@ -1,5 +1,5 @@
 attributes:
-  pwshake_verbosity: Normal # Try to change with any of: Error, Warning, Normal, Verbose, Debug
+  pwshake_verbosity: Normal # Try to change with any of: Quiet, Error, Warning, Information, Verbose, Debug
   hello_to: PWSHAKE
 
 tasks:

examples/1.hello/v1.5/my_pwshake.yaml

@@ -0,0 +1,14 @@
+attributes:
+  pwshake_verbosity: Normal
+  hello_to: PWSHAKE
+  on_error: continue
+  pwshake_log_to_json: true
+
+tasks:
+  say_hello:
+  - echo: "Hello {{hello_to}}!"
+  - pwsh: throw '{{hello_to}} is sick!'
+    on_error: '{{on_error}}'
+
+invoke_tasks:
+- say_hello

examples/4.complex/v1.0/attributes.json

@@ -2,5 +2,6 @@
     "override_to": "unknown",
     "env_name": "undefined",
     "env_path": "{{$env:PATH}}",
-    "artifact_id": "0"
+    "artifact_id": "0",
+    "sql_server": "undefined"
 }

examples/4.complex/v1.0/build.properties

@@ -4,6 +4,7 @@ agent.name=BuildAgent-i-02db5c3ee5174b8dc
 agent.ownPort=9090
 agent.work.dir=C\:\\BuildAgent\\work
 build.number=101
+build.counter=101
 build.vcs.number=299ab897f9caf481992a35efbdf009f3eba0118d
 build.vcs.number.1=299ab897f9caf481992a35efbdf009f3eba0118d
 build.vcs.number.Shake=299ab897f9caf481992a35efbdf009f3eba0118d

examples/5.templates/v1.2/templates/dotnet.yaml

@@ -4,10 +4,10 @@ templates:
     options: --version
     powershell: |
       $cmd = "dotnet"
-      if ($step.command) {
-        $cmd += " $($step.command)"
+      if ($dotnet.command) {
+        $cmd += " $($dotnet.command)"
       }
-      if ($step.options) {
-        $cmd += " $($step.options)"
+      if ($dotnet.options) {
+        $cmd += " $($dotnet.options)"
       }
       $cmd | Cmd-Shell

examples/5.templates/v1.2/templates/python.yaml

@@ -4,9 +4,9 @@ templates:
     inline:
     powershell: |
       if ($step.python) {
-        "python3 $($step.python)" | Cmd-Shell
+        iex "python3 $($step.python)"
       } elseif ($step.options) {
-        "python3 $($step.options)" | Cmd-Shell
+        iex "python3 $($step.options)"
       } elseif ($step.inline) {
         python3 -c $step.inline
       } else {

examples/5.templates/v1.2/templates/transform_xml_file.yaml

@@ -5,7 +5,7 @@ templates:
     updates:
     removes:
     powershell: |
-      $path = Normalize-Path (Coalesce $step.transform_xml_file, $step.path) $config
+      $path = Build-Path (Coalesce $step.transform_xml_file, $step.path) $config
       $appends = $step.appends
       $updates = $step.updates
       $removes = $step.removes

examples/5.templates/v1.4/templates/composed.yaml

@@ -2,10 +2,7 @@ includes:
 - nested.yaml
 
 templates:
-  each:
-    powershell: |
-      (@() + $step.items) | ForEach-Object { Invoke-Expression $step.action }
   steps:
     each:
-      items: $[[$step.steps]]
-      action: Invoke-Step $config $_
+      items: $[[$_.steps]]
+      action: $_ | Invoke-Step

examples/5.templates/v1.4/templates/sparse-checkout.yaml

@@ -2,5 +2,5 @@ templates:
   sparse-checkout:
     powershell: |
       foreach ($directory in (Coalesce $step.directories, $step['sparse-checkout'])) {
-        Invoke-Step $config @{echo=$directory}
+        @{echo=$directory} | Invoke-Step
       }

examples/7.resources/v1.4/resources_pwshake.yaml

@@ -8,4 +8,11 @@ resources:
     target: .old_repo
 
 includes:
-- .old_repo/examples/hello_pwshake.yaml
+- .old_repo/examples/hello_pwshake.yaml
+
+tasks:
+    clean-up:
+    - pwsh: Remove-Item '{{pwshake_path}}/.old_repo' -Recurse -Force
+
+invoke_tasks:
+- clean-up

examples/8.filters/v1.5/context_pwshake.yaml

@@ -0,0 +1,35 @@
+attributes:
+  context:
+    Key: PWSHAKE
+    Value: '[[.Key]]'
+    Files:
+    - '[[.Key]].txt'
+    - '[[.Key]].log'
+    - '[[.Key]].key'
+    ListOfFiles: '[[$("$($_.Files)")]]'
+    AppService:
+      Locations: '[[.Files]]'
+      Executable: '[[.Key]]\[[.Value]].exe'
+
+tasks:
+  context:
+  - echo: 'Hello {{context.Key}}!'
+  - pwsh: |-
+      $result = $config.attributes.context | Interpolate-Item | ConvertTo-Yaml
+      Write-Host "`$result`:`n$result"
+      if ($result -notmatch 'ListOfFiles: PWSHAKE\.txt PWSHAKE\.log PWSHAKE\.key') {
+        throw 'Something wrong!'
+      }
+  - each:
+      items: $[[$config.attributes.context]]
+      action:
+        invoke_steps:
+        - pwsh: |-
+            $result = '[[.Key]] files: [[.ListOfFiles]]'
+            Write-Host "`$result = '$result'"
+            if ($result -notmatch 'PWSHAKE files: PWSHAKE\.txt PWSHAKE\.log PWSHAKE\.key') {
+              throw 'Something wrong!'
+            }
+
+invoke_tasks:
+- context