updated doc and example action (ThummeTo#155)

* restored gif fix * restored gif fix * deploy docs to triggering repo * added cleanup before commit * fixes for examples pushing * fixes for doc * minor example ci fixes * example ci fixes * Update Example.yml * doc fixes makeshift * updated make.jl to new standart * updates to match FMI.jl structure & doc fixes * "archived" pluto Hybrid Modeling
0815Creeper · Sep 19, 2024 · 65b6ccf · 65b6ccf
1 parent d6c6d3f
commit 65b6ccf
Show file tree

Hide file tree

Showing 12 changed files with 153 additions and 44 deletions.
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
@@ -14,4 +14,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}   # Deploy key to trigger other actions
-
diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml
@@ -24,13 +24,11 @@ jobs:
     steps:
       - name: "Check out repository"
         uses: actions/checkout@v4
-        with:
-          ref: ''
 
       - name: "Set up Julia"
         uses: julia-actions/setup-julia@v2
         with:
-          version: '1' # 1.6
+          version: '1'
           arch: x64
 
       - name: "Copy examples, readme"
@@ -47,7 +45,7 @@ jobs:
           cp ./README.md ./docs/src/index.md
 
       - name: "Install dependencies"
-        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.update()'
+        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
 
       - name: "Build and deploy"
         env:

diff --git a/.github/workflows/Example.yml b/.github/workflows/Example.yml
@@ -8,23 +8,22 @@ on:
       - main
     paths:
       - 'src/**'
-      - 'examples/jupyter-src/**'
-      - 'examples/pluto-src/**'
+      - 'examples/**'
       - '.github/workflows/Example.yml'
       - 'Project.toml'
-
+  
 jobs:
   jupyter:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest] # , ubuntu-latest]
+        os: [windows-latest] # ubuntu-latest]
         file-name: [simple_hybrid_CS, simple_hybrid_ME, juliacon_2023, modelica_conference_2021]
-        julia-version: ['1.10'] 
+        julia-version: ['1.10']
         julia-arch: [x64]
         experimental: [false]
-        
+
     steps:
       - name: "Check out repository"
         uses: actions/checkout@v4
@@ -34,13 +33,13 @@ jobs:
         with:
             version: ${{ matrix.julia-version }}
             arch: ${{ matrix.julia-arch }}
-            
+
       - name: "Install dependencies"
         run: julia --project=examples/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
 
       - name: "Install packages"
         run: pip install jupyter nbconvert
-
+        
       - name: "Execute notebook"
         env:
           FILE: examples/jupyter-src/${{ matrix.file-name }}.ipynb
@@ -65,9 +64,9 @@ jobs:
         if: success() && github.event_name != 'pull_request' && github.ref_name == 'main'
         uses: nick-fields/retry@v3
         env: 
-          CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.os }}-${{ matrix.file-name }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref }}]
+          CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.file-name }}-${{ matrix.os }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref_name }}]
           CI_COMMIT_AUTHOR: github-actions[bot]
-          EXAMPLES_PATH: examples
+          EXAMPLES_PATH: examples/jupyter-src
         # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch.
         # Reset the branch and remove all current files in the examples folder. 
         # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder.
@@ -79,16 +78,19 @@ jobs:
           command: |
             git fetch --all
             git stash clear
-            git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}
+            git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
             git switch examples
             git reset --hard origin/examples
+            rm -r ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
             git checkout stash -f -- ${{ env.EXAMPLES_PATH }}
             git stash apply --index
             git stash drop
             git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
             git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com"
             git config --global core.autocrlf false
-            git add ${{ env.EXAMPLES_PATH }}
+            git pull
+            git reset
+            git add ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
             git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
             git push origin examples || (git reset --soft HEAD~1 && (exit 1))
 
@@ -103,16 +105,16 @@ jobs:
         with:
             version: '1.10'
 
-      - run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer"); Pkg.add("FMIFlux")'
+      - run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer");'
       - run: julia -e 'using PlutoSliderServer; PlutoSliderServer.export_directory("examples/pluto-src")'
 
       - name: "auto-commit (retry on merge)"
         if: success() && github.event_name != 'pull_request' && github.ref_name == 'main'
         uses: nick-fields/retry@v3
         env: 
-          CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref }}]
+          CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref_name }}]
           CI_COMMIT_AUTHOR: github-actions[bot]
-          EXAMPLES_PATH: examples
+          EXAMPLES_PATH: examples/pluto-src
         # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch.
         # Reset the branch and remove all current files in the examples folder. 
         # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder.
@@ -127,17 +129,19 @@ jobs:
             git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}
             git switch examples
             git reset --hard origin/examples
+            rm -r ${{ env.EXAMPLES_PATH }}/*
             git checkout stash -f -- ${{ env.EXAMPLES_PATH }}
             git stash apply --index
             git stash drop
             git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
             git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com"
             git config --global core.autocrlf false
             git pull
+            git reset
             git add ${{ env.EXAMPLES_PATH }}
             git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
             git push origin examples || (git reset --soft HEAD~1 && (exit 1))
-
+            
   call-docu:
     needs: [jupyter, pluto]
     if: github.event_name != 'pull_request' && github.ref_name == 'main'

diff --git a/.github/workflows/Formatter.yml b/.github/workflows/Formatter.yml
@@ -3,6 +3,8 @@ on:
   pull_request:
     # this argument is not required if you don't use the `suggestion-label` input
     types: [ opened, reopened, synchronize, labeled, unlabeled ]
+  workflow_dispatch:
+
 jobs:
   code-style:
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -28,4 +28,3 @@ LocalPreferences.toml
 tmp/
 build/
 .vscode/
-examples/src/.ipynb_checkpoints/
diff --git a/docs/Project.toml b/docs/Project.toml
@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 
 [compat]
 julia = "1.6"
diff --git a/docs/make.jl b/docs/make.jl
@@ -7,29 +7,98 @@ import Pkg;
 Pkg.develop(path = joinpath(@__DIR__, "../../FMIFlux.jl"));
 using Documenter, FMIFlux
 using Documenter: GitHubActions
+using Suppressor
 
-makedocs(
+example_pages = [
+    "Overview" => joinpath("examples", "overview.md"),
+    "Simple CS-NeuralFMU" => joinpath("examples", "simple_hybrid_CS.md"),
+    "Simple ME-NeuralFMU" => joinpath("examples", "simple_hybrid_ME.md"),
+    "Growing Horizon ME-NeuralFMU" => joinpath("examples", "growing_horizon_ME.md"),
+    "JuliaCon 2023" => joinpath("examples", "juliacon_2023.md"),
+    "MDPI 2022" => joinpath("examples", "mdpi_2022.md"),
+    "Modelica Conference 2021" => joinpath("examples", "modelica_conference_2021.md"),
+    "Pluto Workshops" => joinpath("examples", "workshops.md"),
+]
+
+#check if all md files in examples are included in docs
+for md in readdir(joinpath("docs", "src", "examples"))
+    if endswith(md, ".md") &&
+       !occursin("README", md) &&
+       all([!endswith(file, md) for (x, file) in example_pages])
+        print(
+            string(
+                "::warning title=Example-Warning::example \"",
+                md,
+                "\" is not included in the doc-manual\r\n",
+            ),
+        )
+    end
+end
+
+#remove any example pages, for witch the example can not be found
+# and remove svgs if md building failed
+for (x, md) in deepcopy(example_pages)
+    # check if file is missing
+    if !(any([occursin(file, md) for file in readdir(joinpath("docs", "src", "examples"))]))
+        print(
+            string(
+                "::warning title=Example-Warning::example-page \"",
+                md,
+                "\" is to be included in the doc-manual, but could not be found on the examples branch or in \"docs/src/examples\"\r\n",
+            ),
+        )
+        filter!(e -> e ≠ (x => md), example_pages)
+    else
+        # removal of svgs is here if there is xml data in the md
+        r = open(joinpath("docs", "src", md), "r")
+        s = read(r, String)
+        close(r)
+        if occursin("<svg", s) && occursin("</svg>", s)
+            print(
+                string(
+                    "::warning title=SVG-Warning::example-page \"",
+                    md,
+                    "\" has svg-xml text in it. Most likely, linking of support-files generated by jupyter is broken. The svg-xml text has been removed for the doc-manual, but also no plot will be displayed\r\n",
+                ),
+            )
+            # regex replace exeeds stack limit: s = replace(s, r"\<\?xml(?!<\/svg>)(.|\n)*?<\/svg>" => "")
+            # so take iterative approach:
+            while occursin("<?xml", s) && occursin("</svg>", s)
+                a = findfirst("<?xml", s)[1] - 1
+                b = findfirst("</svg>", s)[end] + 1
+                s = string(s[1:a], s[b:end])
+            end
+            w = open(joinpath("docs", "src", md * "tmp"), "w+")
+            write(w, s)
+            close(w)
+        end
+        if isfile(joinpath("docs", "src", md * "tmp"))
+            mv(
+                joinpath("docs", "src", md * "tmp"),
+                joinpath("docs", "src", md),
+                force = true,
+            )
+        end
+    end
+end
+
+my_makedocs() = makedocs(
     sitename = "FMIFlux.jl",
     format = Documenter.HTML(
         collapselevel = 1,
         sidebar_sitename = false,
         edit_link = nothing,
-        size_threshold_ignore = [joinpath("examples", "juliacon_2023.md")],
+        size_threshold = 512000,
+        size_threshold_ignore = [],
     ),
-    warnonly = true,
+    modules = [FMIFlux],
+    checkdocs = :exports,
+    linkcheck = true,
+    warnonly = :linkcheck,
     pages = Any[
         "Introduction" => "index.md"
-        "Examples" => [
-            "Overview" => "examples/overview.md"
-            "Simple CS-NeuralFMU" => "examples/simple_hybrid_CS.md"
-            "Simple ME-NeuralFMU" => "examples/simple_hybrid_ME.md"
-            "Growing Horizon ME-NeuralFMU" => "examples/growing_horizon_ME.md"
-            "JuliaCon 2023" => "examples/juliacon_2023.md"
-            "MDPI 2022" => "examples/mdpi_2022.md"
-            "Modelica Conference 2021" => "examples/modelica_conference_2021.md"
-            "Pluto Workshops" => "examples/workshops.md"
-        ]
         "FAQ" => "faq.md"
+        "Examples" => example_pages
         "Library Functions" => "library.md"
         "Related Publication" => "related.md"
         "Contents" => "contents.md"
@@ -46,8 +115,25 @@ function deployConfig()
     return GitHubActions(github_repository, github_event_name, github_ref)
 end
 
+output = ""
+try
+    global output = @capture_err begin
+        my_makedocs()
+    end
+catch e
+    my_makedocs() # if it fails, re-run without capturing, so that its stderr appears in the console/logs
+end
+
+# errors = findall(r"Error:.*", output)
+warns = findall(r"Warning:.*", output)
+
+for w in warns
+    s = string("::warning title=Documenter-Warning::", output[w], "\r\n")
+    print(s)
+end
+
 deploydocs(
-    repo = "github.com/ThummeTo/FMIFlux.jl.git",
+    repo = string("github.com/", get(ENV, "GITHUB_REPOSITORY", ""), "git"),
     devbranch = "main",
     deploy_config = deployConfig(),
 )
diff --git a/docs/src/examples/overview.md b/docs/src/examples/overview.md
@@ -16,10 +16,10 @@ The examples show how to combine FMUs with machine learning ("NeuralFMU") and il
 - [__Modelica Conference 2021: NeuralFMUs__](https://thummeto.github.io/FMIFlux.jl/dev/examples/modelica_conference_2021/): Showing basics on how to train a NeuralFMU (Contribution for the *Modelica Conference 2021*).
 
 ## Workshops
-[Pluto](https://plutojl.org/) based notebooks, that can easily be executed on your own Pluto-Setup.
-- [__Scientific Machine Learning using Functional Mock-up Units__](../pluto-src/SciMLUsingFMUs/SciMLUsingFMUs.html): Workshop at JuliaCon 2024 (Eindhoven University, Netherlands)
+[__Pluto workshops__](https://thummeto.github.io/FMI.jl/dev/examples/workshops/): [Pluto](https://plutojl.org/) based notebooks, that can easily be executed on your own Pluto-Setup.
+- [__Scientific Machine Learning using Functional Mock-up Units__](https://thummeto.github.io/FMI.jl/dev/examples/pluto-src/SciMLUsingFMUs/SciMLUsingFMUs.html): Workshop at JuliaCon 2024 (Eindhoven University, Netherlands)
 
 ## Archived
 - [__MDPI 2022: Physics-enhanced NeuralODEs in real-world applications__](https://thummeto.github.io/FMIFlux.jl/dev/examples/mdpi_2022/): An example for a NeuralODE in a real world modeling scenario (Contribution in *MDPI Electronics 2022*).
 - [__Growing Horizon ME-NeuralFMU__](https://thummeto.github.io/FMIFlux.jl/dev/examples/growing_horizon_ME/): Growing horizon training technique for a ME-NeuralFMU.
-- [__HybridModelingUsingFMI__](../pluto-src/HybridModelingUsingFMI/HybridModelingUsingFMI.html): Workshop at MODPROD 2024 (Linköping University, Sweden)
+- [__Hands-on: Hybrid Modeling using FMI__](https://thummeto.github.io/FMI.jl/dev/examples/pluto-src/HybridModelingUsingFMI/HybridModelingUsingFMI.html): Workshop at MODPROD 2024 (Linköping University, Sweden)
diff --git a/docs/src/library.md b/docs/src/library.md
@@ -14,15 +14,14 @@ NeuralFMU
 ## FMI 2 version dependent functions
 
 ```@docs
-fmi2DoStepCS
+fmiDoStepCS
 fmi2EvaluateME
 fmi2InputDoStepCSOutput
 ```
 
 ## FMI version independent functions
 
 ```@docs
-fmiDoStepCS
 fmiEvaluateME
 fmiInputDoStepCSOutput
 ```
@@ -31,5 +30,22 @@ fmiInputDoStepCSOutput
 
 ```@docs
 mse_interpolate
+```
 transferParams!
+
+## TODO: Sort docs
+
+```@docs
+LossAccumulationScheduler
+FMIFlux.transferFlatParams!
+FMIFlux.WorstElementScheduler
+FMIFlux.FMUParameterRegistrator
+FMIFlux.SimultaniousZeroCrossing
+FMIFlux.ParameterRegistrator
+FMIFlux.ScaleShift
+FMIFlux.WorstGrowScheduler
+FMIFlux.SequentialScheduler
+FMIFlux.FMUTimeLayer
+FMIFlux.ShiftScale
+FMIFlux.RandomScheduler
 ```
diff --git a/examples/jupyter-src/.gitignore b/examples/jupyter-src/.gitignore
@@ -1,3 +1,4 @@
+.ipynb_checkpoints/
 params/
 *.png
-*.gif
+./*.gif
diff --git a/examples/jupyter-src/juliacon_2023.ipynb b/examples/jupyter-src/juliacon_2023.ipynb
@@ -919,7 +919,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "plotEnhancements(neuralFMU, fmu, data; filename=joinpath(@__DIR__, \"result.gif\"))"
+    "plotEnhancements(neuralFMU, fmu, data; filename=joinpath(@__DIR__, \"gif_1.gif\"))"
    ]
   },
   {

diff --git a/test/.gitignore b/test/.gitignore
@@ -1,3 +1,6 @@
 *.svg
 *.pdf
-*.gif
+*.gif
+*.jld2
+*.csv
+*.mat