diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index fc333e71..e401e491 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -14,4 +14,3 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} # Deploy key to trigger other actions - diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index 22d4e996..afcc915a 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -24,13 +24,11 @@ jobs: steps: - name: "Check out repository" uses: actions/checkout@v4 - with: - ref: '' - name: "Set up Julia" uses: julia-actions/setup-julia@v2 with: - version: '1' # 1.6 + version: '1' arch: x64 - name: "Copy examples, readme" @@ -47,7 +45,7 @@ jobs: cp ./README.md ./docs/src/index.md - name: "Install dependencies" - run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.update()' + run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: "Build and deploy" env: diff --git a/.github/workflows/Example.yml b/.github/workflows/Example.yml index ed73abc2..ae19d78d 100644 --- a/.github/workflows/Example.yml +++ b/.github/workflows/Example.yml @@ -8,23 +8,22 @@ on: - main paths: - 'src/**' - - 'examples/jupyter-src/**' - - 'examples/pluto-src/**' + - 'examples/**' - '.github/workflows/Example.yml' - 'Project.toml' - + jobs: jupyter: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [windows-latest] # , ubuntu-latest] + os: [windows-latest] # ubuntu-latest] file-name: [simple_hybrid_CS, simple_hybrid_ME, juliacon_2023, modelica_conference_2021] - julia-version: ['1.10'] + julia-version: ['1.10'] julia-arch: [x64] experimental: [false] - + steps: - name: "Check out repository" uses: actions/checkout@v4 @@ -34,13 +33,13 @@ jobs: with: version: ${{ matrix.julia-version }} arch: ${{ matrix.julia-arch }} - + - name: "Install dependencies" run: julia --project=examples/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: "Install packages" run: pip install jupyter nbconvert - + - name: "Execute notebook" env: FILE: examples/jupyter-src/${{ matrix.file-name }}.ipynb @@ -65,9 +64,9 @@ jobs: if: success() && github.event_name != 'pull_request' && github.ref_name == 'main' uses: nick-fields/retry@v3 env: - CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.os }}-${{ matrix.file-name }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref }}] + CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.file-name }}-${{ matrix.os }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref_name }}] CI_COMMIT_AUTHOR: github-actions[bot] - EXAMPLES_PATH: examples + EXAMPLES_PATH: examples/jupyter-src # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch. # Reset the branch and remove all current files in the examples folder. # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder. @@ -79,16 +78,19 @@ jobs: command: | git fetch --all git stash clear - git stash --include-untracked -- ${{ env.EXAMPLES_PATH }} + git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}* git switch examples git reset --hard origin/examples + rm -r ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}* git checkout stash -f -- ${{ env.EXAMPLES_PATH }} git stash apply --index git stash drop git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com" git config --global core.autocrlf false - git add ${{ env.EXAMPLES_PATH }} + git pull + git reset + git add ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}* git commit -m "${{ env.CI_COMMIT_MESSAGE }}" git push origin examples || (git reset --soft HEAD~1 && (exit 1)) @@ -103,16 +105,16 @@ jobs: with: version: '1.10' - - run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer"); Pkg.add("FMIFlux")' + - run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer");' - run: julia -e 'using PlutoSliderServer; PlutoSliderServer.export_directory("examples/pluto-src")' - name: "auto-commit (retry on merge)" if: success() && github.event_name != 'pull_request' && github.ref_name == 'main' uses: nick-fields/retry@v3 env: - CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref }}] + CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref_name }}] CI_COMMIT_AUTHOR: github-actions[bot] - EXAMPLES_PATH: examples + EXAMPLES_PATH: examples/pluto-src # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch. # Reset the branch and remove all current files in the examples folder. # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder. @@ -127,6 +129,7 @@ jobs: git stash --include-untracked -- ${{ env.EXAMPLES_PATH }} git switch examples git reset --hard origin/examples + rm -r ${{ env.EXAMPLES_PATH }}/* git checkout stash -f -- ${{ env.EXAMPLES_PATH }} git stash apply --index git stash drop @@ -134,10 +137,11 @@ jobs: git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com" git config --global core.autocrlf false git pull + git reset git add ${{ env.EXAMPLES_PATH }} git commit -m "${{ env.CI_COMMIT_MESSAGE }}" git push origin examples || (git reset --soft HEAD~1 && (exit 1)) - + call-docu: needs: [jupyter, pluto] if: github.event_name != 'pull_request' && github.ref_name == 'main' diff --git a/.github/workflows/Formatter.yml b/.github/workflows/Formatter.yml index 0155d9b6..14659356 100644 --- a/.github/workflows/Formatter.yml +++ b/.github/workflows/Formatter.yml @@ -3,6 +3,8 @@ on: pull_request: # this argument is not required if you don't use the `suggestion-label` input types: [ opened, reopened, synchronize, labeled, unlabeled ] + workflow_dispatch: + jobs: code-style: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6461217d..1c6b8552 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,3 @@ LocalPreferences.toml tmp/ build/ .vscode/ -examples/src/.ipynb_checkpoints/ diff --git a/docs/Project.toml b/docs/Project.toml index 67a236d2..42c482d4 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,6 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" [compat] julia = "1.6" diff --git a/docs/make.jl b/docs/make.jl index 8d49af6a..2b7f7439 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -7,29 +7,98 @@ import Pkg; Pkg.develop(path = joinpath(@__DIR__, "../../FMIFlux.jl")); using Documenter, FMIFlux using Documenter: GitHubActions +using Suppressor -makedocs( +example_pages = [ + "Overview" => joinpath("examples", "overview.md"), + "Simple CS-NeuralFMU" => joinpath("examples", "simple_hybrid_CS.md"), + "Simple ME-NeuralFMU" => joinpath("examples", "simple_hybrid_ME.md"), + "Growing Horizon ME-NeuralFMU" => joinpath("examples", "growing_horizon_ME.md"), + "JuliaCon 2023" => joinpath("examples", "juliacon_2023.md"), + "MDPI 2022" => joinpath("examples", "mdpi_2022.md"), + "Modelica Conference 2021" => joinpath("examples", "modelica_conference_2021.md"), + "Pluto Workshops" => joinpath("examples", "workshops.md"), +] + +#check if all md files in examples are included in docs +for md in readdir(joinpath("docs", "src", "examples")) + if endswith(md, ".md") && + !occursin("README", md) && + all([!endswith(file, md) for (x, file) in example_pages]) + print( + string( + "::warning title=Example-Warning::example \"", + md, + "\" is not included in the doc-manual\r\n", + ), + ) + end +end + +#remove any example pages, for witch the example can not be found +# and remove svgs if md building failed +for (x, md) in deepcopy(example_pages) + # check if file is missing + if !(any([occursin(file, md) for file in readdir(joinpath("docs", "src", "examples"))])) + print( + string( + "::warning title=Example-Warning::example-page \"", + md, + "\" is to be included in the doc-manual, but could not be found on the examples branch or in \"docs/src/examples\"\r\n", + ), + ) + filter!(e -> e ≠ (x => md), example_pages) + else + # removal of svgs is here if there is xml data in the md + r = open(joinpath("docs", "src", md), "r") + s = read(r, String) + close(r) + if occursin("", s) + print( + string( + "::warning title=SVG-Warning::example-page \"", + md, + "\" has svg-xml text in it. Most likely, linking of support-files generated by jupyter is broken. The svg-xml text has been removed for the doc-manual, but also no plot will be displayed\r\n", + ), + ) + # regex replace exeeds stack limit: s = replace(s, r"\<\?xml(?!<\/svg>)(.|\n)*?<\/svg>" => "") + # so take iterative approach: + while occursin("", s) + a = findfirst("", s)[end] + 1 + s = string(s[1:a], s[b:end]) + end + w = open(joinpath("docs", "src", md * "tmp"), "w+") + write(w, s) + close(w) + end + if isfile(joinpath("docs", "src", md * "tmp")) + mv( + joinpath("docs", "src", md * "tmp"), + joinpath("docs", "src", md), + force = true, + ) + end + end +end + +my_makedocs() = makedocs( sitename = "FMIFlux.jl", format = Documenter.HTML( collapselevel = 1, sidebar_sitename = false, edit_link = nothing, - size_threshold_ignore = [joinpath("examples", "juliacon_2023.md")], + size_threshold = 512000, + size_threshold_ignore = [], ), - warnonly = true, + modules = [FMIFlux], + checkdocs = :exports, + linkcheck = true, + warnonly = :linkcheck, pages = Any[ "Introduction" => "index.md" - "Examples" => [ - "Overview" => "examples/overview.md" - "Simple CS-NeuralFMU" => "examples/simple_hybrid_CS.md" - "Simple ME-NeuralFMU" => "examples/simple_hybrid_ME.md" - "Growing Horizon ME-NeuralFMU" => "examples/growing_horizon_ME.md" - "JuliaCon 2023" => "examples/juliacon_2023.md" - "MDPI 2022" => "examples/mdpi_2022.md" - "Modelica Conference 2021" => "examples/modelica_conference_2021.md" - "Pluto Workshops" => "examples/workshops.md" - ] "FAQ" => "faq.md" + "Examples" => example_pages "Library Functions" => "library.md" "Related Publication" => "related.md" "Contents" => "contents.md" @@ -46,8 +115,25 @@ function deployConfig() return GitHubActions(github_repository, github_event_name, github_ref) end +output = "" +try + global output = @capture_err begin + my_makedocs() + end +catch e + my_makedocs() # if it fails, re-run without capturing, so that its stderr appears in the console/logs +end + +# errors = findall(r"Error:.*", output) +warns = findall(r"Warning:.*", output) + +for w in warns + s = string("::warning title=Documenter-Warning::", output[w], "\r\n") + print(s) +end + deploydocs( - repo = "github.com/ThummeTo/FMIFlux.jl.git", + repo = string("github.com/", get(ENV, "GITHUB_REPOSITORY", ""), "git"), devbranch = "main", deploy_config = deployConfig(), ) diff --git a/docs/src/examples/overview.md b/docs/src/examples/overview.md index 8d6175cc..a0a6b8b5 100644 --- a/docs/src/examples/overview.md +++ b/docs/src/examples/overview.md @@ -16,10 +16,10 @@ The examples show how to combine FMUs with machine learning ("NeuralFMU") and il - [__Modelica Conference 2021: NeuralFMUs__](https://thummeto.github.io/FMIFlux.jl/dev/examples/modelica_conference_2021/): Showing basics on how to train a NeuralFMU (Contribution for the *Modelica Conference 2021*). ## Workshops -[Pluto](https://plutojl.org/) based notebooks, that can easily be executed on your own Pluto-Setup. -- [__Scientific Machine Learning using Functional Mock-up Units__](../pluto-src/SciMLUsingFMUs/SciMLUsingFMUs.html): Workshop at JuliaCon 2024 (Eindhoven University, Netherlands) +[__Pluto workshops__](https://thummeto.github.io/FMI.jl/dev/examples/workshops/): [Pluto](https://plutojl.org/) based notebooks, that can easily be executed on your own Pluto-Setup. +- [__Scientific Machine Learning using Functional Mock-up Units__](https://thummeto.github.io/FMI.jl/dev/examples/pluto-src/SciMLUsingFMUs/SciMLUsingFMUs.html): Workshop at JuliaCon 2024 (Eindhoven University, Netherlands) ## Archived - [__MDPI 2022: Physics-enhanced NeuralODEs in real-world applications__](https://thummeto.github.io/FMIFlux.jl/dev/examples/mdpi_2022/): An example for a NeuralODE in a real world modeling scenario (Contribution in *MDPI Electronics 2022*). - [__Growing Horizon ME-NeuralFMU__](https://thummeto.github.io/FMIFlux.jl/dev/examples/growing_horizon_ME/): Growing horizon training technique for a ME-NeuralFMU. -- [__HybridModelingUsingFMI__](../pluto-src/HybridModelingUsingFMI/HybridModelingUsingFMI.html): Workshop at MODPROD 2024 (Linköping University, Sweden) +- [__Hands-on: Hybrid Modeling using FMI__](https://thummeto.github.io/FMI.jl/dev/examples/pluto-src/HybridModelingUsingFMI/HybridModelingUsingFMI.html): Workshop at MODPROD 2024 (Linköping University, Sweden) diff --git a/docs/src/library.md b/docs/src/library.md index cf121108..8afcc7fb 100644 --- a/docs/src/library.md +++ b/docs/src/library.md @@ -14,7 +14,7 @@ NeuralFMU ## FMI 2 version dependent functions ```@docs -fmi2DoStepCS +fmiDoStepCS fmi2EvaluateME fmi2InputDoStepCSOutput ``` @@ -22,7 +22,6 @@ fmi2InputDoStepCSOutput ## FMI version independent functions ```@docs -fmiDoStepCS fmiEvaluateME fmiInputDoStepCSOutput ``` @@ -31,5 +30,22 @@ fmiInputDoStepCSOutput ```@docs mse_interpolate +``` transferParams! + +## TODO: Sort docs + +```@docs +LossAccumulationScheduler +FMIFlux.transferFlatParams! +FMIFlux.WorstElementScheduler +FMIFlux.FMUParameterRegistrator +FMIFlux.SimultaniousZeroCrossing +FMIFlux.ParameterRegistrator +FMIFlux.ScaleShift +FMIFlux.WorstGrowScheduler +FMIFlux.SequentialScheduler +FMIFlux.FMUTimeLayer +FMIFlux.ShiftScale +FMIFlux.RandomScheduler ``` \ No newline at end of file diff --git a/examples/jupyter-src/.gitignore b/examples/jupyter-src/.gitignore index 9448d154..670f8b8f 100644 --- a/examples/jupyter-src/.gitignore +++ b/examples/jupyter-src/.gitignore @@ -1,3 +1,4 @@ +.ipynb_checkpoints/ params/ *.png -*.gif \ No newline at end of file +./*.gif \ No newline at end of file diff --git a/examples/jupyter-src/juliacon_2023.ipynb b/examples/jupyter-src/juliacon_2023.ipynb index 68861ca7..77e267e6 100644 --- a/examples/jupyter-src/juliacon_2023.ipynb +++ b/examples/jupyter-src/juliacon_2023.ipynb @@ -919,7 +919,7 @@ "metadata": {}, "outputs": [], "source": [ - "plotEnhancements(neuralFMU, fmu, data; filename=joinpath(@__DIR__, \"result.gif\"))" + "plotEnhancements(neuralFMU, fmu, data; filename=joinpath(@__DIR__, \"gif_1.gif\"))" ] }, { diff --git a/test/.gitignore b/test/.gitignore index 07a25449..f0a471af 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,3 +1,6 @@ *.svg *.pdf -*.gif \ No newline at end of file +*.gif +*.jld2 +*.csv +*.mat \ No newline at end of file