Merge branch 'dev' for release

INTI-CMNB · Apr 24, 2023 · 147f12d · 147f12d
2 parents b30afdd + 8576771
commit 147f12d
Show file tree

Hide file tree

Showing 352 changed files with 183,411 additions and 912 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -2,11 +2,13 @@
 source = 
     kibot
     src
-omit = */kibot/docopt.py
+omit = */kibot/blender_scripts/*
+       */kibot/docopt.py
        */kibot/kicad/sexpdata.py
        */kibot/mcpy/*
        */kibot/mcpyrate/*
        */kibot/PyPDF2/*
+       */kibot/PcbDraw/*
        */kibot/svgutils/*
 parallel = true
 

diff --git a/.github/workflows/dev_images.yml b/.github/workflows/dev_images.yml
@@ -6,11 +6,10 @@ on:
   workflow_dispatch
 
 jobs:
-  push_to_registry_ki5:
+  push_to_registry:
     strategy:
       matrix:
-        ki_release: [dev_image_k5, dev_image_k6, dev_image_k5f, dev_image_k6f]
-#        ki_release: [dev_image_k5, dev_image_k6, dev_image_k7, dev_image_k5f, dev_image_k6f, dev_image_k7f]
+        ki_release: [dev_image_k5, dev_image_k6, dev_image_k7, dev_image_k5f, dev_image_k6f, dev_image_k7f]
 
     name: Docker ${{ matrix.ki_release }}
     runs-on: ubuntu-latest
@@ -41,3 +40,14 @@ jobs:
       run: |
         cd tools/${{ matrix.ki_release }}
         ./build.sh
+
+  clean_images:
+    name: Triggers the kicad_auto and kicad_auto_test clean
+    runs-on: ubuntu-latest
+    needs: push_to_registry
+    if: github.ref == 'refs/heads/dev'
+    steps:
+    - name: Trigger garbage collector
+      run: |
+        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto/dispatches --data '{"event_type": "clean"}'
+        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto_test/dispatches --data '{"event_type": "clean"}'
diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
@@ -28,8 +28,9 @@ on:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        ki_release: [latest, ki6, ki7]
+        ki_release: [ki5, ki6, ki7]
         w_tests: [g1, g2, g3]
 
     runs-on: ubuntu-latest
@@ -39,12 +40,6 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: recursive
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings.
-        flake8 . --count --exit-zero --statistics
     - name: Test ${{ matrix.ki_release }}
       env:
         KI_RELEASE: ${{ matrix.ki_release }}
@@ -110,10 +105,72 @@ jobs:
         #echo Disabled by now
         coveralls
 
+  test_indep:
+    strategy:
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+    container: ghcr.io/inti-cmnb/kicad_auto_test:latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings.
+        flake8 . --count --exit-zero --statistics
+    - name: Test Independent
+      env:
+        KI_RELEASE: latest
+      run: |
+        rm -rf output
+        rm -f tests/.local
+        # Ensure we start a fresh coverage meassurement
+        python3-coverage erase
+        # Create the caches with macros
+        python3-coverage run src/kibot --help-outputs > /dev/null
+        ./g4.sh
+    - name: Store coverage
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test_Coverage_Independent
+        path: .coverage.*
+    - name: Collect coverage independent
+      run: |
+        python3-coverage combine
+        python3-coverage report
+        python3-coverage html -d output/htmlcov_independent
+    - name: Store results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test_Output_Independent
+        # Important! empty directories are skipped!!!!
+        path: output
+    - name: Upload Coverage
+      # Don't mix stable coverage with development coverage
+      if: github.ref == 'refs/heads/dev'
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        COVERALLS_FLAG_NAME: independent
+        COVERALLS_PARALLEL: true
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        # apt-get update
+        # apt-get -y install python3-pip
+        # pip3 install coveralls
+        #coveralls --service=github
+        #echo Disabled by now
+        coveralls
+
   create_images:
     name: Triggers the docker images creation
     runs-on: ubuntu-latest
-    needs: test
+    needs: [test, test_indep]
     if: github.ref == 'refs/heads/dev'
     steps:
     - name: Trigger `dev` images build
@@ -123,8 +180,8 @@ jobs:
   consolidate_coverage:
     name: Consolidate coverage data
     runs-on: ubuntu-latest
-    container: setsoft/kicad_auto_test:ki6
-    needs: test
+    container: setsoft/kicad_auto_test:ki7
+    needs: [test, test_indep]
     # Don't mix stable coverage with development coverage
     if: github.ref == 'refs/heads/dev'
 
@@ -136,17 +193,17 @@ jobs:
     - name: Download coverage k5 g1
       uses: actions/download-artifact@v3
       with:
-        name: Test_Coverage_latest_g1
+        name: Test_Coverage_ki5_g1
 
     - name: Download coverage k5 g2
       uses: actions/download-artifact@v3
       with:
-        name: Test_Coverage_latest_g2
+        name: Test_Coverage_ki5_g2
 
     - name: Download coverage k5 g3
       uses: actions/download-artifact@v3
       with:
-        name: Test_Coverage_latest_g3
+        name: Test_Coverage_ki5_g3
 
     - name: Download coverage k6 g1
       uses: actions/download-artifact@v3
@@ -163,6 +220,11 @@ jobs:
       with:
         name: Test_Coverage_ki6_g3
 
+    - name: Download coverage independent
+      uses: actions/download-artifact@v3
+      with:
+        name: Test_Coverage_Independent
+
     - name: Combine coverage
       run: |
         pip3 install --upgrade coverage==5.1
@@ -179,7 +241,7 @@ jobs:
 
   coveralls:
     name: Finish Coveralls
-    needs: test
+    needs: [test, test_indep]
     runs-on: ubuntu-latest
     # Don't mix stable coverage with development coverage
     if: github.ref == 'refs/heads/dev'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -88,4 +88,7 @@ jobs:
         run: |
           curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto/dispatches --data '{"event_type": "build_k5"}'
           curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto/dispatches --data '{"event_type": "build_k6"}'
-
+          curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto/dispatches --data '{"event_type": "build_k7"}'
+          curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto_test/dispatches --data '{"event_type": "build_k5"}'
+          curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto_test/dispatches --data '{"event_type": "build_k6"}'
+          curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/INTI-CMNB/kicad_auto_test/dispatches --data '{"event_type": "build_k7"}'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -33,7 +33,7 @@ repos:
       - id: mixed-line-ending
       - id: check-builtin-literals
       - id: check-merge-conflict
-      - id: check-docstring-first
+      # - id: check-docstring-first
       - id: fix-byte-order-marker
       - id: check-case-conflict
       - id: check-executables-have-shebangs

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,74 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.2] - 2023-04-24
+### Added
+- General:
+  - Support for time stamp in the date (i.e. 2023-04-02T09:22-03:00)
+  - Support to pass variables to the 3D models download URL (#414)
+  - Support for netclass flags (#418)
+  - Export KICADn_* environment variables for the older versions
+    So you can use KICAD6_* variables on KiCad 7.
+- Expansion patterns:
+  - **%M** directory where the pcb/sch resides. Only the last component
+    i.e. /a/b/c/name.kicad_pcb -> c  (#421)
+- Command line:
+  - `--banner N` Option to display a banner
+  - `--log FILE` Option to log to a file, in addition to the stderr
+- Global options:
+  - `colored_tht_resistors` to disable the 3D colored resistors.
+  - `field_tolerance` field/s to look for resistor tolerance.
+  - `default_resistor_tolerance` which tolerance to use when none found.
+  - `cache_3d_resistors` to avoid generating them all the time.
+  - `resources_dir` to specify fonts and colors to install (CI/CD)
+- 3D: colored 3D models for THT resistors
+- Blender export:
+  - Better default light
+  - More light options
+- Datasheet download: now the warnings mention which reference failed.
+- Plot related outputs and PCB_Print:
+  - `individual_page_scaling`: to control if the center of the page is computed
+    using all pages or individually.
+- Plot related outputs:
+  - All outputs now support scaling.
+- BoM:
+  - Support for extra information in the *Value* field.
+    Currently just parsed, not rejected.
+- PCB/SCH parity test:
+  - Check for value and fields/properties.
+- SCH print:
+  - Support for title change
+- VRML:
+  - Option to use the auxiliary origin as reference. (#420)
+
+### Fixed
+- Makefile: don't skip all preflights on each run, just the ones we generate
+  as targets. (#405)
+- KiKit present: problems when no board was specified. (#402)
+- Datasheet download:
+  - Avoid interruptions when too many redirections is detected (#408)
+- PcbDraw:
+  - KiCad 7.0.1 polygons used as board edge. (yaqwsx/PcbDraw#142)
+- PCB Print:
+  - Interference between the visible layers in the PRL file and the results
+    when scaling. (#407)
+  - Problems with images in the WKS (KiCad 5/6)
+- Diff:
+  - Problems when using an output and no variant specified.
+- PCB/SCH parity test:
+  - Workaround for bogus net codes generated by KiCad (#410)
+- 3D Models:
+  - Problems to download KiCad 7 models (#417)
+  - Added workaround for KiCad 7 failing to export VRMLs for PCBs using paths
+    relative to the footprint. (See #417)
+- VRML:
+  - ref_y coordinate not used. (#419)
+
+### Changed:
+- Some R, L and C values that were rejected are accepted now. You just get a
+  warning about what part of the value was discarded.
+
+
 ## [1.6.1] - 2023-03-16
 ### Added
 - KiCad 7.0.1 support

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/inti-cmnb/kicad6_auto:1.6.0-1_k6.0.10_d11.6
+FROM ghcr.io/inti-cmnb/kicad7_auto_full:latest
 LABEL AUTHOR Salvador E. Tropea <stropea@inti.gob.ar>
 LABEL Description="Export various files from KiCad projects (KiCad 7)"
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -11,6 +11,7 @@ include kibot/resources/config_templates/panelize/*.yaml
 include kibot/resources/config_templates/*.yaml
 include kibot/resources/images/*.svg
 include kibot/resources/images/*.ico
+include kibot/resources/parsers/*.lark
 include kibot/resources/pcbdraw/styles/*.json
 include kibot/resources/pcbdraw/templates/*.handlebars
 include kibot/blender_scripts/*.py

diff --git a/Makefile b/Makefile
@@ -48,7 +48,7 @@ test: lint
 	rm -f tests/.local
 	$(PY_COV) erase
 	# python3-pytest-xdist
-	$(PYTEST) -m "not slow" -n 2 --test_dir=output
+	$(PYTEST) -m "not slow" -n 4 --test_dir=output
 	$(PYTEST) -m "slow" --test_dir=output
 	$(PY_COV) combine
 	$(PY_COV) report
@@ -80,7 +80,7 @@ test_docker_local_1:
 	# Run in the same directory to make the __pycache__ valid
 	# Also change the owner of the files to the current user (we run as root like in GitHub)
 	#docker run --rm -it -v $(CWD):$(CWD) --workdir="$(CWD)" setsoft/kicad_auto_test:latest '/bin/bash'
-	docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" setsoft/kicad_auto_test:latest  \
+	docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" ghcr.io/inti-cmnb/kicad_auto_test:ki5  \
 		/bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs > /dev/null; pytest-3 --log-cli-level debug -k '$(SINGLE_TEST)' --test_dir=output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage.* htmlcov/"
 	#$(PY_COV) report
 	#x-www-browser htmlcov/index.html
@@ -106,14 +106,24 @@ test_docker_local_1_ki7:
 	docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" ghcr.io/inti-cmnb/kicad_auto_test:ki7 \
 		/bin/bash -c "python3-coverage run src/kibot --help-outputs > /dev/null; pytest-3 --log-cli-level debug -k '$(SINGLE_TEST)' --test_dir=output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage htmlcov/"
 
+t1k7: test_docker_local_1_ki7
+
+test_docker_local_1_n:
+	rm -rf output
+	rm -f tests/.local
+	docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" ghcr.io/inti-cmnb/kicad_auto_test:nightly \
+		/bin/bash -c "src/kibot --help-outputs > /dev/null ; pytest-3 --log-cli-level debug -k '$(SINGLE_TEST)' --test_dir=output ; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage htmlcov/"
+
+t1n: test_docker_local_1_n
+
 test_docker_local:
 	rm -rf output
 	rm -f tests/.local
 	$(PY_COV) erase
 	# Run in the same directory to make the __pycache__ valid
 	# Also change the owner of the files to the current user (we run as root like in GitHub)
 	docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" ghcr.io/inti-cmnb/kicad_auto_test:latest \
-		/bin/bash -c "flake8 . --count --statistics ; python3-coverage run src/kibot --help-outputs ; pytest-3 --test_dir=output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ .coverage htmlcov/"
+		/bin/bash -c "flake8 . --count --statistics ; python3-coverage run src/kibot --help-outputs ; pytest-3 --test_dir=output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ .coverage.* htmlcov/"
 	$(PY_COV) combine
 	$(PY_COV) report
 	x-www-browser htmlcov/index.html