diff --git a/.cookiecutter.json b/.cookiecutter.json new file mode 100644 index 00000000..55941943 --- /dev/null +++ b/.cookiecutter.json @@ -0,0 +1,35 @@ +{ + "cookiecutter": { + "codeowner_github_usernames": "@mzbroch @scetron @glennmatthews @chadell", + "full_name": "Network to Code, LLC", + "email": "info@networktocode.com", + "github_org": "nautobot", + "plugin_name": "nautobot_device_onboarding", + "verbose_name": "Device Onboarding", + "plugin_slug": "nautobot-device-onboarding", + "project_slug": "nautobot-plugin-device-onboarding", + "repo_url": "https://github.com/nautobot/nautobot-plugin-device-onboarding", + "base_url": "nautobot-device-onboarding", + "min_nautobot_version": "2.0.3", + "max_nautobot_version": "2.9999", + "camel_name": "NautobotDeviceOnboarding", + "project_short_description": "Device Onboarding", + "model_class_name": "None", + "open_source_license": "Apache-2.0", + "docs_base_url": "https://docs.nautobot.com", + "docs_app_url": "https://docs.nautobot.com/projects/device-onboarding/en/latest", + "_drift_manager": { + "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git", + "template_dir": "nautobot-app", + "template_ref": "develop", + "cookie_dir": "", + "branch_prefix": "drift-manager", + "pull_request_strategy": "create", + "post_actions": [ + "black" + ], + "draft": true, + "baked_commit_ref": "094719b3173bd24329c250c54c586b699be5f0c1" + } + } +} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..2270f496 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +# Docker related +development/Dockerfile +development/docker-compose*.yml +development/*.env +*.env +environments/ + +# Python +**/*.pyc +**/*.pyo +**/__pycache__/ +**/.pytest_cache/ +**/.venv/ + + +# Other +docs/_build +FAQ.md +.git/ +.gitignore +.github +tasks.py +LICENSE +**/*.log +**/.vscode/ +invoke*.yml +tasks.py diff --git a/.flake8 b/.flake8 index 1587fc6c..c9f5e84d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,10 @@ [flake8] -# E501: Line length is enforced by Black, so flake8 doesn't need to check it -# W503: Black disagrees with this rule, as does PEP 8; Black wins -ignore = E501, W503 -exclude = - .venv +ignore = + E501, # Line length is enforced by Black, so flake8 doesn't need to check it + W503 # Black disagrees with this rule, as does PEP 8; Black wins +exclude = + migrations, + __pycache__, + manage.py, + settings.py, + .venv diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f3bfd631..2a252e5a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,10 +4,17 @@ about: Report a reproducible bug in the current release of nautobot-device-onboa --- ### Environment -* Python version: -* Nautobot version: +* Python version: +* Nautobot version: * nautobot-device-onboarding version: + +### Expected Behavior + + + +### Observed Behavior + -### Expected Behavior - - - -### Observed Behavior \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index ba609ddd..23b3b625 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -5,7 +5,7 @@ about: Propose a new feature or enhancement --- ### Environment -* Nautobot version: +* Nautobot version: * nautobot-device-onboarding version: + +# Closes: # + +## What's Changed + + + +## To Do + + +- [ ] Explanation of Change(s) +- [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/#creating-changelog-fragments)) +- [ ] Attached Screenshots, Payload Example +- [ ] Unit, Integration Tests +- [ ] Documentation Updates (when adding/changing features) +- [ ] Example Plugin Updates (when adding/changing features) +- [ ] Outline Remaining Work, Constraints from Design diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54e64d6a..d9c7844b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,56 +17,67 @@ env: jobs: black: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: black" run: "poetry run invoke black" bandit: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: bandit" run: "poetry run invoke bandit" pydocstyle: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: pydocstyle" run: "poetry run invoke pydocstyle" flake8: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: flake8" run: "poetry run invoke flake8" + poetry: + runs-on: "ubuntu-22.04" + env: + INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Checking: poetry lock file" + run: "poetry run invoke lock --check" yamllint: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: yamllint" @@ -76,27 +87,28 @@ jobs: - "bandit" - "pydocstyle" - "flake8" + - "poetry" - "yamllint" - "black" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" strategy: fail-fast: true matrix: - python-version: ["3.10"] - nautobot-version: ["2.0.0"] + python-version: ["3.11"] + nautobot-version: ["2.0.3"] env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}" INVOKE_NAUTOBOT_DEVICE_ONBOARDING_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -113,30 +125,80 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" + check-migrations: + needs: + - "bandit" + - "pydocstyle" + - "flake8" + - "poetry" + - "yamllint" + - "black" + runs-on: "ubuntu-22.04" + strategy: + fail-fast: true + matrix: + python-version: ["3.11"] + nautobot-version: ["2.0.3"] + env: + INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}" + INVOKE_NAUTOBOT_DEVICE_ONBOARDING_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Set up Docker Buildx" + id: "buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Build" + uses: "docker/build-push-action@v5" + with: + builder: "${{ steps.buildx.outputs.name }}" + context: "./" + push: false + load: true + tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + file: "./development/Dockerfile" + cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + build-args: | + NAUTOBOT_VER=${{ matrix.nautobot-version }} + PYTHON_VER=${{ matrix.python-version }} + - name: "Copy credentials" + run: "cp development/creds.example.env development/creds.env" + - name: "Checking: migrations" + run: "poetry run invoke check-migrations" unittest: needs: - "pylint" + - "check-migrations" strategy: fail-fast: true matrix: - python-version: ["3.10"] + python-version: ["3.8", "3.11"] db-backend: ["postgresql"] - nautobot-version: ["2.0.0"] - - runs-on: "ubuntu-20.04" + nautobot-version: ["stable"] + include: + - python-version: "3.11" + db-backend: "postgresql" + nautobot-version: "2.0.3" + # - python-version: "3.11" + # db-backend: "mysql" + # nautobot-version: "stable" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}" INVOKE_NAUTOBOT_DEVICE_ONBOARDING_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -160,15 +222,15 @@ jobs: needs: - "unittest" name: "Publish to GitHub" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: - python-version: "3.10" + python-version: "3.11" - name: "Install Python Packages" run: "pip install poetry" - name: "Set env" @@ -189,15 +251,15 @@ jobs: needs: - "unittest" name: "Push Package to PyPI" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v2" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: - python-version: "3.10" + python-version: "3.11" - name: "Install Python Packages" run: "pip install poetry" - name: "Set env" @@ -215,7 +277,7 @@ jobs: needs: - "publish_gh" - "publish_pypi" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" SLACK_MESSAGE: >- @@ -228,7 +290,7 @@ jobs: # ENVs cannot be used directly in job.if. This is a workaround to check # if SLACK_WEBHOOK_URL is present. if: "env.SLACK_WEBHOOK_URL != ''" - uses: "slackapi/slack-github-action@v1.17.0" + uses: "slackapi/slack-github-action@v1" with: payload: | { diff --git a/.github/workflows/rebake.yml b/.github/workflows/rebake.yml new file mode 100644 index 00000000..13d1e3a0 --- /dev/null +++ b/.github/workflows/rebake.yml @@ -0,0 +1,118 @@ +--- +name: "Rebake Cookie" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "latest" + workflow_dispatch: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "latest" +jobs: + rebake: + runs-on: "ubuntu-22.04" + permissions: + actions: "write" + contents: "write" + packages: "read" + pull-requests: "write" + container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:${{ github.event.inputs.drift-manager-tag }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + steps: + - name: "Configure Rebake Arguments" + id: "config" + shell: "bash" + run: | + ARGS='--push' + + if [[ '${{ github.event.inputs.draft }}' == 'true' ]]; then + ARGS="$ARGS --draft" + elif [[ '${{ github.event.inputs.draft }}' == 'false' ]]; then + ARGS="$ARGS --no-draft" + elif [[ '${{ github.event.inputs.draft }}' == '' ]]; then + echo "Using repo default value for --draft" + else + echo "ERROR: Invalid value for draft: '${{ github.event.inputs.draft }}'" + exit 1 + fi + + if [[ '${{ github.event.inputs.pull-request }}' != '' ]]; then + ARGS="$ARGS --pull-request='${{ github.event.inputs.pull-request }}'" + fi + + if [[ '${{ github.event.inputs.template }}' != '' ]]; then + ARGS="$ARGS --template='${{ github.event.inputs.template }}'" + fi + + if [[ '${{ github.event.inputs.template-dir }}' != '' ]]; then + ARGS="$ARGS --template-dir='${{ github.event.inputs.template-dir }}'" + fi + + if [[ '${{ github.event.inputs.template-ref }}' != '' ]]; then + ARGS="$ARGS --template-ref='${{ github.event.inputs.template-ref }}'" + fi + + if [[ '${{ github.event.inputs.cookie }}' == '' ]]; then + ARGS="$ARGS '${{ github.repositoryUrl }}'" + else + ARGS="$ARGS '${{ github.event.inputs.cookie }}'" + fi + + echo "args=$ARGS" >> $GITHUB_OUTPUT + - name: "Rebake" + run: | + python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.args }} diff --git a/.github/workflows/upstream_testing.yml b/.github/workflows/upstream_testing.yml index 5ea5de79..8f382be4 100644 --- a/.github/workflows/upstream_testing.yml +++ b/.github/workflows/upstream_testing.yml @@ -10,4 +10,4 @@ jobs: uses: "nautobot/nautobot/.github/workflows/plugin_upstream_testing_base.yml@develop" with: # Below could potentially be collapsed into a single argument if a concrete relationship between both is enforced invoke_context_name: "NAUTOBOT_DEVICE_ONBOARDING" - plugin_name: "nautobot-device-onboarding" + plugin_name: "nautobot-plugin-device-onboarding" diff --git a/.gitignore b/.gitignore index b63d7ed1..99bbe8a9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ # Swap files *.swp -dist # Byte-compiled / optimized / DLL files __pycache__/ @@ -304,6 +303,7 @@ development/*.txt invoke.yml # Docs -docs/README.md -docs/CHANGELOG.md public +/compose.yaml +/dump.sql +/nautobot_device_onboarding/static/nautobot_device_onboarding/docs diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a9d358ef..9a0a64c7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,7 +14,7 @@ build: mkdocs: configuration: "mkdocs.yml" - # fail_on_warning: true + fail_on_warning: true # Use our docs/requirements.txt during installation. python: diff --git a/.yamllint.yml b/.yamllint.yml index b49e490c..8cc3e9a9 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -10,3 +10,4 @@ rules: quote-type: "double" ignore: | .venv/ + compose.yaml diff --git a/LICENSE b/LICENSE index 27a9f15b..d46cc975 100644 --- a/LICENSE +++ b/LICENSE @@ -1,15 +1,15 @@ -Copyright 2021 Network to Code -Network to Code, LLC +Apache Software License 2.0 + +Copyright (c) 2023, Network to Code, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/README.md b/README.md index bffe47d3..082ce25a 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,15 @@ ## Overview -The `nautobot-device-onboarding` plugin is using the [netmiko](https://github.com/ktbyers/netmiko) and [NAPALM](https://napalm.readthedocs.io/en/latest/) libraries to simplify the onboarding process of a new device into Nautobot down to, in many cases, an *IP Address* and a *Site*. In some cases, the user may also have to specify a specific *Device Platform* and *Device Port*. +The `nautobot-device-onboarding` plugin is using the [netmiko](https://github.com/ktbyers/netmiko) and [NAPALM](https://napalm.readthedocs.io/en/latest/) libraries to simplify the onboarding process of a new device into Nautobot down to, in many cases, an *IP Address* and a *Location*. In some cases, the user may also have to specify a specific *Device Platform* and *Device Port*. -Regardless, the Onboarding Plugin greatly simplifies the onboarding process -by allowing the user to specify a small amount of info and having the plugin populate a much larger amount of device data in Nautobot. +Regardless, the Onboarding Plugin greatly simplifies the onboarding process by allowing the user to specify a small amount of info and having the app populate a much larger amount of device data in Nautobot. ### Screenshots -![Overview](https://raw.githubusercontent.com/nautobot/nautobot-plugin-device-onboarding/develop/docs/images/onboarding_tasks_view.png) +Device Onboarding is a Job that allows you to provide a few required pieces of information and onboard the device. + +![job input](https://raw.githubusercontent.com/nautobot/nautobot-plugin-device-onboarding/develop/docs/images/do_job_inputs.png) ## Try it out! diff --git a/development/Dockerfile b/development/Dockerfile index 5d6a229a..6b2a9793 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -6,11 +6,11 @@ # ------------------------------------------------------------------------------------- # !!! USE CAUTION WHEN MODIFYING LINES BELOW -# Accepts a desired Nautobot version as build argument, default to 1.4.10 -ARG NAUTOBOT_VER="2.0.0" +# Accepts a desired Nautobot version as build argument, default to 2.0.3 +ARG NAUTOBOT_VER="2.0.3" -# Accepts a desired Python version as build argument, default to 3.8 -ARG PYTHON_VER="3.8" +# Accepts a desired Python version as build argument, default to 3.11 +ARG PYTHON_VER="3.11" # Retrieve published development image of Nautobot base which should include most CI dependencies FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} @@ -19,20 +19,21 @@ FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} ARG NAUTOBOT_ROOT=/opt/nautobot ENV prometheus_multiproc_dir=/prom_cache -ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT} +ENV NAUTOBOT_ROOT=${NAUTOBOT_ROOT} +ENV INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL=true # Install Poetry manually via its installer script; # We might be using an older version of Nautobot that includes an older version of Poetry # and CI and local development may have a newer version of Poetry # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary # We also don't need virtual environments in container -RUN curl -sSL https://install.python-poetry.org | python3 - && \ +RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ poetry config virtualenvs.create false # !!! USE CAUTION WHEN MODIFYING LINES ABOVE # ------------------------------------------------------------------------------------- # App-specifc system build/test dependencies. -# +# # Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. # ------------------------------------------------------------------------------------- # --> Start safe to modify section @@ -68,11 +69,13 @@ RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_ # Install all local project as editable, constrained on Nautobot version, to get any additional # direct dependencies of the app -RUN pip install -c constraints.txt -e . +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -e .[all] # Install any dev dependencies frozen from Poetry # Can be improved in Poetry 1.2 which allows `poetry install --only dev` -RUN pip install -c constraints.txt -r poetry_freeze_dev.txt +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -r poetry_freeze_dev.txt COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py # !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/development.env b/development/development.env index 1945f95f..54f0b870 100644 --- a/development/development.env +++ b/development/development.env @@ -7,8 +7,6 @@ NAUTOBOT_BANNER_TOP="Local" NAUTOBOT_CHANGELOG_RETENTION=0 NAUTOBOT_DEBUG=True -NAUTOBOT_DJANGO_EXTENSIONS_ENABLED=True -NAUTOBOT_DJANGO_TOOLBAR_ENABLED=True NAUTOBOT_LOG_LEVEL=DEBUG NAUTOBOT_METRICS_ENABLED=True NAUTOBOT_NAPALM_TIMEOUT=5 diff --git a/development/docker-compose.base.yml b/development/docker-compose.base.yml index 087df10f..01abcedb 100644 --- a/development/docker-compose.base.yml +++ b/development/docker-compose.base.yml @@ -22,13 +22,13 @@ services: db: condition: "service_healthy" <<: - - *nautobot-build - *nautobot-base + - *nautobot-build worker: entrypoint: - "sh" - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env - - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" ## $$ because of docker-compose + - "nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" ## $$ because of docker-compose depends_on: - "nautobot" healthcheck: @@ -38,3 +38,13 @@ services: retries: 3 test: ["CMD", "bash", "-c", "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"] ## $$ because of docker-compose <<: *nautobot-base + beat: + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env + - "nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL" ## $$ because of docker-compose + depends_on: + - "nautobot" + healthcheck: + disable: true + <<: *nautobot-base diff --git a/development/docker-compose.dev.yml b/development/docker-compose.dev.yml index a0294b33..735afa66 100644 --- a/development/docker-compose.dev.yml +++ b/development/docker-compose.dev.yml @@ -12,25 +12,32 @@ services: volumes: - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" - "../:/source" + healthcheck: + test: ["CMD", "true"] # Due to layering, disable: true won't work. Instead, change the test docs: entrypoint: "mkdocs serve -v -a 0.0.0.0:8080" ports: - "8001:8080" volumes: - - "../docs:/source/docs:ro" - - "../mkdocs.yml:/source/mkdocs.yml:ro" + - "../:/source" image: "nautobot-device-onboarding/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" healthcheck: disable: true tty: true worker: + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env + - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" ## $$ because of docker-compose volumes: - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" - "../:/source" - # To expose postgres or redis to the host uncomment the following - # db: - # ports: - # - "5432:5432" - # redis: - # ports: - # - "6379:6379" + healthcheck: + test: ["CMD", "true"] # Due to layering, disable: true won't work. Instead, change the test +# To expose postgres or redis to the host uncomment the following +# postgres: +# ports: +# - "5432:5432" +# redis: +# ports: +# - "6379:6379" diff --git a/development/docker-compose.mysql.yml b/development/docker-compose.mysql.yml index c7fa6a1f..062ada94 100644 --- a/development/docker-compose.mysql.yml +++ b/development/docker-compose.mysql.yml @@ -20,6 +20,7 @@ services: image: "mysql:8" command: - "--default-authentication-plugin=mysql_native_password" + - "--max_connections=1000" env_file: - "development.env" - "creds.env" @@ -27,7 +28,12 @@ services: volumes: - "mysql_data:/var/lib/mysql" healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + test: + - "CMD" + - "mysqladmin" + - "ping" + - "-h" + - "localhost" timeout: "20s" retries: 10 volumes: diff --git a/development/docker-compose.postgres.yml b/development/docker-compose.postgres.yml index 55afdb70..12d1de31 100644 --- a/development/docker-compose.postgres.yml +++ b/development/docker-compose.postgres.yml @@ -7,11 +7,13 @@ services: - "NAUTOBOT_DB_ENGINE=django.db.backends.postgresql" db: image: "postgres:13-alpine" + command: + - "-c" + - "max_connections=200" env_file: - "development.env" - "creds.env" volumes: - # - "./nautobot.sql:/tmp/nautobot.sql" - "postgres_data:/var/lib/postgresql/data" healthcheck: test: "pg_isready --username=$$POSTGRES_USER --dbname=$$POSTGRES_DB" diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 91b0643a..315261a3 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -1,11 +1,24 @@ """Nautobot development configuration file.""" -# pylint: disable=invalid-envvar-default import os import sys -from nautobot.core.settings import * # noqa: F403 -from nautobot.core.settings_funcs import parse_redis_connection +from nautobot.core.settings import * # noqa: F403 # pylint: disable=wildcard-import,unused-wildcard-import +from nautobot.core.settings_funcs import is_truthy, parse_redis_connection +# +# Debug +# + +DEBUG = is_truthy(os.getenv("NAUTOBOT_DEBUG", False)) +_TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" + +if DEBUG and not _TESTING: + DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: True} + + if "debug_toolbar" not in INSTALLED_APPS: # noqa: F405 + INSTALLED_APPS.append("debug_toolbar") # noqa: F405 + if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: # noqa: F405 + MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa: F405 # # Misc. settings @@ -14,6 +27,9 @@ ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS", "").split(" ") SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "") +# +# Database +# nautobot_db_engine = os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql") default_db_settings = { @@ -43,18 +59,28 @@ DATABASES["default"]["OPTIONS"] = {"charset": "utf8mb4"} # -# Debug +# Redis # -DEBUG = True +# The django-redis cache is used to establish concurrent locks using Redis. +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": parse_redis_connection(redis_database=0), + "TIMEOUT": 300, + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + } +} -# Django Debug Toolbar -DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING} +# Redis Cacheops +CACHEOPS_REDIS = parse_redis_connection(redis_database=1) -if DEBUG and "debug_toolbar" not in INSTALLED_APPS: # noqa: F405 - INSTALLED_APPS.append("debug_toolbar") # noqa: F405 -if DEBUG and "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: # noqa: F405 - MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa: F405 +# +# Celery settings are not defined here because they can be overloaded with +# environment variables. By default they use `CACHES["default"]["LOCATION"]`. +# # # Logging @@ -62,10 +88,8 @@ LOG_LEVEL = "DEBUG" if DEBUG else "INFO" -TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" - # Verbose logging during normal development operation, but quiet logging during unit test execution -if not TESTING: +if not _TESTING: LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -101,41 +125,14 @@ } # -# Redis -# - -# The django-redis cache is used to establish concurrent locks using Redis. The -# django-rq settings will use the same instance/database by default. -# -# This "default" server is now used by RQ_QUEUES. -# >> See: nautobot.core.settings.RQ_QUEUES -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": parse_redis_connection(redis_database=0), - "TIMEOUT": 300, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, - } -} - -# RQ_QUEUES is not set here because it just uses the default that gets imported -# up top via `from nautobot.core.settings import *`. - -# Redis Cacheops -CACHEOPS_REDIS = parse_redis_connection(redis_database=1) - -# -# Celery settings are not defined here because they can be overloaded with -# environment variables. By default they use `CACHES["default"]["LOCATION"]`. +# Apps # -# Enable installed plugins. Add the name of each plugin to the list. +# Enable installed Apps. Add the name of each App to the list. PLUGINS = ["nautobot_device_onboarding"] -# Plugins configuration settings. These settings are used by various plugins that the user may have installed. -# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# Apps configuration settings. These settings are used by various Apps that the user may have installed. +# Each key in the dictionary is the name of an installed App and its value is a dictionary of settings. # PLUGINS_CONFIG = { # 'nautobot_device_onboarding': { # 'foo': 'bar', diff --git a/docs/admin/compatibility_matrix.md b/docs/admin/compatibility_matrix.md index 0b615e24..d7cf5d30 100644 --- a/docs/admin/compatibility_matrix.md +++ b/docs/admin/compatibility_matrix.md @@ -6,3 +6,4 @@ | 1.1.2 | 1.2.0 | 1.99 | | 1.2.0 | 1.4.0 | 1.99 | | 2.0.0 | 2.0.0 | 2.99 | +| 3.0.0 | 2.0.3 | 2.99 | diff --git a/docs/admin/install.md b/docs/admin/install.md index 86ef9dc7..21470b62 100644 --- a/docs/admin/install.md +++ b/docs/admin/install.md @@ -1,8 +1,10 @@ # Installing the App in Nautobot +Here you will find detailed instructions on how to **install** and **configure** the App within your Nautobot environment. + ## Prerequisites -- The plugin is compatible with Nautobot 1.2.0 and higher. +- The app is compatible with Nautobot 2.0.3 and higher. - Databases supported: PostgreSQL, MySQL !!! note @@ -12,22 +14,23 @@ #### NAPALM Credentials -The Onboarding Plugin uses NAPALM. You can configure a default NAPALM username and password in `nautobot_config.py`. +The Onboarding App uses NAPALM. You can configure a default NAPALM username and password in `nautobot_config.py`. -When `NAPALM_USERNAME` and `NAPALM_PASSWORD` are configured in `nautobot_config.py`, the user does not have to specify the `Username` and `Password` fields in the Device Onboarding Task, unless they wish to override the values in `nautobot_config.py`: +When `NAPALM_USERNAME`, `NAPALM_PASSWORD` and `NAPALM_ARGS` are configured in `nautobot_config.py`, the user does not have to use the Credentials/SecretGroup fields in the Device Onboarding job, unless they wish to override the values in `nautobot_config.py`: ```python # Credentials that Nautobot will use to authenticate to devices when connecting via NAPALM. NAPALM_USERNAME = "" NAPALM_PASSWORD = "" +NAPALM_ARGS = {"secret": ""} ``` ## Install Guide !!! note - Plugins can be installed manually or using Python's `pip`. See the [nautobot documentation](https://nautobot.readthedocs.io/en/latest/plugins/#install-the-package) for more details. The pip package name for this plugin is [`nautobot-device-onboarding`](https://pypi.org/project/nautobot-device-onboarding/). + App can be installed manually or using Python's `pip`. See the [nautobot documentation](https://nautobot.readthedocs.io/en/latest/plugins/#install-the-package) for more details. The pip package name for this plugin is [`nautobot-device-onboarding`](https://pypi.org/project/nautobot-device-onboarding/). -The plugin is available as a Python package via PyPI and can be installed with `pip`: +The app is available as a Python package via PyPI and can be installed with `pip`: ```shell pip install nautobot-device-onboarding @@ -39,7 +42,7 @@ To ensure Device Onboarding is automatically re-installed during future upgrades echo nautobot-device-onboarding >> local_requirements.txt ``` -Once installed, the plugin needs to be enabled in your Nautobot configuration. The following block of code below shows the additional configuration required to be added to your `nautobot_config.py` file: +Once installed, the app needs to be enabled in your Nautobot configuration. The following block of code below shows the additional configuration required to be added to your `nautobot_config.py` file: - Append `"nautobot_device_onboarding"` to the `PLUGINS` list. - Append the `"nautobot_device_onboarding"` dictionary to the `PLUGINS_CONFIG` dictionary and override any defaults. @@ -55,7 +58,7 @@ PLUGINS = ["nautobot_device_onboarding"] # } ``` -Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache. +Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache: ```shell nautobot-server post_upgrade diff --git a/docs/admin/release_notes/version_1.2.md b/docs/admin/release_notes/version_1.2.md index 0a176883..1ab6b3d8 100644 --- a/docs/admin/release_notes/version_1.2.md +++ b/docs/admin/release_notes/version_1.2.md @@ -10,9 +10,10 @@ This document describes all new features and changes in `nautobot-device-onboard ### Added ### Changed + - [#53](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/53) - Documentation updates - [#87](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/87) - Add OnboardingTask status and failure reason metrics -- ### Fixed + - [#75](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/75) - Fix reverse migration for migrations 0002, 0003 diff --git a/docs/admin/release_notes/version_2.0.md b/docs/admin/release_notes/version_2.0.md index 7a2c0235..f1e512a5 100644 --- a/docs/admin/release_notes/version_2.0.md +++ b/docs/admin/release_notes/version_2.0.md @@ -2,6 +2,12 @@ ## Release Overview +## v2.0.2 (2023-10-16) + +### Fixed + +* Migrations for new + existing installs + ## v2.0.1 (2023-10-11) ### Fixed diff --git a/docs/admin/release_notes/version_3.0.md b/docs/admin/release_notes/version_3.0.md new file mode 100644 index 00000000..659715f2 --- /dev/null +++ b/docs/admin/release_notes/version_3.0.md @@ -0,0 +1,22 @@ +# v3.0 Release Notes + +!!! warning + Nautobot Device Onboarding v2.0.0-2.0.2 contains a vulnerability where the credentials used to log into a device may be visible in clear text on the Job Results page under the Additional Data tab. It is recommended to review all OnbaordingTasks from the affected releases, delete any affeccted JobResults, and upgrade to v3.0.0. For more information please see the full write up on the issue which is available as [a security advisory](https://github.com/nautobot/nautobot-plugin-device-onboarding/security/advisories/GHSA-qf3c-rw9f-jh7v) on the repo. Nautobot Device Onboarding app versions v2.0.0-2.0.2 have been removed for PyPI to ensure all gaps are closed. v2.0.3 is published with disabled functionality and banner message encouraging to upgrade to v3.0.0. [CVE-2023-48700](https://www.cve.org/CVERecord?id=CVE-2023-48700) has been issued for this vulnerability. + +## Release Overview + +## v3.0.0 2023-11-21 + +### Changed + +- [#124](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/124) - Device onboarding is now provided via a Nautobot Job. +- [#124](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/124) - CSV import has changed to a comma separated list of IPs/FQDNs as a job input + +### Fixed + +- [#124](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/124) - Leaking of device credentials if username & password were provided on creation of an instance of the `OnboardingTask` object. + +### Removed + +- [#124](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/124) - Removed all models, UI Views, and API Views from app +- [#124](https://github.com/nautobot/nautobot-plugin-device-onboarding/pull/124) - All data for instances of `OnboardingTask` & `OnboardingDevice` will be removed on upgrade, affected `JobResults` from tasks created while on affected versions should be reviewed & deleted. diff --git a/docs/admin/uninstall.md b/docs/admin/uninstall.md index 0a43c7d2..d32fe859 100644 --- a/docs/admin/uninstall.md +++ b/docs/admin/uninstall.md @@ -1,11 +1,15 @@ # Uninstall the App from Nautobot -## Uninstall Guide - -Remove the configuration you added in `nautobot_config.py` from `PLUGINS` & `PLUGINS_CONFIG`. +Here you will find any steps necessary to cleanly remove the App from your Nautobot environment. ## Database Cleanup -Drop all tables from the plugin: `nautobot_plugin_device_onboarding*`. +Prior to removing the plugin from the `nautobot_config.py`, run the following command to roll back any migration specific to this plugin. + +```shell +nautobot-server migrate nautobot_plugin_device_onboarding zero +``` -Any cleanup operations to ensure the database is clean after the app is removed. +## Remove App configuration + +Remove the configuration you added in `nautobot_config.py` from `PLUGINS` & `PLUGINS_CONFIG`. diff --git a/docs/admin/upgrade.md b/docs/admin/upgrade.md index b0710090..2d358dcc 100644 --- a/docs/admin/upgrade.md +++ b/docs/admin/upgrade.md @@ -1,5 +1,7 @@ # Upgrading the App +Here you will find any steps necessary to upgrade the App in your Nautobot environment. + ## Upgrade Guide -When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this plugin. Execute the command `nautobot-server migrate` from the Nautobot install nautobot/ directory after updating the package. +When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this plugin. Execute the command `nautobot-server post-upgrade` within the runtime environment of your Nautobot installation after updating the `nautobot-device-onboarding` package via `pip`. diff --git a/docs/assets/extra.css b/docs/assets/extra.css index 50884f4a..dfe2e4b1 100644 --- a/docs/assets/extra.css +++ b/docs/assets/extra.css @@ -18,6 +18,15 @@ font-size: 0.7rem; } +/* +* The default max-width is 61rem which does not provide nearly enough space to present code examples or larger tables +*/ +.md-grid { + margin-left: auto; + margin-right: auto; + max-width: 95%; +} + .md-tabs__link { font-size: 0.8rem; } @@ -39,7 +48,7 @@ } img.logo { - height: 100px; + height: 200px; } img.copyright-logo { diff --git a/docs/assets/overrides/partials/copyright.html b/docs/assets/overrides/partials/copyright.html index e0b77e5f..b92cf5e3 100644 --- a/docs/assets/overrides/partials/copyright.html +++ b/docs/assets/overrides/partials/copyright.html @@ -1,3 +1,4 @@ +