Added ingested images #759
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Pytest on Branches | |
| concurrency: | |
| group: ${{github.workflow}}-${{github.ref}} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_STORAGE_ACCOUNT_URL: ${{ secrets.AZURE_STORAGE_ACCOUNT_URL }} | |
| AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} | |
| PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | |
| SLACK_ACCESS_KEY: ${{ secrets.SLACK_ACCESS_KEY }} | |
| DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }} | |
| DROPBOX_APP_SECRET: ${{secrets.DROPBOX_APP_SECRET}} | |
| DROPBOX_REFRESH_TOKEN: ${{secrets.DROPBOX_REFRESH_TOKEN}} | |
| USERNAME_GITHUB: ${{secrets.USERNAME_GITHUB}} | |
| ACCESS_TOKEN_GITHUB: ${{secrets.ACCESS_TOKEN_GITHUB}} | |
| REPOSITORY_NAME_GITHUB: ${{secrets.REPOSITORY_NAME_GITHUB}} | |
| DRIVE_SCOPES: ${{secrets.DRIVE_SCOPES}} | |
| DRIVE_REFRESH_TOKEN: ${{secrets.DRIVE_REFRESH_TOKEN}} | |
| DRIVE_TOKEN: ${{secrets.DRIVE_TOKEN}} | |
| DRIVE_CLIENT_ID: ${{secrets.DRIVE_CLIENT_ID}} | |
| DRIVE_CLIENT_SECRET: ${{secrets.DRIVE_CLIENT_SECRET}} | |
| IMAP_PASSWORD: ${{secrets.IMAP_PASSWORD}} | |
| JIRA_API_TOKEN: ${{secrets.JIRA_API_TOKEN}} | |
| OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} | |
| NEWS_API_KEY: ${{secrets.NEWS_API_KEY}} | |
| GOOGLE_BUCKET_PRIVATE_KEY_ID: ${{secrets.GOOGLE_BUCKET_PRIVATE_KEY_ID}} | |
| GOOGLE_BUCKET_PRIVATE_KEY: ${{secrets.GOOGLE_BUCKET_PRIVATE_KEY}} | |
| GOOGLE_BUCKET_CLIENT_ID: ${{secrets.GOOGLE_BUCKET_CLIENT_ID}} | |
| jobs: | |
| pytest: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Check if requirements.txt has changed | |
| id: check-requirements | |
| run: | | |
| git fetch origin main | |
| git diff --quiet origin/main -- requirements.txt || echo "Requirements changed" | |
| continue-on-error: true | |
| - name: Cache Python dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| if: steps.check-requirements.outputs['check-requirements'] == 'Requirements changed' | |
| - name: Apt install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt install tesseract-ocr | |
| sudo apt install libtesseract-dev | |
| sudo apt-get install ffmpeg | |
| sudo apt install antiword | |
| sudo apt install libreoffice | |
| - id: "auth" | |
| name: "Authenticate to Google Cloud" | |
| uses: "google-github-actions/auth@v1" | |
| with: | |
| credentials_json: "${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}" | |
| - name: "Copy to tmp" | |
| run: | | |
| mkdir -p /tmp/.config/gcloud | |
| cp $GOOGLE_APPLICATION_CREDENTIALS /tmp/.config/gcloud/application_default_credentials.json | |
| - name: "Set up Google Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v0.2.0" | |
| with: | |
| project_id: "protocolstreams-ai" | |
| service_account_key: "${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}" | |
| export_default_credentials: true | |
| - name: Install curl | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install curl -y | |
| sudo apt-get install -y jq | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Cache NLTK punkt | |
| id: cache-nltk | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/nltk_data/tokenizers/punkt | |
| key: nltk-punkt-${{ runner.os }} | |
| restore-keys: | | |
| nltk-punkt-${{ runner.os }} | |
| - name: Download NLTK punkt | |
| run: | | |
| python -m nltk.downloader punkt | |
| python -m nltk.downloader wordnet | |
| python -m nltk.downloader stopwords | |
| - name: Cache spaCy large model | |
| id: cache-spacy | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/spacy | |
| key: spacy-large-${{ runner.os }} | |
| restore-keys: | | |
| spacy-large-${{ runner.os }} | |
| - name: Download spaCy large model | |
| run: | | |
| python -m spacy download en_core_web_lg | |
| # - name: Run Pytest | |
| # run: python -B -m pytest -v --disable-warnings . | |
| # env: | |
| # PYTHONDONTWRITEBYTECODE: 1 | |
| - name: Test Code Coverage | |
| run: pytest -s --cov=querent -vv --durations=0 | |
| env: | |
| PYTHONDONTWRITEBYTECODE: 1 | |
| package: | |
| runs-on: ubuntu-latest | |
| needs: pytest | |
| if: github.ref == 'refs/heads/main' && github.event_name == 'push' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Package | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install --upgrade wheel setuptools | |
| python setup.py sdist bdist_wheel | |
| - name: Publish to PyPI | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install --upgrade twine | |
| python -m twine upload dist/* --skip-existing | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} |