Adiciona novo workflow de raspagem fatiada anualmente (#1316)

.github/workflows/daily_crawl.yaml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  schedule-jobs:
+  daily-crawl:
     runs-on: ubuntu-latest
     env:
       SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}

.github/workflows/deploy.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  deploy_to_scrapy_cloud:
+  deploy:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2

.github/workflows/monthly_crawl.yaml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  schedule-jobs:
+  monthly-crawl:
     runs-on: ubuntu-latest
     env:
       SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}

.github/workflows/schedule_spider.yaml

@@ -14,7 +14,7 @@ on:
         required: false
 
 jobs:
-  schedule:
+  full-crawl:
     runs-on: ubuntu-latest
     env:
       SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}

.github/workflows/schedule_spider_by_date.yaml

@@ -8,7 +8,7 @@ on:
         required: true
 
 jobs:
-  schedule:
+  full-crawl-by-date:
     runs-on: ubuntu-latest
     env:
       SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}

.github/workflows/schedule_spider_yearly.yaml

@@ -0,0 +1,76 @@
+name: Schedule Spider Crawl Split Per Year
+
+on:
+  workflow_dispatch:
+    inputs:
+      spider_name:
+        description: 'Spider to be scheduled'
+        required: true
+      start_date:
+        description: 'Start date (YYYY-MM-Dd")'
+        required: true
+      end_date:
+        description: 'End date (YYYY-MM-Dd")'
+        required: false
+
+jobs:
+  full-crawl-yearly:
+    runs-on: ubuntu-latest
+    env:
+      SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}
+      SCRAPY_CLOUD_PROJECT_ID: ${{ secrets.SCRAPY_CLOUD_PROJECT_ID }}
+      FILES_STORE: ${{ secrets.FILES_STORE }}
+      QUERIDODIARIO_DATABASE_URL: ${{ secrets.QUERIDODIARIO_DATABASE_URL }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
+      AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
+      SPIDERMON_DISCORD_FAKE: ${{ secrets.SPIDERMON_DISCORD_FAKE }}
+      SPIDERMON_DISCORD_WEBHOOK_URL: ${{ secrets.SPIDERMON_DISCORD_WEBHOOK_URL }}
+      ZYTE_SMARTPROXY_APIKEY: ${{ secrets.ZYTE_SMARTPROXY_APIKEY }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+    - name: Prepare environment
+      run: |
+        python -m pip install --upgrade pip
+        pip install click python-decouple scrapinghub SQLAlchemy psycopg2     
+    - name: Schedule full crawl per year
+      if: ${{ !github.event.inputs.end_date }}
+      run: |
+        cd data_collection/
+        spider=${{ github.event.inputs.spider_name }}
+        start_date=${{ github.event.inputs.start_date }}
+        int_start_date=$(date -d $start_date +"%Y%m%d")
+        int_end_date=$(date --date="today" +"%Y%m%d")
+        while [[ $int_start_date -lt $int_end_date ]]; do
+            int_date_to=$(date -d"$int_start_date + 1 year" +"%Y%m%d")
+            if [[ $int_date_to -ge $int_end_date ]]; then
+                int_date_to="$int_end_date"
+            fi
+            date_from=$(date -d"$int_start_date" +"%Y-%m-%d")
+            date_to=$(date -d"$int_date_to" +"%Y-%m-%d")
+            python scheduler.py schedule-spider --spider_name="$spider" --start_date="$date_from" --end_date="$date_to"
+            int_start_date="$int_date_to"
+        done
+    - name: Schedule partial crawl per year
+      if: ${{ github.event.inputs.end_date }}
+      run: |
+        cd data_collection/
+        spider=${{ github.event.inputs.spider_name }}
+        start_date=${{ github.event.inputs.start_date }}
+        end_date=${{ github.event.inputs.end_date }}
+        int_start_date=$(date -d $start_date +"%Y%m%d")
+        int_end_date=$(date -d $end_date +"%Y%m%d")
+        while [[ $int_start_date -lt $int_end_date ]]; do
+            int_date_to=$(date -d"$int_start_date + 1 year" +"%Y%m%d")
+            if [[ $int_date_to -ge $int_end_date ]]; then
+                int_date_to="$int_end_date"
+            fi
+            date_from=$(date -d"$int_start_date" +"%Y-%m-%d")
+            date_to=$(date -d"$int_date_to" +"%Y-%m-%d")
+            python scheduler.py schedule-spider --spider_name="$spider" --start_date="$date_from" --end_date="$date_to"
+            int_start_date="$int_date_to"
+        done

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: Run tests
 on: [ push, pull_request ]
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

.github/workflows/update_spider_status.yaml

@@ -15,7 +15,7 @@ on:
         required: true
 
 jobs:
-  update_status:
+  update-status:
     runs-on: ubuntu-latest
     env:
       QUERIDODIARIO_DATABASE_URL: ${{ secrets.QUERIDODIARIO_DATABASE_URL }}