fix: python templates use SDKv2 and Crawlee templates are added (#288)

Merging is blocked until SDK v2 is released. Also do not forget to revert tmp changes in `test/templates.test.js`.
apify · Sep 10, 2024 · cef15b5 · cef15b5
1 parent e350073
commit cef15b5
Show file tree

Hide file tree

Showing 84 changed files with 3,356 additions and 651 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,22 @@
+# IDE
 .idea
 .DS_Store
 .vscode
-node_modules
+
+# General
+templates/**/storage
 tmp
-templates/**/package-lock.json
 test-actors
+
+# JS
+node_modules
+templates/**/package-lock.json
+
+# Python
+__pycache__
+.mypy_cache
+.pytest_cache
+.ruff_cache
+pyproject.toml
+.venv
+venv
diff --git a/templates/manifest.json b/templates/manifest.json
@@ -10,7 +10,6 @@
             "technologies": [
                 "scrapy"
             ],
-            "skipTests": true,
             "description": "This example Scrapy spider scrapes page titles from URLs defined in input parameter. It shows how to use Apify SDK for Python and Scrapy pipelines to save results.",
             "messages": {
                 "postCreate": "To install additional Python packages, you need to activate the virtual environment in the \".venv\" folder in the actor directory."
@@ -147,7 +146,7 @@
         },
         {
             "id": "python-empty",
-            "name": "python_empty",
+            "name": "python-empty",
             "label": "Empty Python project",
             "category": "python",
             "description": "Empty template with basic structure for the Actor with Apify SDK that allows you to easily add your own functionality.",
@@ -167,7 +166,7 @@
         },
         {
             "id": "python-standby",
-            "name": "python_standby",
+            "name": "python-standby",
             "label": "Standby Python project",
             "category": "python",
             "description": "Template with basic structure for an Actor using Standby mode that allows you to easily add your own functionality.",
@@ -186,6 +185,68 @@
                 "STARTER"
             ]
         },
+        {
+            "id": "python-crawlee-beautifulsoup",
+            "name": "python-crawlee-beautifulsoup",
+            "label": "Start with Python Crawlee and BeautifulSoup",
+            "category": "python",
+            "technologies": [
+                "crawlee",
+                "beautifulsoup"
+            ],
+            "description": "Crawl and scrape websites using Crawlee and BeautifulSoup. Start from a given start URLs, and store results to Apify dataset.",
+            "messages": {
+                "postCreate": "To install additional Python packages, you need to activate the virtual environment in the \".venv\" folder in the actor directory."
+            },
+            "archiveUrl": "https://github.com/apify/actor-templates/blob/master/dist/templates/python-crawlee-beautifulsoup.zip?raw=true",
+            "defaultRunOptions": {
+                "build": "latest",
+                "memoryMbytes": 1024,
+                "timeoutSecs": 3600
+            },
+            "aliases": [
+                "getting_started_crawlee_beautifulsoup"
+            ],
+            "showcaseFiles": [
+                "src/main.py",
+                "src/__main__.py"
+            ],
+            "useCases": [
+                "STARTER",
+                "WEB_SCRAPING"
+            ]
+        },
+        {
+            "id": "python-crawlee-playwright",
+            "name": "python-crawlee-playwright",
+            "label": "Start with Python Crawlee and Playwright",
+            "category": "python",
+            "technologies": [
+                "crawlee",
+                "playwright"
+            ],
+            "description": "Crawl and scrape websites using Crawlee and Playwright. Start from a given start URLs, and store results to Apify dataset.",
+            "messages": {
+                "postCreate": "To install additional Python packages, you need to activate the virtual environment in the \".venv\" folder in the actor directory."
+            },
+            "archiveUrl": "https://github.com/apify/actor-templates/blob/master/dist/templates/python-crawlee-playwright.zip?raw=true",
+            "defaultRunOptions": {
+                "build": "latest",
+                "memoryMbytes": 1024,
+                "timeoutSecs": 3600
+            },
+            "aliases": [
+                "getting_started_crawlee_playwright"
+            ],
+            "showcaseFiles": [
+                "src/main.py",
+                "src/__main__.py"
+            ],
+            "useCases": [
+                "STARTER",
+                "WEB_SCRAPING"
+            ]
+        },
         {
             "id": "js-start",
             "name": "getting_started_node",

diff --git a/templates/python-beautifulsoup/.actor/Dockerfile b/templates/python-beautifulsoup/.actor/Dockerfile
@@ -1,7 +1,7 @@
 # First, specify the base Docker image.
 # You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
 # You can also use any other image from Docker Hub.
-FROM apify/actor-python:3.11
+FROM apify/actor-python:3.12
 
 # Second, copy just requirements.txt into the Actor image,
 # since it should be the only file that affects the dependency install in the next step,

diff --git a/templates/python-beautifulsoup/.actor/actor.json b/templates/python-beautifulsoup/.actor/actor.json
@@ -4,39 +4,10 @@
     "title": "Getting started with Python and BeautifulSoup",
     "description": "Scrapes titles of websites using BeautifulSoup.",
     "version": "0.0",
+    "buildTag": "latest",
     "meta": {
         "templateId": "python-beautifulsoup"
     },
     "input": "./input_schema.json",
-    "dockerfile": "./Dockerfile",
-    "storages": {
-        "dataset": {
-            "actorSpecification": 1,
-            "title": "URLs and their titles",
-            "views": {
-                "titles": {
-                    "title": "URLs and their titles",
-                    "transformation": {
-                        "fields": [
-                            "url",
-                            "title"
-                        ]
-                    },
-                    "display": {
-                        "component": "table",
-                        "properties": {
-                            "url": {
-                                "label": "URL",
-                                "format": "text"
-                            },
-                            "title": {
-                                "label": "Title",
-                                "format": "text"
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
+    "dockerfile": "./Dockerfile"
 }
diff --git a/templates/python-beautifulsoup/.dockerignore b/templates/python-beautifulsoup/.dockerignore
@@ -1,13 +1,155 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
+.git
+.mise.toml
+.nvim.lua
 storage
 
-# installed files
+# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
 .venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
 
-# git folder
-.git
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
diff --git a/templates/python-beautifulsoup/.editorconfig b/templates/python-beautifulsoup/.editorconfig