feat: Improve project bootstrapping

src/crawlee/_cli.py

@@ -1,7 +1,7 @@
 # ruff: noqa: TRY301, FBT002, UP007
 from __future__ import annotations
 
-import os
+import json
 from pathlib import Path
 from typing import Annotated, Optional, cast
 
@@ -16,6 +16,11 @@
 
 cli = typer.Typer(no_args_is_help=True)
 
+cookiecutter_json = json.load((Path().parent.parent.parent / 'templates' / 'crawler' / 'cookiecutter.json').open())
+crawler_choices = cookiecutter_json['crawler_type']
+package_manager_choices = cookiecutter_json['package_manager']
+default_start_url = cookiecutter_json['start_url']
+
 
 @cli.callback(invoke_without_command=True)
 def callback(
@@ -64,25 +69,42 @@ def _prompt_for_project_name(initial_project_name: str | None) -> str:
         return project_name
 
 
-def _prompt_for_template() -> str:
-    """Prompt the user to select a template from a list."""
-    # Fetch available templates
-    response = httpx.get(
-        TEMPLATE_LIST_URL,
-        timeout=httpx.Timeout(10),
-        headers=[('Authorization', f'Bearer {os.environ["GH_TOKEN"]}')] if 'GH_TOKEN' in os.environ else [],
+def _prompt_text(message: str, default: str) -> str:
+    return cast(
+        str,
+        ConsoleRender().render(
+            inquirer.Text(
+                name='text',
+                message=message,
+                default=default,
+                validate=lambda _, value: bool(value.strip()),
+            ),
+        ),
     )
-    response.raise_for_status()
-    template_choices = [item['name'] for item in response.json() if item['type'] == 'dir']
 
-    # Prompt for template choice
+
+def _prompt_choice(message: str, choices: list[str]) -> str:
+    """Prompt the user to pick one from a list of choices."""
     return cast(
         str,
         ConsoleRender().render(
             inquirer.List(
-                name='template',
-                message='Please select the template for your new Crawlee project',
-                choices=[(choice[0].upper() + choice[1:], choice) for choice in template_choices],
+                name='choice',
+                message=message,
+                choices=[(choice[0].upper() + choice[1:], choice) for choice in choices],
+            ),
+        ),
+    )
+
+
+def _prompt_bool(message: str, *, default: bool) -> bool:
+    return cast(
+        bool,
+        ConsoleRender().render(
+            inquirer.Confirm(
+                name='confirm',
+                message=message,
+                default=default,
             ),
         ),
     )
@@ -92,26 +114,63 @@ def _prompt_for_template() -> str:
 def create(
     project_name: Optional[str] = typer.Argument(
         default=None,
+        show_default=False,
         help='The name of the project and the directory that will be created to contain it. '
         'If none is given, you will be prompted.',
+    ),
+    crawler_type: Optional[str] = typer.Option(
+        None,
+        '--crawler-type',
+        '--template',
+        show_default=False,
+        help='The library that will be used for crawling in your crawler. If none is given, you will be prompted.',
+    ),
+    package_manager: Optional[str] = typer.Option(
+        default=None,
+        show_default=False,
+        help='Package manager to be used in the new project. If none is given, you will be prompted.',
+    ),
+    start_url: Optional[str] = typer.Option(
+        default=None,
         show_default=False,
+        help='The URL where crawling should start. If none is given, you will be prompted.',
     ),
-    template: Optional[str] = typer.Option(
+    enable_apify_integration: Optional[bool] = typer.Option(
         default=None,
-        help='The template to be used to create the project. If none is given, you will be prompted.',
         show_default=False,
+        help='Should Apify integration be set up for you? If not given, you will be prompted.',
     ),
 ) -> None:
     """Bootstrap a new Crawlee project."""
     try:
         # Prompt for project name if not provided.
         project_name = _prompt_for_project_name(project_name)
 
-        # Prompt for template choice if not provided.
-        if template is None:
-            template = _prompt_for_template()
-
-        if project_name and template:
+        # Prompt for crawler_type if not provided.
+        if crawler_type is None:
+            crawler_type = _prompt_choice('Please select the Crawler type', crawler_choices)
+
+        # Prompt for package manager if not provided.
+        if package_manager is None:
+            package_manager = _prompt_choice('Please select the package manager', package_manager_choices)
+
+        # Prompt for start URL
+        if start_url is None:
+            start_url = _prompt_text('Please specify the start URL', default=default_start_url)
+
+        # Ask about Apify integration if not explicitly configured
+        if enable_apify_integration is None:
+            enable_apify_integration = _prompt_bool('Should Apify integration be set up for you?', default=False)
+
+        if all(
+            [
+                project_name,
+                crawler_type,
+                package_manager,
+                start_url,
+                enable_apify_integration is not None,
+            ]
+        ):
             # Start the bootstrap process.
             with Progress(
                 SpinnerColumn(),
@@ -121,9 +180,14 @@ def create(
                 progress.add_task(description='Bootstrapping...', total=None)
                 cookiecutter(
                     template='gh:apify/crawlee-python',
-                    directory=f'templates/{template}',
+                    directory='templates/crawler',
                     no_input=True,
-                    extra_context={'project_name': project_name},
+                    extra_context={
+                        'project_name': project_name,
+                        'package_manager': package_manager,
+                        'crawler_type': crawler_type,
+                        'enable_apify_integration': enable_apify_integration,
+                    },
                 )
 
             typer.echo(f'Your project "{project_name}" was created.')

templates/crawler/cookiecutter.json

@@ -0,0 +1,12 @@
+{
+    "project_name": "crawlee-python-beautifulsoup-project",
+    "__package_name": "{{ cookiecutter.project_name|lower|replace('-', '_') }}",
+    "crawler_type": ["beautifulsoup", "parsel", "playwright"],
+    "package_manager": ["poetry", "pip"],
+    "enable_apify_integration": false,
+    "start_url": "https://crawlee.dev",
+    "_jinja2_env_vars": {
+        "line_statement_prefix": "# %"
+    },
+    "_extensions": ["jinja2.ext.do"]
+}

templates/crawler/templates/main.py

@@ -0,0 +1,32 @@
+# % if cookiecutter.enable_apify_integration
+from apify import Actor
+# % endif
+# % block import required
+# % endblock
+
+from .routes import router
+
+
+async def main() -> None:
+    """The crawler entry point."""
+    # % filter truncate(0, end='')
+    # % block instantiation required
+    # % endblock
+    # % endfilter
+
+    # % if cookiecutter.enable_apify_integration
+    async with Actor:
+        # % filter indent(width=8, first=False)
+        {{ self.instantiation() }}
+        # % endfilter
+    # % else
+        # % filter indent(width=4, first=False)
+    {{ self.instantiation() }}
+        # % endfilter
+    # % endif
+
+    await crawler.run(
+        [
+            '{{ cookiecutter.start_url }}',
+        ]
+    )

templates/crawler/templates/main_beautifulsoup.py

@@ -0,0 +1,12 @@
+# % extends 'main.py'
+
+# % block import
+from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler
+# % endblock
+
+# % block instantiation
+crawler = BeautifulSoupCrawler(
+    request_handler=router,
+    max_requests_per_crawl=50,
+)
+# % endblock

templates/crawler/templates/main_parsel.py

@@ -0,0 +1,12 @@
+# % extends 'main.py'
+
+# % block import
+from crawlee.parsel_crawler import ParselCrawler
+# % endblock
+
+# % block instantiation
+crawler = ParselCrawler(
+    request_handler=router,
+    max_requests_per_crawl=50,
+)
+# % endblock

templates/crawler/templates/main_playwright.py

@@ -0,0 +1,13 @@
+# % extends 'main.py'
+
+# % block import
+from crawlee.playwright_crawler import PlaywrightCrawler
+# % endblock
+
+# % block instantiation
+crawler = PlaywrightCrawler(
+    request_handler=router,
+    headless=True,
+    max_requests_per_crawl=50,
+)
+# % endblock

templates/crawler/templates/routes_beautifulsoup.py

@@ -0,0 +1,19 @@
+from crawlee.beautifulsoup_crawler import BeautifulSoupCrawlingContext
+from crawlee.router import Router
+
+router = Router[BeautifulSoupCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = context.soup.find('title')
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': title.text if title else None,
+        }
+    )
+
+    await context.enqueue_links()

templates/crawler/templates/routes_parsel.py

@@ -0,0 +1,19 @@
+from crawlee.parsel_crawler import ParselCrawlingContext
+from crawlee.router import Router
+
+router = Router[ParselCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: ParselCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = context.selector.xpath('//title/text()').get()
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': title,
+        }
+    )
+
+    await context.enqueue_links()

templates/crawler/templates/routes_playwright.py

@@ -0,0 +1,19 @@
+from crawlee.playwright_crawler import PlaywrightCrawlingContext
+from crawlee.router import Router
+
+router = Router[PlaywrightCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: PlaywrightCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = await context.page.query_selector('title')
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': await title.inner_text() if title else None,
+        }
+    )
+
+    await context.enqueue_links()

templates/crawler/{{cookiecutter.project_name}}/Dockerfile

@@ -0,0 +1,62 @@
+# First, specify the base Docker image.
+# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
+# You can also use any other image from Docker Hub.
+# % if cookiecutter.crawler_type == 'playwright'
+FROM apify/actor-python-playwright:3.12
+# % else
+FROM apify/actor-python:3.12
+# % endif
+
+RUN apt install -yq git && rm -rf /var/lib/apt/lists/*
+
+# % if cookiecutter.package_manager == 'poetry'
+RUN pip install -U pip setuptools \
+    && pip install poetry \
+    && poetry self add poetry-plugin-export
+
+# Second, copy just poetry.lock and pyproject.toml into the Actor image,
+# since those should be the only files that affects the dependency install in the next step,
+# in order to speed up the build
+COPY pyproject.toml ./
+COPY poetry.lock ./
+
+# Install the dependencies
+RUN echo "Python version:" \
+ && python --version \
+ && echo "Installing dependencies:" \
+ # Export packages from poetry.lock
+ && poetry export -f requirements.txt --without-hashes | \
+ # Replace playwright version so that it matches whatever is pre-installed in the image
+    sed "s/^playwright==.*/playwright==$(playwright --version | cut -d ' ' -f 2)/" | \
+ # Install everything using pip (ignore dependency checks - the lockfile is correct, period)
+    pip install -r /dev/stdin --no-dependencies \
+ && echo "All installed Python packages:" \
+ && pip freeze
+# % elif cookiecutter.package_manager == 'pip'
+RUN pip install -U pip setuptools
+
+# Second, copy just pyproject.toml into the Actor image,
+# since it should be the only file that affects the dependency install in the next step,
+# in order to speed up the build
+COPY pyproject.toml ./
+
+# Install the dependencies
+RUN echo "Python version:" \
+ && python --version \
+ && echo "Installing dependencies:" \
+ # Install everything using pip, set playwright version so that it matches whatever is pre-installed in the image
+    pip install . playwright==$(playwright --version | cut -d ' ' -f 2) \
+ && echo "All installed Python packages:" \
+ && pip freeze
+# % endif
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after installing the dependencies, quick build will be really fast
+# for most source file changes.
+COPY . ./
+
+# Use compileall to ensure the runnability of the Actor Python code.
+RUN python -m compileall -q .
+
+# Specify how to launch the source code of your Actor.
+CMD ["python", "-m", "{{ cookiecutter.__package_name }}"]