Annotate typer.Option types as Optional if they have a default of Non…

…e. ignore use of Optional in ruff config since typer relies on it.
CybersecurityForDemocracy · Aug 19, 2024 · dc2e560 · dc2e560
1 parent 00757ef
commit dc2e560
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 71 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -96,6 +96,8 @@ select = [
 ignore=[
   # Do not suggest ternary operator
   "SIM108",
+  # Allow use of Optional[...] type, since Typer relies on it
+  "UP007",
 ]
 
 [tool.hatch.build.targets.sdist]

diff --git a/src/tiktok_research_api_helper/cli/custom_argument_types.py b/src/tiktok_research_api_helper/cli/custom_argument_types.py
@@ -21,7 +21,7 @@
 ]
 
 DBFileType = Annotated[
-    Path,
+    Path | None,
     typer.Option(
         exists=False,
         file_okay=True,
@@ -30,17 +30,18 @@
     ),
 ]
 
-DBUrlType = Annotated[str, typer.Option(help="database URL for storing API results")]
+DBUrlType = Annotated[str | None, typer.Option(help="database URL for storing API results")]
 
 JsonQueryFileType = Annotated[
-    Path,
+    Path | None,
     typer.Option(
         exists=True,
         file_okay=True,
         dir_okay=False,
         help=(
             "Path to file query as JSON. File contents will be parsed as JSON and used directly "
-            "in query of API requests."
+            "in query of API requests. Can be used multiple times to run multiple queries (in "
+            "serial)"
         ),
     ),
 ]
@@ -79,10 +80,10 @@
     ),
 ]
 
-RegionCodeListType = Annotated[list[SupportedRegions], typer.Option()]
+RegionCodeListType = Annotated[list[SupportedRegions] | None, typer.Option()]
 
 IncludeAnyHashtagListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of hashtag names. Will query API for videos that have "
@@ -92,7 +93,7 @@
 ]
 
 ExcludeAnyHashtagListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of hashtag names. Will query API for videos that DO NOT "
@@ -102,7 +103,7 @@
 ]
 
 IncludeAllHashtagListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of hashtag names. Will query API for videos that have "
@@ -112,7 +113,7 @@
 ]
 
 ExcludeAllHashtagListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of hashtag names. Will query API for videos that DO NOT "
@@ -122,7 +123,7 @@
 ]
 
 IncludeAnyKeywordListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of keywords. Will query API for videos that have "
@@ -132,7 +133,7 @@
 ]
 
 ExcludeAnyKeywordListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of keywords. Will query API for videos that DO NOT "
@@ -142,7 +143,7 @@
 ]
 
 IncludeAllKeywordListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of keywords. Will query API for videos that have "
@@ -152,7 +153,7 @@
 ]
 
 ExcludeAllKeywordListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of keywords. Will query API for videos that DO NOT "
@@ -162,7 +163,7 @@
 ]
 
 OnlyUsernamesListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of usernames. Will query API for videos that have "
@@ -172,7 +173,7 @@
 ]
 
 ExcludeUsernamesListType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "A comma separated list of usernames. Will query API for videos that DO NOT "
@@ -181,10 +182,10 @@
     ),
 ]
 
-VideoIdType = Annotated[int, typer.Option(help="ID of specific video to query for.")]
+VideoIdType = Annotated[int | None, typer.Option(help="ID of specific video to query for.")]
 
 CrawlTagType = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "Extra metadata for tagging the crawl of the data with a name (e.g. "
@@ -218,7 +219,7 @@
 ]
 
 MaxApiRequests = Annotated[
-    int,
+    int | None,
     typer.Option(
         help=(
             "maximum number of requests to send to the API. If this limit is reached crawl/fetch "
@@ -231,7 +232,7 @@
 EnableDebugLoggingFlag = Annotated[bool, typer.Option(help="Enable debug level logging")]
 
 CatchupFromStartDate = Annotated[
-    str,
+    str | None,
     typer.Option(
         help=(
             "Date from which to attempt to catch up from for run-repeated. IE start crawling at "

diff --git a/src/tiktok_research_api_helper/cli/main.py b/src/tiktok_research_api_helper/cli/main.py
@@ -149,7 +149,7 @@ def test(
         "API client config: %s\nVideo query config: %s", api_client_config, video_query_config
     )
 
-    driver_single_day(api_client_config, video_query_config)
+    main_driver(api_client_config, video_query_config)
 
 
 def get_query_file_json(query_file: Path):
@@ -189,17 +189,17 @@ def validate_region_code_flag_value(region_code_list: Sequence[str] | None):
 @APP.command()
 def print_query(
     region: RegionCodeListType = None,
-    include_any_hashtags: IncludeAnyHashtagListType | None = None,
-    exclude_any_hashtags: ExcludeAnyHashtagListType | None = None,
-    include_all_hashtags: IncludeAllHashtagListType | None = None,
-    exclude_all_hashtags: ExcludeAllHashtagListType | None = None,
-    include_any_keywords: IncludeAnyKeywordListType | None = None,
-    exclude_any_keywords: ExcludeAnyKeywordListType | None = None,
-    include_all_keywords: IncludeAllKeywordListType | None = None,
-    exclude_all_keywords: ExcludeAllKeywordListType | None = None,
-    only_from_usernames: OnlyUsernamesListType | None = None,
-    exclude_from_usernames: ExcludeUsernamesListType | None = None,
-    video_id: VideoIdType | None = None,
+    include_any_hashtags: IncludeAnyHashtagListType = None,
+    exclude_any_hashtags: ExcludeAnyHashtagListType = None,
+    include_all_hashtags: IncludeAllHashtagListType = None,
+    exclude_all_hashtags: ExcludeAllHashtagListType = None,
+    include_any_keywords: IncludeAnyKeywordListType = None,
+    exclude_any_keywords: ExcludeAnyKeywordListType = None,
+    include_all_keywords: IncludeAllKeywordListType = None,
+    exclude_all_keywords: ExcludeAllKeywordListType = None,
+    only_from_usernames: OnlyUsernamesListType = None,
+    exclude_from_usernames: ExcludeUsernamesListType = None,
+    video_id: VideoIdType = None,
 ) -> None:
     """Prints to stdout the query generated from flags. Useful for creating a base from which to
     build more complex custom JSON queries."""
@@ -307,29 +307,29 @@ def run_repeated(
         ),
     ] = 1,
     repeat_interval: Annotated[int, typer.Option(help="How many days between crawls.")] = 1,
-    db_file: DBFileType | None = None,
-    db_url: DBUrlType | None = None,
-    crawl_tag: CrawlTagType | None = None,
-    raw_responses_output_dir: RawResponsesOutputDir | None = None,
-    query_file_json: JsonQueryFileType | None = None,
+    db_file: DBFileType = None,
+    db_url: DBUrlType = None,
+    crawl_tag: CrawlTagType = None,
+    raw_responses_output_dir: RawResponsesOutputDir = None,
+    query_file_json: JsonQueryFileType = None,
     api_credentials_file: ApiCredentialsFileType = _DEFAULT_CREDENTIALS_FILE_PATH,
     rate_limit_wait_strategy: ApiRateLimitWaitStrategyType = (
         ApiRateLimitWaitStrategy.WAIT_FOUR_HOURS
     ),
     region: RegionCodeListType = None,
-    include_any_hashtags: IncludeAnyHashtagListType | None = None,
-    exclude_any_hashtags: ExcludeAnyHashtagListType | None = None,
-    include_all_hashtags: IncludeAllHashtagListType | None = None,
-    exclude_all_hashtags: ExcludeAllHashtagListType | None = None,
-    include_any_keywords: IncludeAnyKeywordListType | None = None,
-    exclude_any_keywords: ExcludeAnyKeywordListType | None = None,
-    include_all_keywords: IncludeAllKeywordListType | None = None,
-    exclude_all_keywords: ExcludeAllKeywordListType | None = None,
-    only_from_usernames: OnlyUsernamesListType | None = None,
-    exclude_from_usernames: ExcludeUsernamesListType | None = None,
-    fetch_user_info: FetchUserInfoFlag | None = None,
-    fetch_comments: FetchCommentsFlag | None = None,
-    catch_up_from_start_date: CatchupFromStartDate | None = None,
+    include_any_hashtags: IncludeAnyHashtagListType = None,
+    exclude_any_hashtags: ExcludeAnyHashtagListType = None,
+    include_all_hashtags: IncludeAllHashtagListType = None,
+    exclude_all_hashtags: ExcludeAllHashtagListType = None,
+    include_any_keywords: IncludeAnyKeywordListType = None,
+    exclude_any_keywords: ExcludeAnyKeywordListType = None,
+    include_all_keywords: IncludeAllKeywordListType = None,
+    exclude_all_keywords: ExcludeAllKeywordListType = None,
+    only_from_usernames: OnlyUsernamesListType = None,
+    exclude_from_usernames: ExcludeUsernamesListType = None,
+    fetch_user_info: FetchUserInfoFlag = False,
+    fetch_comments: FetchCommentsFlag = False,
+    catch_up_from_start_date: CatchupFromStartDate = None,
     debug: EnableDebugLoggingFlag = False,
 ) -> None:
     """
@@ -439,32 +439,32 @@ def run(
     # breaks the documentation of CLI Arguments for some reason
     start_date_str: TikTokStartDateFormat,
     end_date_str: TikTokEndDateFormat,
-    db_file: DBFileType | None = None,
-    db_url: DBUrlType | None = None,
+    db_file: DBFileType = None,
+    db_url: DBUrlType = None,
     stop_after_one_request: StopAfterOneRequestFlag = False,
-    max_api_requests: MaxApiRequests | None = None,
+    max_api_requests: MaxApiRequests = None,
     max_days_per_query: MaxDaysPerQueryType = _MAX_DAYS_PER_QUERY_DEFAULT,
-    crawl_tag: CrawlTagType | None = None,
-    raw_responses_output_dir: RawResponsesOutputDir | None = None,
-    query_file_json: JsonQueryFileType | None = None,
+    crawl_tag: CrawlTagType = None,
+    raw_responses_output_dir: RawResponsesOutputDir = None,
+    query_file_json: JsonQueryFileType = None,
     api_credentials_file: ApiCredentialsFileType = _DEFAULT_CREDENTIALS_FILE_PATH,
     rate_limit_wait_strategy: ApiRateLimitWaitStrategyType = (
         ApiRateLimitWaitStrategy.WAIT_FOUR_HOURS
     ),
-    region: RegionCodeListType | None = None,
-    include_any_hashtags: IncludeAnyHashtagListType | None = None,
-    exclude_any_hashtags: ExcludeAnyHashtagListType | None = None,
-    include_all_hashtags: IncludeAllHashtagListType | None = None,
-    exclude_all_hashtags: ExcludeAllHashtagListType | None = None,
-    include_any_keywords: IncludeAnyKeywordListType | None = None,
-    exclude_any_keywords: ExcludeAnyKeywordListType | None = None,
-    include_all_keywords: IncludeAllKeywordListType | None = None,
-    exclude_all_keywords: ExcludeAllKeywordListType | None = None,
-    only_from_usernames: OnlyUsernamesListType | None = None,
-    exclude_from_usernames: ExcludeUsernamesListType | None = None,
-    video_id: VideoIdType | None = None,
-    fetch_user_info: FetchUserInfoFlag | None = None,
-    fetch_comments: FetchCommentsFlag | None = None,
+    region: RegionCodeListType = None,
+    include_any_hashtags: IncludeAnyHashtagListType = None,
+    exclude_any_hashtags: ExcludeAnyHashtagListType = None,
+    include_all_hashtags: IncludeAllHashtagListType = None,
+    exclude_all_hashtags: ExcludeAllHashtagListType = None,
+    include_any_keywords: IncludeAnyKeywordListType = None,
+    exclude_any_keywords: ExcludeAnyKeywordListType = None,
+    include_all_keywords: IncludeAllKeywordListType = None,
+    exclude_all_keywords: ExcludeAllKeywordListType = None,
+    only_from_usernames: OnlyUsernamesListType = None,
+    exclude_from_usernames: ExcludeUsernamesListType = None,
+    video_id: VideoIdType = None,
+    fetch_user_info: FetchUserInfoFlag = False,
+    fetch_comments: FetchCommentsFlag = False,
     debug: EnableDebugLoggingFlag = False,
     # Skips logging init/setup. Hidden because this is intended for other commands that setup
     # logging and then call this as a function.
@@ -597,8 +597,6 @@ def run(
             exclude_from_usernames=exclude_from_usernames,
         )
 
-    logging.log(logging.INFO, f"VideoQuery: {query}")
-
     if db_url:
         engine = get_engine_and_create_tables(db_url)
     elif db_file: