diff --git a/pyproject.toml b/pyproject.toml index cda8040..c284471 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,8 @@ select = [ ignore=[ # Do not suggest ternary operator "SIM108", + # Allow use of Optional[...] type, since Typer relies on it + "UP007", ] [tool.hatch.build.targets.sdist] diff --git a/src/tiktok_research_api_helper/cli/custom_argument_types.py b/src/tiktok_research_api_helper/cli/custom_argument_types.py index 8ffdd6b..6df872c 100644 --- a/src/tiktok_research_api_helper/cli/custom_argument_types.py +++ b/src/tiktok_research_api_helper/cli/custom_argument_types.py @@ -21,7 +21,7 @@ ] DBFileType = Annotated[ - Path, + Path | None, typer.Option( exists=False, file_okay=True, @@ -30,17 +30,18 @@ ), ] -DBUrlType = Annotated[str, typer.Option(help="database URL for storing API results")] +DBUrlType = Annotated[str | None, typer.Option(help="database URL for storing API results")] JsonQueryFileType = Annotated[ - Path, + Path | None, typer.Option( exists=True, file_okay=True, dir_okay=False, help=( "Path to file query as JSON. File contents will be parsed as JSON and used directly " - "in query of API requests." + "in query of API requests. Can be used multiple times to run multiple queries (in " + "serial)" ), ), ] @@ -79,10 +80,10 @@ ), ] -RegionCodeListType = Annotated[list[SupportedRegions], typer.Option()] +RegionCodeListType = Annotated[list[SupportedRegions] | None, typer.Option()] IncludeAnyHashtagListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of hashtag names. Will query API for videos that have " @@ -92,7 +93,7 @@ ] ExcludeAnyHashtagListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of hashtag names. Will query API for videos that DO NOT " @@ -102,7 +103,7 @@ ] IncludeAllHashtagListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of hashtag names. Will query API for videos that have " @@ -112,7 +113,7 @@ ] ExcludeAllHashtagListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of hashtag names. Will query API for videos that DO NOT " @@ -122,7 +123,7 @@ ] IncludeAnyKeywordListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of keywords. Will query API for videos that have " @@ -132,7 +133,7 @@ ] ExcludeAnyKeywordListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of keywords. Will query API for videos that DO NOT " @@ -142,7 +143,7 @@ ] IncludeAllKeywordListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of keywords. Will query API for videos that have " @@ -152,7 +153,7 @@ ] ExcludeAllKeywordListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of keywords. Will query API for videos that DO NOT " @@ -162,7 +163,7 @@ ] OnlyUsernamesListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of usernames. Will query API for videos that have " @@ -172,7 +173,7 @@ ] ExcludeUsernamesListType = Annotated[ - str, + str | None, typer.Option( help=( "A comma separated list of usernames. Will query API for videos that DO NOT " @@ -181,10 +182,10 @@ ), ] -VideoIdType = Annotated[int, typer.Option(help="ID of specific video to query for.")] +VideoIdType = Annotated[int | None, typer.Option(help="ID of specific video to query for.")] CrawlTagType = Annotated[ - str, + str | None, typer.Option( help=( "Extra metadata for tagging the crawl of the data with a name (e.g. " @@ -218,7 +219,7 @@ ] MaxApiRequests = Annotated[ - int, + int | None, typer.Option( help=( "maximum number of requests to send to the API. If this limit is reached crawl/fetch " @@ -231,7 +232,7 @@ EnableDebugLoggingFlag = Annotated[bool, typer.Option(help="Enable debug level logging")] CatchupFromStartDate = Annotated[ - str, + str | None, typer.Option( help=( "Date from which to attempt to catch up from for run-repeated. IE start crawling at " diff --git a/src/tiktok_research_api_helper/cli/main.py b/src/tiktok_research_api_helper/cli/main.py index 8ab0af3..16f9cbb 100644 --- a/src/tiktok_research_api_helper/cli/main.py +++ b/src/tiktok_research_api_helper/cli/main.py @@ -149,7 +149,7 @@ def test( "API client config: %s\nVideo query config: %s", api_client_config, video_query_config ) - driver_single_day(api_client_config, video_query_config) + main_driver(api_client_config, video_query_config) def get_query_file_json(query_file: Path): @@ -189,17 +189,17 @@ def validate_region_code_flag_value(region_code_list: Sequence[str] | None): @APP.command() def print_query( region: RegionCodeListType = None, - include_any_hashtags: IncludeAnyHashtagListType | None = None, - exclude_any_hashtags: ExcludeAnyHashtagListType | None = None, - include_all_hashtags: IncludeAllHashtagListType | None = None, - exclude_all_hashtags: ExcludeAllHashtagListType | None = None, - include_any_keywords: IncludeAnyKeywordListType | None = None, - exclude_any_keywords: ExcludeAnyKeywordListType | None = None, - include_all_keywords: IncludeAllKeywordListType | None = None, - exclude_all_keywords: ExcludeAllKeywordListType | None = None, - only_from_usernames: OnlyUsernamesListType | None = None, - exclude_from_usernames: ExcludeUsernamesListType | None = None, - video_id: VideoIdType | None = None, + include_any_hashtags: IncludeAnyHashtagListType = None, + exclude_any_hashtags: ExcludeAnyHashtagListType = None, + include_all_hashtags: IncludeAllHashtagListType = None, + exclude_all_hashtags: ExcludeAllHashtagListType = None, + include_any_keywords: IncludeAnyKeywordListType = None, + exclude_any_keywords: ExcludeAnyKeywordListType = None, + include_all_keywords: IncludeAllKeywordListType = None, + exclude_all_keywords: ExcludeAllKeywordListType = None, + only_from_usernames: OnlyUsernamesListType = None, + exclude_from_usernames: ExcludeUsernamesListType = None, + video_id: VideoIdType = None, ) -> None: """Prints to stdout the query generated from flags. Useful for creating a base from which to build more complex custom JSON queries.""" @@ -307,29 +307,29 @@ def run_repeated( ), ] = 1, repeat_interval: Annotated[int, typer.Option(help="How many days between crawls.")] = 1, - db_file: DBFileType | None = None, - db_url: DBUrlType | None = None, - crawl_tag: CrawlTagType | None = None, - raw_responses_output_dir: RawResponsesOutputDir | None = None, - query_file_json: JsonQueryFileType | None = None, + db_file: DBFileType = None, + db_url: DBUrlType = None, + crawl_tag: CrawlTagType = None, + raw_responses_output_dir: RawResponsesOutputDir = None, + query_file_json: JsonQueryFileType = None, api_credentials_file: ApiCredentialsFileType = _DEFAULT_CREDENTIALS_FILE_PATH, rate_limit_wait_strategy: ApiRateLimitWaitStrategyType = ( ApiRateLimitWaitStrategy.WAIT_FOUR_HOURS ), region: RegionCodeListType = None, - include_any_hashtags: IncludeAnyHashtagListType | None = None, - exclude_any_hashtags: ExcludeAnyHashtagListType | None = None, - include_all_hashtags: IncludeAllHashtagListType | None = None, - exclude_all_hashtags: ExcludeAllHashtagListType | None = None, - include_any_keywords: IncludeAnyKeywordListType | None = None, - exclude_any_keywords: ExcludeAnyKeywordListType | None = None, - include_all_keywords: IncludeAllKeywordListType | None = None, - exclude_all_keywords: ExcludeAllKeywordListType | None = None, - only_from_usernames: OnlyUsernamesListType | None = None, - exclude_from_usernames: ExcludeUsernamesListType | None = None, - fetch_user_info: FetchUserInfoFlag | None = None, - fetch_comments: FetchCommentsFlag | None = None, - catch_up_from_start_date: CatchupFromStartDate | None = None, + include_any_hashtags: IncludeAnyHashtagListType = None, + exclude_any_hashtags: ExcludeAnyHashtagListType = None, + include_all_hashtags: IncludeAllHashtagListType = None, + exclude_all_hashtags: ExcludeAllHashtagListType = None, + include_any_keywords: IncludeAnyKeywordListType = None, + exclude_any_keywords: ExcludeAnyKeywordListType = None, + include_all_keywords: IncludeAllKeywordListType = None, + exclude_all_keywords: ExcludeAllKeywordListType = None, + only_from_usernames: OnlyUsernamesListType = None, + exclude_from_usernames: ExcludeUsernamesListType = None, + fetch_user_info: FetchUserInfoFlag = False, + fetch_comments: FetchCommentsFlag = False, + catch_up_from_start_date: CatchupFromStartDate = None, debug: EnableDebugLoggingFlag = False, ) -> None: """ @@ -439,32 +439,32 @@ def run( # breaks the documentation of CLI Arguments for some reason start_date_str: TikTokStartDateFormat, end_date_str: TikTokEndDateFormat, - db_file: DBFileType | None = None, - db_url: DBUrlType | None = None, + db_file: DBFileType = None, + db_url: DBUrlType = None, stop_after_one_request: StopAfterOneRequestFlag = False, - max_api_requests: MaxApiRequests | None = None, + max_api_requests: MaxApiRequests = None, max_days_per_query: MaxDaysPerQueryType = _MAX_DAYS_PER_QUERY_DEFAULT, - crawl_tag: CrawlTagType | None = None, - raw_responses_output_dir: RawResponsesOutputDir | None = None, - query_file_json: JsonQueryFileType | None = None, + crawl_tag: CrawlTagType = None, + raw_responses_output_dir: RawResponsesOutputDir = None, + query_file_json: JsonQueryFileType = None, api_credentials_file: ApiCredentialsFileType = _DEFAULT_CREDENTIALS_FILE_PATH, rate_limit_wait_strategy: ApiRateLimitWaitStrategyType = ( ApiRateLimitWaitStrategy.WAIT_FOUR_HOURS ), - region: RegionCodeListType | None = None, - include_any_hashtags: IncludeAnyHashtagListType | None = None, - exclude_any_hashtags: ExcludeAnyHashtagListType | None = None, - include_all_hashtags: IncludeAllHashtagListType | None = None, - exclude_all_hashtags: ExcludeAllHashtagListType | None = None, - include_any_keywords: IncludeAnyKeywordListType | None = None, - exclude_any_keywords: ExcludeAnyKeywordListType | None = None, - include_all_keywords: IncludeAllKeywordListType | None = None, - exclude_all_keywords: ExcludeAllKeywordListType | None = None, - only_from_usernames: OnlyUsernamesListType | None = None, - exclude_from_usernames: ExcludeUsernamesListType | None = None, - video_id: VideoIdType | None = None, - fetch_user_info: FetchUserInfoFlag | None = None, - fetch_comments: FetchCommentsFlag | None = None, + region: RegionCodeListType = None, + include_any_hashtags: IncludeAnyHashtagListType = None, + exclude_any_hashtags: ExcludeAnyHashtagListType = None, + include_all_hashtags: IncludeAllHashtagListType = None, + exclude_all_hashtags: ExcludeAllHashtagListType = None, + include_any_keywords: IncludeAnyKeywordListType = None, + exclude_any_keywords: ExcludeAnyKeywordListType = None, + include_all_keywords: IncludeAllKeywordListType = None, + exclude_all_keywords: ExcludeAllKeywordListType = None, + only_from_usernames: OnlyUsernamesListType = None, + exclude_from_usernames: ExcludeUsernamesListType = None, + video_id: VideoIdType = None, + fetch_user_info: FetchUserInfoFlag = False, + fetch_comments: FetchCommentsFlag = False, debug: EnableDebugLoggingFlag = False, # Skips logging init/setup. Hidden because this is intended for other commands that setup # logging and then call this as a function. @@ -597,8 +597,6 @@ def run( exclude_from_usernames=exclude_from_usernames, ) - logging.log(logging.INFO, f"VideoQuery: {query}") - if db_url: engine = get_engine_and_create_tables(db_url) elif db_file: