From ace253e1017a2800a4925e96fa53afd05cdc781e Mon Sep 17 00:00:00 2001 From: Gianfranco Rossi Date: Wed, 21 Aug 2024 12:27:59 -0500 Subject: [PATCH] refactor(cl_scrape_opinions): renamed variable used in logging --- cl/scrapers/management/commands/cl_back_scrape_citations.py | 2 ++ .../management/commands/cl_back_scrape_oral_arguments.py | 2 +- cl/scrapers/management/commands/cl_scrape_opinions.py | 6 +++--- cl/scrapers/management/commands/cl_scrape_oral_arguments.py | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cl/scrapers/management/commands/cl_back_scrape_citations.py b/cl/scrapers/management/commands/cl_back_scrape_citations.py index 8975461217..b2da0a4581 100644 --- a/cl/scrapers/management/commands/cl_back_scrape_citations.py +++ b/cl/scrapers/management/commands/cl_back_scrape_citations.py @@ -23,6 +23,8 @@ class Command(cl_back_scrape_opinions.Command): + scrape_target_descr = "citations" + def scrape_court( self, site, diff --git a/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py b/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py index b1105f01e0..299a091597 100644 --- a/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py +++ b/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py @@ -5,7 +5,7 @@ class Command(cl_scrape_oral_arguments.Command): - def parse_and_scrape_site(self, mod, full_crawl): + def parse_and_scrape_site(self, mod, options: dict): court_str = mod.__name__.split(".")[-1].split("_")[0] logger.info(f'Using court_str: "{court_str}"') diff --git a/cl/scrapers/management/commands/cl_scrape_opinions.py b/cl/scrapers/management/commands/cl_scrape_opinions.py index f1547d8f7d..9e7854824b 100644 --- a/cl/scrapers/management/commands/cl_scrape_opinions.py +++ b/cl/scrapers/management/commands/cl_scrape_opinions.py @@ -219,7 +219,7 @@ def save_everything( class Command(VerboseCommand): help = "Runs the Juriscraper toolkit against one or many jurisdictions." - object_type = "opinions" # for logging purposes + scrape_target_descr = "opinions" # for logging purposes def __init__(self, stdout=None, stderr=None, no_color=False): super().__init__(stdout=None, stderr=None, no_color=False) @@ -288,7 +288,7 @@ def scrape_court( if site.cookies: logger.info("Using cookies: %s", site.cookies) - logger.debug("#%s opinions found.", len(site)) + logger.debug("#%s %s found.", len(site), self.scrape_target_descr) added = 0 for i, item in enumerate(site): @@ -313,7 +313,7 @@ def scrape_court( site.court_id, added, len(site), - self.object_type, + self.scrape_target_descr, ) if not full_crawl: # Only update the hash if no errors occurred. diff --git a/cl/scrapers/management/commands/cl_scrape_oral_arguments.py b/cl/scrapers/management/commands/cl_scrape_oral_arguments.py index a2f09dbae4..33da02925d 100644 --- a/cl/scrapers/management/commands/cl_scrape_oral_arguments.py +++ b/cl/scrapers/management/commands/cl_scrape_oral_arguments.py @@ -106,7 +106,7 @@ def make_objects( class Command(cl_scrape_opinions.Command): - object_type = "oral arguments" + scrape_target_descr = "oral arguments" def ingest_a_case( self, @@ -132,7 +132,9 @@ def ingest_a_case( ) logger.info( - "Adding new document found at: %s", item["download_urls"].encode() + "Adding new %s found at: %s", + self.scrape_target_descr, + item["download_urls"].encode(), ) dup_checker.reset()