Fixed return code problems and code cleanups

miha42-github · miha42-github · commit 485f4ae44bc3 · 2024-04-19T19:24:56.000-07:00
diff --git a/company_dns.py b/company_dns.py
@@ -62,8 +62,12 @@ async def general_query(request):
     try:
         gq.query = request.path_params['company_name']
         # Log the query request as a debug message
-        logger.debug(f'Querying for general data for {request.path_params["company_name"]}')
+        logger.debug(f'Performing general query for company name: [{request.path_params["company_name"]}]')
         company_wiki_data = gq.get_firmographics_wikipedia()
+        if company_wiki_data['code'] != 200:
+            logger.error(f'There were [0] results for resource [company_name].')
+            return JSONResponse(company_wiki_data)
+        
         general_company_data = gq.merge_data(company_wiki_data['data'], company_wiki_data['data']['cik'])
         # Call check_status_and_return to check the status of the data and return the data or an error message
         checked_data = _check_status_and_return(general_company_data, request.path_params['company_name'])
@@ -79,16 +83,20 @@ async def general_query(request):
 
 # -------------------------------------------------------------- #
 # BEGIN: Helper functions
-def _check_status_and_return(data, resource_name):
-    if data.get('code') != 200:
+# TODO: This function may not be needed as it is and the code could be moved into _handle_request.  Essentially we're checking for a 200 status code and returning the data that includes the error message.  The change would likely be to log the error message and return the data as is.  This would mean this funtion would be removed.
+def _check_status_and_return(result_data, resource_name):
+    return_code = result_data.get('code')
+    result_count = result_data.get('total')
+    return_msg = result_data.get('message')
+    if return_code != 200:
         # Log the error message
-        logger.error(f'Data for resource {resource_name} not found')
+        logger.error(f'There were [{result_count}] results for resource [{resource_name}].')
         # Return an error message that the data was not found using the resource name
-        return {'error': f'Data for resource {resource_name} not found', 'code': 404}
-    return data
+        return {'message': return_msg, 'code': return_code, 'data': result_data}
+    return result_data
 
-def _prepare_logging(log_level=logging.DEBUG):
-    logging.basicConfig(format='%(levelname)s:\t%(asctime)s [module: %(name)s] %(message)s', level=log_level)
+def _prepare_logging(log_level=logging.INFO):
+    logging.basicConfig(format='%(levelname)s:\t%(asctime)s [module: %(name)s] %(message)s', level=logging.DEBUG)
     return logging.getLogger(__file__)
 
 def _handle_request(request, handler, func, path_param, *args, **kwargs):
@@ -195,6 +203,6 @@ async def dispatch(self, request, call_next):
 
 if __name__ == "__main__": 
     try:
-        uvicorn.run(app, host='0.0.0.0', port=8000, log_level="debug", lifespan='off')
+        uvicorn.run(app, host='0.0.0.0', port=8000, log_level='info', lifespan='off')
     except KeyboardInterrupt:
         logger.info("Server was shut down by the user.")
diff --git a/lib/edgar.py b/lib/edgar.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 import sqlite3
 import sys
 import requests
@@ -90,13 +89,13 @@ def __init__(
             'Accept-Encoding': 'gzip, deflate'
         }
 
+        # What we are are to query
+        self.query = None
+
         # Command line naming helpers
         self.NAME = name
         self.DESC = description
 
-        # What we are are to query
-        self.query = None
-
         # Define the form type we're after
         self.form_type = '10-'
 
diff --git a/lib/firmographics.py b/lib/firmographics.py
@@ -1,21 +1,16 @@
-#!/usr/local/bin/python3
-
 from . import edgar
 from . import wikipedia
-import argparse
-import pprint
 import sys
 import urllib.parse as url_parse
 import logging
 from geopy.geocoders import ArcGIS
 
 __author__ = "Michael Hay"
-__copyright__ = "Copyright 2023, Mediumroast, Inc. All rights reserved."
+__copyright__ = "Copyright 2024, Mediumroast, Inc. All rights reserved."
 __license__ = "Apache 2.0"
 __version__ = "2.1.0"
 __maintainer__ = "Michael Hay"
-__status__ = "Alpha"
-__date__ = '2023-April-1'
+__status__ = "Production"
 __contact__ = 'https://github.com/miha42-github/company_dns'
 
 #### Globals ####
@@ -51,60 +46,23 @@ def __init__(
         self,
         database=None,
         name='general', 
-        description='A module and simple CLI too to search for company data in Wikipedia, EDGAR, and also a merger of the two data sources.'):
+        description='A module to search for company data in Wikipedia, EDGAR, and also a merger of the two data sources.'):
         
         # Construct the object to determine lat long pairs
         self.locator = ArcGIS(timeout=2, user_agent="company_dns")
 
         # Contains the company name or CIK
         self.query = None
 
-        # Command line naming helpers
-        self.NAME = name
-        self.DESC = description
-
         # Define the db_cache location
         self.db_file='./company_dns.db' if database is None else database
 
+        # Naming helpers
+        self.NAME = name
+        self.DESC = description
+
         # Set up the logging
         self.logger = logging.getLogger(self.NAME)
-
-    def get_cli_args(self):
-        """Parse common CLI arguments including system configs and behavior switches.
-        """
-        # Set up the argument parser
-        parser = argparse.ArgumentParser(prog=self.NAME, description=self.DESC)
-
-        # Setup the command line switches
-        parser.add_argument(
-            '--query',
-            help="Company name to search for in Wikipedia or EDGAR",
-            type=str,
-            dest='query',
-            required=True
-        )
-        parser.add_argument(
-            "--debug",
-            help="Turn on debugging",
-            dest="debug",
-            type=int,
-            default=0,
-        )
-        parser.add_argument(
-            '--operation',
-            help="Company name to search for in Wikipedia.",
-            type=str,
-            dest='operation',
-            choices=['merge', 'ciks', 'details', 'summaries', 'firmographics_wiki', 'firmographics_edgar'],
-            default='merge',
-            required=True
-        )
-
-        # Parse the CLI
-        cli_args = parser.parse_args()
-
-        # Return parsed arguments
-        return cli_args
     
     def locate (self, place):
         # Log the place to locate via a debug message
@@ -277,30 +235,4 @@ def merge_data(self, wiki_data, cik, company_name=None):
             'module': my_class + '-> ' + my_function,
             'data': final_company,
             'dependencies': DEPENDENCIES
-        }
-
-
-
-if __name__ == '__main__':
-    query = GeneralQueries('../company_dns.db')
-    cli_args = query.get_cli_args()
-    query.query = cli_args.query
-    DEBUG = cli_args.debug
-    
-    results = dict()
-    if cli_args.operation == 'ciks':
-        results = query.get_all_ciks()
-    elif cli_args.operation == 'details':
-        results = query.get_all_details()
-    elif cli_args.operation == 'summaries':
-        results = query.get_all_summaries()
-    elif cli_args.operation == 'firmographics_wiki':
-        results = query.get_firmographics_wikipedia()
-    elif cli_args.operation == 'firmographics_edgar':
-        results = query.get_firmographics_edgar()
-    elif cli_args.operation == 'merge':
-        wiki_data = query.get_firmographics_wikipedia()
-        if wiki_data['code'] != 200: results = wiki_data
-        results = query.merge_data(wiki_data, wiki_data['cik'])
-    
-    if not DEBUG: pprint.pprint(results)
+        }
diff --git a/lib/sic.py b/lib/sic.py
@@ -1,20 +1,13 @@
-#!/usr/bin/env python3
-
-import argparse
-import pprint
 import sys
-import pprint
 import sqlite3
-import re
-import urllib.parse as url_parse
+import logging
 
 __author__ = "Michael Hay"
-__copyright__ = "Copyright 2023, Mediumroast, Inc. All rights reserved."
+__copyright__ = "Copyright 2024, Mediumroast, Inc. All rights reserved."
 __license__ = "Apache 2.0"
-__version__ = "1.1.0"
+__version__ = "2.0.0"
 __maintainer__ = "Michael Hay"
-__status__ = "Beta"
-__date__ = '2023-April-1'
+__status__ = "Production"
 __contact__ = 'https://github.com/miha42-github/company_dns'
 
 #### Globals ####
@@ -67,56 +60,22 @@ def __init__(
         self, 
         db_file='./company_dns.db', 
         name='sic', 
-        description='A module and simple CLI too to lookup SIC data.'):
+        description='A module to lookup SIC data.'):
 
         # The SQLite database connection and cursor
         self.e_conn = sqlite3.connect(db_file)
         self.ec = self.e_conn.cursor()
         self.db_file = db_file
 
-        # Command line naming helpers
+        # Naming helpers
         self.NAME = name
         self.DESC = description
 
         # Query object
         self.query = None
 
-    def get_cli_args(self):
-        """Parse common CLI arguments including system configs and behavior switches.
-        """
-        # Set up the argument parser
-        parser = argparse.ArgumentParser(prog=self.NAME, description=self.DESC)
-
-        # Setup the command line switches
-        parser.add_argument(
-            '--query',
-            help="Description of the SIC to search for in data cache.",
-            type=str,
-            dest='query',
-            required=True
-        )
-        parser.add_argument(
-            '--operation',
-            help="Type of details to search for.",
-            type=str,
-            dest='operation',
-            choices=['description', 'code'],
-            default='description',
-            required=True
-        )
-        parser.add_argument(
-            "--debug",
-            help="Turn on debugging",
-            dest="debug",
-            type=int,
-            default=0,
-        )
-
-        # Parse the CLI
-        cli_args = parser.parse_args()
-
-        # Return parsed arguments
-        return cli_args
+        # Set up the logging
+        self.logger = logging.getLogger(self.NAME)
 
     def get_all_sic_by_no(self):
         """Using a query string find and return a dictionary containing all SICs with additional metadata. 
@@ -237,13 +196,17 @@ def get_all_sic_by_name(self):
         }
         tmp_sics = {}
 
+        self.logger.debug('Querying db cache for [' + self.query + ']')
+
         # Define the SQL Query
         sql_query = "SELECT sic.division, sic.major_group, sic.industry_group, sic.sic, sic.description, " + \
                     "major_groups.description a FROM sic INNER JOIN major_groups ON major_groups.major_group = sic.major_group WHERE sic.description LIKE '%" + self.query + "%' "
 
         # Issue the query
         for row in self.ec.execute(sql_query):
 
+            self.logger.debug('Processing row [' + row + ']')
+
             # Get the fields in a structure we can manipulate
             sic_code = str(row[SICS])
             sic_desc = str(row[SICS_DESC])
@@ -278,7 +241,11 @@ def get_all_sic_by_name(self):
         
         final_sics['sics'] = tmp_sics
         final_sics['total'] = len(tmp_sics)
+
+        self.logger.info('Found a total of [' + str(final_sics['total']) + '] SICs returning data.')
+
         if final_sics['total'] == 0:
+            final_sics['sics'] = []
             return {
                 'code': 404, 
                 'message': 'No SICs found for query [' + self.query + '].',
@@ -497,20 +464,3 @@ def get_division_desc_by_id(self):
                 'data': final_descs,
                 'dependencies': DEPENDENCIES
             }
-
-
-
-if __name__ == '__main__':
-    q = SICQueries(db_file='../company_dns.db')
-    cli_args = q.get_cli_args()
-    q.query = cli_args.query
-    DEBUG = cli_args.debug
-    
-    results = dict()
-    
-    if cli_args.operation == 'description':
-        results = q.get_all_sic_by_name()
-    elif cli_args.operation == 'code':
-        results = q.get_all_sic_by_no()
-    
-    if not DEBUG: pprint.pprint(results)
diff --git a/lib/wikipedia.py b/lib/wikipedia.py
@@ -95,9 +95,13 @@ def get_firmographics(self):
         lookup_error = {
                 'code': 404,
                 'message': 'Unable to find a company by the name [' + self.query + ']. Maybe you should try an alternative structure like [' + self.query + ' Inc.,' + self.query + ' Corp., or ' + self.query + ' Corporation].',
-                'errorType': 'LookupError',
+                'error': 'LookupError',
                 'module': my_class + '-> ' + my_function,
-                'dependencies': DEPENDENCIES
+                'dependencies': DEPENDENCIES,
+                'data': {
+                    'total': 0,
+                    'results': []
+                }
             }
 
         # TODO try to do the right thing by trying different common combinations like Company, Inc.; Company Corp, etc.
@@ -106,8 +110,9 @@ def get_firmographics(self):
             self.logger.info('Starting retrieval of firmographics for [' + self.query + '] via its wikipedia page.')
             company_page = wptools.page(self.query, silent=True)
             # Log the completion of the page creation
-            self.logger.info('Completed firmographics retrieval [' + self.query + '] via its wikipedia page.')
+            self.debug(f'Page results for [{self.query}]: {company_page}')
         except:
+            self.logger.error('A wikipedia page for [' + self.query + '] was not found.')
             return lookup_error
 
         # Prepare to get the infoblox for the company
@@ -185,7 +190,7 @@ def get_firmographics(self):
 
         # City
         firmographics['city'] = self._get_item(company_info, ['location_city', 'hq_location_city', 'location'], r'\[\[\]\]', 0)
-        firmographics['city'] = firmographics['city'].replace('[[', '').replace(']]', '') if re.search('(\[)|(\])', firmographics['city']) else firmographics['city']
+        firmographics['city'] = firmographics['city'].replace('[[', '').replace(']]', '') if re.search(r'(\[)|(\])', firmographics['city']) else firmographics['city']
         firmographics['city'] = firmographics['city'].replace('<br>', ', ') if re.search('<br>', firmographics['city']) else firmographics['city']
 
 
