Lock down dependency versions and minor Dockerfile cleanup

Also applied some minor formatting/text changes to python scripts

Signed-off-by: Jeremy Ho <jujaga@gmail.com>

shared/geospatial/convert_geometry.py

@@ -93,4 +93,4 @@
 end = time.time()
 
 print("Geometry conversion completed successfully.")
-print("The time of execution of the program is:", (end - start), "seconds")
+print(f"The time of execution of the program is: {(end - start)} seconds")

shared/ora2pg/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

shared/ora2pg/Dockerfile

@@ -12,17 +12,15 @@ RUN apt-get update && apt-get install -y libaio1 wget unzip \
     && echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf \
     && ldconfig
 
-WORKDIR /app
-
 # PostgreSQL library
 
 RUN apt-get update \
     && apt-get -y install libpq-dev gcc \
     && pip install psycopg2
 
-ADD *.py .
+WORKDIR /app
 
-COPY requirements.txt requirements.txt
+COPY . .
 
 RUN pip3 install -r requirements.txt
 

shared/ora2pg/data_replication_ora2pg.py

@@ -67,7 +67,7 @@ def del_audit_entries_rerun(current_date):
     postgres_cursor = postgres_connection.cursor()
     del_sql = f"""
   DELETE FROM {mstr_schema}.{audit_table} c
-  where application_name='{app_name}' and batch_run_date='{current_date}'
+  WHERE application_name='{app_name}' and batch_run_date='{current_date}'
   """
     postgres_cursor.execute(del_sql)
     postgres_connection.commit()
@@ -86,7 +86,7 @@ def audit_batch_status_insert(table_name, status):
         print(audit_batch_status_query)
         postgres_cursor.execute(audit_batch_status_query)
         postgres_connection.commit()
-        print(f"Record inserted into audit batch status table")
+        print("Record inserted into audit batch status table")
         return None
     except Exception as e:
         print(
@@ -106,9 +106,9 @@ def get_active_tables(mstr_schema, app_name):
     postgres_cursor = postgres_connection.cursor()
     list_sql = f"""
   SELECT application_name,source_schema_name,source_table_name,target_schema_name,target_table_name,truncate_flag,cdc_flag,full_inc_flag,cdc_column,replication_order,customsql_ind,customsql_query
-  from {mstr_schema}.cdc_master_table_list c
-  where  active_ind = 'Y' and application_name='{app_name}'
-  order by replication_order, source_table_name
+  FROM {mstr_schema}.cdc_master_table_list c
+  WHERE active_ind = 'Y' and application_name='{app_name}'
+  ORDER BY replication_order, source_table_name
   """
     with postgres_connection.cursor() as curs:
         curs.execute(list_sql)
@@ -240,4 +240,4 @@ def load_data_from_src_tgt(table_name, source_schema, target_schema, customsql_i
     PgresPool.closeall()
 
     print("ETL process completed successfully.")
-    print("The time of execution of the program is:", (end - start), "secs")
+    print(f"The time of execution of the program is: {(end - start)} secs")

shared/ora2s3/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

shared/ora2s3/Dockerfile

@@ -1,4 +1,4 @@
-FROM python
+FROM python:3.12.5-slim-bookworm
 
 WORKDIR /opt/oracle
 RUN apt-get update && apt-get install -y libaio1 wget unzip \
@@ -12,9 +12,7 @@ RUN apt-get update && apt-get install -y libaio1 wget unzip \
 
 WORKDIR /app
 
-ADD data_replication_ora2s3.py .
-
-COPY requirements.txt requirements.txt
+COPY . .
 
 RUN pip3 install -r requirements.txt
 

shared/ora2s3/requirements.txt

@@ -1,4 +1,4 @@
-boto3
-oracledb
-pandas
-xlsxwriter
+boto3==1.34.147
+oracledb==2.3.0
+pandas==2.2.2
+xlsxwriter==3.2.0

shared/pg2pg/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

shared/pg2pg/Dockerfile

@@ -8,9 +8,7 @@ RUN apt-get update \
     && apt-get -y install libpq-dev gcc \
     && pip install psycopg2
 
-ADD *.py .
-
-COPY requirements.txt requirements.txt
+COPY . .
 
 RUN pip3 install -r requirements.txt
 

shared/pg2pg/data_replication_pg2pg.py

@@ -88,7 +88,7 @@ def del_audit_entries_rerun(current_date):
     try:
         del_sql = f"""
     DELETE FROM {mstr_schema}.{audit_table} c
-    where application_name='{app_name}' and batch_run_date='{current_date}'
+    WHERE application_name='{app_name}' AND batch_run_date='{current_date}'
     """
         postgres_cursor.execute(del_sql)
         postgres_connection.commit()
@@ -133,9 +133,9 @@ def get_active_tables(mstr_schema, app_name):
     postgres_cursor = postgres_connection.cursor()
     list_sql = f"""
   SELECT application_name,source_schema_name,source_table_name,target_schema_name,target_table_name,truncate_flag,cdc_flag,full_inc_flag,cdc_column,replication_order,customsql_ind,customsql_query
-  from {mstr_schema}.cdc_master_table_list c
-  where  active_ind = 'Y' and application_name='{app_name}' AND replication_source = 'postgresql'
-  order by replication_order, source_table_name
+  FROM {mstr_schema}.cdc_master_table_list c
+  WHERE active_ind = 'Y' and application_name='{app_name}' AND replication_source = 'postgresql'
+  ORDER BY replication_order, source_table_name
   """
     try:
         with postgres_connection.cursor() as curs:
@@ -244,8 +244,8 @@ def check_failed_tables(mstr_schema, app_name, current_date):
     postgres_cursor = postgres_connection.cursor()
     list_sql = f"""
   SELECT object_name
-  from {mstr_schema}.audit_batch_status c
-  where application_name='{app_name}' and batch_run_date='{current_date}' and object_execution_status='failed'
+  FROM {mstr_schema}.audit_batch_status c
+  WHERE application_name='{app_name}' and batch_run_date='{current_date}' AND object_execution_status='failed'
   """
     try:
         with postgres_connection.cursor() as curs:
@@ -339,18 +339,18 @@ def get_table_comments_from_source(table):
     srcpostgres_connection_c = SrcPgresPool.getconn()
     srcpostgres_cursor_c = srcpostgres_connection_c.cursor()
     srcpostgres_cursor_c.execute("""
-        select
+        SELECT
             'COMMENT ON COLUMN ' || %s||'.'||c.table_name||'.'||c.column_name||' IS '''||pgd.description||''';' as comments
-             from pg_catalog.pg_statio_all_tables as st
-            inner join pg_catalog.pg_description pgd on (
+            FROM pg_catalog.pg_statio_all_tables as st
+            INNER JOIN pg_catalog.pg_description pgd on (
                 pgd.objoid = st.relid
                 )
-            inner join information_schema.columns c on (
+            INNER JOIN information_schema.columns c on (
             pgd.objsubid   = c.ordinal_position and
             c.table_schema = st.schemaname and
             c.table_name   = st.relname
             )
-            where c.table_schema=%s and c.table_name=%s;
+            WHERE c.table_schema=%s and c.table_name=%s;
     """, (table[3], table[1], table[0]))
     comments = srcpostgres_cursor_c.fetchall()
     print(comments)
@@ -506,6 +506,6 @@ def create_table_comments(row, table):
     PgresPool.closeall()
 
     print("ETL process completed successfully.")
-    print("The time of execution of the program is:", (end - start), "secs")
+    print(f"The time of execution of the program is: {(end - start)} secs")
 
 sys.exit(0)

shared/tools/meltano/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

shared/tools/meltano/Dockerfile

@@ -1,3 +1,9 @@
-FROM meltano/meltano
+FROM meltano/meltano:v3.5-python3.12
+
+RUN bash -c 'mkdir -p /app/ods-dev/{analyze,extract,load,notebook,orchestrate,output,transform}'
+
+WORKDIR /app
+
+COPY ods-dev .
 
 CMD ["--help"]

shared/tools/trino/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

shared/tools/trino/Dockerfile

@@ -1,4 +1,4 @@
-FROM trinodb/trino
+FROM trinodb/trino:452
 
 USER trino