Code migration texas

scripts/transform_pipeline.py

@@ -32,14 +32,18 @@
 input_directory.mkdir(parents=True, exist_ok=True)
 output_directory.mkdir(parents=True, exist_ok=True)
 
-individuals_output_path = output_directory / "individuals_table.csv"
-organizations_output_path = output_directory / "organizations_table.csv"
-transactions_output_path = output_directory / "transactions_table.csv"
+individuals_output_path = output_directory / "individuals_table_TX.csv"
+organizations_output_path = output_directory / "organizations_table_TX.csv"
+transactions_output_path = output_directory / "transactions_table_TX.csv"
+id_table_output_path = output_directory / "id_map.csv"
 (
     complete_individuals_table,
     complete_organizations_table,
     complete_transactions_table,
+    complete_id_table,
 ) = transform_and_merge()
 complete_individuals_table.to_csv(individuals_output_path)
 complete_organizations_table.to_csv(organizations_output_path)
 complete_transactions_table.to_csv(transactions_output_path)
+complete_id_table.to_csv(id_table_output_path)
+print("pipeline finished and save data to csv.")

src/score/match_score.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+individuals_df = pd.read_csv('src/score/individuals_table_TX.csv')
+score_df = pd.read_csv('src/score/score_TX.csv')
+
+individuals_df['FIRST_NAME'] = individuals_df['FIRST_NAME'].str.title()
+individuals_df['LAST_NAME'] = individuals_df['LAST_NAME'].str.title()
+score_df['First Name'] = score_df['First Name'].str.title()
+score_df['Last Name'] = score_df['Last Name'].str.title()
+
+merged_df = pd.merge(individuals_df, score_df, how='inner', left_on=['FIRST_NAME', 'LAST_NAME'], right_on=['First Name', 'Last Name'])
+
+output_columns = ['ID', 'ENTITY_TYPE_SPECIFIC', 'ENTITY_TYPE_GENERAL', 'FULL_NAME', 'LAST_NAME', 'FIRST_NAME', 'ADDRESS_LINE_1', 'ADDRESS_LINE_2', 'CITY', 'STATE', 'ZIP_CODE', 'PHONE_NUMBER', 'OFFICE_SOUGHT', 'DISTRICT', 'ENTITY_TYPE', 'ORIGINAL_ID', 'EMPLOYER', 'OCCUPATION', 'Chamber', 'Party']
+result_df = merged_df[output_columns]
+
+result_df.to_csv('src/score/score_match.csv', index=False)
+
+print("Matched records have been saved to 'score_match.csv'.")

src/score/score_TX.csv

@@ -0,0 +1,179 @@
+First Name,Last Name,Chamber,Party
+Yvonne,Davis,House of Representatives,D
+William,Metcalf,House of Representatives,R
+Vikki,Goodwin,House of Representatives,D
+Victoria,Neave Criado,House of Representatives,D
+Venton,Jones,House of Representatives,D
+Valoree,Swanson,House of Representatives,R
+Trey,Martinez Fischer,House of Representatives,D
+Trent,Ashby,House of Representatives,R
+Travis,Clardy,House of Representatives,R
+Tracy,King,House of Representatives,D
+Toni,Rose,House of Representatives,D
+Tom,Craddick,House of Representatives,R
+Tom,Oliverson,House of Representatives,R
+Todd,Hunter,House of Representatives,R
+Thresa,Meza,House of Representatives,D
+Terry,Canales,House of Representatives,D
+Terry,Wilson,House of Representatives,R
+Terri,Leo-Wilson,House of Representatives,D
+Suleman,Lalani,House of Representatives,D
+Steve,Toth,House of Representatives,R
+Stephen,Allison,House of Representatives,R
+Stephanie,Klick,House of Representatives,R
+Stan,Gerdes,House of Representatives,R
+Stan,Kitzman,House of Representatives,R
+Stan,Lambert,House of Representatives,R
+Sheryl,Cole,House of Representatives,D
+Shelby,Slawson,House of Representatives,R
+Shawn,Thierry,House of Representatives,D
+Sergio,Munoz,House of Representatives,D
+Senfronia,Thompson,House of Representatives,D
+Sarah,Eckhardt,Senate,D
+Sam,Harless,House of Representatives,R
+Salman,Bhojani,House of Representatives,D
+Ryan,Guillen,House of Representatives,R
+Royce,West,Senate,D
+Ron,Reynolds,House of Representatives,D
+Roland,Gutierrez,Senate,D
+Robert,Guerra,House of Representatives,D
+Robert,Nichols,Senate,R
+Richard,Hayes,House of Representatives,R
+Richard,Raymond,House of Representatives,D
+Rhett,Bowers,House of Representatives,D
+Reggie,Smith,House of Representatives,R
+Ray,Lopez,House of Representatives,D
+Ramon,Romero,House of Representatives,D
+Rafael,Anchia,House of Representatives,D
+Philip,Cortez,Senate,R
+Phil,King,Senate,R
+Peter,Flores,Senate,R
+Penny,Morales Shaw,House of Representatives,D
+Paul,Bettencourt,Senate,R
+Oscar,Longoria,House of Representatives,D
+Nicole,Collier,House of Representatives,D
+Nathaniel,Parker,Senate,R
+Nathan,Johnson,Senate,D
+Nate,Schatzline,House of Representatives,R
+Morgan,LaMantia,Senate,D
+Morgan,Meyer,House of Representatives,R
+Mihaela,Plesa,House of Representatives,D
+Michael,Schofield,House of Representatives,R
+Matthew,Shaheen,House of Representatives,R
+Matt,Schaefer,House of Representatives,R
+Mary,Gonzalez,House of Representatives,D
+Mary,Perez,House of Representatives,D
+Mark,Dorazio,House of Representatives,R
+Maria,Flores,House of Representatives,D
+Mano,DeAyala,House of Representatives,R
+Lynn,Stucky,House of Representatives,R
+Lois,Kolkhorst,Senate,R
+Lacey,Hull,House of Representatives,R
+Kyle,Kacal,House of Representatives,R
+Kronda,Thimesch,House of Representatives,R
+Kevin,Sparks,Senate,R
+Ken,King,House of Representatives,R
+Kelly,Hancock,Senate,R
+Keith,Bell,House of Representatives,R
+Justin,Holland,House of Representatives,R
+Julie,Johnson,House of Representatives,D
+Judith,Zaffirini,Senate,D
+Juan,Hinojosa,Senate,D
+Josey,Garcia,House of Representatives,D
+Joseph,Moody,House of Representatives,D
+Jose,Menendez,Senate,D
+Jon,Rosenthal,House of Representatives,D
+Jolanda,Jones,House of Representatives,D
+John,Bryant,House of Representatives,D
+John,Bucy,House of Representatives,D
+John,Kuempel,House of Representatives,R
+John,Lujan,House of Representatives,R
+John,Raney,House of Representatives,R
+John,Smithie,House of Representatives,R
+John,Whitmire,Senate,D
+Joan,Huffman,Senate,R
+Jessica,Gonzalez,House of Representatives,D
+Jeff,Leach,House of Representatives,R
+Jay,Dean,House of Representatives,R
+Jarvis,Johnson,House of Representatives,D
+Jared,Patterson,House of Representatives,R
+Janie,Lopez,House of Representatives,R
+James,Frank,House of Representatives,R
+James,Talarico,House of Representatives,D
+Jacey,Jetton,House of Representatives,R
+J.M.,Lozano,House of Representatives,R
+Hugh,Shine,House of Representatives,R
+Hubert,Vo,House of Representatives,D
+Harold,Dutton,House of Representatives,D
+Greg,Bonnen,House of Representatives,R
+Glenn,Rogers,House of Representatives,R
+Giovanni,Capriglione,House of Representatives,R
+Gina,Hinojosa,House of Representatives,D
+Geanie,Morrison,House of Representatives,R
+Gary,Gates,House of Representatives,R
+Gary,Vandeaver,House of Representatives,R
+Frederick,Frazier,House of Representatives,R
+Four,Price,House of Representatives,R
+Evelina,Ortega,House of Representatives,D
+Eugene,Wu,House of Representatives,D
+Ernest,Bailes,House of Representatives,R
+Erin,Zwiener,House of Representatives,D
+Ellen,Troxclair,House of Representatives,R
+Elizabeth,Campos,House of Representatives,D
+Eddie,Morales,House of Representatives,D
+Ed,Thompson,House of Representatives,R
+Dustin,Burrows,House of Representatives,R
+Drew,Darby,House of Representatives,R
+Drew,Springer,Senate,R
+Donna,Campbell,Senate,R
+Donna,Howard,House of Representatives,D
+Dewayne,Burns,House of Representatives,R
+Dennis,Paul,House of Representatives,R
+David,Cook,House of Representatives,R
+David,Middleton,Senate,R
+David,Spiller,House of Representatives,R
+Dade,Phelan,House of Representatives,R
+Craig,Goldman,House of Representatives,R
+Cole,Hefner,House of Representatives,R
+Cody,Harris,House of Representatives,R
+Cody,Vasut,House of Representatives,R
+Claudia,Ordaz Perez,House of Representatives,D
+Christina,Morales,House of Representatives,D
+Christian,Hayes,House of Representatives,D
+Chris,Turner,House of Representatives,D
+Charlie,Geren,House of Representatives,R
+Charles,Anderson,House of Representatives,R
+Charles,Creighton,Senate,R
+Charles,Cunningham,House of Representatives,R
+Charles,Perry,Senate,R
+Charles,Schwertner,Senate,R
+Cesar,Blanco,Senate,D
+Cecil,Bell,House of Representatives,R
+Carrie,Isaac,House of Representatives,R
+Caroline,Harris,House of Representatives,R
+Carol,Alvarado,Senate,D
+Carl,Sherman,House of Representatives,D
+Carl,Tepper,House of Representatives,R
+Candy,Noble,House of Representatives,R
+Bryan,Hughes,Senate,R
+Bryan,Slaton,House of Representatives,R
+Brooks,Landgraf,House of Representatives,R
+Briscoe,Cain,House of Representatives,R
+Brian,Birdwell,Senate,R
+Brian,Harrison,House of Representatives,R
+Bradley,Buckley,House of Representatives,R
+Borris,Miles,Senate,D
+Bob,Hall,Senate,R
+Ben,Bumgarner,House of Representatives,R
+Barbara,Gervin-Hawkins,House of Representatives,D
+Armando,Martinez,House of Representatives,D
+Armando,Walle,House of Representatives,D
+Ann,Johnson,House of Representatives,D
+Angie,Button,House of Representatives,R
+Angelina,Orr,House of Representatives,R
+Angela,Paxton,Senate,R
+Andrew,Murr,House of Representatives,R
+Ana-Maria,Ramos,House of Representatives,D
+Ana,Hernandez,House of Representatives,D
+Alma,Allen,House of Representatives,D
+Abel,Herrero,House of Representatives,D