#65 add new abbreviations and retrain models

hygia-org · Feb 8, 2023 · 975cc18 · 975cc18
1 parent b40eae5
commit 975cc18
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 25 deletions.
diff --git a/data/dicts/mexico_abbreviations.csv b/data/dicts/mexico_abbreviations.csv
@@ -3,6 +3,8 @@ AV,AVENUE
 NO,NUMBER
 NUM,NUMBER
 PO,POST OFFICE
+P.O,POST OFFICE
 BLVD,BOULEVARD
 LT,LOTE
-MZ,MANZANA
+CDMX,Ciudad de México
+DF,Distrito Federal
diff --git a/data/models/RandomForest_Ksmash_Regex_Enrichments_Normalization.pkl b/data/models/RandomForest_Ksmash_Regex_Enrichments_Normalization.pkl
diff --git a/data/models/RandomForest_Ksmash_Word_Embedding_Regex_Enrichments_Normalization.pkl b/data/models/RandomForest_Ksmash_Word_Embedding_Regex_Enrichments_Normalization.pkl
diff --git a/examples/MEXICO_predict_example_no_embeddings copy.ipynb b/examples/MEXICO_predict_example_no_embeddings copy.ipynb
@@ -49,8 +49,7 @@
    "outputs": [],
    "source": [
     "file_path = '../data/tmp/AI_LATA_ADDRESS_MEX_modificado.csv'\n",
-    "# df = pd.read_csv(file_path, sep='¨', nrows=None, engine='python')\n",
-    "df = pd.read_csv(file_path, sep='¨', nrows=5000, engine='python')"
+    "df = pd.read_csv(file_path, sep='¨', nrows=None, engine='python')"
    ]
   },
   {
@@ -176,8 +175,8 @@
     {
      "data": {
       "text/plain": [
-       "0.0    4984\n",
-       "1.0      16\n",
+       "0.0    2518008\n",
+       "1.0       2288\n",
        "Name: prediction_is_key_smash, dtype: int64"
       ]
      },
@@ -201,7 +200,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [

diff --git a/examples/MEXICO_retrain_predict_example.ipynb b/examples/MEXICO_retrain_predict_example.ipynb
@@ -270,8 +270,8 @@
     {
      "data": {
       "text/plain": [
-       "valid                             1344182\n",
-       "key_smash                             661\n",
+       "valid                             1344254\n",
+       "key_smash                             657\n",
        "contains_email                        569\n",
        "contains_exactly_the_word_test        177\n",
        "only_special_characters               144\n",
@@ -312,9 +312,9 @@
     {
      "data": {
       "text/plain": [
-       "valid                             2511507\n",
+       "valid                             2511527\n",
        "contains_context_invalid_words       3079\n",
-       "key_smash                            1514\n",
+       "key_smash                            1494\n",
        "only_special_characters              1291\n",
        "contains_email                       1048\n",
        "contains_exactly_the_word_test        667\n",
@@ -357,10 +357,10 @@
       "\u001b[33mtranning model...\u001b[37m\n",
       "\u001b[32mdone\u001b[37m\n",
       "\u001b[33mget model score...\u001b[37m\n",
-      "\u001b[1maccuracy -> \u001b[22m0.9910256410256411\n",
-      "\u001b[1mprecision -> \u001b[22m0.9776536312849162\n",
-      "\u001b[1mrecall -> \u001b[22m0.9831460674157303\n",
-      "\u001b[1mf1 -> \u001b[22m0.9803921568627452\n"
+      "\u001b[1maccuracy -> \u001b[22m0.9922879177377892\n",
+      "\u001b[1mprecision -> \u001b[22m0.9903846153846154\n",
+      "\u001b[1mrecall -> \u001b[22m0.9809523809523809\n",
+      "\u001b[1mf1 -> \u001b[22m0.985645933014354\n"
      ]
     }
    ],
@@ -391,8 +391,8 @@
     {
      "data": {
       "text/plain": [
-       "0.0    1343185\n",
-       "1.0       2936\n",
+       "0.0    1342240\n",
+       "1.0       3949\n",
        "Name: prediction, dtype: int64"
       ]
      },

diff --git a/examples/MEXICO_retrain_predict_example_no_embedding.ipynb b/examples/MEXICO_retrain_predict_example_no_embedding.ipynb
@@ -247,8 +247,8 @@
     {
      "data": {
       "text/plain": [
-       "valid                             1344182\n",
-       "key_smash                             661\n",
+       "valid                             1344254\n",
+       "key_smash                             657\n",
        "contains_email                        569\n",
        "contains_exactly_the_word_test        177\n",
        "only_special_characters               144\n",
@@ -289,9 +289,9 @@
     {
      "data": {
       "text/plain": [
-       "valid                             2511507\n",
+       "valid                             2511527\n",
        "contains_context_invalid_words       3079\n",
-       "key_smash                            1514\n",
+       "key_smash                            1494\n",
        "only_special_characters              1291\n",
        "contains_email                       1048\n",
        "contains_exactly_the_word_test        667\n",
@@ -334,10 +334,10 @@
       "\u001b[33mtranning model...\u001b[37m\n",
       "\u001b[32mdone\u001b[37m\n",
       "\u001b[33mget model score...\u001b[37m\n",
-      "\u001b[1maccuracy -> \u001b[22m1.0\n",
-      "\u001b[1mprecision -> \u001b[22m1.0\n",
+      "\u001b[1maccuracy -> \u001b[22m0.9987146529562982\n",
+      "\u001b[1mprecision -> \u001b[22m0.9946524064171123\n",
       "\u001b[1mrecall -> \u001b[22m1.0\n",
-      "\u001b[1mf1 -> \u001b[22m1.0\n"
+      "\u001b[1mf1 -> \u001b[22m0.9973190348525469\n"
      ]
     }
    ],
@@ -368,8 +368,8 @@
     {
      "data": {
       "text/plain": [
-       "0.0    1345148\n",
-       "1.0        973\n",
+       "0.0    1345413\n",
+       "1.0        776\n",
        "Name: prediction, dtype: int64"
       ]
      },