update sort script to rewrite w/ normalized arabic

TechForPalestine · Feb 11, 2024 · bfee7e1 · bfee7e1
1 parent 9439757
commit bfee7e1
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 22 deletions.
diff --git a/scripts/data/common/killed-in-gaza/data/dict_ar_en.csv b/scripts/data/common/killed-in-gaza/data/dict_ar_en.csv
@@ -26,7 +26,7 @@ ar,en
 أبوشماله,Aboshmalah
 أبولبن,Abolbn
 أبومحيسن,Abomuhaisan
-أُبي,Abi
+أبي,Abi
 أثار,Athar
 أثال,Athal
 أحلام,Ahlam
@@ -1514,7 +1514,7 @@ ar,en
 ايمان,eman
 ايمن,ayman
 ايناس,Ainas
-اّيه,Aّiah
+ايه,Aّiah
 ايه,ayah
 ايهاب,ihab
 ايهم,Aihm
@@ -2094,7 +2094,7 @@ ar,en
 رياطي,Raiatai
 ريام,Raiam
 ريان,Raian
-ريّان,Riّan
+ريان,Riّan
 ريتا,rita
 ريتاج,Ritaj
 ريتال,Rital
@@ -2612,7 +2612,7 @@ ar,en
 عبير,abeer
 عتيق,atiq
 عثمان,Athman
-عُثمان,Aُthman
+عثمان,Aُthman
 عجور,Ajoar
 عدس,Ads
 عدله,Adlah
@@ -2685,7 +2685,7 @@ ar,en
 عكيلة,Akilah
 عكيله,Akilah
 علا,Alaa
-عُلا,Ula
+علا,Ula
 علاء,Alaaa
 علاءالدين,AlaaaAl-Deen
 علام,Alaam
@@ -3083,7 +3083,7 @@ ar,en
 محضيه,Muhdhiah
 محفوظ,Mahfoz
 محمد,muhammad
-مُحمد,Muhammad
+محمد,Muhammad
 محمدد,Muhamdd
 محمدرياض,Muhamdraiadh
 محمدعوض,Muhamduaodh
@@ -3514,7 +3514,7 @@ ar,en
 يسر,Yosr
 يسرى,Ysura
 يسري,Ysri
-ُيسري,Ysri
+يسري,Ysri
 يعقوب,Yaqob
 يقين,Yqain
 يمامه,Yamamuah

diff --git a/scripts/utils/sort-csv.ts b/scripts/utils/sort-csv.ts
@@ -6,6 +6,9 @@ const headerRow = "original,cleaned";
 const sortCsv = (repoFilePath: string) => {
   const csv = fs.readFileSync(repoFilePath).toString();
 
+  const uniqueArParts = new Set<string>();
+  const duplicates = new Set<string>();
+
   const sortedRows = csv
     .split("\n")
     .sort((aRaw, bRaw) => {
@@ -21,21 +24,19 @@ const sortCsv = (repoFilePath: string) => {
       const b = new ArabicClass(bRaw).normalize();
       return a.localeCompare(b);
     })
-    .map((row) => row.replace(/\s*,\s*/, ",").trim())
-    .filter((row) => !!row);
+    .map((row) => {
+      const [arKey, cleanedValue] = row.split(/\s*,\s*/);
+      const normalizedArKey = new ArabicClass(arKey.trim()).normalize();
 
-  const uniqueArParts = new Set<string>();
-  const duplicates = new Set<string>();
-  sortedRows.forEach((row) => {
-    const [arRaw] = row.split(",");
-    const ar = new ArabicClass(arRaw).normalize();
+      if (uniqueArParts.has(normalizedArKey)) {
+        duplicates.add(normalizedArKey);
+      } else {
+        uniqueArParts.add(normalizedArKey);
+      }
 
-    if (uniqueArParts.has(ar)) {
-      duplicates.add(ar);
-    } else {
-      uniqueArParts.add(ar);
-    }
-  });
+      return [normalizedArKey, cleanedValue.trim()].join(",");
+    })
+    .filter((row) => !!row);
 
   console.log(
     `${filePath} sorted alphabetically by arabic name column (${uniqueArParts.size} names)`
@@ -51,8 +52,12 @@ const sortCsv = (repoFilePath: string) => {
 };
 
 const filePath = process.argv.slice().pop();
-if (typeof filePath !== "string" || filePath.endsWith("sort-csv.ts")) {
-  console.log("requires a repo file path argument");
+if (
+  typeof filePath !== "string" ||
+  filePath.endsWith("sort-csv.ts") ||
+  filePath.includes("ar_") === false
+) {
+  console.log("requires a repo file path argument for ar_* dict csv");
   process.exit(1);
 }