Skip to content

Commit

Permalink
handle CI reading RTL csv wrong way
Browse files Browse the repository at this point in the history
  • Loading branch information
sterlingwes committed Feb 12, 2024
1 parent d99eda1 commit 911424b
Showing 1 changed file with 15 additions and 12 deletions.
27 changes: 15 additions & 12 deletions scripts/data/common/killed-in-gaza/generate_killed_list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,17 @@ const pwd = "scripts/data/common/killed-in-gaza";
const arRawNameColumnLabel = "name_ar_raw";
const arEnNameColumnLabel = "name_en";

const readCsv = (repoPath: string, rtl: boolean) => {
const readCsv = (repoPath: string) => {
const csvString = fs.readFileSync(repoPath).toString();
return csvString.split(/\r?\n/g).map((row) => {
if (rtl) {
console.log(">>", row);
}
const ltrRow = row.replace(/\u200f/u, "");
return ltrRow.split(",");
});
return csvString.split(/\r?\n/g).map((row) => row.split(","));
};

/**
* read a CSV file and return an object lookup ("dict") with keys
* as the first CSV column value, and values as the second CSV column
*/
const readCsvToDict = (repoPath: string, rtl = false) => {
return readCsv(repoPath, rtl).reduce(
const readCsvToDict = (repoPath: string) => {
return readCsv(repoPath).reduce(
(dict, row) => ({
...dict,
[row[0]]: row[1],
Expand All @@ -30,10 +24,19 @@ const readCsvToDict = (repoPath: string, rtl = false) => {
);
};

const rawList = readCsv(`${pwd}/data/raw.csv`, false);
const arToAr = readCsvToDict(`${pwd}/data/dict_ar_ar.csv`, true);
const rawList = readCsv(`${pwd}/data/raw.csv`);
let arToAr = readCsvToDict(`${pwd}/data/dict_ar_ar.csv`);
const arToEn = readCsvToDict(`${pwd}/data/dict_ar_en.csv`);

// if this matches, our ar->ar dict was read backwards and we need to flip it
if (arToAr["ابوالليل"]) {
console.log("⚠️ inverting ar->ar which was read LTR");
arToAr = Object.entries(arToAr).reduce(
(flipped, [key, value]) => ({ ...flipped, [value]: key }),
{}
);
}

const [rawHeaderRow, ...rawListRows] = rawList;
const arRawColumn = rawHeaderRow.indexOf(arRawNameColumnLabel);
if (arRawColumn === -1) {
Expand Down

0 comments on commit 911424b

Please sign in to comment.