Add multiprocess wiki-extractor argument

howl-anderson · Jul 10, 2018 · 644523e · 644523e
1 parent 3ce35a5
commit 644523e
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/extract_wikipedia_json_corpus.bash b/extract_wikipedia_json_corpus.bash
@@ -1,3 +1,5 @@
 #!/bin/bash
 
-WikiExtractor.py --json raw_data/zhwiki-latest-pages-articles.xml.bz2 -o extracted_json_data
+cpu_count=`nproc --all`
+process_count=$(expr $cpu_count - 1)
+WikiExtractor.py --json raw_data/zhwiki-latest-pages-articles.xml.bz2 -o extracted_json_data --processes ${process_count}