make versions and add lists without comments

Author: aurelberra

make_lists.R

@@ -8,7 +8,7 @@ source("~/Documents/github/r-dev/helpers.R")
 # GREEK
 
 # Set version number
-version_greek <- "2.4"
+version_greek <- "2.5"
 
 # Convert current JSON list to TXT with Markdown headings
 greek_json <- read_file("stopwords_greek.json")
@@ -54,17 +54,18 @@ greek_metadata <- paste0(
 stopwords_greek <- paste0(greek_metadata, greek_raw)
 stopwords_greek <- utf8::utf8_normalize(stopwords_greek)
 write_file(stopwords_greek, "stopwords_greek.txt")
+write_file(stopwords_greek, paste("./versions/stopwords_greek_v", str_replace(version_greek, "\\.", "_"), ".txt", sep = ""))
 
 # Make file without categories as comments
 greek_raw %>%
-    str_replace_all("#.+\n", "\n") %>%
-    str_replace_all("\n\n", "\n") %>%
+    str_replace_all("#.+\n", "") %>%
+    str_replace_all("\n+", "\n") %>%
     write_file("./test/test_json_txt/stopwords_greek_no_comments.txt")
 
 # LATIN
 
 # Set version number
-version_latin <- "2.3"
+version_latin <- "2.4"
 
 # Convert current JSON list to TXT with Markdown headings
 latin_json <- read_file("stopwords_latin.json")
@@ -109,9 +110,10 @@ latin_metadata <- paste0(
 )
 stopwords_latin <- paste0(latin_metadata, latin_raw)
 write_file(stopwords_latin, "stopwords_latin.txt")
+write_file(stopwords_latin, paste("./versions/stopwords_latin_v", str_replace(version_latin, "\\.", "_"), ".txt", sep = ""))
 
 # Make file without categories as comments
 latin_raw %>%
-    str_replace_all("#.+\n", "\n") %>%
-    str_replace_all("\n\n", "\n") %>%
+    str_replace_all("#.+\n", "") %>%
+    str_replace_all("\n+", "\n") %>%
     write_file("./test/test_json_txt/stopwords_latin_no_comments.txt")

rationale.md

@@ -17,16 +17,18 @@ For version 2 (January-February 2018) I rebased the lists on corpus statistics a
 Total number of items (tokens or symbols):
 
 * Latin
-    * 4009 items in [stopwords_latin_v2_3.txt](versions/stopwords_latin_v2_3.txt)
-    * 4008 items in [stopwords_latin_v2_2.txt](versions/stopwords_latin_v2_2.txt)
-    * 3844 items in [stopwords_latin_v2_1.txt](versions/stopwords_latin_v2_1.txt)
+    * 4010 items in [stopwords_latin_v2_4.txt](versions/stopwords_latin_v2_4.txt)   
+    * 4010 items in [stopwords_latin_v2_3.txt](versions/stopwords_latin_v2_3.txt)
+    * 4009 items in [stopwords_latin_v2_2.txt](versions/stopwords_latin_v2_2.txt)
+    * 3845 items in [stopwords_latin_v2_1.txt](versions/stopwords_latin_v2_1.txt)
     * 3839 items in [stopwords_latin_v2_0.txt](versions/stopwords_latin_v2_0.txt)
     * 0144 items in [stopwords_latin_v1_0.txt](versions/stopwords_latin_v1_0.txt)
 * Greek
-    * 6694 items in [stopwords_greek_v2_4.txt](versions/stopwords_greek_v2_4.txt)
-    * 6693 items in [stopwords_greek_v2_3.txt](versions/stopwords_greek_v2_3.txt)
-    * 6529 items in [stopwords_greek_v2_2.txt](versions/stopwords_greek_v2_2.txt)
-    * 6517 items in [stopwords_greek_v2_1.txt](versions/stopwords_greek_v2_1.txt)
+    * 6695 items in [stopwords_greek_v2_5.txt](versions/stopwords_greek_v2_5.txt)
+    * 6695 items in [stopwords_greek_v2_4.txt](versions/stopwords_greek_v2_4.txt)
+    * 6694 items in [stopwords_greek_v2_3.txt](versions/stopwords_greek_v2_3.txt)
+    * 6530 items in [stopwords_greek_v2_2.txt](versions/stopwords_greek_v2_2.txt)
+    * 6518 items in [stopwords_greek_v2_1.txt](versions/stopwords_greek_v2_1.txt)
     * 7573 items in [stopwords_greek_v2_0.txt](versions/stopwords_greek_v2_0.txt)
     * 0262 items in [stopwords_greek_v1_0.txt](versions/stopwords_greek_v1_0.txt)
 
@@ -136,6 +138,10 @@ I added abbreviations commonly found in critical apparatus and notes. They don'
 
 I sorted both lists alphabetically (except for numerals) with James Tauber's [Pyuca](https://github.com/jtauber/pyuca/) software ("Python Unicode Collation Algorithm implementation"). This was mostly useful for Greek as polytonic Unicode is not handled correctly by default. I also reinserted a number ("79") missing in Roman numerals.
 
+### Latin version 2.4 and Greek version 2.5: Fixing counts
+
+I only corrected the item counts. The function I used was counting the octothorpe sign as a comment in the list of typographical symbols.
+
 ## Feedback?
 
 Questions, comments and advice are most welcome.

stopwords_greek.txt

@@ -1,12 +1,12 @@
 # Ancient Greek stopwords
-# version 2.4
-# 2018-02-10
+# version 2.5
+# 2018-02-13
 # Aurélien Berra
 # 
 # Ancient Greek stopwords for textual analysis
 # language: Ancient Greek (grc)
 # type: dataset
-# items count: 6694
+# items count: 6695
 # https://github.com/aurelberra/stopwords
 # rights: CC-BY-NC-SA
 

stopwords_latin.txt

@@ -1,12 +1,12 @@
 # Ancient Latin stopwords
-# version 2.3
-# 2018-02-10
+# version 2.4
+# 2018-02-13
 # Aurélien Berra
 # 
 # Ancient Latin stopwords for textual analysis
 # language: Latin (la, lat)
 # type: dataset
-# items count: 4009
+# items count: 4010
 # https://github.com/aurelberra/stopwords
 # rights: CC-BY-NC-SA