Remove trailing whitespace from paragraphs

jncraton · Jan 23, 2025 · 03391d6 · 03391d6
1 parent 1e22202
commit 03391d6
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/languagemodels/preprocess.py b/languagemodels/preprocess.py
@@ -54,7 +54,7 @@ def handle_data(self, data):
                     self.paras.append(data)
 
         def get_plain(self):
-            return "\n\n".join([p for p in self.paras if len(p) > 140])
+            return "\n\n".join([p.rstrip() for p in self.paras if len(p) > 140])
 
     extractor = ParagraphExtractor()
     extractor.feed(src)