Adding stopwords for Yorùbá

README.md

@@ -73,6 +73,7 @@ Arrays of stopwords for the following  23 languages are supplied:
 * `ru` - Russian
 * `sv` - Swedish
 * `sw` - Swahili
+* `yo` - Yorùbá
 * `zh` - Chinese Simplified
 
 ```javascript

lib/stopword.js

@@ -33,4 +33,5 @@ exports.pt = require('./stopwords_pt.js').words
 exports.ru = require('./stopwords_ru.js').words
 exports.sv = require('./stopwords_sv.js').words
 exports.sw = require('./stopwords_sw.js').words
+exports.yo = require('./stopwords_yo.js').words
 exports.zh = require('./stopwords_zh.js').words

lib/stopwords_yo.js

@@ -0,0 +1,32 @@
+/* Copyright 2016 Liam Doherty
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* A list of commonly used words that have little meaning and can be excluded
+from analysis.
+This list is frequency sorted. That means it can be sliced from the bottom
+and be less agressive in excluding stopwords */
+
+var words = [
+  'ó', 'ní', 'ìjàpá', 'ṣe', 'rẹ̀', 'tí', 'àwọn', 'sí', 'ni', 'náà', 'anansi',
+  'láti', 'kan', 'ti', 'ń', 'lọ', 'o', 'bí', 'padà', 'sì', 'wá', 'wangari',
+  'lè', 'wà', 'kí', 'púpọ̀', 'odò', 'mi', 'wọ́n', 'pẹ̀lú', 'a', 'ṣùgbọ́n',
+  'fún', 'jẹ́', 'fẹ́', 'oúnjẹ', 'rí', 'igi', 'kò', 'ilé', 'jù', 'olóńgbò',
+  'pé', 'é', 'gbogbo', 'iṣu', 'inú', 'bẹ̀rẹ̀', 'jẹ', 'fi', 'dúró', 'alẹ́',
+  'ọjọ́', 'nítorí', 'nǹkan', 'ọ̀rẹ́', 'àkókò', 'sínú', 'ṣ', 'yìí'
+]
+
+
+// Tell the world about the noise words.
+exports.words = words

package.json

@@ -1,7 +1,7 @@
 {
   "name": "stopword",
-  "version": "0.1.18",
-  "description": "A module for node.js that takes in text and returns text that is stripped of stopwords. Has pre-defined stopword lists for 23 languages and also takes lists with custom stopwords as input.",
+  "version": "0.1.19",
+  "description": "A module for node.js that takes in text and returns text that is stripped of stopwords. Has pre-defined stopword lists for 24 languages and also takes lists with custom stopwords as input.",
   "main": "lib/stopword.js",
   "scripts": {
     "test": "mocha"

test/test.js

@@ -158,6 +158,12 @@ describe('general stopwordiness:', function () {
     newString.should.eql(['1502','kristoffer','kolumbus','saapui','hondurasin','rannikolle','antoi','sille','nykyisen','nimen'])
   })
 
+  it('should remove Yorùbá stopwords', function () {
+    const oldString = 'jẹ́ ẹ̀gbà ọrùn tí wọ́n gbẹ́ lére tí ó sì jẹ́ àwòrán akọni obìrin tí a mọ̀ si ìyá wa olorì idia ti ọ̀rundún mẹ́rìndínlógún ṣẹ́yìn'.split(' ')
+    const newString = sw.removeStopwords(oldString, sw.yo)
+    newString.should.eql(['ẹ̀gbà', 'ọrùn', 'gbẹ́', 'lére', 'àwòrán', 'akọni', 'obìrin', 'mọ̀', 'si', 'ìyá', 'wa', 'olorì', 'idia', 'ọ̀rundún', 'mẹ́rìndínlógún', 'ṣẹ́yìn'])
+  })
+
 
 
   // Right to Left languages