Merge pull request #2023 from openzim/count-graphemes

Measure Metadata length by grapheme

package.json

@@ -103,6 +103,7 @@
     "rimraf": "^4.4.1",
     "semver": "^7.3.8",
     "sharp": "^0.32.6",
+    "split-by-grapheme": "^1.0.1",
     "swig-templates": "^2.0.3",
     "typescript": "^4.9.4",
     "utf8-binary-cutter": "^0.9.2",

src/util/metaData.ts

@@ -1,4 +1,6 @@
 import AjvModule from 'ajv'
+import type { KeywordCxt } from 'ajv'
+import { byGrapheme } from 'split-by-grapheme'
 
 const Ajv = AjvModule.default
 const ajv = new Ajv({ allErrors: true })
@@ -18,18 +20,31 @@ ajv.addKeyword({
   },
 })
 
+ajv.addKeyword({
+  keyword: 'uMaxLength',
+  type: 'string',
+  validate: (max_length: number, value) => {
+    return value.split(byGrapheme).length <= max_length
+  },
+  error: {
+    message: (cxt: KeywordCxt): string => {
+      return `must NOT have more than ${cxt.schemaValue} graphemes`
+    },
+  },
+})
+
 const schema = {
   type: 'object',
   properties: {
     Name: { type: 'string', minLength: 1 },
     Creator: { type: 'string', minLength: 1 },
-    Description: { type: 'string', maxLength: 80, minLength: 1 },
+    Description: { type: 'string', uMaxLength: 80, minLength: 1 },
     Language: { type: 'string', minLength: 1, pattern: '^\\w{3}(,\\w{3})*$' },
     Publisher: { type: 'string', minLength: 1 },
-    Title: { type: 'string', maxLength: 30, minLength: 1 },
+    Title: { type: 'string', uMaxLength: 30, minLength: 1 },
     Date: { type: 'string', maxLength: 10, minLength: 10 },
     'Illustration_48x48@1': { checkRegexFromBuffer: '^\x89\x50\x4e\x47\x0d\x0a\x1a\x0a.+' },
-    LongDescription: { type: 'string', maxLength: 4000 },
+    LongDescription: { type: 'string', uMaxLength: 4000 },
     License: { type: 'string' },
     Tags: { type: 'string' },
     Relation: { type: 'string' },

test/unit/util.test.ts

@@ -422,7 +422,7 @@ describe('Utils', () => {
       Description: 'test Description',
       Language: 'eng,ita',
       Publisher: 'test Publisher',
-      Title: 'test Title',
+      Title: 'विकी मेड मेडिकल इनसाइक्लोपीडिया हिंदी मेंfविविविमे',
       'Illustration_48x48@1': pngImage,
     }
 
@@ -459,15 +459,15 @@ describe('Utils', () => {
         ...minimumValidMetadata,
         Description: 'test Description test Description test Description test Description test Description test Description ',
       }
-      expect(() => validateMetadata(metaData)).toThrow('MetaData Description: must NOT have more than 80 characters')
+      expect(() => validateMetadata(metaData)).toThrow('MetaData Description: must NOT have more than 80 graphemes')
     })
 
     test('validate long Title', () => {
       const metaData = {
         ...minimumValidMetadata,
-        Title: 'test Title test Title test Title',
+        Title: 'विकी मेड मेडिकल इनसाइक्लोपीडिया हिंदी मेंfविविविमेविमेवि',
       }
-      expect(() => validateMetadata(metaData)).toThrow('MetaData Title: must NOT have more than 30 characters')
+      expect(() => validateMetadata(metaData)).toThrow('MetaData Title: must NOT have more than 30 graphemes')
     })
 
     test('validate string with line brake', () => {