backporting some changes from readability

Author: masylum

package.json

@@ -1,6 +1,6 @@
 {
     "name": "cheer-reader",
-    "version": "0.1.1",
+    "version": "0.1.2",
     "description": "A cheerio port of readability.js",
     "main": "index.js",
     "types": "dist/index.d.ts",

src/index.ts

@@ -957,29 +957,18 @@ export class Readability {
                 parseSuccessful = false
                 $page.html(pageCacheHtml)
 
+                this.attempts.push({
+                    articleContent: $articleContent,
+                    textLength,
+                })
+
                 if (this.flagIsActive(FLAG_STRIP_UNLIKELYS)) {
                     this.removeFlag(FLAG_STRIP_UNLIKELYS)
-                    this.attempts.push({
-                        articleContent: $articleContent,
-                        textLength,
-                    })
                 } else if (this.flagIsActive(FLAG_WEIGHT_CLASSES)) {
                     this.removeFlag(FLAG_WEIGHT_CLASSES)
-                    this.attempts.push({
-                        articleContent: $articleContent,
-                        textLength,
-                    })
                 } else if (this.flagIsActive(FLAG_CLEAN_CONDITIONALLY)) {
                     this.removeFlag(FLAG_CLEAN_CONDITIONALLY)
-                    this.attempts.push({
-                        articleContent: $articleContent,
-                        textLength,
-                    })
                 } else {
-                    this.attempts.push({
-                        articleContent: $articleContent,
-                        textLength,
-                    })
                     // No luck after removing flags, just return the longest text we found during the different loops
                     this.attempts.sort(function (a, b) {
                         return b.textLength - a.textLength

src/regexes.ts

@@ -8,7 +8,7 @@ export const okMaybeItsACandidate =
 export const positive =
     /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i
 export const negative =
-    /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i
+    /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i
 export const extraneous =
     /print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|utility/i
 export const replaceFonts = /<(\/?)font[^>]*>/gi

src/unescapeHtmlEntities.ts

@@ -18,8 +18,18 @@ export function unescapeHtmlEntities(str: string | null | undefined) {
             const index = tag as keyof typeof HTML_ESCAPE_MAP
             return HTML_ESCAPE_MAP[index]
         })
-        .replace(/&#(?:x([0-9a-z]{1,4})|([0-9]{1,4}));/gi, (_, hex, numStr) => {
-            const num = parseInt(hex || numStr, hex ? 16 : 10)
-            return String.fromCharCode(num)
+        .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function (_, hex, numStr) {
+            let num = parseInt(hex || numStr, hex ? 16 : 10)
+
+            // these character references are replaced by a conforming HTML parser
+            if (
+                num == 0 ||
+                num > 0x10ffff ||
+                (num >= 0xd800 && num <= 0xdfff)
+            ) {
+                num = 0xfffd
+            }
+
+            return String.fromCodePoint(num)
         })
 }