fix typo in amazon domains

zytedata · Jul 30, 2024 · f18c190 · f18c190
1 parent 51b1104
commit f18c190
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 4 deletions.
diff --git a/duplicate_url_discarder_rules/queryRemovalExcept/main.json b/duplicate_url_discarder_rules/queryRemovalExcept/main.json
@@ -29,8 +29,8 @@
         "amazon.com/dp",
         "amazon.de/*/dp",
         "amazon.de/dp",
-        "amazon.e/*/dp",
-        "amazon.e/dp",
+        "amazon.es/*/dp",
+        "amazon.es/dp",
         "amazon.fr/*/dp",
         "amazon.fr/dp",
         "amazon.in/*/dp",
@@ -39,8 +39,8 @@
         "amazon.it/dp",
         "amazon.nl/*/dp",
         "amazon.nl/dp",
-        "amazon.p/*/dp",
-        "amazon.p/dp",
+        "amazon.pt/*/dp",
+        "amazon.pt/dp",
         "amazon.sa/*/dp",
         "amazon.sa/dp",
         "amazon.se/*/dp",

diff --git a/duplicate_url_discarder_rules/subpathRemoval/main.json b/duplicate_url_discarder_rules/subpathRemoval/main.json
@@ -19,10 +19,12 @@
         "amazon.com.tr/*/dp",
         "amazon.com/*/dp",
         "amazon.de/*/dp",
+        "amazon.es/*/dp",
         "amazon.fr/*/dp",
         "amazon.in/*/dp",
         "amazon.it/*/dp",
         "amazon.nl/*/dp",
+        "amazon.pt/*/dp",
         "amazon.sa/*/dp",
         "amazon.se/*/dp"
       ]

diff --git a/tests/queryRemovalExcept/test_query_removal_except_main.py b/tests/queryRemovalExcept/test_query_removal_except_main.py
@@ -0,0 +1,55 @@
+from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
+from duplicate_url_discarder.processors import QueryRemovalExceptProcessor
+
+from duplicate_url_discarder_rules import RULE_PATHS
+
+
+def test_query_removal_except_main_rules():
+    rule_path = [
+        path for path in RULE_PATHS if path.endswith("queryRemovalExcept/main.json")
+    ]
+    assert len(rule_path) == 1
+
+    canonicalizer = UrlCanonicalizer(rule_path)
+
+    assert len(canonicalizer.processors) == 1
+    assert isinstance(
+        list(canonicalizer.processors.values())[0], QueryRemovalExceptProcessor
+    )
+
+    domains = [
+        "ae",
+        "ca",
+        "cn",
+        "co.jp",
+        "co.uk",
+        "com.au",
+        "com.br",
+        "com.mx",
+        "com.sg",
+        "com.tr",
+        "com",
+        "de",
+        "es",
+        "fr",
+        "in",
+        "it",
+        "nl",
+        "pt",
+        "sa",
+        "se",
+    ]
+
+    for domain in domains:
+        assert (
+            canonicalizer.process_url(
+                f"https://www.amazon.{domain}/some-text/dp/ASIN?p=1&q=2#frag"
+            )
+            == f"https://www.amazon.{domain}/some-text/dp/ASIN#frag"
+        )
+        assert (
+            canonicalizer.process_url(
+                f"https://www.amazon.{domain}/dp/ASIN?p=1&q=2#frag"
+            )
+            == f"https://www.amazon.{domain}/dp/ASIN#frag"
+        )
diff --git a/tests/subpathRemoval/test_subpath_removal_main.py b/tests/subpathRemoval/test_subpath_removal_main.py
@@ -30,10 +30,12 @@ def test_subpath_removal_main_rules():
         "com.tr",
         "com",
         "de",
+        "es",
         "fr",
         "in",
         "it",
         "nl",
+        "pt",
         "sa",
         "se",
     ]
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,10 +30,12 @@ def test_subpath_removal_main_rules(): @@
             "com.tr",
             "com",
             "de",
+            "es",
             "fr",
             "in",
             "it",
             "nl",
+            "pt",
             "sa",
             "se",
         ]
@@ Expand Down @@