updated language for inter wiki link extraction

sven-h · Mar 28, 2019 · 0db26a7 · 0db26a7
1 parent e8a83a0
commit 0db26a7
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 17 deletions.
diff --git a/core/src/main/scala/org/dbpedia/extraction/util/Language.scala b/core/src/main/scala/org/dbpedia/extraction/util/Language.scala
@@ -124,7 +124,7 @@ object Language extends (String => Language)
         for(lang <- wikiLanguageCodes)
         {
           try {
-            languages(lang) = makeDbkwikLanguage(lang, "default")
+            languages(lang) = makeDbkwikLanguage(lang, "default", true)
           }
           catch{
             case mre : MissingResourceException => logger.log(Level.WARNING, "Could not create the language: " + lang)
@@ -134,24 +134,38 @@ object Language extends (String => Language)
       }
 
 
-      def updateOneLanguage(wikiprefix : String, wiki: String):Unit={
-        map(wikiprefix) = makeDbkwikLanguage(wikiprefix,  wiki)
-        English = map("en")
+      def preprocessWikiBase(wikiBase: String): String ={
+        //return new java.net.URI(wikiBase).getScheme()
+        return wikiBase.stripPrefix("http://").stripPrefix("https://").stripSuffix("/$1").stripSuffix("/wiki")
+      }
+
+      def getLanguageFreeWikiBase(wikiBase: String): String ={
+        var base = preprocessWikiBase(wikiBase)
+        var splits = base.split("\\.")
+        if(splits.length > 1){
+          if(wikiLanguageCodes.contains(splits(0))){
+            var test = splits.slice(1, splits.length).mkString(".")
+            return test
+          }
+        }
+        return base
       }
 
       def updateInterwikis(interwikis : scala.collection.Map[String, String]):Unit={
         for ((prefix, url) <- interwikis) {
-          Language.updateOneLanguage(prefix, url.stripPrefix("http://").stripSuffix("/$1").stripSuffix("/wiki"))
+          if(url.contains("fandom") || url.contains("wikia"))
+            map(prefix) = makeDbkwikLanguage(prefix,  preprocessWikiBase(url), false)
         }
-        print("test")
+        English = map("en")
       }
 
-      def updateAllLanguages(wikiBase: String): Unit ={
+      def updateAllLanguages(base: String): Unit ={
         //map.clear()//do not clear because we want to keep "mappings", "wikidata" etc.
+        var wikiBase = getLanguageFreeWikiBase(base)
         for(lang <- wikiLanguageCodes)
         {
           try {
-            map(lang) = makeDbkwikLanguage(lang, wikiBase)
+            map(lang) = makeDbkwikLanguage(lang, wikiBase, true)
           }
           catch{
             case mre : MissingResourceException => logger.log(Level.WARNING, "Could not create the language: " + lang)
@@ -167,9 +181,14 @@ object Language extends (String => Language)
 
 
 
-      def makeDbkwikLanguage(language : String, wikiBase: String): Language = {
+      def makeDbkwikLanguage(language : String, wikiBase: String, modifyBase: Boolean): Language = {
+        var base = wikiBase
+        if(language.equals("en") == false && modifyBase){
+          base = language + "." + base
+        }
+
 
-        val baseDomain = "dbkwik.webdatacommons.org/" + wikiBase
+        val baseDomain = "dbkwik.webdatacommons.org/" + base
 
         val loc = Locale.forLanguageTag(language)
 
@@ -187,8 +206,8 @@ object Language extends (String => Language)
           "http://" + baseDomain,   //val dbpediaUri: String,
           new DBpediaNamespace("http://" + baseDomain + "/resource/"), //val resourceUri: RdfNamespace,
           new DBpediaNamespace("http://" + baseDomain + "/property/"), //val propertyUri: RdfNamespace,
-          "http://"+wikiBase.stripPrefix("http://"),               //val baseUri: String,
-          "https://"+wikiBase.stripPrefix("http://")+"/api.php", //val apiUri: String,
+          "http://"+ base,               //val baseUri: String,
+          "https://"+ base + "/api.php", //val apiUri: String,
           0                                 //val pages: Int
         )
       }

diff --git a/core/src/main/scala/org/dbpedia/extraction/wikiparser/WikiTitle.scala b/core/src/main/scala/org/dbpedia/extraction/wikiparser/WikiTitle.scala
@@ -1,7 +1,9 @@
 package org.dbpedia.extraction.wikiparser
 
+import org.dbpedia.extraction.ontology.RdfNamespace
 import org.dbpedia.extraction.util.RichString.wrapString
-import org.dbpedia.extraction.util.{Language, WikiUtil}
+import org.dbpedia.extraction.util.StringUtils.replacements
+import org.dbpedia.extraction.util.{Language, StringUtils, WikiUtil}
 import org.dbpedia.iri.UriDecoder
 import org.dbpedia.util.text.ParseExceptionIgnorer
 import org.dbpedia.util.text.html.{HtmlCoder, XmlCodes}
@@ -41,7 +43,7 @@ class WikiTitle (
     val encodedWithNamespace = withNamespace(true)
 
     /** page IRI for this page title */
-    val pageIri = language.baseUri+"/wiki/"+encodedWithNamespace
+    val pageIri = language.baseUri+"/wiki/"+StringUtils.escape(encodedWithNamespace, WikiTitle.iriEscapes)
 
     /** resource IRI for this page title */
     val resourceIri = language.resourceUri.append(encodedWithNamespace)
@@ -92,6 +94,12 @@ class WikiTitle (
 
 object WikiTitle
 {
+    // for this list of characters, see RFC 3987 and https://sourceforge.net/mailarchive/message.php?msg_id=28982391
+    private val iriEscapes = {
+      val chars = ('\u0000' to '\u0020').mkString + "\"#%<>?[\\]^`{|}" + ('\u007F' to '\u009F').mkString
+      replacements('%', chars)
+    }
+
     /**
      * Parses a MediaWiki link or title.
      *

diff --git a/core/src/main/scala/org/dbpedia/extraction/wikiparser/impl/wikipedia/Namespaces.scala b/core/src/main/scala/org/dbpedia/extraction/wikiparser/impl/wikipedia/Namespaces.scala
@@ -324,6 +324,6 @@ object Namespaces
         (names.toMap, codes.toMap)
     }
 
-    def names(lang : Language) : Map[Int, String] = names.getOrElse(lang.wikiCode, throw new IllegalArgumentException("no namespace names found for language '"+lang.wikiCode+"'"))
-    def codes(lang : Language) : Map[String, Int] = codes.getOrElse(lang.wikiCode, throw new IllegalArgumentException("no namespace codes found for language '"+lang.wikiCode+"'"))
+    def names(lang : Language) : Map[Int, String] = names.getOrElse(lang.wikiCode, names("en"))
+    def codes(lang : Language) : Map[String, Int] = codes.getOrElse(lang.wikiCode, codes("en"))
 }
diff --git a/dump/extraction.dbkwik.properties b/dump/extraction.dbkwik.properties
@@ -31,7 +31,7 @@ copyrightCheck=false
 
 #parameter for xml-safe properties:
 
-uri-policy.iri=xml-safe-predicates:*;xml-safe-subjects:*
+uri-policy.iri=xml-safe:*
 
 format.ttl=turtle-triples;uri-policy.iri