Dev

.dockerignore

@@ -0,0 +1,34 @@
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.gitignore

@@ -9,3 +9,4 @@ venv
 *log*
 *sh
 src/
+**/.DS_Store

Dockerfile

@@ -1,13 +1,26 @@
-FROM python:3.8-slim-buster
 
-RUN apt-get update && apt-get install -y git
-RUN apt-get install -y procps
+ARG PYTHON_VERSION=3.9.13
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
 
 WORKDIR /app
 
 COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
+RUN pip install -r requirements.txt
 
+# Copy the source code into the container.
 COPY . .
 
-CMD "./script.sh"
+# Expose the port that the application listens on.
+EXPOSE 5000
+
+# Run the application.
+#CMD gunicorn 'venv.lib.python3.11.site-packages.werkzeug.wsgi' --bind=0.0.0.0:5000
+#CMD gunicorn main:app --access-logfile ./server.log --timeout 300
+CMD python main.py

Dockerfile.old

@@ -0,0 +1,13 @@
+FROM python:3.8-slim-buster
+
+RUN apt-get update && apt-get install -y git
+RUN apt-get install -y procps
+
+WORKDIR /app
+
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+
+COPY . .
+
+CMD "./script.sh"

compose.yaml

@@ -0,0 +1,7 @@
+services:
+  server:
+    build:
+      context: .
+    ports:
+      - 5000:5000
+    restart: always

import_one.py

@@ -1,7 +1,7 @@
 # configuration for pywikibot
 import sys
 
-import pywikibot
+from util.util import import_one
 
 # connect to the wikibase
 wikibase = pywikibot.Site("my", "my")
@@ -19,13 +19,4 @@
 # import a single item or property
 arg = sys.argv[1]
 print(f"Importing {arg}")
-if arg.startswith("Q"):
-    print("before get")
-    wikidata_item = pywikibot.ItemPage(wikidata_repo, arg)
-    wikidata_item.get()
-    print("after get")
-    wikibase_importer.change_item(wikidata_item, wikibase_repo, True)
-elif arg.startswith("P"):
-    wikidata_property = pywikibot.PropertyPage(wikidata_repo, arg)
-    wikidata_property.get()
-    wikibase_importer.change_property(wikidata_property, wikibase_repo, True)
+import_one(arg)

import_recent_changes.py

@@ -27,8 +27,7 @@
 identifier.get(wikibase_repo)
 print('Wikidata Item Identifier',identifier.itemIdentifier)
 
-idSparql = IdSparql(app_config.get('wikibase', 'sparqlEndPoint'), identifier.itemIdentifier, identifier.propertyIdentifier)
-idSparql.load()
+idSparql = IdSparql()
 
 #grab all entities that changed
 recent = get_wikidata_changes(None, 15)

main.py

@@ -0,0 +1,91 @@
+import socket
+import sys
+
+import requests
+from flask import Flask, request
+from flask_restful import Api, Resource
+from flask_cors import CORS, cross_origin
+from threading import Thread
+
+from util.util import import_one
+
+user_config = __import__("user-config")
+
+app = Flask(__name__)
+CORS(app)
+
+api = Api(app)
+
+class Index(Resource):
+    def get(self):
+        return {"data": "Welcome to the index"}
+
+# class that imports all statements from Wikidata
+class Sync(Resource):
+    def get(self):
+        q_id = request.args.get('q_id')
+        api_key = request.args.get('api_key')
+        if is_authorised(api_key):
+            import_one(q_id).getID()
+            payload = {"status_code": 200, "completed": True, "message": "Import process complete"}
+        else:
+            payload = {"status_code": 403, "completed": False, "message": "Unauthorised Access"}
+        return payload
+
+# import one
+class ImportOne(Resource):
+    def get(self):
+        q_id = request.args.get('q_id')
+        api_key = request.args.get('api_key')
+
+        if is_authorised(api_key):
+            response = import_one(q_id, import_statements = False).getID()
+            #   print(response)
+            if response:
+                payload = {"status_code": 200, "message": "Import successful", "pid": response}
+            else:
+                payload = {"status_code": 500,
+                           "message": "Import could not be completed"}
+        else:
+            payload = {"status_code": 403, "completed": False, "message": "Unathorised Access"}
+        return payload
+
+
+class WikiDataQuery(Resource):
+    def get(self):
+        query_string = request.args.get('query_string')
+        query_type = request.args.get('query_type')
+        api_key = request.args.get('api_key')
+
+        url = "https://www.wikidata.org/w/api.php?action=wbsearchentities&search=" + \
+              query_string + "&format=json&errorformat=plaintext&language=en&uselang=en&type=" + query_type
+
+        if is_authorised(api_key):
+            response = requests.get(url)
+            response = response.json()
+            if response:
+                payload = {"status_code": 200, "response": response}
+            else:
+                payload = {"status_code": 500,
+                           "message": "Import could not be completed"}
+        else:
+            payload = {"status_code": 403, "completed": False, "message": "Unathorised Access"}
+        return payload
+
+
+def is_authorised(api_key):
+    if str(user_config.apiKey) == api_key:
+        return True
+    else:
+        return False
+
+
+# ROUTES
+api.add_resource(Index, "/")
+api.add_resource(Sync, "/sync")
+api.add_resource(ImportOne, "/import-wikidata-item")
+api.add_resource(WikiDataQuery, "/remote-wikidata-query")
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', debug=True)

requirements.txt

@@ -10,5 +10,10 @@ six==1.13.0
 SPARQLWrapper==1.8.4
 urllib3==1.25.7
 
+Flask~=2.0.2
+Flask-Restful==0.3.9
+Flask-Cors==3.0.10
 pywikibot~=7.7.2
-mwparserfromhell>=0.5.0
+mwparserfromhell>=0.5.0
+gunicorn==20.1.0
+werkzeug==2.3.7

user-config.py

@@ -1,4 +1,4 @@
-from pywikibot.config2 import usernames
+from pywikibot.config import usernames
 
 user_families_paths = ['./config']
 mylang = "wikidata"
@@ -9,3 +9,4 @@
 maxthrottle = 0
 max_retries = 100
 #verbose_output = True
+apiKey = 123456

user-password.py

@@ -1 +1 @@
-(u'admin', BotPassword(u'WikidataUpdater', u'BotPassword'))
+(u'admin', BotPassword(u'WikidataUpdater', u'BotPassword'))

util/IdSparql.py

@@ -1,37 +1,56 @@
 # this class makes the correspondence between Wikidata entities and entities in the Wikibase using the external
 # identifier for Wikidata
+import pywikibot
 from SPARQLWrapper import SPARQLWrapper, JSON
 import configparser
 
+from util.PropertyWikidataIdentifier import PropertyWikidataIdentifier
+
 
 class IdSparql:
-    def __init__(self, endpoint, item_identifier, property_identifier):
+    _instance = None
+
+    def __new__(class_, *args, **kwargs):
+        if not isinstance(class_._instance, class_):
+            class_._instance = object.__new__(class_, *args, **kwargs)
+
+        return class_._instance
+
+    def __init__(self):
         self.mapEntity = {}
         self.mapProperty = {}
-        self.endpoint = endpoint
-        self.item_identifier = item_identifier
-        self.property_identifier = property_identifier
+        wikibase = pywikibot.Site("my", "my")
+        wikibase_repo = wikibase.data_repository()
+        wikibase_repo.login()
+        identifier = PropertyWikidataIdentifier()
+        identifier.get(wikibase_repo)
+        self.item_identifier = identifier.itemIdentifier
+        self.property_identifier = identifier.propertyIdentifier
         self.app_config = configparser.ConfigParser()
         self.app_config.read('config/application.config.ini')
+        self.endpoint = self.app_config.get('wikibase', 'sparqlEndPoint')
+        self.load()
 
     def load(self):
         sparql = SPARQLWrapper(self.endpoint)
         query = """
                             select ?item ?id where {
-                                ?item <""" + self.app_config.get('wikibase','propertyUri') + """/direct/""" + self.item_identifier + """> ?id
+                                ?item <""" + self.app_config.get('wikibase',
+                                                                 'propertyUri') + """/direct/""" + self.item_identifier + """> ?id
                             }
                         """
         sparql.setQuery(query)
         sparql.setReturnFormat(JSON)
         results = sparql.query().convert()
         for result in results['results']['bindings']:
             split = result['item']['value'].split('/')
-            id = split[len(split)-1]
+            id = split[len(split) - 1]
             if id.startswith('Q'):
                 self.mapEntity[result['id']['value']] = id
         query = """
                     select ?item ?id where {
-                        ?item <""" + self.app_config.get('wikibase','propertyUri') + """/direct/""" + self.property_identifier + """> ?id
+                        ?item <""" + self.app_config.get('wikibase',
+                                                         'propertyUri') + """/direct/""" + self.property_identifier + """> ?id
                     }
                 """
         sparql.setQuery(query)
@@ -45,26 +64,26 @@ def load(self):
             else:
                 print("This should not happen")
 
-    def get_id(self,id):
+    def get_id(self, id):
         if id.startswith("Q"):
             return self.mapEntity[id]
         elif id.startswith("P"):
             return self.mapProperty[id]
         else:
             raise NameError('This should not happen')
 
-    def save_id(self,id,new_id):
+    def save_id(self, id, new_id):
         if id.startswith("Q"):
             self.mapEntity[id] = str(new_id)
         elif id.startswith("P"):
             self.mapProperty[id] = str(new_id)
         else:
             raise NameError('This should not happen')
 
-    def contains_id(self,id):
+    def contains_id(self, id):
         if id.startswith("Q"):
             return id in self.mapEntity
         elif id.startswith("P"):
             return id in self.mapProperty
         else:
-            print('This should not happen')
+            print('This should not happen')