T358307 - Update Release Variables Script

build.sh

@@ -17,6 +17,17 @@ source ./versions.inc.sh
 
 DOCKER_BUILD_CACHE_OPT=""
 
+# ℹ️ Update Commit Hashes
+function update_commit_hashes {
+    docker compose \
+    --file test/docker-compose.yml \
+    --env-file test/test-runner.env \
+    run --rm --build test-runner -c "
+    cd ..
+    python3 -m pip install requests bs4 lxml
+    python3 update_commits.py
+    "
+}
 
 # wikibase/wdqs -> wdqs
 function image_url_to_image_name {
@@ -205,6 +216,9 @@ for arg in "$@"; do
             build_all
             build_target_set=true
             ;;
+        update_hashes)
+            update_commit_hashes
+            ;;
         -n|--no-cache)
             DOCKER_BUILD_CACHE_OPT="--no-cache"
             ;;

update_commits.py

@@ -0,0 +1,109 @@
+import re, requests, json
+from bs4 import BeautifulSoup
+
+
+def get_commit(variable: str, url: str, parse_commit: callable, previous_commit: str):
+    print(f"Variable:\t{variable}")
+    print(f"\tURL:\t{url}")
+    try:
+        response = requests.get(url)
+        commit = parse_commit(response)
+        if previous_commit != commit:
+            print(f"\tOld Commit:\t{previous_commit}")
+            print(f"\tNew Commit:\t{commit}")
+            return commit
+        else:
+            print(f"\tCommit:\t{commit}")
+            return False
+    except Exception as exc:
+        print(f"\tError:\t{exc}")
+        return False
+
+
+gerrit_pattern = re.compile(
+    r"# (https://gerrit.*)[ \t\r\n]*([A-Z]+_COMMIT)=([0-9a-f]+)"
+)
+
+
+def parse_gerrit_commit(response: requests.Response) -> str:
+    """Parse webpage using BeautifulSoup"""
+    soup = BeautifulSoup(response.content, "lxml")
+    return soup.find("th", text="commit").next_sibling.text
+
+
+github_pattern = re.compile(
+    r"# (https://github\.com/(.*/commits.*))[ \t\r\n]*([A-Z]+_COMMIT)=([0-9a-f]+)"
+)
+
+
+def parse_github_commit(response: requests.Response) -> str:
+    """Fetch from API"""
+    data = json.loads(response.content)
+    return data["sha"]
+
+
+bitbucket_pattern = re.compile(
+    r"# (https://bitbucket\.org/(.*/commits)/branch/master)[ \t\r\n]*([A-Z]+_COMMIT)=([0-9a-f]+)"
+)
+
+
+def parse_bitbucket_commit(response: requests.Response) -> str:
+    """Fetch from API"""
+    data = json.loads(response.content)
+    return data["values"][0]["hash"]
+
+
+def run():
+    with open("variables.env", "r") as variable_file:
+        variable_contents = variable_file.read()
+
+    mediawiki_match = re.search(r"MEDIAWIKI_VERSION=(\d+)\.(\d+)", variable_contents)
+    rel = f"REL{mediawiki_match.group(1)}_{mediawiki_match.group(2)}"
+    print(f"Mediawiki Version:\t{mediawiki_match.group(1)}.{mediawiki_match.group(2)}")
+    variable_contents = re.sub(r"\bREL\d+_\d+", rel, variable_contents)
+
+    for gerrit_commit in re.findall(gerrit_pattern, variable_contents):
+        if commit := get_commit(
+            gerrit_commit[1],
+            gerrit_commit[0],
+            parse_gerrit_commit,
+            gerrit_commit[2],
+        ):
+            variable_contents = re.sub(
+                f"{gerrit_commit[1]}=[0-9a-f]+",
+                f"{gerrit_commit[1]}={commit}",
+                variable_contents,
+            )
+
+    for github_commit in re.findall(github_pattern, variable_contents):
+        if commit := get_commit(
+            github_commit[2],
+            f"https://api.github.com/repos/{github_commit[1]}",
+            parse_github_commit,
+            github_commit[3],
+        ):
+            variable_contents = re.sub(
+                f"{github_commit[2]}=[0-9a-f]+",
+                f"{github_commit[2]}={commit}",
+                variable_contents,
+            )
+
+    for bitbucket_commit in re.findall(bitbucket_pattern, variable_contents):
+        if commit := get_commit(
+            bitbucket_commit[2],
+            f"https://bitbucket.org/!api/2.0/repositories/{bitbucket_commit[1]}",
+            parse_bitbucket_commit,
+            bitbucket_commit[3],
+        ):
+            variable_contents = re.sub(
+                f"{bitbucket_commit[2]}=[0-9a-f]+",
+                f"{bitbucket_commit[2]}={commit}",
+                variable_contents,
+            )
+
+    with open("variables.env", "w") as variable_file:
+        variable_file.write(variable_contents)
+
+
+if __name__ == "__main__":
+    run()

variables.env

@@ -76,15 +76,15 @@ VISUALEDITOR_COMMIT=58d7d809829b4e326f1b8a0b69f83bae2ab851f7
 WIKIBASECIRRUSSEARCH_COMMIT=0e3e82e125254e86e262e6620c53e07ddf75c888
 # https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseManifest/+/refs/heads/REL1_40
 WIKIBASEMANIFEST_COMMIT=10f0ddcb7eac28f56999f983f169cf7b1c6e240a
-# https://github.com/ProfessionalWiki/WikibaseLocalMedia/tree/master
+# https://github.com/ProfessionalWiki/WikibaseLocalMedia/commits/master
 WIKIBASELOCALMEDIA_COMMIT=74c3f70893ffd76feefa9f1e3d77052db20ec3ac
-# https://github.com/ProfessionalWiki/WikibaseEdtf/tree/master
+# https://github.com/ProfessionalWiki/WikibaseEdtf/commits/master
 WIKIBASEEDTF_COMMIT=5596403a27d8e4a5ac044257c49be3e0685318a6
 # https://github.com/magnusmanske/quickstatements/commits/master
 QUICKSTATEMENTS_COMMIT=c4b2c6b086b319aa32dcdd7a323edf188faaa66d
 # https://bitbucket.org/magnusmanske/magnustools/commits/branch/master
 MAGNUSTOOLS_COMMIT=e4abdb6b3bedfdde0325df513b272e1972b1af36
-# https://gerrit.wikimedia.org/r/wikidata/query/gui
+# https://gerrit.wikimedia.org/r/plugins/gitiles/wikidata/query/gui/+/refs/heads/master
 WDQSQUERYGUI_COMMIT=eb1825b3431f3adf47cd774bf52344024e2f11e9
 
 # Image URLs for the docker images built by us.