diff --git a/.github/ISSUE_TEMPLATE/bug_template.yml b/.github/ISSUE_TEMPLATE/bug_template.yml index 6135b249..a3232a0c 100644 --- a/.github/ISSUE_TEMPLATE/bug_template.yml +++ b/.github/ISSUE_TEMPLATE/bug_template.yml @@ -36,8 +36,7 @@ body: - cpan/flatpak-cpan-generator.pl - dotnet/flatpak-dotnet-generator.py - dub/flatpak-dub-generator.py - - go-get/flatpak-go-get-generator.py - - go-get/flatpak-go-vendor-generator.py + - go/flatpak-go-deps.py - node/flatpak-node-generator.py - npm/flatpak-npm-generator.py - pip/flatpak-pip-generator diff --git a/go-get/README.md b/go-get/README.md deleted file mode 100644 index ff3977ad..00000000 --- a/go-get/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Flatpak Go Get Generator -Tool to automatically create the source list for a Go module. - -The script does not require Go in the host system. - -## Usage -1. In the manifest, give the Go module network access and set GOPATH to $PWD. - - Example manifest module (json): -```json -{ - "name": "writeas-cli", - "buildsystem": "simple", - "build-options": { - "env": { - "GOBIN": "/app/bin/" - }, - "build-args": [ - "--share=network" - ] - }, - "build-commands": [ - ". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; go get github.com/writeas/writeas-cli/cmd/writeas" - ] -} -``` - - Example manifest (yaml): -```yaml -app-id: writeas-cli -runtime: org.freedesktop.Platform -runtime-version: '21.08' -sdk: org.freedesktop.Sdk -sdk-extensions: - - org.freedesktop.Sdk.Extension.golang -command: echo "Done" -modules: - - name: writeas - buildsystem: simple - build-options: - append-path: /usr/lib/sdk/golang/bin - env: - GOBIN: /app/bin - GO111MODULE: off - GOPATH: /run/build/writeas - build-args: - - --share=network - build-commands: - - go get github.com/writeas/writeas-cli/cmd/writeas -``` - -2. Run flatpak-builder with `--keep-build-dirs`. -3. Run `go-get/flatpak-go-get-generator.py ` with build-dir pointing the the build directory in `.flatpak-builder/build`. -4. Convert the source list to YAML if necessary. -5. Add the list to the sources field of the Go module in the manifest. -6. Change build command from `go get` to `go install`. -7. Remove network access. - -**The script assumes the networked built was run with `GOPATH=$PWD`.** - -## Example final module -```json -{ - "name": "writeas-cli", - "buildsystem": "simple", - "build-options": { - "env": { - "GOBIN": "/app/bin/" - } - }, - "build-commands": [ - ". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; go install github.com/writeas/writeas-cli/cmd/writeas" - ], - "sources": [ - { - "type": "git", - "url": "https://github.com/atotto/clipboard", - "commit": "aa9549103943c05f3e8951009cdb6a0bec2c8949", - "dest": "src/github.com/atotto/clipboard" - }, - ... - ] - } -``` - diff --git a/go-get/flatpak-go-get-generator.py b/go-get/flatpak-go-get-generator.py deleted file mode 100755 index 7597312a..00000000 --- a/go-get/flatpak-go-get-generator.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2018 Çağatay Yiğit Şahin -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from pathlib import Path -from typing import List, Dict -import subprocess -import argparse -import json - -def is_git_repository(p): - is_git_repo = p.is_dir() and (p / ".git").is_dir() - return is_git_repo - -def repo_paths(build_dir: Path) -> List[Path]: - src_dir = build_dir / 'src' - repo_paths: List[Path] = [] - - domains = src_dir.iterdir() - for domain in domains: - domain_users = domain.iterdir() - for user in domain_users: - if is_git_repository(user): - repo_paths.append(user) - else: - user_repos = user.iterdir() - for ur in user_repos: - if is_git_repository(ur): - repo_paths.append(ur) - return repo_paths - -def repo_source(repo_path: Path) -> Dict[str, str]: - def current_commit(repo_path: Path) -> str: - output = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=repo_path).decode('ascii').strip() - return output - - def remote_url(repo_path: Path) -> str: - output = subprocess.check_output( - ['git', 'remote', 'get-url', 'origin'], - cwd=repo_path).decode('ascii').strip() - return output - - repo_path_str = str(repo_path) - dest_path = repo_path_str[repo_path_str.rfind('src/'):] - source_object = {'type': 'git', 'url': remote_url(repo_path), 'commit': current_commit(repo_path), 'dest': dest_path} - return source_object - -def sources(build_dir: Path) -> List[Dict[str, str]]: - return list(map(repo_source, repo_paths(build_dir))) - -def main(): - def directory(string: str) -> Path: - path = Path(string) - if not path.is_dir(): - msg = 'build-dir should be a directory.' - raise argparse.ArgumentTypeError(msg) - return path - - parser = argparse.ArgumentParser(description='For a Go module’s dependencies, output array of sources in flatpak-manifest format.') - parser.add_argument('build_dir', help='Build directory of the module in .flatpak-builder/build', type=directory) - parser.add_argument('-o', '--output', dest='output_file', help='The file to write the source list to. Default is -sources.json', type=str) - args = parser.parse_args() - source_list = sources(args.build_dir) - - output_file = args.output_file - if output_file is None: - output_file = args.build_dir.absolute().name + '-sources.json' - - with open(output_file, 'w') as out: - json.dump(source_list, out, indent=2) - -if __name__ == '__main__': - main() diff --git a/go-get/flatpak-go-vendor-generator.py b/go-get/flatpak-go-vendor-generator.py deleted file mode 100755 index 680a7e92..00000000 --- a/go-get/flatpak-go-vendor-generator.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -""" -This is a very pragmatic (i.e. simple) tool for using Go vendor -with flatpak. - -To make use of the tool you need to produce a vendor/modules.txt -through `go mod vendor`. One approach is to modify your manifest to -include - -build-options: - build-args: - - --share=network - -and run `go mod vendor` just before you start to build. - -Once that is done, you should see a "vendors/modules.txt" which -you can point this tool at. - -This tool has a few rough edges, such as special-casing a few things. -For example, it assumes that everything is git-clonable. -Except for certain URLs which are rewritten. - -The real solution is https://github.com/golang/go/issues/35922 -""" -import json -import logging -import sys -import urllib.request -from html.parser import HTMLParser - -import attr - -log = logging.getLogger(__name__) - -@attr.s -class GoModule: - name = attr.ib() - version = attr.ib() - revision = attr.ib() - -def parse_modules(fh): - for line in (l.strip() for l in fh if l.strip()): - log.debug("Read line: %s", line) - if line.startswith("# "): - splits = line.split(" ") - name, line_version = splits[-2], splits[-1] - if '-' in line_version: - log.debug("Parsing version: %s", line_version) - _version, date_revision = line_version.strip().split("-", 1) - try: - log.debug("Splitting %s", date_revision) - date, revision = date_revision.split('-') - except ValueError: - log.debug("no further split of %s", date_revision) - date = None - version = revision = line_version - else: - version = _version - - log.debug("Parsed version into: %s %s %s", version, date, revision) - else: - revision = None - version = line_version - - m = GoModule(name, version, revision) - yield m - -def get_go_redirect(html_data): - class GoImportParser(HTMLParser): - _repo = None - - def handle_starttag(self, tag, attrs): - if self._repo is not None: - return - - # Make a dict of the attribute name/values - # since it's easier to work with and understand. - _attrs = {} - for attr, value in attrs: - _attrs[attr] = value - - name_attr = _attrs.get('name') - if name_attr != 'go-import': - return - content = _attrs.get('content') - if content is not None: - self._repo = content.split(' ')[-1] - - def get_repo(self): - return self._repo - - parser = GoImportParser() - parser.feed(html_data) - return parser.get_repo() - - -def go_module_to_flatpak(m): - if not m.name.startswith("github.com"): - url = m.name - else: - splits = m.name.split('/') - if len(splits) > 3: - url = '/'.join(splits[:3]) - else: - url = m.name - url = "https://" + url - - print('Checking {}...'.format(url), file=sys.stderr) - - try: - with urllib.request.urlopen(url + '?go-get=1') as response: - page_contents = str(response.read()) - except urllib.request.URLError as e: - print('Failed to check {}: {}'.format(url, e), file=sys.stderr) - sys.exit(1) - else: - repo = get_go_redirect(page_contents) - url_found = repo - if url_found != url: - print(' got {}'.format(url_found), file=sys.stderr) - else: - print(' done', file=sys.stderr) - url = url_found - - if not '+' in m.version: - tag = m.version - else: - splits = m.version.split('+') - log.debug(f"Splitting version for {url}: {m.version} {splits}") - tag = splits[0] - - rev = m.revision - source = { - "type": "git", - "url": url, - "tag": tag, - "dest": "vendor/" + m.name, - } - if m.revision: - del source["tag"] - source["commit"] = m.revision - - return source - -def main(): - modules_file = sys.argv[1] - fh = open(modules_file) - fp_modules = [go_module_to_flatpak(m) for m in parse_modules(fh)] - print (json.dumps(fp_modules, indent=4)) - -if __name__ == "__main__": - main() diff --git a/go/README.md b/go/README.md new file mode 100644 index 00000000..94e58a39 --- /dev/null +++ b/go/README.md @@ -0,0 +1,184 @@ +# Flatpak Go Generator + +This script requires Go and git to be installed. You also need the Python in [requirements.txt](./requirements.txt). + +This script works by creating a new Go module in a temporary folder, add the given Go package as a dependency, and then runs `go list -m all` to get the full list of Go modules. For each module, it uses `go list -m -json ` to get detailed information. And then finally, it outputs the module in YAML format. + +## Usage + +``` +./flatpak-go-deps.py +``` + +For example, here's how you'd get Flatpak manifests for some Tor pluggable transports: + +``` +./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client --version v0.38.0 +./flatpak-go-deps.py git.torproject.org/pluggable-transports/snowflake.git/client --version v2.6.0 +./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy --version obfs4proxy-0.0.14 +``` + +If the deps are hosted on GitHub or GitLab, it uses the GitHub and GitLab API to find the commit IDs, as this is much quicker than git cloning the full repos. However these APIs have rate limits, and it falls back to git cloning. You can optionally pass in `--github_api_token` or `--gitlab_api_token` if you want to avoid the rate limits. + +This is what the output looks like: + +``` +$ ./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client --version v0.38.0 +✨ Creating temporary Go module +go: creating new go.mod: module tempmod +✨ Cloning the target repository +Cloning into 'src/meek-client'... +warning: redirecting to https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/meek.git/ +remote: Enumerating objects: 2676, done. +remote: Counting objects: 100% (658/658), done. +remote: Compressing objects: 100% (281/281), done. +remote: Total 2676 (delta 372), reused 658 (delta 372), pack-reused 2018 +Receiving objects: 100% (2676/2676), 549.97 KiB | 1.02 MiB/s, done. +Resolving deltas: 100% (1546/1546), done. +✨ Checking out version v0.38.0 +Note: switching to 'v0.38.0'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 3be00b7 programVersion = "0.38.0" +✨ Found 10 dependencies +✨ Module: git.torproject.org/pluggable-transports/goptlib.git v1.1.0 +✨ Git URL: https://git.torproject.org/pluggable-transports/goptlib.git +✨ Cloning https://git.torproject.org/pluggable-transports/goptlib.git@v1.1.0 to find commit ID +Cloning into bare repository '/tmp/tmp4auq0b5l'... +warning: redirecting to https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib.git/ +remote: Enumerating objects: 920, done. +remote: Counting objects: 100% (26/26), done. +remote: Compressing objects: 100% (20/20), done. +remote: Total 920 (delta 8), reused 9 (delta 2), pack-reused 894 +Receiving objects: 100% (920/920), 169.14 KiB | 347.00 KiB/s, done. +Resolving deltas: 100% (473/473), done. +✨ Found commit ID: 781a46c66d2ddbc3509354ae7f1fccab74cb9927 +✨ Module: github.com/andybalholm/brotli v1.0.4 +✨ Git URL: https://github.com/andybalholm/brotli +✨ Used GitHub API to find commit ID: 1d750214c25205863625bb3eb8190a51b2cef26d +✨ Module: github.com/klauspost/compress v1.15.9 +✨ Git URL: https://github.com/klauspost/compress +✨ Used GitHub API to find commit ID: 4b4f3c94fdf8c3a6c725e2ff110d9b44f88823ed +✨ Module: github.com/refraction-networking/utls v1.1.5 +✨ Git URL: https://github.com/refraction-networking/utls +✨ Used GitHub API to find commit ID: 7a37261931c6d4ab67fec65e73a3cc68df4ef84a +✨ Module: golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 +✨ Found short_commit_id: c86fa9a7ed90 +✨ Git URL: https://go.googlesource.com/crypto +✨ Cloning https://go.googlesource.com/crypto to find long commit ID version of c86fa9a7ed90 +Cloning into bare repository '/tmp/tmp_v2n6aqt'... +remote: Finding sources: 100% (6/6) +remote: Total 6906 (delta 3826), reused 6901 (delta 3826) +Receiving objects: 100% (6906/6906), 7.07 MiB | 12.10 MiB/s, done. +Resolving deltas: 100% (3826/3826), done. +✨ Found commit ID: c86fa9a7ed909e2f2a8ab8298254fca727aba16a +✨ Module: golang.org/x/net v0.0.0-20220909164309-bea034e7d591 +✨ Found short_commit_id: bea034e7d591 +✨ Git URL: https://go.googlesource.com/net +✨ Cloning https://go.googlesource.com/net to find long commit ID version of bea034e7d591 +Cloning into bare repository '/tmp/tmpnuomejb1'... +remote: Finding sources: 100% (38/38) +remote: Total 12487 (delta 7842), reused 12476 (delta 7842) +Receiving objects: 100% (12487/12487), 13.68 MiB | 13.91 MiB/s, done. +Resolving deltas: 100% (7842/7842), done. +✨ Found commit ID: bea034e7d591acfddd606603cf48fae48bbdd340 +✨ Module: golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 +✨ Found short_commit_id: 3c1f35247d10 +✨ Git URL: https://go.googlesource.com/sys +✨ Cloning https://go.googlesource.com/sys to find long commit ID version of 3c1f35247d10 +Cloning into bare repository '/tmp/tmpq5awv07s'... +remote: Total 14830 (delta 10172), reused 14830 (delta 10172) +Receiving objects: 100% (14830/14830), 24.51 MiB | 18.52 MiB/s, done. +Resolving deltas: 100% (10172/10172), done. +✨ Found commit ID: 3c1f35247d107ad3669216fc09e75d66fa146363 +✨ Module: golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 +✨ Found short_commit_id: 03fcf44c2211 +✨ Git URL: https://go.googlesource.com/term +✨ Cloning https://go.googlesource.com/term to find long commit ID version of 03fcf44c2211 +Cloning into bare repository '/tmp/tmp047hj5wu'... +remote: Total 385 (delta 212), reused 385 (delta 212) +Receiving objects: 100% (385/385), 115.10 KiB | 1.55 MiB/s, done. +Resolving deltas: 100% (212/212), done. +✨ Found commit ID: 03fcf44c2211dcd5eb77510b5f7c1fb02d6ded50 +✨ Module: golang.org/x/text v0.3.7 +✨ Git URL: https://go.googlesource.com/text +✨ Cloning https://go.googlesource.com/text@v0.3.7 to find commit ID +Cloning into bare repository '/tmp/tmp83wmuo0u'... +remote: Total 6627 (delta 3942), reused 6627 (delta 3942) +Receiving objects: 100% (6627/6627), 24.28 MiB | 14.28 MiB/s, done. +Resolving deltas: 100% (3942/3942), done. +✨ Found commit ID: 383b2e75a7a4198c42f8f87833eefb772868a56f +✨ Module: golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e +✨ Found short_commit_id: 90fa682c2a6e +✨ Git URL: https://go.googlesource.com/tools +✨ Cloning https://go.googlesource.com/tools to find long commit ID version of 90fa682c2a6e +Cloning into bare repository '/tmp/tmps_tqnqmg'... +remote: Total 81895 (delta 50074), reused 81895 (delta 50074) +Receiving objects: 100% (81895/81895), 51.39 MiB | 20.37 MiB/s, done. +Resolving deltas: 100% (50074/50074), done. +✨ Found commit ID: 90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d +✨ 🌟 ✨ +build-commands: +- . /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go + install git.torproject.org/pluggable-transports/meek.git/meek.git +build-options: + env: + GOBIN: /app/bin/ +buildsystem: simple +name: meek-client +sources: +- commit: 781a46c66d2ddbc3509354ae7f1fccab74cb9927 + dest: src/git/torproject/org/pluggable-transports/goptlib/git + type: git + url: https://git.torproject.org/pluggable-transports/goptlib.git +- commit: 1d750214c25205863625bb3eb8190a51b2cef26d + dest: src/github/com/andybalholm/brotli + type: git + url: https://github.com/andybalholm/brotli +- commit: 4b4f3c94fdf8c3a6c725e2ff110d9b44f88823ed + dest: src/github/com/klauspost/compress + type: git + url: https://github.com/klauspost/compress +- commit: 7a37261931c6d4ab67fec65e73a3cc68df4ef84a + dest: src/github/com/refraction-networking/utls + type: git + url: https://github.com/refraction-networking/utls +- commit: c86fa9a7ed909e2f2a8ab8298254fca727aba16a + dest: src/golang/org/x/crypto + type: git + url: https://go.googlesource.com/crypto +- commit: bea034e7d591acfddd606603cf48fae48bbdd340 + dest: src/golang/org/x/net + type: git + url: https://go.googlesource.com/net +- commit: 3c1f35247d107ad3669216fc09e75d66fa146363 + dest: src/golang/org/x/sys + type: git + url: https://go.googlesource.com/sys +- commit: 03fcf44c2211dcd5eb77510b5f7c1fb02d6ded50 + dest: src/golang/org/x/term + type: git + url: https://go.googlesource.com/term +- commit: 383b2e75a7a4198c42f8f87833eefb772868a56f + dest: src/golang/org/x/text + type: git + url: https://go.googlesource.com/text +- commit: 90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d + dest: src/golang/org/x/tools + type: git + url: https://go.googlesource.com/tools +``` \ No newline at end of file diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py new file mode 100755 index 00000000..36e46b17 --- /dev/null +++ b/go/flatpak-go-deps.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import subprocess +import sys +import json +import os +import re +import tempfile + +import yaml +import requests +import click +from bs4 import BeautifulSoup + + +def extract_commit_id(module_version): + # Regex to match formats like: v0.0.0-20190819201941-24fa4b261c55 + complex_format_regex = re.compile( + r"v\d+\.\d+\.\d+-\d{14}-(?P[a-fA-F0-9]{12,40})" + ) + + match = complex_format_regex.search(module_version) + if match: + return match.group("commit") + + # If the version is just a simple version like v1.4.0 or v0.13.0, return None + return None + + +def get_commit_id_from_git( + git_url, + version=None, + short_commit_id=None, + github_api_token=None, + gitlab_api_token=None, +): + # If short_commit_id is provided, simply expand it + if short_commit_id: + print( + f"✨ Cloning {git_url} to find long commit ID version of {short_commit_id}" + ) + with tempfile.TemporaryDirectory() as tmp_dir: + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + result = subprocess.run( + ["git", "rev-parse", short_commit_id], + cwd=tmp_dir, + check=True, + capture_output=True, + text=True, + ) + commit_id = result.stdout.strip() + print(f"✨ Found commit ID: {commit_id}") + return commit_id + + # If it's a GitHub URL, use the GitHub API + if "github.com" in git_url: + repo_parts = git_url.replace("https://github.com/", "").split("/") + if len(repo_parts) == 2: + owner, repo = repo_parts + tag_url = ( + f"https://api.github.com/repos/{owner}/{repo}/git/refs/tags/{version}" + ) + + headers = {} + if github_api_token: + headers["Authorization"] = f"token {github_api_token}" + + response = requests.get(tag_url, headers=headers) + if response.status_code == 200: + json_data = response.json() + commit_id = json_data["object"]["sha"] + print(f"✨ Used GitHub API to find commit ID: {commit_id}") + return commit_id + + # If it's a GitLab URL, use the GitLab API + elif "gitlab.com" in git_url: + repo_parts = ( + git_url.replace("https://gitlab.com/", "").rstrip(".git").split("/") + ) + if len(repo_parts) >= 2: + tag_url = f"https://gitlab.com/api/v4/projects/{'%2F'.join(repo_parts)}/repository/tags/{version}" + + headers = {} + if gitlab_api_token: + headers["Private-Token"] = gitlab_api_token + + response = requests.get(tag_url) + if response.status_code == 200: + json_data = response.json() + commit_id = json_data["commit"]["id"] + print(f"✨ Used GitHub API to find commit ID: {commit_id}") + return commit_id + + # Otherwise, clone the git repo to find the commit id + with tempfile.TemporaryDirectory() as tmp_dir: + try: + if version: + print(f"✨ Cloning {git_url}@{version} to find commit ID") + subprocess.run( + ["git", "clone", "--bare", "-b", version, git_url, tmp_dir], + check=True, + ) + else: + print(f"✨ Cloning {git_url} to find commit ID") + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + except subprocess.CalledProcessError: + # If cloning with a specific tag fails, fall back to default branch + if version: + print(f"✨ Tag {version} not found. Cloning {git_url} default branch...") + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=tmp_dir, + check=True, + capture_output=True, + text=True, + ) + commit_id = result.stdout.strip() + print(f"✨ Found commit ID: {commit_id}") + return commit_id + except subprocess.CalledProcessError: + return None + + +def get_module_info(module_name): + result = subprocess.run( + ["go", "list", "-m", "-json", module_name], + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout) + + +def get_git_url(module_name): + # Remove the version suffix, if present + module_name = re.sub(r"/v\d+$", "", module_name) + + # Remove the subdirectory, if present (e.g. github.com/foo/bar/subdir -> github.com/foo/bar) + if "gitlab.com" in module_name or "github.com" in module_name: + url_parts = module_name.split("/") + if len(url_parts) > 3: + module_name = "/".join(url_parts[:3]) + + if "gitlab.com" in module_name: + return f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}" + elif "github.com" in module_name: + return f"https://github.com/{module_name.replace('github.com/', '')}" + elif "git.torproject.org" in module_name: + return f"https://{module_name}" + else: + response = requests.get(f"https://{module_name}/?go-get=1") + if response.status_code != 200: + return None + soup = BeautifulSoup(response.content, "html.parser") + meta_tag = soup.find("meta", {"name": "go-import"}) + if meta_tag: + url = meta_tag["content"].split()[2] + r = requests.get(url, allow_redirects=True) + if r.history: + return r.url + else: + return url + + return None + + +@click.command() +@click.argument("repo_and_folder", type=str) +@click.option("--version", default=None, help="Version of the repository.") +@click.option("--github_api_token", default=None, help="GitHub API Token.") +@click.option("--gitlab_api_token", default=None, help="GitLab API Token.") +def main(repo_and_folder, version, github_api_token, gitlab_api_token): + """Flatpak Go Generator""" + + if "/" in repo_and_folder: + repo, folder = repo_and_folder.rsplit("/", 1) + repo_name = folder + else: + repo = repo_and_folder + folder = "" + repo_name = os.path.basename(repo).replace(".git", "") + + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + + print("✨ Creating temporary Go module") + subprocess.run(["go", "mod", "init", "tempmod"], check=True) + + try: + print("✨ Cloning the target repository") + subprocess.run( + ["git", "clone", f"https://{repo}", f"src/{repo_name}"], check=True + ) + os.chdir(f"src/{repo_name}") + + if version: + print(f"✨ Checking out version {version}") + subprocess.run(["git", "checkout", version], check=True) + + os.chdir(temp_dir) + + if folder: + os.chdir(f"src/{repo_name}/{folder}") + except subprocess.CalledProcessError: + print(f"✨ Error fetching {sys.argv[1]}") + sys.exit(1) + + result = subprocess.run( + ["go", "list", "-m", "all"], + check=True, + capture_output=True, + text=True, + ) + + modules = result.stdout.strip().split("\n") + modules = modules[1:] # Skip the first module, which is the current module + + print(f"✨ Found {len(modules)} dependencies") + + sources = [] + + for module in modules: + module_name, module_version = module.split(" ", 1) + print(f"✨ Module: {module}") + + short_commit_id = extract_commit_id(module_version) + if short_commit_id: + print(f"✨ Found short_commit_id: {short_commit_id}") + + info = get_module_info(module_name) + path = info.get("Path") + version = info.get("Version") + if version.endswith("+incompatible"): + version = version[:-13] + if not version: + continue + + git_url = get_git_url(module_name) + if not git_url: + git_url = f"https://{module_name}.git" + + print(f"✨ Git URL: {git_url}") + + commit_id = get_commit_id_from_git( + git_url, version, short_commit_id, github_api_token, gitlab_api_token + ) + + if not commit_id: + print( + f"✨ Error: Could not retrieve commit ID for {module_name}@{version}." + ) + continue + + sources.append( + { + "type": "git", + "url": git_url, + "commit": commit_id, + "dest": f"src/{path.replace('.', '/')}", + } + ) + + yaml_data = { + "name": repo_name, + "buildsystem": "simple", + "build-options": {"env": {"GOBIN": "/app/bin/"}}, + "build-commands": [ + f". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go install {repo}/{os.path.basename(repo)}" + ], + "sources": sources, + } + + print("✨ 🌟 ✨") + print(yaml.dump(yaml_data)) + + +if __name__ == "__main__": + main() diff --git a/go/requirements.txt b/go/requirements.txt new file mode 100644 index 00000000..33607f42 --- /dev/null +++ b/go/requirements.txt @@ -0,0 +1,4 @@ +pyyaml +requests +bs4 +click \ No newline at end of file