diff --git a/.github/workflows/package-index-sync.yaml b/.github/workflows/package-index-sync.yaml new file mode 100644 index 00000000..e507d884 --- /dev/null +++ b/.github/workflows/package-index-sync.yaml @@ -0,0 +1,106 @@ +name: Sync Package Index + +# Below is the configuration and variables that you need to set up. +# 1. A environment named `ruyi-sync` is required. +# 2. Some variables are required to be set in the environments: +# - GHO_TOKEN: is described in [docs](https://github.com/ruyisdk/support-matrix/blob/main/assets/docs/ruyi_index_updator.md) as GITHUB_TOKEN (rename due to a same name default var in CI env) +# - PACKAGE_INDEX_OWNER: decide where the pr goes: blank defaults to `ruyisdk` +# - SSH_PRIVATE: A private SSH key of the bot account + +on: + pull_request: + workflow_dispatch: + inputs: + makepr: + description: 'Make a PR to sync package index' + required: false + default: 'false' + debuginfo: + description: 'Output Debug' + required: false + default: 'false' + # push: # Uncomment this line to enable push event, but it is not recommended as it may cause unnecessary PRs. Use workflow_dispatch instead. + +jobs: + build: + name: Generate and Upload + runs-on: ubuntu-latest + environment: ruyi-sync + env: + GITHUB_TOKEN: ${{ secrets.GHO_TOKEN }} + CACHE_DIR: ~/cache + PACKAGE_INDEX_OWNER: ${{ vars.PACKAGE_INDEX_OWNER }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + - uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE }} + - name: Load Cache + uses: actions/cache@v4 + with: + path: | + ~/cache + key: ${{ runner.os }}-package-index-sync + - name: Install Dependencies + run: | + sudo apt-get update + pip install -r assets/requirements_ruyinv.txt + - name: Run tool to generate and upload + if: ${{ ( github.event_name == 'workflow_dispatch' && inputs.makepr == false ) || github.event_name != 'pull_request' || ( github.event_name == 'pull_request' && github.event.pull_request.merged == false ) }} + run: | + echo "Generate Only" > $RUNNER_TEMP/type.txt + export CI_RUN_ID=${{ github.run_id }} + export CI_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + python assets/renew_ruyi_index.py -c assets/config.toml -p . -i $RUNNER_TEMP/cache --log $RUNNER_TEMP/log.txt --warn $RUNNER_TEMP/warn.txt + - name: Run tool to generate and upload and PR + if: ${{ ( github.event_name == 'workflow_dispatch' && inputs.makepr == true ) || ( github.event_name == 'pull_request' && github.event.pull_request.merged == true ) }} + run: | + echo "Generate and PR" > $RUNNER_TEMP/type.txt + export CI_RUN_ID=${{ github.run_id }} + export CI_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + python assets/renew_ruyi_index.py -c assets/config.toml -p . -i $RUNNER_TEMP/cache --log $RUNNER_TEMP/log.txt --warn $RUNNER_TEMP/warn.txt --pr + - name: Output Summary + run: | + echo "# Package Index Sync Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "[CI #${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) powered by [${{ github.repository }}](${{ github.server_url }}/${{ github.repository }})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Type: $(cat $RUNNER_TEMP/type.txt)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "CI has built on commit [${{ steps.truecommit.outputs.shortid }}](${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.truecommit.outputs.longid }}). The commit message is shown below." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY + echo "$(git log -1)" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Warn Log" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY + cat $RUNNER_TEMP/warn.txt >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Build Log" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
Click to expand" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY + cat $RUNNER_TEMP/log.txt >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Output Debug Info + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debuginfo }} + run: | + echo "## Debug Info" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY + echo "${{ toJSON( github ) }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY diff --git a/assets/renew_ruyi_index.py b/assets/renew_ruyi_index.py index 0b7a664a..35bc4532 100644 --- a/assets/renew_ruyi_index.py +++ b/assets/renew_ruyi_index.py @@ -2,6 +2,7 @@ Automatically check the index of ruyi and renew it. """ +import os import shutil import argparse import tempfile @@ -11,14 +12,6 @@ from src.matrix_parser import Systems from src.ruyi_index_updator import RuyiDiff, RuyiGitRepo -handler = colorlog.StreamHandler() -handler.setFormatter(colorlog.ColoredFormatter( - '%(log_color)s[%(relativeCreated)d %(levelname)s]%(reset)s: %(message)s')) - -logging.basicConfig(level=logging.INFO, handlers=[handler]) - -logger = logging.getLogger(__name__) - def main(): """ @@ -36,8 +29,40 @@ def main(): arg.add_argument( '--pr', help='create a PR for the update', action='store_true' ) + arg.add_argument( + '--log', help='output the log to the file', default=None + ) + arg.add_argument( + '--warn', help='output the warn to the file', default=None + ) args = arg.parse_args() + handler = colorlog.StreamHandler() + handler.setFormatter(colorlog.ColoredFormatter( + '%(log_color)s[%(relativeCreated)d %(levelname)s]%(reset)s: %(message)s')) + + handlers = [handler] + + if args.log is not None: + p = os.path.abspath(args.log) + log_handler = logging.FileHandler(p) + log_handler.setFormatter(logging.Formatter( + '%(relativeCreated)d %(levelname)s: %(message)s')) + log_handler.setLevel(logging.INFO) + handlers.append(log_handler) + + if args.warn is not None: + p = os.path.abspath(args.warn) + warn_handler = logging.FileHandler(p) + warn_handler.setFormatter(logging.Formatter( + '%(relativeCreated)d %(levelname)s: %(message)s')) + warn_handler.setLevel(logging.WARNING) + handlers.append(warn_handler) + + logging.basicConfig(level=logging.INFO, handlers=handlers) + + logger = logging.getLogger() + index_path = args.index if index_path is None: index_path = tempfile.mkdtemp() @@ -51,15 +76,7 @@ def main(): if pr is None: continue if not args.pr: - logger.info("""\ -PR info: - Title: %s - - Body: -%s - - Branch: %s -> upstream/%s -""", pr.title, pr.body, pr.self_branch, pr.upstream_branch) + logger.info("%s", repr(pr)) continue repo.create_wrapped_pr(pr) diff --git a/assets/src/matrix_parser/matrix_parser.py b/assets/src/matrix_parser/matrix_parser.py index 0a62a974..deaa78c1 100644 --- a/assets/src/matrix_parser/matrix_parser.py +++ b/assets/src/matrix_parser/matrix_parser.py @@ -284,6 +284,7 @@ class Systems: '.git', '.vscode', '__pycache__', + '~', # ? ] def should_exclude(self, path): diff --git a/assets/src/ruyi_index_parser/parse_board_img.py b/assets/src/ruyi_index_parser/parse_board_img.py index 437bbdde..cbab51ff 100644 --- a/assets/src/ruyi_index_parser/parse_board_img.py +++ b/assets/src/ruyi_index_parser/parse_board_img.py @@ -171,10 +171,8 @@ class BoardImages: @property def version(self) -> str: """ - return the version, following bot identifier + return the version """ - if self.is_bot_created: - return f"{self.raw_version}-matrix.bot" return self.raw_version @version.setter diff --git a/assets/src/ruyi_index_updator/index_handler.py b/assets/src/ruyi_index_updator/index_handler.py index 0fb8d6fc..cff957fe 100644 --- a/assets/src/ruyi_index_updator/index_handler.py +++ b/assets/src/ruyi_index_updator/index_handler.py @@ -11,9 +11,13 @@ logger = logging.getLogger(__name__) -PACKAGE_INDEX_OWNER = "ruyisdk" +PACKAGE_INDEX_OWNER = os.getenv("PACKAGE_INDEX_OWNER", "ruyisdk") +if len(PACKAGE_INDEX_OWNER) == 0 or PACKAGE_INDEX_OWNER is None: + PACKAGE_INDEX_OWNER = "ruyisdk" PACKAGE_INDEX_REPO = "packages-index" +CI_RUN_ID = os.getenv("CI_RUN_ID", None) +CI_RUN_URL = os.getenv("CI_RUN_URL", None) class PrWrapper: """ @@ -33,6 +37,7 @@ def __repr__(self) -> str: Body: {self.body} + From: {self.self_branch} To: {self.upstream_branch} """ @@ -68,13 +73,33 @@ def check_pr_exist(self, identifer: str) -> bool: return True return False + def check_pr_updated(self, head: str, base: str) -> bool: + """ + Check if the PR with head and base exist. + Different from check_pr_exist, this senerio is for the PR is already exist, + but identifier is not the same. + Maybe the PR is manually created, or something went wrong. + Neverthless, manual interraction is needed. + """ + prs = self.upstream.get_pulls(state="open") + for pr in prs: + if pr.head.ref == head and pr.base.ref == base: + return True + return False + def __create_pr(self, wrapper: PrWrapper): """ Create a pull request. """ head = f"{self.user.login}:{wrapper.self_branch}" + base = wrapper.upstream_branch + if self.check_pr_updated(head, base): + logger.error( + "PR already exist for %s -> %s, please check manually", head, base) + logger.error("New PR info: %s", repr(wrapper)) + return pr = self.upstream.create_pull( - title=wrapper.title, body=wrapper.body, head=head, base=wrapper.upstream_branch) + title=wrapper.title, body=wrapper.body, head=head, base=base) logger.info("PR created: %s at %s", pr.title, pr.html_url) def create_wrapped_pr(self, wrapper: PrWrapper): @@ -91,11 +116,38 @@ def create_pr(self, title: str, body: str, self_branch: str, upstream_branch: st title, body, self_branch, upstream_branch)) def __reset_to_upstream(self): - self.local_repo.remote().set_url(self.upstream.ssh_url) - self.local_repo.remote().fetch() + # self.local_repo.remote().set_url(self.upstream.ssh_url) + # self.local_repo.remote().fetch() + + # self.local_repo.remote().refs['main'].checkout() + # self.local_repo.remote().set_url(self.repo.ssh_url) + flag = False + for ref in self.local_repo.remotes: + if ref.name == "upstream": + flag = True + break + if not flag: + self.local_repo.git.execute( + ["git", "remote", "add", "upstream", self.upstream.ssh_url] + ) - self.local_repo.remote().refs['main'].checkout() - self.local_repo.remote().set_url(self.repo.ssh_url) + self.local_repo.git.execute( + ["git", "remote", "set-url", "upstream", self.upstream.ssh_url] + ) + self.local_repo.git.execute( + ["git", "fetch", "upstream"] + ) + self.local_repo.git.execute( + ["git", "reset", "--hard", "upstream/main"] + ) + self.local_repo.git.execute( + ["git", "clean", "-xdf"] + ) + + def __clean(self): + self.local_repo.git.execute( + ["git", "clean", "-xdf"] + ) def __init__(self, repo_dir: str) -> None: github_token = os.getenv('GITHUB_TOKEN', None) @@ -133,6 +185,7 @@ def local_checkout(self, branch: str): self.__reset_to_upstream() head = self.local_repo.create_head(branch) head.checkout() + self.__clean() logger.info("Checkout to %s", branch) def local_commit(self, message: str): @@ -158,17 +211,28 @@ def add_image(self, image: BoardImageWrapper): """ Add a image index to the repo. """ - index_name = image.new_index_name() + file_name = image.new_index_name() index_file = os.path.join( self.local_repo.working_dir, "manifests", "board-image", - index_name + image.index_name, + file_name ) with open(index_file, "w", encoding="utf-8") as f: f.write(image.new_index_toml()) - self.local_repo.index.add([index_file]) - logger.info("Add %s", index_name) + if image.index.is_bot_created and CI_RUN_ID is None: + f.write("\n# This file is created by program renew_ruyi_index in support-matrix\n") + f.write("# Run: In local\n") + elif image.index.is_bot_created: + f.write("\n# This file is created by CI Sync Package Index inside support-matrix\n") + f.write(f"# Run ID: {CI_RUN_ID}\n") + f.write(f"# Run URL: {CI_RUN_URL}\n") + # self.local_repo.index.add([index_file]) + self.local_repo.git.execute( + ["git", "add", index_file] + ) + logger.info("Add %s", file_name) def upload_image(self, image: BoardImageWrapper): """ diff --git a/assets/src/ruyi_index_updator/upload_plugin_base.py b/assets/src/ruyi_index_updator/upload_plugin_base.py index 7982047d..951c61a9 100644 --- a/assets/src/ruyi_index_updator/upload_plugin_base.py +++ b/assets/src/ruyi_index_updator/upload_plugin_base.py @@ -57,7 +57,8 @@ def handle_version(self, vinfo: VInfo) -> str: raise NotImplementedError @abstractmethod - def handle_report(self, vinfo: VInfo, index: str, last_index: list[BoardImages]) -> BoardImages | None: + def handle_report(self, vinfo: VInfo, + index: str, last_index: list[BoardImages]) -> BoardImages | None: """ Handle the report data from the system. """ @@ -131,6 +132,7 @@ def gen_distfile(self, file: str, url: str) -> BoardIndexDistfiles: } }) + def register() -> UploadPluginBase | None: """ Register the plugin to the system diff --git a/assets/src/ruyi_index_updator/version_diff.py b/assets/src/ruyi_index_updator/version_diff.py index 1a9c6965..95a320dd 100644 --- a/assets/src/ruyi_index_updator/version_diff.py +++ b/assets/src/ruyi_index_updator/version_diff.py @@ -84,7 +84,7 @@ def gen_hash(self) -> str: Generate the hash of the new index, used to identify the pr """ h = hashlib.sha224( - self.new_index_toml().encode("utf-8") + self.new_index_toml().encode("utf-8") + self.new_index_name().encode("utf-8") ) return h.hexdigest()