Add sync framework with ruyi packages-index

.github/workflows/table.yaml

@@ -39,7 +39,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y ninja-build
           sudo pip3 install --upgrade meson
-          pip install -r assets/requirements.txt
+          pip install -r assets/requirements_svg.txt
           mkdir assets/output
           python assets/generate_svgimage.py -p . -o assets/output --html https://${{ github.repository_owner }}.github.io/support-matrix | tee build.log
           python assets/generate_svgimage.py -p . -o assets/output --html https://${{ github.repository_owner }}.github.io/support-matrix -l zh | tee -a build.log

LicheePi4A/README.md

@@ -1,4 +1,5 @@
 ---
+vendor: sipeed_licheepi4a
 product: LicheePi 4A
 cpu: TH1520
 cpu_core: XuanTie C910 + XuanTie C906 + XuanTie E902
@@ -59,4 +60,4 @@ cpu_core: XuanTie C910 + XuanTie C906 + XuanTie E902
 [openKylin]: ./openKylin/README.md
 [OpenWRT]: ./OpenWRT/README.md
 [ArchLinux]: ./ArchLinux/README.md
-[Deepin]: ./Deepin/README.md
+[Deepin]: ./Deepin/README.md

assets/check_update.py

@@ -0,0 +1,56 @@
+"""
+Check for updates in the matrix
+"""
+
+import sys
+import argparse
+import logging
+from src.matrix_parser import Systems
+from src.version_checker import gen_oldver, run_nvchecker, vinfo_dict_to_dict, filter_newer
+
+logger = logging.getLogger(__name__)
+
+def main():
+    """
+    Main function
+    """
+
+    arg = argparse.ArgumentParser()
+    arg.add_argument('-c', '--config', help='config file',
+                     default='config.toml')
+    arg.add_argument(
+        '-p', '--path', help='path to the support matrix', default='../')
+    args = arg.parse_args()
+
+    matrix = Systems(args.path)
+    old = gen_oldver(matrix, args.config)
+
+    new, fail = run_nvchecker(oldvers=old)
+    if fail:
+        logger.exception("Failed to run nvchecker: %s", fail)
+        sys.exit(-1)
+    old = vinfo_dict_to_dict(old)
+
+    upd = filter_newer(old, new)
+
+    has_update = False
+    for prod, ver in upd.items():
+        if ver['old'] is None:
+            logger.info(
+                "Found new version for %s: %s", prod, ver['new']['version'])
+            has_update = True
+        elif ver['new'] is None:
+            logger.info(
+                "No new version found for %s", prod)
+        else:
+            logger.info(
+                "Can be updated for %s: %s -> %s",
+                prod, ver['old']['version'], ver['new']['version'])
+            has_update = True
+
+    if has_update:
+        logger.info("Update available")
+
+
+if __name__ == '__main__':
+    main()

assets/config.toml

@@ -0,0 +1,9 @@
+[__config__]
+verify_cert = false
+sort_version_key = "awesomeversion"
+user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+
+[sipeed_licheepi4a-debian-null] # RevyOS is seen as Debian
+source = "regex"
+url = "https://mirror.iscas.ac.cn/revyos/extra/images/lpi4a/"
+regex = '<a href="(\d{8})/" title="\1">\1/</a>'

assets/docs/matrix_parser_en.md

@@ -0,0 +1,29 @@
+## Matrix Parser
+
+A tool used for parsing metadata. The support_matrix repo can be divided into the following structure:
+
+```plaintext
+Support Matrix
+ |
+ |--- Board
+    |
+    |--- README.md # Documentation for the board, containing the basic metadata of the board
+    |
+    |--- Various Systems
+    |    |
+    |    |--- README.md # Documentation for the main variant of the system, containing the basic metadata of the system
+    |    |
+    |    |--- README_Variant.md # Documentation for other variants of the system, containing the metadata of the variant
+    |    |
+    |    |--- README(_Variant)_lang.yaml # Translations in various languages, **does not include metadata!**
+    |
+    |--- others.yml # Metadata for systems that do not have documentation
+```
+
+Based on the structure above, the data structure are:
+- `class Systems`: The entire project, containing a series of development boards (`class Board`)
+- `class Board`: A development board, containing the board's metadata and a series of systems (`class System`)
+- `class System`: A system, with the system's ID and a series of variants (`class SystemVar`)
+- `class SystemVar`: A variant, containing the metadata of the variant
+
+By passing the project's root directory to `Systems`, you can parse the metadata of the entire project.

assets/docs/ruyi_index_updator.md

@@ -0,0 +1,72 @@
+## ruyi index updator
+
+自动更新 ruyi 工具中镜像的索引文件，同步支持矩阵与 ruyi index。
+
+### 使用方法
+
+你需要设置环境变量 `GITHUB_TOKEN`，并且该 token 需要有对应仓库的读写权限和 pr 权限。或者你可以同时允许 fork 仓库的权限，让工具自动 fork 仓库。
+
+**注意！**你直接从设置中获取到的 token 都是 personal token，但是向组织仓库进行 pr 时，该 token 是会被拒绝的。你需要使用一个 OAuth app token。最简单的辨别方法是，personal token 的格式是 `ghp` 开头，OAuth app token 的格式是 `gho` 开头。
+
+临时测试可以通过 `gh` 命令行工具来获取一个。 **请注意不要泄漏，从 `gh` 工具获得的 token 具有您账号的全部权限。**；创建一个 OAuth app 并使用见下文。
+
+默认工具会在临时文件夹中运行，每次可能会下载一大堆镜像文件。你可以设置环境变量 `CACHE_DIR` 来指定缓存目录，这样工具会将所有的东西都缓存到这个目录中。
+
+renew_ruyi_index.py 里面封装了所有的操作，你可以直接运行这个脚本。各个参数的含义如下：
+- `-c` 或 `--config`：配置文件的路径，可以与 nvchecker 共用。主要是要它的 `[vendor-system-variance]` 这个字段。
+- `-p` 或 `--path`：support matrix 的路径。
+- `-i` 或 `--index`：上传的新 ruyi index 的路径（不包含`ruyi-index`部分）。不指定的话会临时文件夹中 clone 一个新的。
+- `--pr`：是否直接提交 pr。不指定的话会在控制台中显示 pr 的内容。**由于 pr 中包含用于识别重复 pr 的字段，请至少包含该 identifier！**
+
+### 插件开发说明
+
+由于镜像映射情况较为复杂（一份报告对应多个文件乃至于多个 index），且版本号存在需要自行处理的问题，现阶段为每个镜像写一些插件是必需的。后续可能可以考虑通用插件的开发。
+
+新增一个插件，即在 `ruyi_index_updator/upload_plugin` 中新建一个文件即可。你可以直接引入当前文件夹下的 `prelude` 来获取一些 typing。
+
+文件中，一个`register`函数是必须的，若其返回 Null，则说明该插件不应该被加载（你可以自行加逻辑判断）。否则，返回一个`UploadPluginBase`实例。
+
+`UploadPluginBase`类中你需要实现以下方法：
+- `get_name`：static 方法，返回插件的名字。
+- `can_handle`：判断是否能处理当前镜像（以 vendor-system-variance 三元组存储，当然包含原报告在`raw_data`中）。
+- `is_mapped_ruyi_index`：判断当前的 ruyi index 中的一个镜像是否可以由该报告生成。
+- `handle_version`：从镜像版本 map 到 ruyi 使用的语义化版本号。更多定义问 ruyi。
+- `handle_report`：使用当前报告，为 index 生成一份新的 image index。
+
+其中包含的一些 helper 函数可自行查看。一份示例是给 lpi4a 中 ruyi 系统的插件，包含了 RevyOS 和两个 u-boot 镜像，RevyOS 镜像中又有多个文件，应该能较好的代表所有情况。
+
+## 如何使用一个 OAuth APP
+
+以下将会介绍如何使用一个 OAuth APP 并获取一个 token。
+
+### 确认您想使用的账户的类型
+
+首先，由于 Github 对权限的处理，所有的权限是 **绑定在用户上的**（包括组织），这也代表着您无法是一个组织有一个 token，而是您的账号有一个 token。因此，对于您授权的 Github 账户，请确认其类型是一个用户账户，而非组织账户。
+
+如果您不想使用您的主账户，请放心的去创建一个新的账户。Github 为 bot 账户提供了特例，不会违反 Github 一人一号的政策：
+> User accounts are intended for humans, but you can create accounts to automate activity on GitHub. This type of account is called a machine user. For example, you can create a machine user account to automate continuous integration (CI) workflows.
+
+### 创建一个 OAuth APP
+
+打开您账户的设置，找到 Developer settings：
+![1](img/1.png)
+
+进入，并选择 OAuth Apps：
+![2](img/2.png)
+
+点击 New OAuth App：
+![3](img/3.png)
+
+自行填写信息，其中 Authorization callback URL 可以随便填写，如
+ `http://127.0.0.1:12345`, **务必勾选 Enable Device Flow**：
+![4](img/4.png)
+
+注册 APP，记录下其中的 Client ID：
+![5](img/5.png)
+
+### 使用它
+
+在 `src/ruyi_index_updator/github_auth.py` 是一个单独的脚本，其用途就是获取一个 token。直接运行它，按照提示输入即可。
+```python
+python assets/src/ruyi_index_updator/github_auth.py
+```

assets/docs/ruyi_index_updator_en.md

@@ -0,0 +1,71 @@
+## Ruyi Index Updater
+
+This tool automatically updates the index file for mirrors in the ruyi tool, synchronizing the support matrix with the ruyi index.
+
+### How to Use
+
+You need to set the environment variable `GITHUB_TOKEN`, and this token must have read, write, and PR permissions for the relevant repository. Alternatively, you can enable permissions to allow the tool to fork the repository automatically.
+
+**Note!** Tokens get directly from Github settings are personal tokens, which will be rejected when making PRs to organization repositories. You need to use an OAuth app token. The easiest way to distinguish them is by format: personal tokens start with `ghp`, while OAuth app tokens start with `gho`. 
+
+If want to test only, you can use the `gh` command-line tool to get one. **Be careful not to leak the token, it has all permission of your account if you get it from `gh` tool.**; Create an OAuth app and use it as described below.
+
+By default, the tool operates in a temporary directory, which might involve downloading many mirror files each time. You can set the environment variable `CACHE_DIR` to specify a cache directory, and the tool will store everything there.
+
+The `renew_ruyi_index.py` script encapsulates all operations, and you can run this script directly. The parameters are as follows:
+- `-c` or `--config`: Path to the configuration file, which can be shared with nvchecker. The primary focus is on the `[vendor-system-variance]` field.
+- `-p` or `--path`: Path to the support matrix.
+- `-i` or `--index`: Path to the new ruyi index to be uploaded (excluding the `ruyi-index` part). If not specified, a new one will be cloned in the temporary folder.
+- `--pr`: Whether to directly submit a PR. If not specified, the PR content will be displayed in the console. **Since the PR includes an identifier to prevent duplicate PRs, please make sure to include this identifier!**
+
+### Plugin Development Guide
+
+Due to the complexity of mirror mapping (a single report might correspond to multiple files or indices) and version numbers that need manual handling, creating plugins for each mirror is currently necessary. In the future, generic plugin development may be considered.
+
+To add a new plugin, simply create a new file in the `ruyi_index_updator/upload_plugin` directory. You can directly import the `prelude` from the current folder for some typing helpers.
+
+A `register` function is mandatory in the file. If it returns `Null`, it indicates that the plugin should not be loaded (you can add logic checks if needed). Otherwise, it should return an instance of `UploadPluginBase`.
+
+In the `UploadPluginBase` class, you need to implement the following methods:
+- `get_name`: A static method that returns the name of the plugin.
+- `can_handle`: Determines if the current mirror can be handled (stored as a vendor-system-variance triplet, including the original report in `raw_data`).
+- `is_mapped_ruyi_index`: Checks if a mirror in the current ruyi index can be generated from the report.
+- `handle_version`: Maps the mirror version to the semantic version used by ruyi. For more definitions, refer to ruyi documentation.
+- `handle_report`: Uses the current report to generate a new image index for the ruyi index.
+
+You can review the provided helper functions as needed. An example plugin is provided for the LPI4A system in ruyi, which includes RevyOS and two U-Boot mirrors. The RevyOS mirror contains multiple files, making it a good representation of all scenarios.
+
+## How to Use an OAuth APP
+
+The following explains how to use an OAuth APP and obtain a token.
+
+### Confirm the Type of Account You Want to Use
+
+Due to Github's permission limitation, all permissions are **bound to users** (including organizations). This means you cannot have a token for an organization but only for your user account. Therefore, for the Github account you authorize, make sure it is a user account, not an organization account.
+
+If you do not want to use your main account, feel free to create a new account. Github provides a special exception for bot accounts, which does not violate the one account per person policy:
+> User accounts are intended for humans, but you can create accounts to automate activity on GitHub. This type of account is called a machine user. For example, you can create a machine user account to automate continuous integration (CI) workflows.
+
+### Create an OAuth APP
+
+Open your account settings and find Developer settings:
+![1](img/1.png)
+
+Enter and select OAuth Apps:
+![2](img/2.png)
+
+Click New OAuth App:
+![3](img/3.png)
+
+Fill in the information as needed. The Authorization callback URL can be anything, such as `http://127.0.0.1:12345`. **Be sure to check Enable Device Flow**:
+![4](img/4.png)
+
+Register the APP, get the Client ID:
+![5](img/5.png)
+
+### Use the OAuth APP
+
+In `src/ruyi_index_updator/github_auth.py` is a separate script that is used to obtain a token. Simply run it and follow the prompts to input the required information.
+```python
+python assets/src/ruyi_index_updator/github_auth.py
+```

assets/docs/svg_gen_en.md

@@ -0,0 +1,51 @@
+## SVG Generator
+
+An entire SVG can be seen as a sequence of objects drawn based on absolute coordinates starting from the top-left corner. This sequence can be considered as a tree, where the root is the entire SVG, and the leaves are individual objects.
+
+During drawing, an absolute cursor is used to track the current drawing position. Each object is drawn relative to the cursor's position. The drawing order follows a post-order traversal: after an object is drawn, the cursor moves to the **top-right corner** of that object.
+
+Certain special objects can modify the cursor's behavior:
+- `SvgLF`: Line feed, moving to a new line. The line height is determined by the tallest object in the current line.
+- `SvgCR`: Resets the cursor to the beginning of the line, which is 0.
+- `SvgAdvanced`: A custom object that does not draw but modifies the cursor, useful for setting line height or moving the cursor to a relative position (e.g., `dx=-3 -> xpos-=3`).
+- `SvgMoveTo`: Ignores all the above rules and directly moves the cursor to a specified position, starting a new line.
+
+Here is an example of a "Hello World" SVG:
+
+```python
+from svg_gen import *
+
+xml = SvgXml()
+svg = SvgSvg()
+
+text = SvgText("Hello World!")
+text_bg = SvgRectContainer(fill='rgb(128, 128, 128)')
+text_bg.add(text)
+
+text_height = text_bg.height
+text_width = text_bg.width + 2 # 2 is the sum of width of borders
+
+border_top = SvgLine(text_width, 0)
+svg.add(border_top)
+svg.add(SvgCR())
+svg.add(SvgLF())
+
+border_left = SvgLine(0, text_height)
+svg.add(border_left)
+
+svg.add(text_bg)
+border_right = SvgLine(0, text_height)
+svg.add(border_right)
+
+svg.add(SvgCR())
+svg.add(SvgLF())
+
+border_bottom = SvgLine(text_width, 0)
+svg.add(border_bottom)
+
+xml.add(svg)
+
+xml.generate()
+```
+
+This example demonstrates the creation of an SVG with a "Hello World!" text, including borders around it.

assets/renew_ruyi_index.py

@@ -0,0 +1,71 @@
+"""
+Automatically check the index of ruyi and renew it.
+"""
+
+import shutil
+import argparse
+import tempfile
+import logging
+import colorlog
+
+from src.matrix_parser import Systems
+from src.ruyi_index_updator import RuyiDiff, RuyiGitRepo
+
+handler = colorlog.StreamHandler()
+handler.setFormatter(colorlog.ColoredFormatter(
+    '%(log_color)s[%(relativeCreated)d %(levelname)s]%(reset)s: %(message)s'))
+
+logging.basicConfig(level=logging.INFO, handlers=[handler])
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """
+    Main function
+    """
+
+    arg = argparse.ArgumentParser()
+    arg.add_argument('-c', '--config', help='config file',
+                     default='assets/config.toml')
+    arg.add_argument(
+        '-p', '--path', help='path to the support matrix', default='.')
+    arg.add_argument(
+        '-i', '--index', help='path clone of ruyi index, default to a temp dir', default=None
+    )
+    arg.add_argument(
+        '--pr', help='create a PR for the update', action='store_true'
+    )
+    args = arg.parse_args()
+
+    index_path = args.index
+    if index_path is None:
+        index_path = tempfile.mkdtemp()
+
+    matrix = Systems(args.path)
+    diffs = RuyiDiff(matrix, args.config)
+    repo = RuyiGitRepo(index_path)
+
+    for diff in diffs.gen_diff():
+        pr = repo.upload_image(diff)
+        if pr is None:
+            continue
+        if not args.pr:
+            logger.info("""\
+PR info:
+    Title: %s
+
+    Body: 
+%s
+    <Body End>
+    Branch: %s -> upstream/%s
+""", pr.title, pr.body, pr.self_branch, pr.upstream_branch)
+            continue
+        repo.create_wrapped_pr(pr)
+
+    if args.index is None:
+        shutil.rmtree(index_path)
+
+
+if __name__ == '__main__':
+    main()

assets/requirements_nv.txt

@@ -0,0 +1,9 @@
+awesomeversion==24.6.0
+lxml==5.3.0
+nvchecker==2.15.1
+platformdirs==4.3.6
+pycurl==7.45.3
+pytoml==0.1.21
+structlog==24.4.0
+toml==0.10.2
+tornado==6.4.1