From 572a12359ad6fe318823022354394d0dfb7d8bde Mon Sep 17 00:00:00 2001 From: EthanC <16727756+EthanC@users.noreply.github.com> Date: Sat, 9 Nov 2024 05:57:45 -0600 Subject: [PATCH] Save files to stacks directory, Add GLOB_PATTERNS --- README.md | 2 ++ pyproject.toml | 2 +- salvage.py | 95 +++++++++++++++++++++++++++++++------------------- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 177db8d..5887dc7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Regardless of your chosen setup method, Salvage is intended for use with a task - `GITHUB_ACCESS_TOKEN` (Required): [Personal Access Token (Classic)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic) for GitHub. - `GITHUB_REPOSITORY` (Required): Name of the private GitHub repository to store backups. - `DISCORD_WEBHOOK_URL`: [Discord Webhook](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks) URL to receive Portainer Stack notifications. +- `GLOB_PATTERNS`: Comma-separated [pathname pattern(s)](https://docs.python.org/3/library/glob.html) to match in local file discovery. Default is `**/compose.yaml`. ### Docker (Recommended) @@ -39,6 +40,7 @@ services: GITHUB_ACCESS_TOKEN: XXXXXXXX GITHUB_REPOSITORY: XXXXXXXX DISCORD_WEBHOOK_URL: https://discord.com/api/webhooks/XXXXXXXX/XXXXXXXX + GLOB_PATTERNS: **/compose.yaml,**/config.json volumes: - /home/username/stacks:/salvage/stacks:ro ``` diff --git a/pyproject.toml b/pyproject.toml index 0c6627a..93b8a76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "salvage" -version = "2.1.0" +version = "3.0.0" description = "Backup Docker Compose files to GitHub and notify about changes." readme = "README.md" requires-python = ">=3.13" diff --git a/salvage.py b/salvage.py index 9236d54..7d36c4c 100644 --- a/salvage.py +++ b/salvage.py @@ -28,7 +28,7 @@ def Start() -> None: if dotenv.load_dotenv(): logger.success("Loaded environment variables") - logger.trace(environ) + logger.trace(f"{environ=}") if level := environ.get("LOG_LEVEL"): logger.remove() @@ -44,7 +44,7 @@ def Start() -> None: ) logger.success(f"Enabled logging to Discord webhook") - logger.trace(url) + logger.trace(f"{url=}") local: dict[str, dict[str, str]] = GetLocalFiles() @@ -74,61 +74,82 @@ def Start() -> None: def GetLocalFiles() -> dict[str, dict[str, str]]: """Return a dictionary containing files from the local stacks directory.""" - stacks: dict[str, dict[str, str]] = {} + results: dict[str, dict[str, str]] = {} directory: Path = Path("./stacks") if not directory.exists(): logger.error(f"Failed to locate local stacks directory {directory.resolve()}") - return stacks + return results - for file in directory.glob("**/compose.yaml"): - stack: str = file.relative_to("./stacks").parts[0] + # Default pattern if GLOB_PATTERN environment variable is not set + patterns: list[str] = ["**/compose.yaml"] - stacks[stack] = { - "stack": stack, - "filename": file.name, - "filepath": str(file.relative_to("./stacks")).replace("\\", "/"), - "content": file.read_text(), - } + if custom := environ.get("GLOB_PATTERNS"): + patterns = custom.split(",") + + logger.trace(f"{patterns=}") - logger.debug(f"Found file {stacks[stack]["filepath"]} in local stacks") - logger.trace(file.resolve()) - logger.trace(stacks[stack]) + for pattern in patterns: + logger.trace(f"{pattern=}") - logger.info(f"Found {len(stacks):,} files in local stacks") + for file in directory.glob(pattern): + logger.trace(f"{file=}") - return stacks + stack: str = file.relative_to("./stacks").parts[0] + filename: str = file.name + + results[f"{stack}/{filename}"] = { + "stack": stack, + "filename": filename, + "filepath": str(file).replace("\\", "/"), + "content": file.read_text(), + } + + logger.debug( + f"Found file {results[f"{stack}/{filename}"]["filepath"]} in local stacks" + ) + logger.trace(file.resolve()) + logger.trace(f"{results[f"{stack}/{filename}"]=}") + + logger.info(f"Found {len(results):,} files in local stacks") + logger.trace(f"{results=}") + + return results def GetRemoteFiles(repo: Repository) -> dict[str, dict[str, str]]: """Return a dictionary containing files from the remote stacks directory.""" - stacks: dict[str, dict[str, str]] = {} + results: dict[str, dict[str, str]] = {} files: list[ContentFile] = GetFiles(repo) for file in files: - stack: str = file.path.split("/")[0] + logger.trace(f"{file=}") + + stack: str = file.path.split("/")[1] + filename: str = file.name - stacks[stack] = { + results[f"{stack}/{filename}"] = { "stack": stack, - "filename": file.name, + "filename": filename, "filepath": file.path, "content": base64.b64decode(file.content).decode("UTF-8"), "sha": file.sha, } logger.debug( - f"Found file {stacks[stack]["filepath"]} in GitHub repository {repo.full_name} stacks" + f"Found file {results[f"{stack}/{filename}"]["filepath"]} in GitHub repository {repo.full_name} stacks" ) - logger.trace(file.html_url) - logger.trace(stacks[stack]) + logger.trace(f"{file.html_url=}") + logger.trace(f"{results[f"{stack}/{filename}"]=}") logger.info( - f"Found {len(stacks):,} files in GitHub repository {repo.full_name} stacks" + f"Found {len(results):,} files in GitHub repository {repo.full_name} stacks" ) + logger.trace(f"{results=}") - return stacks + return results def CompareFiles( @@ -144,9 +165,11 @@ def CompareFiles( remote files that are not present locally will be deleted. """ - for stack in local: - new: dict[str, str] = local[stack] - old: dict[str, str] | None = remote.get(stack) + for file in local: + new: dict[str, str] = local[file] + old: dict[str, str] | None = remote.get(file) + + logger.trace(f"{file=} {new=} {old=}") if not old: url: str | None = SaveFile(repo, new["filepath"], new["content"]) @@ -178,19 +201,21 @@ def CompareFiles( f"Modified file {new["filepath"]} in GitHub repository {repo.full_name}" ) - for stack in remote: - if not local.get(stack): + for file in remote: + logger.trace(f"{file=}") + + if not local.get(file): url: str | None = DeleteFile( - repo, remote[stack]["filepath"], remote[stack]["sha"] + repo, remote[file]["filepath"], remote[file]["sha"] ) if url: - remote[stack]["url"] = url + remote[file]["url"] = url - Notify(remote[stack], "Deleted") + Notify(remote[file], "Deleted") logger.success( - f"Deleted file {remote[stack]["filepath"]} in GitHub repository {repo.full_name}" + f"Deleted file {remote[file]["filepath"]} in GitHub repository {repo.full_name}" )