From 8e4d24e20a8f59b9d7ceaeaa9bc142534a938c0f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 19 Nov 2024 22:04:04 +0100 Subject: [PATCH 01/12] Rename module, update LICENSE, rework Dockerfile --- Dockerfile | 28 +++++++++++++++++++++++++ LICENSE | 3 ++- Makefile | 7 +++++-- docker/builder/Dockerfile | 8 ------- docker/graphite-ch-optimizer/Dockerfile | 28 ------------------------- go.mod | 2 +- 6 files changed, 36 insertions(+), 40 deletions(-) create mode 100644 Dockerfile delete mode 100644 docker/builder/Dockerfile delete mode 100644 docker/graphite-ch-optimizer/Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b6f4407 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# +# Image which contains the binary artefacts +# +FROM golang:alpine AS builder + +ENV GOPATH=/go + +RUN apk add make git --no-cache + +WORKDIR /go/src/github.com/go-graphite/graphite-ch-optimizer +COPY . . +RUN make -e CGO_ENABLED=0 build + +# +# Application image +# +FROM alpine:latest + +RUN apk --no-cache add ca-certificates tzdata && mkdir /graphite-ch-optimizer + +WORKDIR /graphite-ch-optimizer + +COPY --from=builder \ + /go/src/github.com/go-graphite/graphite-ch-optimizer/graphite-ch-optimizer \ + /go/src/github.com/go-graphite/graphite-ch-optimizer/LICENSE \ + . + +ENTRYPOINT ["./graphite-ch-optimizer"] diff --git a/LICENSE b/LICENSE index d804495..164361c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ MIT License -Copyright (c) 2019 InnoGames GmbH +Copyright (c) 2019-2024 InnoGames GmbH +Copyright (c) 2024 go-graphite Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 2e3dd0b..80119e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ NAME = graphite-ch-optimizer +MODULE = github.com/go-graphite/$(NAME) VERSION = $(shell git describe --long --tags 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') VENDOR = "System Administration " -URL = https://github.com/innogames/$(NAME) +URL = https://$(MODULE) define DESC = 'Service to optimize stale GraphiteMergeTree tables This software looking for tables with GraphiteMergeTree engine and evaluate if some of partitions should be optimized. It could work both as one-shot script and background daemon.' @@ -9,9 +10,11 @@ endef GO_FILES = $(shell find -name '*.go') PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm SUM_FILES = build/sha256sum build/md5sum -MODULE = github.com/innogames/$(NAME) GO ?= go +ifeq ("$(CGO_ENABLED)", "0") + GOFLAGS += -ldflags=-extldflags=-static +endif export GO111MODULE := on .PHONY: all clean docker test version diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile deleted file mode 100644 index 850e18e..0000000 --- a/docker/builder/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# -# Image for building packages in docker -# innogames/graphite-ch-optimizer:builder on docker-hub -# -FROM golang:1.13-alpine as builder - -RUN apk --no-cache add ruby ruby-dev ruby-etc make gcc g++ rpm git tar && \ - gem install --no-user-install --no-document fpm diff --git a/docker/graphite-ch-optimizer/Dockerfile b/docker/graphite-ch-optimizer/Dockerfile deleted file mode 100644 index 3ade606..0000000 --- a/docker/graphite-ch-optimizer/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# -# Image which contains the binary artefacts -# -FROM innogames/graphite-ch-optimizer:builder AS build -COPY . ./graphite-ch-optimizer - -WORKDIR ./graphite-ch-optimizer - -RUN make test && \ - make -e CGO_ENABLED=0 build && \ - make -e CGO_ENABLED=0 packages - -# This one will return tar stream of binary artefacts to unpack on the local file system -CMD ["/usr/bin/tar", "-c", "--exclude=build/pkg", "build", "graphite-ch-optimizer"] - - -# -# Application image -# -FROM alpine:latest - -RUN apk --no-cache add ca-certificates tzdata && mkdir /graphite-ch-optimizer - -WORKDIR /graphite-ch-optimizer - -COPY --from=build /go/graphite-ch-optimizer/graphite-ch-optimizer . - -ENTRYPOINT ["./graphite-ch-optimizer"] diff --git a/go.mod b/go.mod index aba7387..ca6025d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/innogames/graphite-ch-optimizer +module github.com/go-graphite/graphite-ch-optimizer go 1.22.0 From 84ffe14a8b7e7d241471221b7e3d9914ca6c82e1 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Nov 2024 01:09:29 +0100 Subject: [PATCH 02/12] Migrate to nfpm --- .gitignore | 2 +- Makefile | 99 ++++++++++++++++++++++-------------------------------- nfpm.yaml | 34 +++++++++++++++++++ 3 files changed, 76 insertions(+), 59 deletions(-) create mode 100644 nfpm.yaml diff --git a/.gitignore b/.gitignore index 55af516..36a336d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ graphite-ch-optimizer -build/ +out/ artifact/ diff --git a/Makefile b/Makefile index 80119e0..08a2f1f 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,36 @@ NAME = graphite-ch-optimizer MODULE = github.com/go-graphite/$(NAME) -VERSION = $(shell git describe --long --tags 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') -VENDOR = "System Administration " -URL = https://$(MODULE) -define DESC = +VERSION = $(shell git describe --always --tags 2>/dev/null | sed 's:^v::; s/\([^-]*-g\)/c\1/; s|-|.|g') +define DESCRIPTION = 'Service to optimize stale GraphiteMergeTree tables This software looking for tables with GraphiteMergeTree engine and evaluate if some of partitions should be optimized. It could work both as one-shot script and background daemon.' endef -GO_FILES = $(shell find -name '*.go') -PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm -SUM_FILES = build/sha256sum build/md5sum +PKG_FILES = $(wildcard out/*$(VERSION)*.deb out/*$(VERSION)*.rpm ) +SUM_FILES = out/sha256sum out/md5sum GO ?= go +GO_VERSION = -ldflags "-X 'main.version=$(VERSION)'" ifeq ("$(CGO_ENABLED)", "0") GOFLAGS += -ldflags=-extldflags=-static endif export GO111MODULE := on +SRCS:=$(shell find . -name '*.go') + .PHONY: all clean docker test version -all: build +all: $(NAME) +build: $(NAME) +$(NAME): $(SRCS) + $(GO) build $(GO_VERSION) -o $@ . version: @echo $(VERSION) clean: rm -rf artifact - rm -rf build rm -rf $(NAME) + rm -rf out rebuild: clean all @@ -36,15 +39,20 @@ test: $(GO) vet $(MODULE) $(GO) test $(MODULE) -build: | $(NAME) - mkdir build - docker: - docker build -t innogames/$(NAME):builder -f docker/builder/Dockerfile . - docker build -t innogames/$(NAME):latest -f docker/$(NAME)/Dockerfile . + docker build -t ghcr.io/go-graphite/$(NAME):latest . + +# we need it static +.PHONY: gox-build +gox-build: + @CGO_ENABLED=0 $(MAKE) out/$(NAME)-linux-amd64 out/$(NAME)-linux-arm64 -$(NAME): $(NAME).go - $(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ . +out/$(NAME)-linux-%: $(SRCS) | out + GOOS=linux GOARCH=$* $(GO) build $(GO_VERSION) -o $@ $(MODULE) + +out: out/done +out/done: + mkdir -p out/done ######################################################### # Prepare artifact directory and set outputs for upload # @@ -57,7 +65,7 @@ artifact: # Link artifact to directory with setting step output to filename artifact/%: ART=$(notdir $@) artifact/%: TYPE=$(lastword $(subst ., ,$(ART))) -artifact/%: build/% | artifact +artifact/%: out/% | artifact cp -l $< $@ @echo '::set-output name=$(TYPE)::$(ART)' @@ -70,54 +78,29 @@ artifact/%: build/% | artifact ############# # Prepare everything for packaging -.ONESHELL: -build/pkg: build/$(NAME) build/config.toml.example - cd build - mkdir -p pkg/etc/$(NAME) - mkdir -p pkg/usr/bin - cp -l $(NAME) pkg/usr/bin/ - cp -l config.toml.example pkg/etc/$(NAME) +out/config.toml.example: $(NAME) | out + ./$(NAME) --print-defaults > $@ -build/$(NAME): $(NAME).go - GOOS=linux GOARCH=amd64 $(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ . +nfpm: nfpm-deb nfpm-rpm -build/config.toml.example: build/$(NAME) - ./build/$(NAME) --print-defaults > $@ +PKG_ARCH = amd64 arm64 -packages: $(PKG_FILES) $(SUM_FILES) +nfpm-%: out/config.toml.example + $(MAKE) out/done/$(NAME)-$(VERSION)-amd64-$* ARCH=amd64 PACKAGER=$* + $(MAKE) out/done/$(NAME)-$(VERSION)-arm64-$* ARCH=arm64 PACKAGER=$* + +.ONESHELL: +out/done/$(NAME)-$(VERSION)%: nfpm.yaml | out/done gox-build + @NAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $(PACKAGER) --target out/ + @touch $@ + +packages: nfpm $(SUM_FILES) # md5 and sha256 sum-files for packages $(SUM_FILES): COMMAND = $(notdir $@) $(SUM_FILES): PKG_FILES_NAME = $(notdir $(PKG_FILES)) -.ONESHELL: -$(SUM_FILES): $(PKG_FILES) - cd build - $(COMMAND) $(PKG_FILES_NAME) > $(COMMAND) - -deb: $(word 1, $(PKG_FILES)) - -rpm: $(word 2, $(PKG_FILES)) - -# Set TYPE to package suffix w/o dot -$(PKG_FILES): TYPE = $(subst .,,$(suffix $@)) -$(PKG_FILES): build/pkg - fpm --verbose \ - -s dir \ - -a x86_64 \ - -t $(TYPE) \ - --vendor $(VENDOR) \ - -m $(VENDOR) \ - --url $(URL) \ - --description $(DESC) \ - --license MIT \ - -n $(NAME) \ - -v $(VERSION) \ - --after-install packaging/postinst \ - --before-remove packaging/prerm \ - -p build \ - build/pkg/=/ \ - packaging/$(NAME).service=/lib/systemd/system/$(NAME).service - +$(SUM_FILES): nfpm + cd out && $(COMMAND) $(PKG_FILES_NAME) > $(COMMAND) ####### # END # ####### diff --git a/nfpm.yaml b/nfpm.yaml new file mode 100644 index 0000000..34393a0 --- /dev/null +++ b/nfpm.yaml @@ -0,0 +1,34 @@ +--- +name: ${NAME} +description: ${DESCRIPTION} + +# Common packages config +arch: "${ARCH}" # amd64, arm64 +platform: "linux" +version: "${VERSION_STRING}" +maintainer: &m "Mikhail f. Shiryaev " +vendor: *m +homepage: "https://github.com/go-graphite/${NAME}" +license: "MIT" +section: "admin" +priority: "optional" + +contents: + - src: "packaging/${NAME}.service" + dst: "/lib/systemd/system/${NAME}.service" + expand: true + - src: out/config.toml.example + dst: /etc/${NAME}/config.toml.example + type: config|noreplace + expand: true + - src: "out/${NAME}-linux-${ARCH}" + dst: /usr/bin/${NAME} + expand: true + # docs + - src: LICENSE + dst: /usr/share/doc/${NAME}/LICENSE + expand: true + +scripts: + postinstall: packaging/postinst + preremove: packaging/prerm From a0732c134e7b9f0a46eb6ae8f6b191adcb113122 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Nov 2024 11:46:02 +0100 Subject: [PATCH 03/12] Add `static` target to make --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 08a2f1f..9083ac1 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ test: $(GO) vet $(MODULE) $(GO) test $(MODULE) +static: + CGO_ENABLED=0 $(MAKE) $(NAME) + docker: docker build -t ghcr.io/go-graphite/$(NAME):latest . From 980b56887965a99421e6d533475ca16b76913c79 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Nov 2024 12:16:19 +0100 Subject: [PATCH 04/12] Add a label for an automatic image linking to ghcr.io --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9083ac1..430e3a5 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ static: CGO_ENABLED=0 $(MAKE) $(NAME) docker: - docker build -t ghcr.io/go-graphite/$(NAME):latest . + docker build --label 'org.opencontainers.image.source=https://$(MODULE)' -t ghcr.io/go-graphite/$(NAME):latest . # we need it static .PHONY: gox-build From ddb38c7e83e7b64e65a4aa02ae10e29d52e65753 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Nov 2024 13:03:04 +0100 Subject: [PATCH 05/12] Change the image url --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0186a3..a631469 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ To build docker image locally run: `make docker` To launch the container run the following command on the host with a running ClickHouse server: -`docker run --net=host --rm innogames/graphite-ch-optimizer:latest` +`docker run --net=host --rm ghcr.io/go-graphite/graphite-ch-optimizer:latest` + +Other versions are available on the [packages](https://github.com/go-graphite/graphite-ch-optimizer/pkgs/container/graphite-ch-optimizer) page. ## FAQ * The `go` version 1.13 or newer is required From 10dd5f0d1f9c75a77ed557039e0ec6e908496ba2 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 27 Nov 2024 16:23:42 +0100 Subject: [PATCH 06/12] Update codeql --- .github/workflows/codeql-analysis.yml | 120 +++++++++++++++----------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e3f2dc8..d5a50ea 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -3,68 +3,90 @@ # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. -name: "CodeQL" +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" on: push: - branches: [master] + branches: [ "master" ] pull_request: - # The branches below must be a subset of the branches above - branches: [master] + branches: [ "master" ] + schedule: + - cron: '15 15 * * 6' jobs: analyze: - name: Analyze - runs-on: ubuntu-latest + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: 'ubuntu-latest' + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read strategy: fail-fast: false matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['go'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection - + include: + - language: go + build-mode: autobuild + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + - name: Checkout repository + uses: actions/checkout@v4 - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - #- run: | - # make bootstrap - # make release + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 512a52d3a168ae6ab8e74f8101dcb3f610bcd790 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 28 Nov 2024 23:45:33 +0100 Subject: [PATCH 07/12] Update the upload assets --- .github/workflows/upload-assets.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upload-assets.yml b/.github/workflows/upload-assets.yml index fb695ab..f6849a0 100644 --- a/.github/workflows/upload-assets.yml +++ b/.github/workflows/upload-assets.yml @@ -1,6 +1,7 @@ +--- name: Upload Packages to new release -on: +'on': release: types: - published @@ -10,9 +11,6 @@ jobs: name: Build runs-on: ubuntu-latest - container: - image: innogames/graphite-ch-optimizer:builder - outputs: deb: ${{ steps.build.outputs.deb }} rpm: ${{ steps.build.outputs.rpm }} @@ -20,7 +18,7 @@ jobs: md5sum: ${{ steps.build.outputs.md5sum }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 name: Checkout with: # Otherwise there's a risk to not get latest tag From 2ee754f20da5c5b4552b1829e29d598e2e874077 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 19 Dec 2024 15:59:20 +0100 Subject: [PATCH 08/12] Clean PKG_ARCH --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 430e3a5..7ea9e02 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,6 @@ out/config.toml.example: $(NAME) | out nfpm: nfpm-deb nfpm-rpm -PKG_ARCH = amd64 arm64 - nfpm-%: out/config.toml.example $(MAKE) out/done/$(NAME)-$(VERSION)-amd64-$* ARCH=amd64 PACKAGER=$* $(MAKE) out/done/$(NAME)-$(VERSION)-arm64-$* ARCH=arm64 PACKAGER=$* From 72140ff715ffbdc569223c2e9909c1fe2dc2a50c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 19 Dec 2024 16:32:52 +0100 Subject: [PATCH 09/12] Update actions --- .github/workflows/tests.yml | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 243e035..5274e78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,11 @@ +--- name: Tests -on: +"on": push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: @@ -14,24 +15,24 @@ jobs: strategy: matrix: go: - - ^1.15 - - ^1.16 - - ^1.17 + - ^1.21 + - ^1.22 + - ^1.23 - ^1 steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code into the Go module directory + uses: actions/checkout@v4 - - name: Test - run: make test + - name: Test + run: make test - - name: Build and run version - run: | - make VERSION=testing-version - ./graphite-ch-optimizer --version + - name: Build and run version + run: | + make VERSION=testing-version + ./graphite-ch-optimizer --version From e619a35fd0fc2fe44b8446f1e065f6221c0d8df1 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 19 Dec 2024 17:39:35 +0100 Subject: [PATCH 10/12] Add packages publishing --- Makefile | 46 +++++++++++++++++++++++++++++++++++++--------- README.md | 3 +++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 7ea9e02..40254ed 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ export GO111MODULE := on SRCS:=$(shell find . -name '*.go') -.PHONY: all clean docker test version +.PHONY: all clean docker test version _nfpm all: $(NAME) build: $(NAME) @@ -84,16 +84,15 @@ artifact/%: out/% | artifact out/config.toml.example: $(NAME) | out ./$(NAME) --print-defaults > $@ -nfpm: nfpm-deb nfpm-rpm - -nfpm-%: out/config.toml.example - $(MAKE) out/done/$(NAME)-$(VERSION)-amd64-$* ARCH=amd64 PACKAGER=$* - $(MAKE) out/done/$(NAME)-$(VERSION)-arm64-$* ARCH=arm64 PACKAGER=$* - .ONESHELL: -out/done/$(NAME)-$(VERSION)%: nfpm.yaml | out/done gox-build +nfpm: + @$(MAKE) _nfpm ARCH=amd64 PACKAGER=deb + @$(MAKE) _nfpm ARCH=arm64 PACKAGER=deb + @$(MAKE) _nfpm ARCH=amd64 PACKAGER=rpm + @$(MAKE) _nfpm ARCH=arm64 PACKAGER=rpm + +_nfpm: nfpm.yaml out/config.toml.example | out/done gox-build @NAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $(PACKAGER) --target out/ - @touch $@ packages: nfpm $(SUM_FILES) @@ -105,3 +104,32 @@ $(SUM_FILES): nfpm ####### # END # ####### + +############## +# PUBLISHING # +############## + +# Use `go install github.com/mlafeldt/pkgcloud/cmd/pkgcloud-push` + + +.ONESHELL: +packagecloud-push-rpm: $(wildcard out/$(NAME)-$(VERSION)*.rpm) + for repo in el/{7..9}; do + pkgcloud-push $(REPO)/$${repo} $^ || true + done + +.ONESHELL: +packagecloud-push-deb: $(wildcard out/$(NAME)_$(VERSION)*.deb) + for repo in ubuntu/{bionic,focal,jammy,nomble} debian/{buster,bullseye,bookworm}; do + pkgcloud-push $(REPO)/$${repo} $^ || true + done + +packagecloud-push: nfpm + @$(MAKE) packagecloud-push-rpm + @$(MAKE) packagecloud-push-deb + +packagecloud-autobuilds: + $(MAKE) packagecloud-push REPO=go-graphite/autobuilds + +packagecloud-stable: + $(MAKE) packagecloud-push REPO=go-graphite/stable diff --git a/README.md b/README.md index a631469..15b7fa8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![deb](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) +[![rpm](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) + # Service to optimize stale GraphiteMergeTree tables When you use [GraphiteMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/graphitemergetree) in ClickHouse DBMS, it applies retention policies from `system.graphite_retentions` configuration during merge processes. Unfortunately, ClickHouse doesn't launch merges for partitions a) without active inserts or b) with only one part in. It means, that it never will watch for the actual retention scheme applied to partitions. This software looking for tables with GraphiteMergeTree engine and evaluate if some of partitions should be optimized. It could work both as one-shot script and background daemon. From 75852c95611643e203b480f0b33b7df29778670c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 19 Dec 2024 17:44:50 +0100 Subject: [PATCH 11/12] Make release and docker workflows --- .github/workflows/docker.yml | 47 ++++++++++++++++++ .github/workflows/upload-assets.yml | 77 ++++++++++------------------- 2 files changed, 72 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..aa99738 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,47 @@ +name: Upload Docker images to ghcr.io +on: + release: + types: [published] + workflow_dispatch: + inputs: + ref: + description: 'Git tag to push the image' + required: true + type: string +jobs: + docker: + name: Build image + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + context: ${{ inputs.ref && 'git' || 'workflow' }} + images: ghcr.io/${{ github.repository }} + # create latest tag for branch events + flavor: | + latest=${{ inputs.ref && 'false' || 'auto' }} + tags: | + type=semver,pattern={{version}},value=${{inputs.ref}} + type=semver,pattern={{major}}.{{minor}},value=${{inputs.ref}} + type=semver,pattern={{major}}.{{minor}}.{{patch}},value=${{inputs.ref}} + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v5 + with: + # push for non-pr events + push: ${{ github.event_name != 'pull_request' }} + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/upload-assets.yml b/.github/workflows/upload-assets.yml index f6849a0..a973be1 100644 --- a/.github/workflows/upload-assets.yml +++ b/.github/workflows/upload-assets.yml @@ -5,74 +5,47 @@ name: Upload Packages to new release release: types: - published + workflow_dispatch: + inputs: + ref: + description: 'Git tag to push the image' + required: true + type: string jobs: build: name: Build runs-on: ubuntu-latest - outputs: - deb: ${{ steps.build.outputs.deb }} - rpm: ${{ steps.build.outputs.rpm }} - sha256sum: ${{ steps.build.outputs.sha256sum }} - md5sum: ${{ steps.build.outputs.md5sum }} - steps: - uses: actions/checkout@v4 name: Checkout with: # Otherwise there's a risk to not get latest tag # We hope, that the current commit at - # least 50 commits close to the latest release - fetch-depth: 50 + # least 100 commits close to the latest release + fetch-depth: 100 + fetch-tags: ${{ inputs.ref != '' }} + ref: ${{ inputs.ref }} + - name: Set up Go 1 + uses: actions/setup-go@v5 + with: + go-version: ^1 - name: Build packages id: build run: | - # Checkout action doesn't fetch tags - git fetch --tags + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0 make -e CGO_ENABLED=0 packages - make github_artifact - - name: Upload rpm - id: upload-rpm - uses: actions/upload-release-asset@v1 + - name: Upload release assets env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ASSET: ${{ steps.build.outputs.rpm }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: artifact/${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream - - name: Upload sha256sum - id: upload-sha256sum - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ASSET: ${{ steps.build.outputs.sha256sum }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: artifact/${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream - - name: Upload md5sum - id: upload-md5sum - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ASSET: ${{ steps.build.outputs.md5sum }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: artifact/${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream - - name: Upload deb - id: upload-deb - uses: actions/upload-release-asset@v1 + run: | + TAG="${{ inputs.ref && inputs.ref || github.event.release.tag_name }}" + gh release upload --repo ${{ github.repository }} "$TAG" \ + out/*.deb out/*.rpm out/*sum + - name: Upload packages to packagecloud.com env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ASSET: ${{ steps.build.outputs.deb }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: artifact/${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + run: | + go install github.com/mlafeldt/pkgcloud/cmd/pkgcloud-push@e79e9efc + make packagecloud-stable From f418ef59784079225ea39242af6d076670384c88 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 19 Dec 2024 18:46:18 +0100 Subject: [PATCH 12/12] Overwrite existing assets in release --- .github/workflows/upload-assets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-assets.yml b/.github/workflows/upload-assets.yml index a973be1..a81745a 100644 --- a/.github/workflows/upload-assets.yml +++ b/.github/workflows/upload-assets.yml @@ -41,7 +41,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | TAG="${{ inputs.ref && inputs.ref || github.event.release.tag_name }}" - gh release upload --repo ${{ github.repository }} "$TAG" \ + gh release upload --clobber --repo ${{ github.repository }} "$TAG" \ out/*.deb out/*.rpm out/*sum - name: Upload packages to packagecloud.com env: