Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove logic for yaml history database format. #13

Merged
merged 7 commits into from
Dec 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions .goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@ project_name: gpbackup_exporter

builds:
- env:
- CGO_ENABLED=0
- CGO_ENABLED=1
goos:
- linux
- darwin
goarch:
- amd64
- arm64
flags:
- -trimpath
- -mod=vendor
Expand Down
4 changes: 3 additions & 1 deletion Dockerfile.artifacts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM goreleaser/goreleaser:v1.24.0 as builder
FROM --platform=linux/amd64 goreleaser/goreleaser:v1.24.0 as builder
WORKDIR /build
COPY . /build
RUN goreleaser release --snapshot --skip-publish --rm-dist
Expand All @@ -7,6 +7,8 @@ FROM alpine:3.19
COPY --from=builder /build/dist/ /dist/
RUN mkdir -p /artifacts && \
cp /dist/*.tar.gz /artifacts/ && \
cp /dist/*.rpm /artifacts/ && \
cp /dist/*.deb /artifacts/ && \
cp /dist/*.txt /artifacts/ && \
ls -la /artifacts/*
CMD ["sleep", "150"]
30 changes: 16 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ BRANCH_FULL := $(shell git rev-parse --abbrev-ref HEAD)
BRANCH := $(subst /,-,$(BRANCH_FULL))
GIT_REV := $(shell git describe --abbrev=7 --always)
SERVICE_CONF_DIR := /etc/systemd/system
HISTORY_FILE ?= /data/master/gpseg-1/gpbackup_history.yaml
HISTORY_FILE ?= /data/master/gpseg-1/gpbackup_history.db
HTTP_PORT := 19854
ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
MUSL_CROSS := $(shell brew list| grep musl-cross)
DOCKER_CONTAINER_E2E := $(shell docker ps -a -q -f name=$(APP_NAME)_e2e)
HTTP_PORT_E2E := $(shell echo $$((10000 + ($$RANDOM % 10000))))

Expand All @@ -21,32 +22,33 @@ test-e2e:
@if [ -n "$(DOCKER_CONTAINER_E2E)" ]; then docker rm -f "$(DOCKER_CONTAINER_E2E)"; fi;
DOCKER_BUILDKIT=1 docker build --pull -f Dockerfile --build-arg REPO_BUILD_TAG=$(BRANCH)-$(GIT_REV) -t $(APP_NAME)_e2e .
$(call e2e_basic,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db)
$(call e2e_basic,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_empty.yml,false,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_TLS_noAuth.yml,true,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_TLSInLine_noAuth.yml,true,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_TLS_Auth.yml,true,basic)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_noTLS_Auth.yml,false,basic)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_TLS_RequireAnyClientCert.yml,true,cert,"$(ROOT_DIR)/e2e_tests")
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.yaml,/e2e_tests/web_config_TLS_RequireAndVerifyClientCert.yml,true,cert,"$(ROOT_DIR)/e2e_tests")
$(call e2e_basic,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_empty.yml,false,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_TLS_noAuth.yml,true,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_TLSInLine_noAuth.yml,true,false)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_TLS_Auth.yml,true,basic)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_noTLS_Auth.yml,false,basic)
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_TLS_RequireAnyClientCert.yml,true,cert,"$(ROOT_DIR)/e2e_tests")
$(call e2e_tls_auth,$(PWD)/e2e_tests/:/e2e_tests/:ro,/e2e_tests/gpbackup_history.db,/e2e_tests/web_config_TLS_RequireAndVerifyClientCert.yml,true,cert,"$(ROOT_DIR)/e2e_tests")

.PHONY: build
build:
@echo "Build $(APP_NAME)"
@make test
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -mod=vendor -trimpath -ldflags "-X main.version=$(BRANCH)-$(GIT_REV)" -o $(APP_NAME) $(APP_NAME).go

.PHONY: build-arm
build-arm:
.PHONY: build-on-darwin
build-on-darwin:
@echo "Build $(APP_NAME)"
@make test
CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -mod=vendor -trimpath -ldflags "-X main.version=$(BRANCH)-$(GIT_REV)" -o $(APP_NAME) $(APP_NAME).go
@if [ -z "$(MUSL_CROSS)" ]; then echo "musl-cross is not installed"; exit 1; fi;
CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -mod=vendor -trimpath -ldflags "-linkmode external -extldflags -static -X main.version=$(BRANCH)-$(GIT_REV)" -o $(APP_NAME) $(APP_NAME).go

.PHONY: build-darwin
build-darwin:
@echo "Build $(APP_NAME)"
@make test
CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -mod=vendor -trimpath -ldflags "-X main.version=$(BRANCH)-$(GIT_REV)" -o $(APP_NAME) $(APP_NAME).go
CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -mod=vendor -trimpath -ldflags "-X main.version=$(BRANCH)-$(GIT_REV)" -o $(APP_NAME) $(APP_NAME).go

.PHONY: dist
dist:
Expand All @@ -68,7 +70,7 @@ prepare-service:
@echo "Prepare config file $(APP_NAME).service for systemd"
cp $(ROOT_DIR)/$(APP_NAME).service.template $(ROOT_DIR)/$(APP_NAME).service
sed -i.bak "s|/usr/bin|$(ROOT_DIR)|g" $(APP_NAME).service
sed -i.bak "s|/data/master/gpseg-1/gpbackup_history.yaml|$(HISTORY_FILE)|g" $(APP_NAME).service
sed -i.bak "s|/data/master/gpseg-1/gpbackup_history.db|$(HISTORY_FILE)|g" $(APP_NAME).service
rm $(APP_NAME).service.bak

.PHONY: install-service
Expand Down
43 changes: 23 additions & 20 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
[![Coverage Status](https://coveralls.io/repos/github/woblerr/gpbackup_exporter/badge.svg?branch=master)](https://coveralls.io/github/woblerr/gpbackup_exporter?branch=master)
[![Go Report Card](https://goreportcard.com/badge/github.com/woblerr/gpbackup_exporter)](https://goreportcard.com/report/github.com/woblerr/gpbackup_exporter)

Prometheus exporter for collecting metrics from [gpbackup](https://github.com/greenplum-db/gpbackup) history file `gpbackup_history.db`/`gpbackup_history.yaml`.
Prometheus exporter for collecting metrics from [gpbackup](https://github.com/greenplum-db/gpbackup) history file `gpbackup_history.db`.

By default, the metrics are collected for all databases and backups in history file. You need to run exporter or Docker image on the same host where is `gpbackup_history.db`/`gpbackup_history.yaml` file located (Greenplum Master host).
By default, the metrics are collected for all databases and backups in history file. You need to run exporter or Docker image on the same host where is `gpbackup_history.db` file located (Greenplum Master host).

If you are using an old `gpbackup` version that supports only the YAML format `gpbackup_history.yaml` , then use `gpbackup_exporter <= v0.3.0`.

## Collected metrics
### Backup metrics
Expand Down Expand Up @@ -51,11 +53,12 @@ Flags:
--web.endpoint="/metrics" Endpoint used for metrics.
--web.listen-address=:19854 ...
Addresses on which to expose metrics and web interface. Repeatable for multiple addresses.
--web.config.file="" Path to configuration file that can enable TLS or authentication. See: https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md
--web.config.file="" Path to configuration file that can enable TLS or authentication. See:
https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md
--collect.interval=600 Collecting metrics interval in seconds.
--collect.depth=0 Metrics depth collection in days. Metrics for backup older than this interval will not be collected. 0 - disable.
--gpbackup.history-file=""
Path to gpbackup_history.db or gpbackup_history.yaml.
Path to gpbackup_history.db.
--gpbackup.db-include="" ...
Specific db for collecting metrics. Can be specified several times.
--gpbackup.db-exclude="" ...
Expand All @@ -71,7 +74,7 @@ Flags:

#### Additional description of flags.

It's necessary to specify the `gpbackup_history.db` or `gpbackup_history.yaml` file location via `--gpbackup.history-file` flag.
It's necessary to specify the `gpbackup_history.db` file location via `--gpbackup.history-file` flag.

By default, metrics a collected only for active backups. The flag `--gpbackup.collect-deleted ` allows to collect metrics for deleted backups. The flag `--gpbackup.collect-failed ` allows to collect metrics for failed backups.

Expand Down Expand Up @@ -149,9 +152,9 @@ Simple run:
docker run -d --restart=always \
--name gpbackup_exporter \
-e TZ=America/Chicago \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-p 19854:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter
```

Expand All @@ -160,10 +163,10 @@ For specific database:
```bash
docker run -d --restart=always \
--name gpbackup_exporter \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-e DB_INCLUDE=demo1 \
-p 19854:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter
```

Expand All @@ -173,18 +176,18 @@ you can run containers on different ports:
```bash
docker run -d --restart=always \
--name gpbackup_exporter \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-e DB_INCLUDE=demo1 \
-p 19854:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter

docker run -d --restart=always \
--name gpbackup_exporter \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-e DB_INCLUDE=demo2 \
-p 19855:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter
```

Expand All @@ -193,10 +196,10 @@ To exclude specific database:
```bash
docker run -d --restart=always \
--name gpbackup_exporter \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-e DB_EXCLUDE=demo1 \
-p 19854:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter
```

Expand All @@ -205,11 +208,11 @@ For specific backup type not older than 14 days:
```bash
docker run -d --restart=always \
--name gpbackup_exporter \
-e HISTORY_FILE=/data/gpbackup_history.yaml \
-e HISTORY_FILE=/data/gpbackup_history.db \
-e BACKUP_TYPE=full \
-e COLLECT_DEPTH=14 \
-p 19854:19854 \
-v /data/master/gpseg-1/gpbackup_history.yaml:/data/gpbackup_history.yaml:ro \
-v /data/master/gpseg-1/gpbackup_history.db:/data/gpbackup_history.db:ro \
gpbackup_exporter
```

Expand All @@ -218,7 +221,7 @@ docker run -d --restart=always \
* Register `gpbackup_exporter` (already builded, if not - exec `make build` before) as a systemd service:

```bash
make prepare-service HISTORY_FILE="/path/to/gpbackup_history.yaml"
make prepare-service HISTORY_FILE="/path/to/gpbackup_history.db"
```

Validate prepared file `gpbackup_exporter.service` and run:
Expand Down Expand Up @@ -246,7 +249,7 @@ Manual register systemd service:
cp gpbackup_exporter.service.template gpbackup_exporter.service
```

In file `gpbackup_exporter.service.template` replace `/usr/bin/gpbackup_exporter` to full path to `gpbackup_exporter` and `/data/master/gpseg-1/gpbackup_history.yaml` to full path to `gpbackup_history.yaml`.
In file `gpbackup_exporter.service.template` replace `/usr/bin/gpbackup_exporter` to full path to `gpbackup_exporter` and `/data/master/gpseg-1/gpbackup_history.db` to full path to `gpbackup_history.db`.

```bash
sudo cp gpbackup_exporter.service /etc/systemd/system/gpbackup_exporter.service
Expand All @@ -268,7 +271,7 @@ rpm -ql gpbackup_exporter
/usr/bin/gpbackup_exporter
```

After installation RPM/DEB package, you need to set correct path to `gpbackup_history.db`/`gpbackup_history.yaml` in `/etc/systemd/system/gpbackup_exporter.service`.
After installation RPM/DEB package, you need to set correct path to `gpbackup_history.db` in `/etc/systemd/system/gpbackup_exporter.service`.


### Running tests
Expand Down
Loading
Loading