From b2d4bce1e872252e5a7eeb31617d0bc007b32fa6 Mon Sep 17 00:00:00 2001 From: Michael Gattozzi Date: Thu, 14 Mar 2024 12:06:54 -0400 Subject: [PATCH] feat: build binaries and Docker images in CI For releases we need to have Docker images and binary images available for the user to actually run influxdb3. These CI changes will build the binaries on a release tag and the Docker image as well, test, sign, and publish them and make them available for download. Co-Authored-By: Brandon Pfeifer --- .circleci/config.yml | 421 +++++++++++++++--- .circleci/packages/config.yaml | 51 +++ .../packages/influxdb3/control/post-install | 16 + .../packages/influxdb3/control/pre-install | 22 + .../packages/influxdb3/fs/usr/bin/.keepdir | 0 .../influxdb3/fs/usr/share/influxdb3/.keepdir | 0 .../{ => scripts}/docker_build_release.bash | 6 +- .circleci/scripts/package-validation/debian | 8 + .circleci/scripts/package-validation/redhat | 97 ++++ .../scripts/package-validation/tf/main.tf | 114 +++++ .circleci/scripts/package-validation/validate | 90 ++++ .circleci/scripts/publish.bash | 19 + Cargo.toml | 4 +- Dockerfile | 9 +- influxdb3/src/commands/serve.rs | 2 +- influxdb3/src/main.rs | 4 + influxdb3_process/Cargo.toml | 1 + influxdb3_process/src/lib.rs | 23 +- influxdb3_server/src/lib.rs | 3 +- 19 files changed, 815 insertions(+), 75 deletions(-) create mode 100644 .circleci/packages/config.yaml create mode 100755 .circleci/packages/influxdb3/control/post-install create mode 100755 .circleci/packages/influxdb3/control/pre-install create mode 100644 .circleci/packages/influxdb3/fs/usr/bin/.keepdir create mode 100644 .circleci/packages/influxdb3/fs/usr/share/influxdb3/.keepdir rename .circleci/{ => scripts}/docker_build_release.bash (81%) create mode 100755 .circleci/scripts/package-validation/debian create mode 100755 .circleci/scripts/package-validation/redhat create mode 100644 .circleci/scripts/package-validation/tf/main.tf create mode 100755 .circleci/scripts/package-validation/validate create mode 100755 .circleci/scripts/publish.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index b8597819b24..8327ea4578f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,41 @@ version: 2.1 +orbs: + aws-s3: circleci/aws-s3@2.0.0 + terraform: circleci/terraform@2.1.0 + rust: circleci/rust@1.6.1 + +# Unlike when a commit is pushed to a branch, CircleCI does not automatically +# execute a workflow when a tag is pushed to a repository. These filters +# allow the corresponding workflow to execute on any branch or tag. +any_filter: &any_filter + filters: + tags: + only: /.*/ + branches: + only: /.*/ + +release_filter: &release_filter + filters: + tags: + # This regex matches what is found in 'scripts/get-version' with the + # '[[:digit:]]' transformed into '\d'. This also excludes release + # candidate detection, because this filter only matches against + # full releases. + only: /^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/ + branches: + ignore: /.*/ + +nofork_filter: &nofork_filter + filters: + branches: + ignore: /pull\/[0-9]+/ + + commands: + install_rust: + rust_components: description: Verify installed components steps: @@ -46,15 +80,12 @@ commands: rustup show cargo fmt --version cargo clippy --version - - login_to_gcloud: + gcloud-docker-login: steps: - run: - name: Login to gcloud + name: configure-gar command: | - echo "${GCLOUD_SERVICE_ACCOUNT_KEY}" >/tmp/gcloud.json - gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file /tmp/gcloud.json --quiet - rm -f /tmp/gcloud.json + gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file <(echo "${GCLOUD_SERVICE_ACCOUNT_KEY}") gcloud auth configure-docker us-docker.pkg.dev jobs: @@ -95,7 +126,7 @@ jobs: - run: name: Yamllint command: yamllint --config-file .circleci/yamllint.yml --strict . - cargo_audit: + cargo-audit: docker: - image: quay.io/influxdb/rust:ci environment: @@ -163,7 +194,7 @@ jobs: command: cargo test --workspace # end to end tests with Heappy (heap profiling enabled) - test_heappy: + test-heappy: docker: - image: quay.io/influxdb/rust:ci resource_class: xlarge # use of a smaller executor tends crashes on link @@ -187,11 +218,16 @@ jobs: # # Compiles a binary with the default ("dev") cargo profile from the influxdb3 source # using the latest ci_image (influxdb/rust) and ensures various targets compile successfully - build_dev: + # Build a dev binary. + build-dev: docker: - - image: quay.io/influxdb/rust:ci - resource_class: 2xlarge # use of a smaller executor tends crashes on link + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + resource_class: 2xlarge+ # use of a smaller executor tends crashes on link environment: + TARGET: << parameters.target >> # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build @@ -202,83 +238,344 @@ jobs: # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit. CARGO_BUILD_JOBS: "12" + parameters: + target: + type: string steps: - checkout - - rust_components + - run: + name: Install Target + command: rustup target add << parameters.target >> - run: name: Cargo build - command: cargo build --workspace + command: target-env cargo build --target=<< parameters.target >> --workspace + - when: + condition: + not: + equal: [ << parameters.target >>, x86_64-pc-windows-gnu ] + steps: + - run: + name: Check benches compile + command: target-env cargo check --target=<< parameters.target >> --workspace --benches + - run: + name: Check extra features (like prod image) + command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" + - when: + condition: + equal: [ << parameters.target >>, x86_64-pc-windows-gnu ] + steps: + - run: + name: Check extra features (like prod image) + command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console" + + # Compile cargo "release" profile binaries for influxdb3 edge releases + build-release: + docker: + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + resource_class: 2xlarge+ + environment: + TARGET: << parameters.target >> + # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. + CARGO_INCREMENTAL: "0" + # Disable full debug symbol generation to speed up CI build + # "1" means line tables only, which is useful for panic tracebacks. + CARGO_PROFILE_DEV_DEBUG: "1" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on + # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit. + CARGO_BUILD_JOBS: "12" + parameters: + target: + type: string + steps: + - checkout + - run: + name: Install Target + command: rustup target add << parameters.target >> + - run: + name: Cargo release build + command: target-env cargo build --target=<< parameters.target >> --workspace --release + # linking might take a while and doesn't produce CLI output + no_output_timeout: 30m + - run: + name: tar and gzip build artifacts + command: | + mkdir -p artifacts + tar --ignore-failed-read -czvf "${PWD}/artifacts/influxdb3-edge_<< parameters.target >>.tar.gz" -C "${PWD}/target/<< parameters.target >>/release" influxdb3{,.exe} + - store_artifacts: + path: artifacts + - persist_to_workspace: + root: . + paths: + - artifacts + build-packages: + docker: + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - run: packager .circleci/packages/config.yaml + - persist_to_workspace: + root: . + paths: + - artifacts + - store_artifacts: + path: artifacts/ + check_package_deb_amd64: + machine: + image: ubuntu-2204:current + resource_class: medium + steps: + - attach_workspace: + at: /tmp/workspace + - checkout - run: - name: Check benches compile - command: cargo check --workspace --benches + name: Validate Debian Package (AMD64) + command: | + sudo .circleci/scripts/package-validation/debian \ + /tmp/workspace/artifacts/influxdb3*amd64.deb + check_package_deb_arm64: + machine: + image: ubuntu-2204:current + resource_class: arm.medium + steps: + - attach_workspace: + at: /tmp/workspace + - checkout - run: - name: Check extra features (like prod image) - command: cargo check --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" + name: Validate Debian Package (ARM64) + command: | + sudo .circleci/scripts/package-validation/debian \ + /tmp/workspace/artifacts/influxdb3*arm64.deb + check_package_rpm: + machine: + image: ubuntu-2204:current + resource_class: arm.medium + parameters: + arch: + type: string + steps: + - attach_workspace: + at: /tmp/workspace + - add_ssh_keys: + fingerprints: + - 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67 + - checkout + - run: | + AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY \ + .circleci/scripts/package-validation/redhat << parameters.arch >> /tmp/workspace/artifacts/influxdb3*.<< parameters.arch >>.rpm + sign-packages: + circleci_ip_ranges: true + docker: + - image: quay.io/influxdb/rsign:latest + auth: + username: $QUAY_RSIGN_USERNAME + password: $QUAY_RSIGN_PASSWORD + steps: + - add_ssh_keys: + fingerprints: + - fc:7b:6e:a6:38:7c:63:5a:13:be:cb:bb:fa:33:b3:3c + - attach_workspace: + at: /tmp/workspace + - run: | + for target in /tmp/workspace/artifacts/* + do + case "${target}" + in + # rsign is shipped on Alpine Linux which uses "busybox ash" instead + # of bash. ash is somewhat more posix compliant and is missing some + # extensions and niceties from bash. + *.deb|*.rpm|*.tar.gz|*.zip) + rsign "${target}" + ;; + esac - # Compile cargo "release" profile binaries for influxdb3 & data generator - # - # Uses the latest ci_image (influxdb/rust) to build a release binary and - # copies it to a minimal container image based upon `rust:slim-buster`. This - # minimal image is then pushed to `quay.io/influxdb/influxdb3:${BRANCH}`. - build_release: + if [ -f "${target}" ] + then + # Since all artifacts are present, sign them here. This saves Circle + # credits over spinning up another instance just to separate out the + # checksum job. + sha256sum "${target}" >> "/tmp/workspace/artifacts/influxdb3-edge.${CIRCLE_TAG}.digests" + + # write individual checksums + md5sum "${target}" >> "${target}.md5" + sha256sum "${target}" >> "${target}.sha256" + fi + done + - persist_to_workspace: + root: /tmp/workspace + paths: + - artifacts + - store_artifacts: + path: /tmp/workspace/artifacts + publish-packages: + docker: + - image: cimg/python:3.6 + steps: + - attach_workspace: + at: /tmp/workspace + - aws-s3/sync: + arguments: --acl public-read + aws-region: RELEASE_AWS_REGION + aws-access-key-id: RELEASE_AWS_ACCESS_KEY_ID + aws-secret-access-key: RELEASE_AWS_SECRET_ACCESS_KEY + from: /tmp/workspace/artifacts + to: s3://dl.influxdata.com/influxdb/releases + build-docker: # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a # kinda small node) machine: image: default - resource_class: 2xlarge # CPU bound, so make it fast + resource_class: 2xlarge+ # CPU bound, so make it fast steps: - checkout - run: - name: Cargo release build + name: Build the docker image command: | - COMMIT_SHA="$(git rev-parse HEAD)" - - .circleci/docker_build_release.bash \ + .circleci/scripts/docker_build_release.bash \ "influxdb3" \ "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \ - "quay.io/influxdb/influxdb3:$COMMIT_SHA" + "influxdb3-edge:latest" - mkdir /tmp/images - docker save quay.io/influxdb/influxdb3:"$COMMIT_SHA" | gzip > /tmp/images/influxdb3.tar.gz # linking might take a while and doesn't produce CLI output no_output_timeout: 30m - - store_artifacts: - path: /tmp/images + - run: | + docker save influxdb3-edge:latest >influxdb3-edge.tar - persist_to_workspace: - root: /tmp/images + root: . paths: - - "*.tar.gz" + - influxdb3-edge.tar + publish-docker: + docker: + - image: cimg/gcp:2023.02 + resource_class: medium + steps: + - checkout + - setup_remote_docker + - gcloud-docker-login + - attach_workspace: + at: . + - run: | + docker load > jobs: - - fmt - - lint - - cargo_audit - - test - - test_heappy - - build_dev - - doc - - build_release: - filters: - branches: - only: main + - fmt: + <<: *any_filter + - lint: + <<: *any_filter + - cargo-audit: + <<: *any_filter + - test: + <<: *any_filter + - test-heappy: + <<: *any_filter + - build-dev: + <<: *any_filter + name: build-dev-<< matrix.target >> + matrix: + parameters: + target: + - aarch64-apple-darwin + - aarch64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - x86_64-pc-windows-gnu + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - doc: + <<: *any_filter + - build-release: + # <<: *release_filter + <<: *any_filter + name: build-release-<< matrix.target >> + matrix: + parameters: + target: + - aarch64-apple-darwin + - aarch64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - x86_64-pc-windows-gnu + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - build-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-release + - check_package_deb_arm64: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_deb_amd64: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_rpm: + # <<: *nofork_filter + <<: *any_filter + name: + check_package_rpm-<< matrix.arch >> + matrix: + parameters: + arch: [ x86_64, aarch64 ] + requires: + - build-packages - # Force build and push of container image for non-main branch. - # See instructions at the top of this file - release_branch: - when: << pipeline.parameters.release_branch >> - jobs: - - build_release + - sign-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_rpm + - check_package_deb_arm64 + - check_package_deb_amd64 + - test + - publish-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-release + - sign-packages + - test + - test-heappy + - doc + - lint + - fmt + - cargo-audit + - build-docker: + <<: *any_filter + # <<: *release_filter + - publish-docker: + <<: *any_filter + # <<: *release_filter + requires: + - build-docker + - wait-for-docker: + <<: *any_filter + # <<: *release_filter + requires: + - build-docker + - publish-docker diff --git a/.circleci/packages/config.yaml b/.circleci/packages/config.yaml new file mode 100644 index 00000000000..1564c5261cd --- /dev/null +++ b/.circleci/packages/config.yaml @@ -0,0 +1,51 @@ + +version: + release: + match: '^v[0-9]+.[0-9]+.[0-9]+' + value: '{{env.CIRCLE_TAG[1:]}}' + default: + value: '3.x-{{env.CIRCLE_SHA1[:8]}}' + +sources: + - binary: /tmp/workspace/artifacts/influxdb3-edge_x86_64-unknown-linux-musl.tar.gz + target: artifacts/ + arch: amd64 + plat: linux + + - binary: /tmp/workspace/artifacts/influxdb3-edge_aarch64-unknown-linux-musl.tar.gz + target: artifacts/ + arch: arm64 + plat: linux + + - binary: /tmp/workspace/artifacts/influxdb3-edge_aarch64-apple-darwin.tar.gz + target: artifacts/ + arch: amd64 + plat: darwin + + - binary: /tmp/workspace/artifacts/influxdb3-edge_x86_64-pc-windows-gnu.tar.gz + target: artifacts/ + arch: amd64 + plat: windows + +packages: + - name: influxdb3 + description: Monolithic time-series database. + license: MIT/Apache-2.0 + binaries: + - influxdb3 + - influxdb3.exe + extras: + - source: LICENSE-APACHE + target: usr/share/influxdb3/LICENSE-APACHE + + - source: LICENSE-MIT + target: usr/share/influxdb3/LICENSE-MIT + + - source: README.md + target: usr/share/influxdb3/README.md + #perm_overrides: + deb_recommends: + - influxdb3-cli + #conflicts: + #depends: + source: .circleci/packages/influxdb3 diff --git a/.circleci/packages/influxdb3/control/post-install b/.circleci/packages/influxdb3/control/post-install new file mode 100755 index 00000000000..ac08539b04e --- /dev/null +++ b/.circleci/packages/influxdb3/control/post-install @@ -0,0 +1,16 @@ +#!/bin/bash + +BIN_DIR=/usr/bin +DATA_DIR=/var/lib/influxdb3 +LOG_DIR=/var/log/influxdb3 + +# Distribution-specific logic +if [[ -f /etc/debian_version ]]; then + # Ownership for RH-based platforms is set in build.py via the `rmp-attr` option. + # We perform ownership change only for Debian-based systems. + # Moving these lines out of this if statement would make `rmp -V` fail after installation. + chown -R -L influxdb:influxdb $LOG_DIR + chown -R -L influxdb:influxdb $DATA_DIR + chmod 755 $LOG_DIR + chmod 755 $DATA_DIR +fi diff --git a/.circleci/packages/influxdb3/control/pre-install b/.circleci/packages/influxdb3/control/pre-install new file mode 100755 index 00000000000..c2e3e792439 --- /dev/null +++ b/.circleci/packages/influxdb3/control/pre-install @@ -0,0 +1,22 @@ +#!/bin/bash + +DATA_DIR=/var/lib/influxdb3 +USER=influxdb3 +GROUP=influxdb3 +LOG_DIR=/var/log/influxdb3 + +if ! id influxdb3 &>/dev/null; then + useradd --system -U -M influxdb3 -s /bin/false -d $DATA_DIR +fi + +# check if DATA_DIR exists +if [ ! -d "$DATA_DIR" ]; then + mkdir -p $DATA_DIR + chown $USER:$GROUP $DATA_DIR +fi + +# check if LOG_DIR exists +if [ ! -d "$LOG_DIR" ]; then + mkdir -p $LOG_DIR + chown $USER:$GROUP $DATA_DIR +fi diff --git a/.circleci/packages/influxdb3/fs/usr/bin/.keepdir b/.circleci/packages/influxdb3/fs/usr/bin/.keepdir new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.circleci/packages/influxdb3/fs/usr/share/influxdb3/.keepdir b/.circleci/packages/influxdb3/fs/usr/share/influxdb3/.keepdir new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.circleci/docker_build_release.bash b/.circleci/scripts/docker_build_release.bash similarity index 81% rename from .circleci/docker_build_release.bash rename to .circleci/scripts/docker_build_release.bash index 70c5e32fe84..52b373c12a4 100755 --- a/.circleci/docker_build_release.bash +++ b/.circleci/scripts/docker_build_release.bash @@ -10,7 +10,7 @@ RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)" COMMIT_SHA="$(git rev-parse HEAD)" COMMIT_TS="$(env TZ=UTC0 git show --quiet --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" HEAD)" NOW="$(date --utc --iso-8601=seconds)" -REPO_URL="https://github.com/influxdata/influxdb_iox" +REPO_URL="https://github.com/influxdata/influxdb" exec docker buildx build \ --build-arg CARGO_INCREMENTAL="no" \ @@ -22,8 +22,8 @@ exec docker buildx build \ --label org.opencontainers.image.url="$REPO_URL" \ --label org.opencontainers.image.revision="$COMMIT_SHA" \ --label org.opencontainers.image.vendor="InfluxData Inc." \ - --label org.opencontainers.image.title="InfluxDB IOx, '$PACKAGE'" \ - --label org.opencontainers.image.description="InfluxDB IOx production image for package '$PACKAGE'" \ + --label org.opencontainers.image.title="InfluxDB3 Edge" \ + --label org.opencontainers.image.description="InfluxDB3 Edge Image" \ --label com.influxdata.image.commit-date="$COMMIT_TS" \ --label com.influxdata.image.package="$PACKAGE" \ --progress plain \ diff --git a/.circleci/scripts/package-validation/debian b/.circleci/scripts/package-validation/debian new file mode 100755 index 00000000000..f3e16edba74 --- /dev/null +++ b/.circleci/scripts/package-validation/debian @@ -0,0 +1,8 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +"${path}/validate" deb "${1}" diff --git a/.circleci/scripts/package-validation/redhat b/.circleci/scripts/package-validation/redhat new file mode 100755 index 00000000000..db899a9bf66 --- /dev/null +++ b/.circleci/scripts/package-validation/redhat @@ -0,0 +1,97 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +# $1 -> architecture +# $2 -> package path +case ${1} in + x86_64) arch=x86_64 ;; + aarch64) arch=arm64 ;; +esac + +package="$(realpath "${2}")" + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +terraform_init() { + pushd "${path}/tf" &>/dev/null + + # Unfortunately, CircleCI doesn't offer any RPM based machine images. + # This is required to test the functionality of the systemd services. + # (systemd doesn't run within docker containers). This will spawn a + # Amazon Linux instance in AWS. + terraform init + terraform apply \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_free() { + pushd "${path}/tf" &>/dev/null + + terraform destroy \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_ip() { + pushd "${path}/tf" &>/dev/null + + terraform output -raw node_ssh + + popd &>/dev/null +} + + +# This ensures that the associated resources within AWS are released +# upon exit or when encountering an error. This is setup before the +# call to "terraform apply" so even partially initialized resources +# are released. +# shellcheck disable=SC2064 +trap "terraform_free \"${arch}\" \"${package}\"" \ + SIGINT \ + SIGTERM \ + ERR \ + EXIT + +function terraform_setup() +{ + # TODO(bnpfeife): remove this once the executor is updated. + # + # Unfortunately, terraform provided by the CircleCI executor is *terribly* + # out of date. Most Linux distributions are disabling "ssh-rsa" public key + # algorithms which this uses to remote into the ec2 instance . This + # installs the latest version of terraform. + # + # Addendum: the "terraform_version" CircleCI option is broken! +sudo tee /etc/apt/sources.list.d/hashicorp.list </dev/null || true +deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main +EOF + + curl -fL https://apt.releases.hashicorp.com/gpg | gpg --dearmor | \ + sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg >/dev/null + + export DEBIAN_FRONTEND=noninteractive + sudo -E apt-get update + sudo -E apt-get install --yes terraform +} + +terraform_setup + +terraform_init "${arch}" "${package}" + +printf 'Setup complete! Testing %s... (this takes several minutes!)' "${1}" + +# Since terraform *just* created this instance, the host key is not +# known. Therefore, we'll disable StrictHostKeyChecking so ssh does +# not wait for user input. +ssh -o 'StrictHostKeyChecking=no' "ec2-user@$(terraform_ip)" 'sudo ./validate rpm ./influxdb3.rpm' diff --git a/.circleci/scripts/package-validation/tf/main.tf b/.circleci/scripts/package-validation/tf/main.tf new file mode 100644 index 00000000000..fa17c14460b --- /dev/null +++ b/.circleci/scripts/package-validation/tf/main.tf @@ -0,0 +1,114 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 2.70" + } + } +} + +variable "architecture" { + type = string +} + +variable "identifier" { + type = string +} + +variable "package_path" { + type = string +} + +provider "aws" { + region = "us-east-1" +} + +data "aws_ami" "test_ami" { + most_recent = true + + filter { + name = "name" + values = ["al20*-ami-20*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "architecture" + values = [var.architecture] + } + + owners = ["137112412989"] +} + +resource "aws_security_group" "influxdb_test_package_sg" { + ingress { + description = "Allow ssh connection" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "Allow all outgoing" + from_port = 0 + to_port = 0 + protocol = "all" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "test_instance" { + count = 1 + ami = data.aws_ami.test_ami.id + instance_type = var.architecture == "x86_64" ? "t2.micro" : "c6g.medium" + key_name = "circleci-oss-test" + vpc_security_group_ids = [aws_security_group.influxdb_test_package_sg.id] + + tags = { + Name = format("circleci_%s_test_%s", var.identifier, var.architecture) + } + + provisioner "file" { + source = var.package_path + destination = "/home/ec2-user/influxdb3.rpm" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "file" { + source = "../validate" + destination = "/home/ec2-user/validate" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /home/ec2-user/validate", + ] + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } +} + +output "node_ssh" { + value = aws_instance.test_instance.0.public_dns +} diff --git a/.circleci/scripts/package-validation/validate b/.circleci/scripts/package-validation/validate new file mode 100755 index 00000000000..a0f7af8cb15 --- /dev/null +++ b/.circleci/scripts/package-validation/validate @@ -0,0 +1,90 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +usage() { + cat <<'EOF' +usage: validate [type] [path] + +Program: + This application performs sanity checks on the provided InfluxDB + package. InfluxDB should *not* be installed on the system before + running this application. This validates new installations and + performs specific checks relevant only to InfluxDB. + +Options: + type Must be "deb" or "rpm". This option instructs the + application to use the package manager associated + with "type". + path Path to InfluxDB package to validate. +EOF +} + +if [[ ! "${1:-}" ]] || [[ ! "${2:-}" ]] +then + (usage) && exit 1 +fi +PACKAGE_TYPE="${1}" +PACKAGE_PATH="${2}" + +install_deb() { + # When installing the package, ensure that the latest repository listings + # are available. This might be required so that all dependencies resolve. + # Since this needs to be run by CI, we supply "noninteractive" and "-y" + # so no prompts stall the pipeline. + export DEBIAN_FRONTEND=noninteractive + apt-get update + # "apt-get install" should be used instead of "dpkg -i", because "dpkg" + # does not resolve dependencies. "apt-get" requires that the package + # path looks like a path (either fullpath or prefixed with "./"). + apt-get install -y binutils "$(realpath "${PACKAGE_PATH}")" +} + +install_rpm() { + # see "install_deb" for "update" + yum update -y + yum install -y binutils + yum localinstall -y "$(realpath "${PACKAGE_PATH}")" +} + +case ${PACKAGE_TYPE} +in + deb) + (install_deb) + ;; + rpm) + (install_rpm) + ;; +esac + +if ! which influxdb3 &>/dev/null +then + printf 'ERROR: Failed to locate influxdb3 executable!\n' >&2 + exit 2 +fi + +NEEDED="$(readelf -d "$(which influxdb3)" | (grep 'NEEDED' || true ))" + +# shellcheck disable=SC2181 +if [[ ${?} -ne 0 ]] +then + cat <<'EOF' +ERROR: readelf could not analyze the influxdb3 executable! This + might be the consequence of installing a package built + for another platform OR invalid compiler/linker flags. +EOF + exit 2 +fi + +if [[ "${NEEDED:-}" ]] +then + cat <<'EOF' +ERROR: influxdb3 not statically linked! This may prevent all + platforms from running influxdb3 without installing + separate dependencies. +EOF + exit 2 +fi + +printf 'Finished validating influxdb3!\n' diff --git a/.circleci/scripts/publish.bash b/.circleci/scripts/publish.bash new file mode 100755 index 00000000000..cc580e2c124 --- /dev/null +++ b/.circleci/scripts/publish.bash @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +release() { + image_src="${1}:latest" + image_dst="us-docker.pkg.dev/influxdata-team-edge/influxdb3-edge/${1}:${2}" + + if docker pull "${image_dst}" ; then + echo "docker image ${image_dst} already exists" + exit 0 + fi + docker tag "${image_src}" "${image_dst}" + docker push "${image_dst}" +} + +release "${1}" "${CIRCLE_SHA1}" +if [[ "${CIRCLE_BRANCH}" == main ]] ; then + release "${1}" latest +fi diff --git a/Cargo.toml b/Cargo.toml index f42e5064d75..2a327446a7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,9 +152,7 @@ bare_urls = "deny" # This profile optimizes for runtime performance and small binary size at the expense of longer # build times. It's most suitable for final release builds. [profile.release] -codegen-units = 16 -debug = true -lto = "thin" +lto = "fat" [profile.bench] debug = true diff --git a/Dockerfile b/Dockerfile index 68382d4f61f..ccc02dfed78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,9 @@ RUN apt update \ && groupadd --gid 1500 influxdb3 \ && useradd --uid 1500 --gid influxdb3 --shell /bin/bash --create-home influxdb3 +RUN mkdir /usr/local/share/influxdb3 && \ + chown influxdb3:influxdb3 /usr/local/share/influxdb3 + USER influxdb3 RUN mkdir ~/.influxdb3 @@ -54,7 +57,11 @@ ENV PACKAGE=$PACKAGE COPY --from=build "/root/$PACKAGE" "/usr/bin/$PACKAGE" COPY docker/entrypoint.sh /usr/bin/entrypoint.sh -EXPOSE 8080 8082 +EXPOSE 8181 + +# TODO: Make this and other env vars not specific to IOx +ENV INFLUXDB_IOX_OBJECT_STORE=file +ENV INFLUXDB_IOX_DB_DIR=/usr/local/share/influxdb3 ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/influxdb3/src/commands/serve.rs b/influxdb3/src/commands/serve.rs index f2dd26566dc..7ea82e44ebb 100644 --- a/influxdb3/src/commands/serve.rs +++ b/influxdb3/src/commands/serve.rs @@ -40,7 +40,7 @@ use trogging::cli::LoggingConfig; pub const DEFAULT_DATA_DIRECTORY_NAME: &str = ".influxdb3"; /// The default bind address for the HTTP API. -pub const DEFAULT_HTTP_BIND_ADDR: &str = "127.0.0.1:8181"; +pub const DEFAULT_HTTP_BIND_ADDR: &str = "0.0.0.0:8181"; #[derive(Debug, Error)] pub enum Error { diff --git a/influxdb3/src/main.rs b/influxdb3/src/main.rs index 8e00d4457da..f68c68bb4f3 100644 --- a/influxdb3/src/main.rs +++ b/influxdb3/src/main.rs @@ -97,6 +97,7 @@ enum Command { } fn main() -> Result<(), std::io::Error> { + #[cfg(unix)] install_crash_handler(); // attempt to render a useful stacktrace to stderr // load all environment variables from .env before doing anything @@ -208,6 +209,7 @@ fn load_dotenv() { // Based on ideas from // https://github.com/servo/servo/blob/f03ddf6c6c6e94e799ab2a3a89660aea4a01da6f/ports/servo/main.rs#L58-L79 +#[cfg(unix)] fn install_crash_handler() { unsafe { set_signal_handler(libc::SIGSEGV, signal_handler); // handle segfaults @@ -216,6 +218,7 @@ fn install_crash_handler() { } } +#[cfg(unix)] unsafe extern "C" fn signal_handler(sig: i32) { use backtrace::Backtrace; use std::process::abort; @@ -233,6 +236,7 @@ unsafe extern "C" fn signal_handler(sig: i32) { } // based on https://github.com/adjivas/sig/blob/master/src/lib.rs#L34-L52 +#[cfg(unix)] unsafe fn set_signal_handler(signal: libc::c_int, handler: unsafe extern "C" fn(libc::c_int)) { use libc::{sigaction, sigfillset, sighandler_t}; let mut sigset = std::mem::zeroed(); diff --git a/influxdb3_process/Cargo.toml b/influxdb3_process/Cargo.toml index 8580335c7a7..85244488882 100644 --- a/influxdb3_process/Cargo.toml +++ b/influxdb3_process/Cargo.toml @@ -19,6 +19,7 @@ tokio.workspace = true uuid.workspace = true # Optional Dependencies +[target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { version = "0.5.4", optional = true } tikv-jemalloc-sys = { version = "0.5.4", optional = true, features = ["unprefixed_malloc_on_supported_platforms"] } diff --git a/influxdb3_process/src/lib.rs b/influxdb3_process/src/lib.rs index 7d8686b1ff2..854c8ef2ba8 100644 --- a/influxdb3_process/src/lib.rs +++ b/influxdb3_process/src/lib.rs @@ -7,13 +7,20 @@ use once_cell::sync::Lazy; /// The process name on the local OS running `influxdb3` pub const INFLUXDB3_PROCESS_NAME: &str = "influxdb3"; -#[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] +#[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") +))] pub mod jemalloc; #[cfg(tokio_unstable)] use tokio_metrics_bridge::setup_tokio_metrics; -#[cfg(all(not(feature = "heappy"), not(feature = "jemalloc_replacing_malloc")))] +#[cfg(any( + all(not(feature = "heappy"), not(feature = "jemalloc_replacing_malloc")), + target_env = "msvc" +))] pub fn build_malloc_conf() -> String { "system".to_string() } @@ -23,7 +30,11 @@ pub fn build_malloc_conf() -> String { "heappy".to_string() } -#[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] +#[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") +))] pub fn build_malloc_conf() -> String { tikv_jemalloc_ctl::config::malloc_conf::mib() .unwrap() @@ -100,7 +111,11 @@ pub fn setup_metric_registry() -> Arc { .set(PROCESS_START_TIME.timestamp() as u64); // Register jemalloc metrics - #[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] + #[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") + ))] registry.register_instrument("jemalloc_metrics", crate::jemalloc::JemallocMetrics::new); // Register tokio metric for main runtime diff --git a/influxdb3_server/src/lib.rs b/influxdb3_server/src/lib.rs index 2a1f5d268ae..f74ba1a7ba5 100644 --- a/influxdb3_server/src/lib.rs +++ b/influxdb3_server/src/lib.rs @@ -29,7 +29,7 @@ use influxdb3_write::{Persister, WriteBuffer}; use iox_query::QueryNamespaceProvider; use iox_query_params::StatementParams; use iox_time::TimeProvider; -use observability_deps::tracing::{error, info}; +use observability_deps::tracing::error; use service::hybrid; use std::convert::Infallible; use std::fmt::Debug; @@ -205,6 +205,7 @@ where /// This method returns if either are signalled #[cfg(unix)] pub async fn wait_for_signal() { + use observability_deps::tracing::info; use tokio::signal::unix::{signal, SignalKind}; let mut term = signal(SignalKind::terminate()).expect("failed to register signal handler"); let mut int = signal(SignalKind::interrupt()).expect("failed to register signal handler");