diff --git a/.circleci/config.yml b/.circleci/config.yml index b8597819b24..8327ea4578f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,41 @@ version: 2.1 +orbs: + aws-s3: circleci/aws-s3@2.0.0 + terraform: circleci/terraform@2.1.0 + rust: circleci/rust@1.6.1 + +# Unlike when a commit is pushed to a branch, CircleCI does not automatically +# execute a workflow when a tag is pushed to a repository. These filters +# allow the corresponding workflow to execute on any branch or tag. +any_filter: &any_filter + filters: + tags: + only: /.*/ + branches: + only: /.*/ + +release_filter: &release_filter + filters: + tags: + # This regex matches what is found in 'scripts/get-version' with the + # '[[:digit:]]' transformed into '\d'. This also excludes release + # candidate detection, because this filter only matches against + # full releases. + only: /^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/ + branches: + ignore: /.*/ + +nofork_filter: &nofork_filter + filters: + branches: + ignore: /pull\/[0-9]+/ + + commands: + install_rust: + rust_components: description: Verify installed components steps: @@ -46,15 +80,12 @@ commands: rustup show cargo fmt --version cargo clippy --version - - login_to_gcloud: + gcloud-docker-login: steps: - run: - name: Login to gcloud + name: configure-gar command: | - echo "${GCLOUD_SERVICE_ACCOUNT_KEY}" >/tmp/gcloud.json - gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file /tmp/gcloud.json --quiet - rm -f /tmp/gcloud.json + gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file <(echo "${GCLOUD_SERVICE_ACCOUNT_KEY}") gcloud auth configure-docker us-docker.pkg.dev jobs: @@ -95,7 +126,7 @@ jobs: - run: name: Yamllint command: yamllint --config-file .circleci/yamllint.yml --strict . - cargo_audit: + cargo-audit: docker: - image: quay.io/influxdb/rust:ci environment: @@ -163,7 +194,7 @@ jobs: command: cargo test --workspace # end to end tests with Heappy (heap profiling enabled) - test_heappy: + test-heappy: docker: - image: quay.io/influxdb/rust:ci resource_class: xlarge # use of a smaller executor tends crashes on link @@ -187,11 +218,16 @@ jobs: # # Compiles a binary with the default ("dev") cargo profile from the influxdb3 source # using the latest ci_image (influxdb/rust) and ensures various targets compile successfully - build_dev: + # Build a dev binary. + build-dev: docker: - - image: quay.io/influxdb/rust:ci - resource_class: 2xlarge # use of a smaller executor tends crashes on link + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + resource_class: 2xlarge+ # use of a smaller executor tends crashes on link environment: + TARGET: << parameters.target >> # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build @@ -202,83 +238,344 @@ jobs: # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit. CARGO_BUILD_JOBS: "12" + parameters: + target: + type: string steps: - checkout - - rust_components + - run: + name: Install Target + command: rustup target add << parameters.target >> - run: name: Cargo build - command: cargo build --workspace + command: target-env cargo build --target=<< parameters.target >> --workspace + - when: + condition: + not: + equal: [ << parameters.target >>, x86_64-pc-windows-gnu ] + steps: + - run: + name: Check benches compile + command: target-env cargo check --target=<< parameters.target >> --workspace --benches + - run: + name: Check extra features (like prod image) + command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" + - when: + condition: + equal: [ << parameters.target >>, x86_64-pc-windows-gnu ] + steps: + - run: + name: Check extra features (like prod image) + command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console" + + # Compile cargo "release" profile binaries for influxdb3 edge releases + build-release: + docker: + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + resource_class: 2xlarge+ + environment: + TARGET: << parameters.target >> + # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. + CARGO_INCREMENTAL: "0" + # Disable full debug symbol generation to speed up CI build + # "1" means line tables only, which is useful for panic tracebacks. + CARGO_PROFILE_DEV_DEBUG: "1" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on + # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit. + CARGO_BUILD_JOBS: "12" + parameters: + target: + type: string + steps: + - checkout + - run: + name: Install Target + command: rustup target add << parameters.target >> + - run: + name: Cargo release build + command: target-env cargo build --target=<< parameters.target >> --workspace --release + # linking might take a while and doesn't produce CLI output + no_output_timeout: 30m + - run: + name: tar and gzip build artifacts + command: | + mkdir -p artifacts + tar --ignore-failed-read -czvf "${PWD}/artifacts/influxdb3-edge_<< parameters.target >>.tar.gz" -C "${PWD}/target/<< parameters.target >>/release" influxdb3{,.exe} + - store_artifacts: + path: artifacts + - persist_to_workspace: + root: . + paths: + - artifacts + build-packages: + docker: + - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager:latest + auth: + username: _json_key + password: $CISUPPORT_GCS_AUTHORIZATION + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - run: packager .circleci/packages/config.yaml + - persist_to_workspace: + root: . + paths: + - artifacts + - store_artifacts: + path: artifacts/ + check_package_deb_amd64: + machine: + image: ubuntu-2204:current + resource_class: medium + steps: + - attach_workspace: + at: /tmp/workspace + - checkout - run: - name: Check benches compile - command: cargo check --workspace --benches + name: Validate Debian Package (AMD64) + command: | + sudo .circleci/scripts/package-validation/debian \ + /tmp/workspace/artifacts/influxdb3*amd64.deb + check_package_deb_arm64: + machine: + image: ubuntu-2204:current + resource_class: arm.medium + steps: + - attach_workspace: + at: /tmp/workspace + - checkout - run: - name: Check extra features (like prod image) - command: cargo check --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" + name: Validate Debian Package (ARM64) + command: | + sudo .circleci/scripts/package-validation/debian \ + /tmp/workspace/artifacts/influxdb3*arm64.deb + check_package_rpm: + machine: + image: ubuntu-2204:current + resource_class: arm.medium + parameters: + arch: + type: string + steps: + - attach_workspace: + at: /tmp/workspace + - add_ssh_keys: + fingerprints: + - 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67 + - checkout + - run: | + AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY \ + .circleci/scripts/package-validation/redhat << parameters.arch >> /tmp/workspace/artifacts/influxdb3*.<< parameters.arch >>.rpm + sign-packages: + circleci_ip_ranges: true + docker: + - image: quay.io/influxdb/rsign:latest + auth: + username: $QUAY_RSIGN_USERNAME + password: $QUAY_RSIGN_PASSWORD + steps: + - add_ssh_keys: + fingerprints: + - fc:7b:6e:a6:38:7c:63:5a:13:be:cb:bb:fa:33:b3:3c + - attach_workspace: + at: /tmp/workspace + - run: | + for target in /tmp/workspace/artifacts/* + do + case "${target}" + in + # rsign is shipped on Alpine Linux which uses "busybox ash" instead + # of bash. ash is somewhat more posix compliant and is missing some + # extensions and niceties from bash. + *.deb|*.rpm|*.tar.gz|*.zip) + rsign "${target}" + ;; + esac - # Compile cargo "release" profile binaries for influxdb3 & data generator - # - # Uses the latest ci_image (influxdb/rust) to build a release binary and - # copies it to a minimal container image based upon `rust:slim-buster`. This - # minimal image is then pushed to `quay.io/influxdb/influxdb3:${BRANCH}`. - build_release: + if [ -f "${target}" ] + then + # Since all artifacts are present, sign them here. This saves Circle + # credits over spinning up another instance just to separate out the + # checksum job. + sha256sum "${target}" >> "/tmp/workspace/artifacts/influxdb3-edge.${CIRCLE_TAG}.digests" + + # write individual checksums + md5sum "${target}" >> "${target}.md5" + sha256sum "${target}" >> "${target}.sha256" + fi + done + - persist_to_workspace: + root: /tmp/workspace + paths: + - artifacts + - store_artifacts: + path: /tmp/workspace/artifacts + publish-packages: + docker: + - image: cimg/python:3.6 + steps: + - attach_workspace: + at: /tmp/workspace + - aws-s3/sync: + arguments: --acl public-read + aws-region: RELEASE_AWS_REGION + aws-access-key-id: RELEASE_AWS_ACCESS_KEY_ID + aws-secret-access-key: RELEASE_AWS_SECRET_ACCESS_KEY + from: /tmp/workspace/artifacts + to: s3://dl.influxdata.com/influxdb/releases + build-docker: # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a # kinda small node) machine: image: default - resource_class: 2xlarge # CPU bound, so make it fast + resource_class: 2xlarge+ # CPU bound, so make it fast steps: - checkout - run: - name: Cargo release build + name: Build the docker image command: | - COMMIT_SHA="$(git rev-parse HEAD)" - - .circleci/docker_build_release.bash \ + .circleci/scripts/docker_build_release.bash \ "influxdb3" \ "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \ - "quay.io/influxdb/influxdb3:$COMMIT_SHA" + "influxdb3-edge:latest" - mkdir /tmp/images - docker save quay.io/influxdb/influxdb3:"$COMMIT_SHA" | gzip > /tmp/images/influxdb3.tar.gz # linking might take a while and doesn't produce CLI output no_output_timeout: 30m - - store_artifacts: - path: /tmp/images + - run: | + docker save influxdb3-edge:latest >influxdb3-edge.tar - persist_to_workspace: - root: /tmp/images + root: . paths: - - "*.tar.gz" + - influxdb3-edge.tar + publish-docker: + docker: + - image: cimg/gcp:2023.02 + resource_class: medium + steps: + - checkout + - setup_remote_docker + - gcloud-docker-login + - attach_workspace: + at: . + - run: | + docker load > jobs: - - fmt - - lint - - cargo_audit - - test - - test_heappy - - build_dev - - doc - - build_release: - filters: - branches: - only: main + - fmt: + <<: *any_filter + - lint: + <<: *any_filter + - cargo-audit: + <<: *any_filter + - test: + <<: *any_filter + - test-heappy: + <<: *any_filter + - build-dev: + <<: *any_filter + name: build-dev-<< matrix.target >> + matrix: + parameters: + target: + - aarch64-apple-darwin + - aarch64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - x86_64-pc-windows-gnu + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - doc: + <<: *any_filter + - build-release: + # <<: *release_filter + <<: *any_filter + name: build-release-<< matrix.target >> + matrix: + parameters: + target: + - aarch64-apple-darwin + - aarch64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - x86_64-pc-windows-gnu + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - build-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-release + - check_package_deb_arm64: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_deb_amd64: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_rpm: + # <<: *nofork_filter + <<: *any_filter + name: + check_package_rpm-<< matrix.arch >> + matrix: + parameters: + arch: [ x86_64, aarch64 ] + requires: + - build-packages - # Force build and push of container image for non-main branch. - # See instructions at the top of this file - release_branch: - when: << pipeline.parameters.release_branch >> - jobs: - - build_release + - sign-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-packages + - check_package_rpm + - check_package_deb_arm64 + - check_package_deb_amd64 + - test + - publish-packages: + # <<: *release_filter + <<: *any_filter + requires: + - build-release + - sign-packages + - test + - test-heappy + - doc + - lint + - fmt + - cargo-audit + - build-docker: + <<: *any_filter + # <<: *release_filter + - publish-docker: + <<: *any_filter + # <<: *release_filter + requires: + - build-docker + - wait-for-docker: + <<: *any_filter + # <<: *release_filter + requires: + - build-docker + - publish-docker diff --git a/.circleci/packages/config.yaml b/.circleci/packages/config.yaml new file mode 100644 index 00000000000..1564c5261cd --- /dev/null +++ b/.circleci/packages/config.yaml @@ -0,0 +1,51 @@ + +version: + release: + match: '^v[0-9]+.[0-9]+.[0-9]+' + value: '{{env.CIRCLE_TAG[1:]}}' + default: + value: '3.x-{{env.CIRCLE_SHA1[:8]}}' + +sources: + - binary: /tmp/workspace/artifacts/influxdb3-edge_x86_64-unknown-linux-musl.tar.gz + target: artifacts/ + arch: amd64 + plat: linux + + - binary: /tmp/workspace/artifacts/influxdb3-edge_aarch64-unknown-linux-musl.tar.gz + target: artifacts/ + arch: arm64 + plat: linux + + - binary: /tmp/workspace/artifacts/influxdb3-edge_aarch64-apple-darwin.tar.gz + target: artifacts/ + arch: amd64 + plat: darwin + + - binary: /tmp/workspace/artifacts/influxdb3-edge_x86_64-pc-windows-gnu.tar.gz + target: artifacts/ + arch: amd64 + plat: windows + +packages: + - name: influxdb3 + description: Monolithic time-series database. + license: MIT/Apache-2.0 + binaries: + - influxdb3 + - influxdb3.exe + extras: + - source: LICENSE-APACHE + target: usr/share/influxdb3/LICENSE-APACHE + + - source: LICENSE-MIT + target: usr/share/influxdb3/LICENSE-MIT + + - source: README.md + target: usr/share/influxdb3/README.md + #perm_overrides: + deb_recommends: + - influxdb3-cli + #conflicts: + #depends: + source: .circleci/packages/influxdb3 diff --git a/.circleci/packages/influxdb3/control/post-install b/.circleci/packages/influxdb3/control/post-install new file mode 100755 index 00000000000..ac08539b04e --- /dev/null +++ b/.circleci/packages/influxdb3/control/post-install @@ -0,0 +1,16 @@ +#!/bin/bash + +BIN_DIR=/usr/bin +DATA_DIR=/var/lib/influxdb3 +LOG_DIR=/var/log/influxdb3 + +# Distribution-specific logic +if [[ -f /etc/debian_version ]]; then + # Ownership for RH-based platforms is set in build.py via the `rmp-attr` option. + # We perform ownership change only for Debian-based systems. + # Moving these lines out of this if statement would make `rmp -V` fail after installation. + chown -R -L influxdb:influxdb $LOG_DIR + chown -R -L influxdb:influxdb $DATA_DIR + chmod 755 $LOG_DIR + chmod 755 $DATA_DIR +fi diff --git a/.circleci/packages/influxdb3/control/pre-install b/.circleci/packages/influxdb3/control/pre-install new file mode 100755 index 00000000000..c2e3e792439 --- /dev/null +++ b/.circleci/packages/influxdb3/control/pre-install @@ -0,0 +1,22 @@ +#!/bin/bash + +DATA_DIR=/var/lib/influxdb3 +USER=influxdb3 +GROUP=influxdb3 +LOG_DIR=/var/log/influxdb3 + +if ! id influxdb3 &>/dev/null; then + useradd --system -U -M influxdb3 -s /bin/false -d $DATA_DIR +fi + +# check if DATA_DIR exists +if [ ! -d "$DATA_DIR" ]; then + mkdir -p $DATA_DIR + chown $USER:$GROUP $DATA_DIR +fi + +# check if LOG_DIR exists +if [ ! -d "$LOG_DIR" ]; then + mkdir -p $LOG_DIR + chown $USER:$GROUP $DATA_DIR +fi diff --git a/.circleci/packages/influxdb3/fs/usr/bin/.keepdir b/.circleci/packages/influxdb3/fs/usr/bin/.keepdir new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.circleci/packages/influxdb3/fs/usr/share/influxdb3/.keepdir b/.circleci/packages/influxdb3/fs/usr/share/influxdb3/.keepdir new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.circleci/docker_build_release.bash b/.circleci/scripts/docker_build_release.bash similarity index 81% rename from .circleci/docker_build_release.bash rename to .circleci/scripts/docker_build_release.bash index 70c5e32fe84..52b373c12a4 100755 --- a/.circleci/docker_build_release.bash +++ b/.circleci/scripts/docker_build_release.bash @@ -10,7 +10,7 @@ RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)" COMMIT_SHA="$(git rev-parse HEAD)" COMMIT_TS="$(env TZ=UTC0 git show --quiet --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" HEAD)" NOW="$(date --utc --iso-8601=seconds)" -REPO_URL="https://github.com/influxdata/influxdb_iox" +REPO_URL="https://github.com/influxdata/influxdb" exec docker buildx build \ --build-arg CARGO_INCREMENTAL="no" \ @@ -22,8 +22,8 @@ exec docker buildx build \ --label org.opencontainers.image.url="$REPO_URL" \ --label org.opencontainers.image.revision="$COMMIT_SHA" \ --label org.opencontainers.image.vendor="InfluxData Inc." \ - --label org.opencontainers.image.title="InfluxDB IOx, '$PACKAGE'" \ - --label org.opencontainers.image.description="InfluxDB IOx production image for package '$PACKAGE'" \ + --label org.opencontainers.image.title="InfluxDB3 Edge" \ + --label org.opencontainers.image.description="InfluxDB3 Edge Image" \ --label com.influxdata.image.commit-date="$COMMIT_TS" \ --label com.influxdata.image.package="$PACKAGE" \ --progress plain \ diff --git a/.circleci/scripts/package-validation/debian b/.circleci/scripts/package-validation/debian new file mode 100755 index 00000000000..f3e16edba74 --- /dev/null +++ b/.circleci/scripts/package-validation/debian @@ -0,0 +1,8 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +"${path}/validate" deb "${1}" diff --git a/.circleci/scripts/package-validation/redhat b/.circleci/scripts/package-validation/redhat new file mode 100755 index 00000000000..db899a9bf66 --- /dev/null +++ b/.circleci/scripts/package-validation/redhat @@ -0,0 +1,97 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +# $1 -> architecture +# $2 -> package path +case ${1} in + x86_64) arch=x86_64 ;; + aarch64) arch=arm64 ;; +esac + +package="$(realpath "${2}")" + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +terraform_init() { + pushd "${path}/tf" &>/dev/null + + # Unfortunately, CircleCI doesn't offer any RPM based machine images. + # This is required to test the functionality of the systemd services. + # (systemd doesn't run within docker containers). This will spawn a + # Amazon Linux instance in AWS. + terraform init + terraform apply \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_free() { + pushd "${path}/tf" &>/dev/null + + terraform destroy \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_ip() { + pushd "${path}/tf" &>/dev/null + + terraform output -raw node_ssh + + popd &>/dev/null +} + + +# This ensures that the associated resources within AWS are released +# upon exit or when encountering an error. This is setup before the +# call to "terraform apply" so even partially initialized resources +# are released. +# shellcheck disable=SC2064 +trap "terraform_free \"${arch}\" \"${package}\"" \ + SIGINT \ + SIGTERM \ + ERR \ + EXIT + +function terraform_setup() +{ + # TODO(bnpfeife): remove this once the executor is updated. + # + # Unfortunately, terraform provided by the CircleCI executor is *terribly* + # out of date. Most Linux distributions are disabling "ssh-rsa" public key + # algorithms which this uses to remote into the ec2 instance . This + # installs the latest version of terraform. + # + # Addendum: the "terraform_version" CircleCI option is broken! +sudo tee /etc/apt/sources.list.d/hashicorp.list </dev/null || true +deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main +EOF + + curl -fL https://apt.releases.hashicorp.com/gpg | gpg --dearmor | \ + sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg >/dev/null + + export DEBIAN_FRONTEND=noninteractive + sudo -E apt-get update + sudo -E apt-get install --yes terraform +} + +terraform_setup + +terraform_init "${arch}" "${package}" + +printf 'Setup complete! Testing %s... (this takes several minutes!)' "${1}" + +# Since terraform *just* created this instance, the host key is not +# known. Therefore, we'll disable StrictHostKeyChecking so ssh does +# not wait for user input. +ssh -o 'StrictHostKeyChecking=no' "ec2-user@$(terraform_ip)" 'sudo ./validate rpm ./influxdb3.rpm' diff --git a/.circleci/scripts/package-validation/tf/main.tf b/.circleci/scripts/package-validation/tf/main.tf new file mode 100644 index 00000000000..fa17c14460b --- /dev/null +++ b/.circleci/scripts/package-validation/tf/main.tf @@ -0,0 +1,114 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 2.70" + } + } +} + +variable "architecture" { + type = string +} + +variable "identifier" { + type = string +} + +variable "package_path" { + type = string +} + +provider "aws" { + region = "us-east-1" +} + +data "aws_ami" "test_ami" { + most_recent = true + + filter { + name = "name" + values = ["al20*-ami-20*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "architecture" + values = [var.architecture] + } + + owners = ["137112412989"] +} + +resource "aws_security_group" "influxdb_test_package_sg" { + ingress { + description = "Allow ssh connection" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "Allow all outgoing" + from_port = 0 + to_port = 0 + protocol = "all" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "test_instance" { + count = 1 + ami = data.aws_ami.test_ami.id + instance_type = var.architecture == "x86_64" ? "t2.micro" : "c6g.medium" + key_name = "circleci-oss-test" + vpc_security_group_ids = [aws_security_group.influxdb_test_package_sg.id] + + tags = { + Name = format("circleci_%s_test_%s", var.identifier, var.architecture) + } + + provisioner "file" { + source = var.package_path + destination = "/home/ec2-user/influxdb3.rpm" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "file" { + source = "../validate" + destination = "/home/ec2-user/validate" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /home/ec2-user/validate", + ] + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } +} + +output "node_ssh" { + value = aws_instance.test_instance.0.public_dns +} diff --git a/.circleci/scripts/package-validation/validate b/.circleci/scripts/package-validation/validate new file mode 100755 index 00000000000..a0f7af8cb15 --- /dev/null +++ b/.circleci/scripts/package-validation/validate @@ -0,0 +1,90 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +usage() { + cat <<'EOF' +usage: validate [type] [path] + +Program: + This application performs sanity checks on the provided InfluxDB + package. InfluxDB should *not* be installed on the system before + running this application. This validates new installations and + performs specific checks relevant only to InfluxDB. + +Options: + type Must be "deb" or "rpm". This option instructs the + application to use the package manager associated + with "type". + path Path to InfluxDB package to validate. +EOF +} + +if [[ ! "${1:-}" ]] || [[ ! "${2:-}" ]] +then + (usage) && exit 1 +fi +PACKAGE_TYPE="${1}" +PACKAGE_PATH="${2}" + +install_deb() { + # When installing the package, ensure that the latest repository listings + # are available. This might be required so that all dependencies resolve. + # Since this needs to be run by CI, we supply "noninteractive" and "-y" + # so no prompts stall the pipeline. + export DEBIAN_FRONTEND=noninteractive + apt-get update + # "apt-get install" should be used instead of "dpkg -i", because "dpkg" + # does not resolve dependencies. "apt-get" requires that the package + # path looks like a path (either fullpath or prefixed with "./"). + apt-get install -y binutils "$(realpath "${PACKAGE_PATH}")" +} + +install_rpm() { + # see "install_deb" for "update" + yum update -y + yum install -y binutils + yum localinstall -y "$(realpath "${PACKAGE_PATH}")" +} + +case ${PACKAGE_TYPE} +in + deb) + (install_deb) + ;; + rpm) + (install_rpm) + ;; +esac + +if ! which influxdb3 &>/dev/null +then + printf 'ERROR: Failed to locate influxdb3 executable!\n' >&2 + exit 2 +fi + +NEEDED="$(readelf -d "$(which influxdb3)" | (grep 'NEEDED' || true ))" + +# shellcheck disable=SC2181 +if [[ ${?} -ne 0 ]] +then + cat <<'EOF' +ERROR: readelf could not analyze the influxdb3 executable! This + might be the consequence of installing a package built + for another platform OR invalid compiler/linker flags. +EOF + exit 2 +fi + +if [[ "${NEEDED:-}" ]] +then + cat <<'EOF' +ERROR: influxdb3 not statically linked! This may prevent all + platforms from running influxdb3 without installing + separate dependencies. +EOF + exit 2 +fi + +printf 'Finished validating influxdb3!\n' diff --git a/.circleci/scripts/publish.bash b/.circleci/scripts/publish.bash new file mode 100755 index 00000000000..cc580e2c124 --- /dev/null +++ b/.circleci/scripts/publish.bash @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +release() { + image_src="${1}:latest" + image_dst="us-docker.pkg.dev/influxdata-team-edge/influxdb3-edge/${1}:${2}" + + if docker pull "${image_dst}" ; then + echo "docker image ${image_dst} already exists" + exit 0 + fi + docker tag "${image_src}" "${image_dst}" + docker push "${image_dst}" +} + +release "${1}" "${CIRCLE_SHA1}" +if [[ "${CIRCLE_BRANCH}" == main ]] ; then + release "${1}" latest +fi diff --git a/Cargo.toml b/Cargo.toml index f42e5064d75..2a327446a7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,9 +152,7 @@ bare_urls = "deny" # This profile optimizes for runtime performance and small binary size at the expense of longer # build times. It's most suitable for final release builds. [profile.release] -codegen-units = 16 -debug = true -lto = "thin" +lto = "fat" [profile.bench] debug = true diff --git a/Dockerfile b/Dockerfile index 68382d4f61f..ccc02dfed78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,9 @@ RUN apt update \ && groupadd --gid 1500 influxdb3 \ && useradd --uid 1500 --gid influxdb3 --shell /bin/bash --create-home influxdb3 +RUN mkdir /usr/local/share/influxdb3 && \ + chown influxdb3:influxdb3 /usr/local/share/influxdb3 + USER influxdb3 RUN mkdir ~/.influxdb3 @@ -54,7 +57,11 @@ ENV PACKAGE=$PACKAGE COPY --from=build "/root/$PACKAGE" "/usr/bin/$PACKAGE" COPY docker/entrypoint.sh /usr/bin/entrypoint.sh -EXPOSE 8080 8082 +EXPOSE 8181 + +# TODO: Make this and other env vars not specific to IOx +ENV INFLUXDB_IOX_OBJECT_STORE=file +ENV INFLUXDB_IOX_DB_DIR=/usr/local/share/influxdb3 ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/influxdb3/src/commands/serve.rs b/influxdb3/src/commands/serve.rs index f2dd26566dc..7ea82e44ebb 100644 --- a/influxdb3/src/commands/serve.rs +++ b/influxdb3/src/commands/serve.rs @@ -40,7 +40,7 @@ use trogging::cli::LoggingConfig; pub const DEFAULT_DATA_DIRECTORY_NAME: &str = ".influxdb3"; /// The default bind address for the HTTP API. -pub const DEFAULT_HTTP_BIND_ADDR: &str = "127.0.0.1:8181"; +pub const DEFAULT_HTTP_BIND_ADDR: &str = "0.0.0.0:8181"; #[derive(Debug, Error)] pub enum Error { diff --git a/influxdb3/src/main.rs b/influxdb3/src/main.rs index 8e00d4457da..f68c68bb4f3 100644 --- a/influxdb3/src/main.rs +++ b/influxdb3/src/main.rs @@ -97,6 +97,7 @@ enum Command { } fn main() -> Result<(), std::io::Error> { + #[cfg(unix)] install_crash_handler(); // attempt to render a useful stacktrace to stderr // load all environment variables from .env before doing anything @@ -208,6 +209,7 @@ fn load_dotenv() { // Based on ideas from // https://github.com/servo/servo/blob/f03ddf6c6c6e94e799ab2a3a89660aea4a01da6f/ports/servo/main.rs#L58-L79 +#[cfg(unix)] fn install_crash_handler() { unsafe { set_signal_handler(libc::SIGSEGV, signal_handler); // handle segfaults @@ -216,6 +218,7 @@ fn install_crash_handler() { } } +#[cfg(unix)] unsafe extern "C" fn signal_handler(sig: i32) { use backtrace::Backtrace; use std::process::abort; @@ -233,6 +236,7 @@ unsafe extern "C" fn signal_handler(sig: i32) { } // based on https://github.com/adjivas/sig/blob/master/src/lib.rs#L34-L52 +#[cfg(unix)] unsafe fn set_signal_handler(signal: libc::c_int, handler: unsafe extern "C" fn(libc::c_int)) { use libc::{sigaction, sigfillset, sighandler_t}; let mut sigset = std::mem::zeroed(); diff --git a/influxdb3_process/Cargo.toml b/influxdb3_process/Cargo.toml index 8580335c7a7..85244488882 100644 --- a/influxdb3_process/Cargo.toml +++ b/influxdb3_process/Cargo.toml @@ -19,6 +19,7 @@ tokio.workspace = true uuid.workspace = true # Optional Dependencies +[target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { version = "0.5.4", optional = true } tikv-jemalloc-sys = { version = "0.5.4", optional = true, features = ["unprefixed_malloc_on_supported_platforms"] } diff --git a/influxdb3_process/src/lib.rs b/influxdb3_process/src/lib.rs index 7d8686b1ff2..854c8ef2ba8 100644 --- a/influxdb3_process/src/lib.rs +++ b/influxdb3_process/src/lib.rs @@ -7,13 +7,20 @@ use once_cell::sync::Lazy; /// The process name on the local OS running `influxdb3` pub const INFLUXDB3_PROCESS_NAME: &str = "influxdb3"; -#[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] +#[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") +))] pub mod jemalloc; #[cfg(tokio_unstable)] use tokio_metrics_bridge::setup_tokio_metrics; -#[cfg(all(not(feature = "heappy"), not(feature = "jemalloc_replacing_malloc")))] +#[cfg(any( + all(not(feature = "heappy"), not(feature = "jemalloc_replacing_malloc")), + target_env = "msvc" +))] pub fn build_malloc_conf() -> String { "system".to_string() } @@ -23,7 +30,11 @@ pub fn build_malloc_conf() -> String { "heappy".to_string() } -#[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] +#[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") +))] pub fn build_malloc_conf() -> String { tikv_jemalloc_ctl::config::malloc_conf::mib() .unwrap() @@ -100,7 +111,11 @@ pub fn setup_metric_registry() -> Arc { .set(PROCESS_START_TIME.timestamp() as u64); // Register jemalloc metrics - #[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))] + #[cfg(all( + not(feature = "heappy"), + feature = "jemalloc_replacing_malloc", + not(target_env = "msvc") + ))] registry.register_instrument("jemalloc_metrics", crate::jemalloc::JemallocMetrics::new); // Register tokio metric for main runtime diff --git a/influxdb3_server/src/lib.rs b/influxdb3_server/src/lib.rs index 2a1f5d268ae..f74ba1a7ba5 100644 --- a/influxdb3_server/src/lib.rs +++ b/influxdb3_server/src/lib.rs @@ -29,7 +29,7 @@ use influxdb3_write::{Persister, WriteBuffer}; use iox_query::QueryNamespaceProvider; use iox_query_params::StatementParams; use iox_time::TimeProvider; -use observability_deps::tracing::{error, info}; +use observability_deps::tracing::error; use service::hybrid; use std::convert::Infallible; use std::fmt::Debug; @@ -205,6 +205,7 @@ where /// This method returns if either are signalled #[cfg(unix)] pub async fn wait_for_signal() { + use observability_deps::tracing::info; use tokio::signal::unix::{signal, SignalKind}; let mut term = signal(SignalKind::terminate()).expect("failed to register signal handler"); let mut int = signal(SignalKind::interrupt()).expect("failed to register signal handler");