From 1a47b2c0d10dfa9d838b04dbcd6a33ee0098380e Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 00:49:59 +0100 Subject: [PATCH 1/6] Added trace test to debug CI failure --- integration/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/main.rs b/integration/main.rs index 74a74dcb..22518a4f 100644 --- a/integration/main.rs +++ b/integration/main.rs @@ -49,6 +49,7 @@ fn test_server_startup(_: PoolOptions, db_options: PgConnectOptions) { } #[sqlx::test] +#[traced_test] fn test_end_to_end_terraform_flow(_: PoolOptions, db_options: PgConnectOptions) { let config = Args { database_url: db_options, From 312af30cbcda024b8e2227f2287dd2a0f7363452 Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 01:01:33 +0100 Subject: [PATCH 2/6] Added mock access and secret key Also made deployment only on merge to main --- .github/workflows/docs.yml | 2 ++ .github/workflows/rust.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3448eb1f..a75c8265 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,6 +42,7 @@ jobs: run: mdbook build docs - name: Upload artifact + if: github.ref == 'refs/heads/main' uses: actions/upload-pages-artifact@v1 with: path: ./docs/book @@ -53,6 +54,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest needs: build + if: github.ref == 'refs/heads/main' steps: - name: Deploy to GitHub Pages id: deployment diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9625eaab..680920fc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,6 +23,10 @@ jobs: run: docker-compose up -d - name: Run tests run: cargo test --verbose --test integration + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_REGION: us-east-1 - name: Stop containers if: always() run: docker-compose down \ No newline at end of file From 53e0d8be6590785226932ac9edd553c3b5eb46d3 Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 01:18:40 +0100 Subject: [PATCH 3/6] Cleanups and added docker-compose ps for easier CI debugging --- .github/workflows/rust.yml | 2 ++ integration/main.rs | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 680920fc..3891b49a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,6 +21,8 @@ jobs: run: cargo build --verbose - name: Start containers run: docker-compose up -d + - name: Show containers + run: docker-compose ps - name: Run tests run: cargo test --verbose --test integration env: diff --git a/integration/main.rs b/integration/main.rs index 22518a4f..f7d3418f 100644 --- a/integration/main.rs +++ b/integration/main.rs @@ -2,10 +2,9 @@ use std::{ net::{IpAddr, Ipv6Addr, SocketAddr}, process::Stdio, str::from_utf8, - time::{Duration, Instant}, + time::Duration, }; -use httpmock::{Method::GET, MockServer}; use reqwest::StatusCode; use sqlx::{pool::PoolOptions, postgres::PgConnectOptions, Postgres}; use terrashine::{self, config::Args}; @@ -68,7 +67,7 @@ fn test_end_to_end_terraform_flow(_: PoolOptions, db_options: PgConnec cancellation_token.child_token(), tx, )); - let socket = rx.await.unwrap().bind_socket; + let _ = rx.await.unwrap().bind_socket; let mut terraform = tokio::process::Command::new("terraform"); let process = terraform .arg("-chdir=resources/test/terraform/random-import-stack/") From 81ff0221118606466e6600c5271cd2a5c7c82daf Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 02:10:50 +0100 Subject: [PATCH 4/6] Added certificate generation to integration test --- .github/workflows/rust.yml | 2 ++ Dockerfile.certificate | 6 ++++++ docker-compose.yml | 9 +++++++++ scripts/generate-test-certificate.py | 3 ++- 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.certificate diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3891b49a..cbe49a03 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,6 +21,8 @@ jobs: run: cargo build --verbose - name: Start containers run: docker-compose up -d + - name: Install self signed certificate + run: sudo cp ./resources/test/certs/localhost.pem /usr/local/share/ca-certificates/localhost.crt && sudo update-ca-certificates - name: Show containers run: docker-compose ps - name: Run tests diff --git a/Dockerfile.certificate b/Dockerfile.certificate new file mode 100644 index 00000000..266cc406 --- /dev/null +++ b/Dockerfile.certificate @@ -0,0 +1,6 @@ +# Dockerfile to install dependencies for generating test certificates +FROM docker.io/library/python:3.11 +WORKDIR /usr/src/app +RUN pip install --no-cache-dir cryptography +COPY scripts/generate-test-certificate.py scripts/generate-test-certificate.py +CMD ["python3", "scripts/generate-test-certificate.py"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index fb65311f..20437d59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,15 @@ services: AWS_SECRET_ACCESS_KEY: minioadmin AWS_DEFAULT_REGION: us-east-1 + generate-certificates: + build: + context: . + dockerfile: Dockerfile.certificate + volumes: + - source: ./resources/ + target: /usr/src/app/resources/ + type: bind + nginx: image: docker.io/library/nginx network_mode: host diff --git a/scripts/generate-test-certificate.py b/scripts/generate-test-certificate.py index 4757fa0d..493f47f9 100644 --- a/scripts/generate-test-certificate.py +++ b/scripts/generate-test-certificate.py @@ -32,7 +32,6 @@ ).not_valid_before( datetime.utcnow() ).not_valid_after( - # Our certificate will be valid for 10 days datetime.utcnow() + timedelta(days=10) ).add_extension( x509.SubjectAlternativeName([x509.DNSName(u"localhost")]), @@ -49,3 +48,5 @@ format=serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm=serialization.NoEncryption(), )) +with open("resources/test/certs/localhost.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) From 5ca9cb59c97a86b94992cf77d72c828354649bd4 Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 02:18:17 +0100 Subject: [PATCH 5/6] throw in .gitkeep to ensure certs folder exists --- resources/test/certs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/test/certs/.gitkeep diff --git a/resources/test/certs/.gitkeep b/resources/test/certs/.gitkeep new file mode 100644 index 00000000..e69de29b From 21a79330e37b0b0c9acdc45fba498adc77e7a1da Mon Sep 17 00:00:00 2001 From: Isawan Millican Date: Thu, 27 Jul 2023 02:40:15 +0100 Subject: [PATCH 6/6] Changed self-sign cert to use openssl cli Easier generation of certs in CI --- .github/workflows/rust.yml | 2 +- Dockerfile.certificate | 6 ---- docker-compose.yml | 13 ++++--- scripts/generate-test-certificate | 8 +++++ scripts/generate-test-certificate.py | 52 ---------------------------- 5 files changed, 18 insertions(+), 63 deletions(-) delete mode 100644 Dockerfile.certificate create mode 100755 scripts/generate-test-certificate delete mode 100644 scripts/generate-test-certificate.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cbe49a03..39d2e1c8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -22,7 +22,7 @@ jobs: - name: Start containers run: docker-compose up -d - name: Install self signed certificate - run: sudo cp ./resources/test/certs/localhost.pem /usr/local/share/ca-certificates/localhost.crt && sudo update-ca-certificates + run: sudo cp ./resources/test/certs/cert.pem /usr/local/share/ca-certificates/localhost.crt && sudo update-ca-certificates - name: Show containers run: docker-compose ps - name: Run tests diff --git a/Dockerfile.certificate b/Dockerfile.certificate deleted file mode 100644 index 266cc406..00000000 --- a/Dockerfile.certificate +++ /dev/null @@ -1,6 +0,0 @@ -# Dockerfile to install dependencies for generating test certificates -FROM docker.io/library/python:3.11 -WORKDIR /usr/src/app -RUN pip install --no-cache-dir cryptography -COPY scripts/generate-test-certificate.py scripts/generate-test-certificate.py -CMD ["python3", "scripts/generate-test-certificate.py"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 20437d59..3d7900cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,12 +44,15 @@ services: AWS_DEFAULT_REGION: us-east-1 generate-certificates: - build: - context: . - dockerfile: Dockerfile.certificate + image: docker.io/alpine/openssl + entrypoint: /mnt/scripts/generate-test-certificate + working_dir: /mnt/ volumes: - source: ./resources/ - target: /usr/src/app/resources/ + target: /mnt/resources/ + type: bind + - source: ./scripts/ + target: /mnt/scripts/ type: bind nginx: @@ -60,6 +63,8 @@ services: deploy: restart_policy: condition: on-failure + depends_on: + - generate-certificates #terrashine: # image: docker.io/library/rust # build: diff --git a/scripts/generate-test-certificate b/scripts/generate-test-certificate new file mode 100755 index 00000000..d411c8ec --- /dev/null +++ b/scripts/generate-test-certificate @@ -0,0 +1,8 @@ +#!/bin/sh +# Generate a self-signed certificate for testing purposes. +create_dir=resources/test/certs +openssl req -x509 -newkey rsa:2048 -keyout ${create_dir}/key.pem -out ${create_dir}/cert.pem -days 10 -nodes \ + -subj '/CN=localhost' -sha256\ + -addext "subjectAltName = DNS:localhost" + +cat ${create_dir}/cert.pem ${create_dir}/key.pem > ./resources/test/nginx/localhost.pem diff --git a/scripts/generate-test-certificate.py b/scripts/generate-test-certificate.py deleted file mode 100644 index 493f47f9..00000000 --- a/scripts/generate-test-certificate.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography import x509 -from cryptography.x509.oid import NameOID -from cryptography.hazmat.primitives import hashes -from datetime import datetime, timedelta - -# Generate our key - -key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, -) - -subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"California"), - x509.NameAttribute(NameOID.LOCALITY_NAME, u"San Francisco"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"My Company"), - x509.NameAttribute(NameOID.COMMON_NAME, u"localhost"), -]) - -cert = x509.CertificateBuilder().subject_name( - subject -).issuer_name( - issuer -).public_key( - key.public_key() -).serial_number( - x509.random_serial_number() -).not_valid_before( - datetime.utcnow() -).not_valid_after( - datetime.utcnow() + timedelta(days=10) -).add_extension( - x509.SubjectAlternativeName([x509.DNSName(u"localhost")]), - critical=False, -# Sign our certificate with our private key -).sign(key, hashes.SHA256()) - - -# Write certificate bundle to disk -with open("resources/test/nginx/localhost.pem", "wb") as f: - f.write(cert.public_bytes(serialization.Encoding.PEM)) - f.write(key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - )) -with open("resources/test/certs/localhost.pem", "wb") as f: - f.write(cert.public_bytes(serialization.Encoding.PEM))