diff --git a/.aws/start-page-task-definition.json b/.aws/start-page-task-definition.json new file mode 100644 index 000000000..1c6566cf3 --- /dev/null +++ b/.aws/start-page-task-definition.json @@ -0,0 +1,58 @@ +{ + "containerDefinitions": [ + { + "name": "thunderbird-start-page", + "image": "latest", + "cpu": 0, + "portMappings": [ + { + "name": "thunderbird-start-page-80-tcp", + "containerPort": 80, + "hostPort": 80, + "protocol": "tcp", + "appProtocol": "http" + } + ], + "essential": true, + "environment": [], + "environmentFiles": [], + "mountPoints": [], + "volumesFrom": [] + } + ], + "family": "thunderbird-start-page", + "executionRoleArn": "arn:aws:iam::768512802988:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "revision": 2, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "2048", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "tags": [] +} diff --git a/.aws/website-task-definition.json b/.aws/website-task-definition.json new file mode 100644 index 000000000..e487cad69 --- /dev/null +++ b/.aws/website-task-definition.json @@ -0,0 +1,58 @@ +{ + "containerDefinitions": [ + { + "name": "thunderbird-website", + "image": "latest", + "cpu": 0, + "portMappings": [ + { + "name": "thunderbird-website-80-tcp", + "containerPort": 80, + "hostPort": 80, + "protocol": "tcp", + "appProtocol": "http" + } + ], + "essential": true, + "environment": [], + "environmentFiles": [], + "mountPoints": [], + "volumesFrom": [] + } + ], + "family": "thunderbird-website", + "executionRoleArn": "arn:aws:iam::768512802988:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "revision": 2, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "2048", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "tags": [] +} diff --git a/.github/scripts/delete-comments.js b/.github/scripts/delete-comments.js new file mode 100644 index 000000000..b435641b0 --- /dev/null +++ b/.github/scripts/delete-comments.js @@ -0,0 +1,26 @@ +module.exports = async ({github, context}) => { + + const response_data = await github.paginate(github.rest.issues.listComments, { + issue_number: context.payload.number, // Pr number + owner: context.repo.owner, + repo: context.repo.repo, + } + ); + + if (response_data) { + for (const comment of response_data) { + // Find the bots previous comments + if (comment.user.login === 'github-actions[bot]' && comment.body.indexOf('preview environments') !== -1) { + // If we have an existing comment, delete it + // Reference: https://octokit.github.io/rest.js/v19#issues-delete-comment + github.rest.issues.deleteComment({ + comment_id: comment.id, + issue_number: context.payload.number, // Pr number + owner: context.repo.owner, + repo: context.repo.repo, + }); + } + } + } + +}; \ No newline at end of file diff --git a/.github/scripts/delete-tasks.js b/.github/scripts/delete-tasks.js new file mode 100644 index 000000000..59b208f8c --- /dev/null +++ b/.github/scripts/delete-tasks.js @@ -0,0 +1,111 @@ +module.exports = async ({ + github, + context, + require, + exec, + partial_value, + pr_id_key, + pr_id_value, + ecs_cluster, + aws_listener_arn, + aws_load_balancer_arn + }) => { + + // Needed for aws output + let output = ''; + const options = {}; + options.listeners = { + stdout: (data) => { + output += data.toString(); + }, + }; + + // Remove the load balancer stuff first + // Rules + await exec.exec(`aws elbv2 describe-rules --listener-arn ${aws_listener_arn} --query "Rules" --output json`, [], options); + const rules = JSON.parse(output); + output = ''; + + const rule_arns_to_destroy = []; + + for (const listener_rule of rules) { + + // describe doesn't include tags... + await exec.exec(`aws elbv2 describe-tags --resource-arns ${listener_rule['RuleArn']} --query "TagDescriptions[0].Tags" --output json`, [], options); + const tags = JSON.parse(output); + output = ''; + + for (const tag of tags) { + // Look for our PR_ID_KEY, and if we're doing a partial string match, see if PR_ID_VALUE is in value, otherwise do an exact match + if (tag['Key'] === pr_id_key && ((partial_value && tag['Value'].indexOf(pr_id_value) === 0) || (!partial_value && tag['Value'] === pr_id_value))) { + rule_arns_to_destroy.push(listener_rule['RuleArn']); + } + } + + for (const rule of rule_arns_to_destroy) { + await exec.exec(`aws elbv2 delete-rule --rule-arn "${rule}"`, [], options); + output = ''; + } + } + + // Target groups + const target_groups_to_destroy = []; + + await exec.exec(`aws elbv2 describe-target-groups --load-balancer-arn ${aws_load_balancer_arn} --query "TargetGroups" --output json`, [], options); + const target_groups = JSON.parse(output); + output = ''; + + for (const target_group of target_groups) { + + // describe doesn't include tags... + await exec.exec(`aws elbv2 describe-tags --resource-arns ${target_group['TargetGroupArn']} --query "TagDescriptions[0].Tags" --output json`, [], options); + const tags = JSON.parse(output); + output = ''; + + + for (const tag of tags) { + // Look for our PR_ID_KEY, and if we're doing a partial string match, see if PR_ID_VALUE is in value, otherwise do an exact match + if (tag['Key'] === pr_id_key && ((partial_value && tag['Value'].indexOf(pr_id_value) === 0) || (!partial_value && tag['Value'] === pr_id_value))) { + target_groups_to_destroy.push(target_group['TargetGroupArn']); + } + } + + for (const target_group_arn of target_groups_to_destroy) { + await exec.exec(`aws elbv2 delete-target-group --target-group-arn "${target_group_arn}"`, [], options); + output = ''; + } + } + // -- + + // Grab a list of tasks + await exec.exec(`aws ecs list-tasks --cluster ${ecs_cluster} --query "taskArns" --output json`, [], options); + const tasks = JSON.parse(output); + output = ''; + + let task_arns_to_destroy = []; + + // Look for tasks created by this PR which are identified by our pr_id_key and pr_id_value + // There shouldn't ever be more than one task per PR, but y'know just in case... + for (const task_arn of tasks) { + await exec.exec(`aws ecs describe-tasks --cluster ${ecs_cluster} --task "${task_arn}" --query "tasks[0].tags" --include TAGS --output json`, [], options); + const tags = JSON.parse(output); + output = ''; + + for (const tag of tags) { + // Look for our PR_ID_KEY, and if we're doing a partial string match, see if PR_ID_VALUE is in value, otherwise do an exact match + if (tag['key'] === pr_id_key && ((partial_value && tag['value'].indexOf(pr_id_value) === 0) || (!partial_value && tag['value'] === pr_id_value))) { + task_arns_to_destroy.push(task_arn); + break; + } + } + } + + // Delete them all! + for (const task_arn of task_arns_to_destroy) { + await exec.exec(`aws ecs stop-task --cluster ${ecs_cluster} --task "${task_arn}" --output json`, [], options); + output = ''; + } + + const delete_comments = await require('./.github/scripts/delete-comments.js'); + await delete_comments({github: github, context: context}); +}; \ No newline at end of file diff --git a/.github/scripts/post-comment-with-link.js b/.github/scripts/post-comment-with-link.js new file mode 100644 index 000000000..23eb85eef --- /dev/null +++ b/.github/scripts/post-comment-with-link.js @@ -0,0 +1,41 @@ +module.exports = async ({core, github, context, require, website_url_file, start_page_url_file}) => { + let website_url = null; + let start_page_url = null; + try { + website_url = await require(website_url_file); + } catch { + // Ignore + } + try { + start_page_url = await require(start_page_url_file); + } catch { + // Ignore + } + + if (!website_url && !start_page_url) { + core.setFailed("No urls to post!"); + return; + } + + // Plural depending on if we have both urls available. + const messages = [ + website_url && start_page_url ? 'Your preview environments have been started and will be available shortly at:' : + 'Your preview environment has started and will be available shortly at:' + ]; + + if (website_url) { + messages.push(`thunderbird.net: ${website_url}`); + } + if (start_page_url) { + messages.push(`start.thunderbird.net: ${start_page_url}`); + } + + // Add a check that succeeds with output url :) + // Reference: https://octokit.github.io/rest.js/v19#issues-create-comment + github.rest.issues.createComment({ + issue_number: context.payload.number, // Pr number + owner: context.repo.owner, + repo: context.repo.repo, + body: messages.join('\n') + }); +}; diff --git a/.github/scripts/run-task-and-set-url.js b/.github/scripts/run-task-and-set-url.js new file mode 100644 index 000000000..87b7e02a3 --- /dev/null +++ b/.github/scripts/run-task-and-set-url.js @@ -0,0 +1,165 @@ +module.exports = async ({ + core, + exec, + require, + task_def_arn_file, + cache_file_path, + pr_id_key, + pr_id_value, + ecs_cluster, + aws_security_groups, + aws_subnets, + aws_vpc_id, + aws_avail_region, + aws_listener_arn, + preview_is_secure = '1', + preview_host_url = 'preview.thunderbird.dev' + }) => { + const fs = require('fs'); + const {uniqueNamesGenerator, adjectives, colors} = require('unique-names-generator'); + const task_def_arn = await require(task_def_arn_file); + + // Needed for aws output + let output = ''; + const options = {}; + options.listeners = { + stdout: (data) => { + output += data.toString(); + }, + }; + + function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + /** + * Run an aws command (sleeping in between runs) and wait until condition_callback(value) returns true + * @param aws_command Command to exec + * @param condition_callback Callback conditional that we're waiting to return true + * @param max_wait_s Max wait time in seconds + * @returns {Promise} + */ + async function waitFor(aws_command, condition_callback, max_wait_s = 30) { + let value = null; + let retry_count = 0; + output = ''; + + // Wait until max_wait_s or until condition_callback is truthy + while (retry_count < max_wait_s) { + await exec.exec(aws_command, [], options); + value = output; + output = ''; + + if (value) { + value = JSON.parse(value); + } + + // Check with our callback to see if we've got the goods + if (condition_callback(value)) { + break; + } + + retry_count++; + await sleep(1000); + } + + return value; + } + + // Start a fargate spot task tagged with our PR_ID_KEY/VALUE, under the specified cluster, with networking settings + // Reference: https://docs.aws.amazon.com/cli/latest/reference/ecs/run-task.html + await exec.exec(`aws ecs run-task --tags "key=${pr_id_key},value=${pr_id_value}" --cluster ${ecs_cluster} --task-definition ${task_def_arn} --capacity-provider-strategy capacityProvider="FARGATE_SPOT",weight=0,base=0 --network-configuration awsvpcConfiguration={subnets=[${aws_subnets}],securityGroups=[${aws_security_groups}],assignPublicIp="ENABLED"} --query "tasks[0].taskArn" --output json`, [], options); + const task_arn = JSON.parse(output); + output = ''; + + if (!task_arn) { + core.setFailed("TaskArn was not found - The fargate instance probably failed to launch!"); + return; + } + + // Lookup the task we just ran, and wait until the network interface id is available + // Reference: https://docs.aws.amazon.com/cli/latest/reference/ecs/describe-tasks.html + const private_ip_array = await waitFor( + `aws ecs describe-tasks --tasks "${task_arn}" --cluster ${ecs_cluster} --query "tasks[0].attachments[0].details[?name == 'privateIPv4Address'].value" --output json`, (value) => { + return value && value.length > 0; + } + ); + + if (!private_ip_array || private_ip_array.length === 0) { + core.setFailed("Private IP was not found"); + return; + } + + const private_ip = private_ip_array[0]; + + // List of fun bird names + const bird_names = ["albatross", "cassowary", "chicken", "crane", "condor", "dove", "duck", "emu", "falcon", "flamingo", "frogmouth", "grebe", "hawk", "heron", "hoatzin", "hornbill", "hummingbird", "ibis", "kagu", "kingfisher", "kiwi", "loon", "mesite", "mousebird", "nightjar", "oilbird", "ostrich", "owl", "parrot", "pelican", "penguin", "petrel", "pigeon", "potoo", "quetzal", "rhea", "sandgrouse", "shortbird", "stork", "sunbittern", "swift", "tinamou", "treeswift", "tropicbird", "turaco", "vulture", "wader", "woodpecker"]; + + const subdomain = uniqueNamesGenerator({ + dictionaries: [adjectives, colors, bird_names], + separator: '-', + length: 3, + }); + + const port = preview_is_secure === '1' ? 443 : 80; + const protocol = preview_is_secure === '1' ? 'HTTPS' : 'HTTP'; + + // Reference: https://docs.aws.amazon.com/cli/latest/reference/elbv2/create-target-group.html + const target_group_array = await waitFor( + `aws elbv2 create-target-group --tags "Key=${pr_id_key},Value=${pr_id_value}" --name pe-${subdomain.slice(0, 28)} --protocol ${protocol} --port ${port} --target-type ip --vpc-id ${aws_vpc_id} --output json`, (value) => { + return value && value !== {}; + } + ); + + if (!target_group_array || target_group_array.length === 0) { + core.setFailed("Target group could not be created"); + return; + } + + const target_group = target_group_array['TargetGroups'][0]['TargetGroupArn'] ?? null; + + if (!target_group) { + core.setFailed("Target group could not be found, but it was created?"); + return; + } + + // Reference: https://docs.aws.amazon.com/cli/latest/reference/elbv2/register-targets.html + const target = await waitFor( + `aws elbv2 register-targets --target-group-arn ${target_group} --targets Id=${private_ip},Port=80,AvailabilityZone=${aws_avail_region} --output json`, (value) => { + // There's no output for this + return true; + } + ); + + + // Grab all of the fules + await exec.exec(`aws elbv2 describe-rules --listener-arn ${aws_listener_arn} --query "Rules" --output json`, [], options); + const rules = JSON.parse(output); + output = ''; + + // We need to find a unique priority. The number doesn't matter because it we have custom matching rules on it.. + let priority = 5; + const priorities = []; + for (const rule of rules) { + priorities.push(rule['Priority']); + } + + // Uh...replace this with something nicer + for (let i = 2; i < 1000; i++) { + if (priorities.indexOf(`${i}`) === -1) { + priority = i; + break; + } + } + + const conditions = `Field=host-header,HostHeaderConfig={"Values"=["${subdomain}.${preview_host_url}"]}`; + const actions = `Type=forward,TargetGroupArn=${target_group}`; + const rule = await waitFor( + `aws elbv2 create-rule --tags "Key=${pr_id_key},Value=${pr_id_value}" --listener-arn ${aws_listener_arn} --priority ${priority} --conditions ${conditions} --actions ${actions} --output json`, (value) => { + return !!value && value !== 'None'; + } + ); + + // Write the url to file, so we can cache it between jobs + fs.writeFileSync(cache_file_path, JSON.stringify(`${protocol.toLowerCase()}://${subdomain}.${preview_host_url}`)); +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..22a853a3c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,85 @@ +name: Re-usable Build + +on: + workflow_call: + inputs: + AWS_REGION: + required: true + type: string + ECR_REPOSITORY: + required: true + type: string + ECS_TASK_DEFINITION: + required: true + type: string + ECS_CLUSTER: + required: true + type: string + CONTAINER: + required: true + type: string + CACHE_FILE_ID: + required: true + type: string + DOCKER_FILE: + required: true + type: string + +jobs: + build: + name: Build Site + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push to Amazon ECR + id: build + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ inputs.CONTAINER }}-website-${{ github.sha }} + run: | + # Build a docker container and + # push it to ECR so that it can + # be deployed to ECS. + docker build -t $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG . -f ${{ inputs.DOCKER_FILE }} + docker push $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG + echo "image=$ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: ${{ inputs.ECS_TASK_DEFINITION }} + container-name: ${{ inputs.CONTAINER }} + image: ${{ steps.build.outputs.image }} + + - name: Deploy Amazon ECS task definition + id: task-def-deploy + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + cluster: ${{ inputs.ECS_CLUSTER }} + wait-for-service-stability: true + + - name: Write Task Definition ARN to File + run: echo '"${{ steps.task-def-deploy.outputs.task-definition-arn }}"' >> ./${{ inputs.CONTAINER }}-task-def-arn-${{ inputs.CACHE_FILE_ID }}.json + + - name: Cache Task Definition ARN + id: cache-task-arn + uses: actions/cache/save@v3 + with: + path: ./${{ inputs.CONTAINER }}-task-def-arn-${{ inputs.CACHE_FILE_ID }}.json + key: ${{ inputs.CONTAINER }}-task-def-arn \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..3fc91f47d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,112 @@ +name: Re-usable Deploy + +on: + workflow_call: + inputs: + AWS_REGION: + required: true + type: string + CONTAINER: + required: true + type: string + CACHE_FILE_ID: + required: true + type: string + DOCKER_FILE: + required: true + type: string + PR_ID_KEY: + required: true + type: string + PR_ID_VALUE: + required: true + type: string + ECS_CLUSTER: + required: true + type: string + +jobs: + deploy: + name: Deploy Site + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get Cached Task Definition ARN + id: cache-task-arn-get + uses: actions/cache/restore@v3 + with: + fail-on-cache-miss: true + path: ./${{ inputs.CONTAINER }}-task-def-arn-${{ inputs.CACHE_FILE_ID }}.json + key: ${{ inputs.CONTAINER }}-task-def-arn + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Install Unique Names Generator + run: npm i unique-names-generator + + - uses: actions/github-script@v6 + name: Clean up existing Tasks + with: + script: | + const script = require('${{github.workspace}}/.github/scripts/delete-tasks.js') + const options = { + github: github, + context: context, + exec: exec, + partial_value: true, + require: require, + pr_id_key: "${{ inputs.PR_ID_KEY }}", + pr_id_value: "${{ inputs.PR_ID_VALUE }}", + ecs_cluster: "${{ inputs.ECS_CLUSTER }}", + aws_load_balancer_arn: '${{ secrets.AWS_LOAD_BALANCER_ARN }}', + aws_listener_arn: '${{ secrets.AWS_LISTENER_ARN }}', + } + await script(options) + + - uses: actions/github-script@v6 + name: Deploy Task + env: + AWS_SECURITY_GROUPS: ${{ secrets.AWS_SECURITY_GROUPS }} + AWS_SUBNETS: ${{ secrets.AWS_SUBNETS }} + AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }} + with: + script: | + const script = require('${{github.workspace}}/.github/scripts/run-task-and-set-url.js') + const options = { + core: core, + github: github, + exec: exec, + require: require, + task_def_arn_file: "./${{ inputs.CONTAINER }}-task-def-arn-${{ inputs.CACHE_FILE_ID }}.json", + cache_file_path: "./${{ inputs.CONTAINER }}-url-${{ inputs.CACHE_FILE_ID }}.json", + pr_id_key: "${{ inputs.PR_ID_KEY }}", + pr_id_value: "${{ inputs.PR_ID_VALUE }}", + ecs_cluster: "${{ inputs.ECS_CLUSTER }}", + aws_security_groups: '${{ secrets.AWS_SECURITY_GROUPS }}', /* The value uses double quotes, so we need to single quote this */ + aws_subnets: '${{ secrets.AWS_SUBNETS }}', /* The value uses double quotes, so we need to single quote this */ + aws_vpc_id: '${{ secrets.AWS_VPC_ID }}', + aws_avail_region: '${{ secrets.AWS_AVAIL_REGION }}', + aws_listener_arn: '${{ secrets.AWS_LISTENER_ARN }}', + preview_is_secure: '${{ secrets.PREVIEW_IS_SECURE }}', + preview_host_url: '${{ secrets.PREVIEW_HOST_URL }}', + } + await script(options) + + - name: Cache URL + id: cache-url + uses: actions/cache/save@v3 + with: + path: ./${{ inputs.CONTAINER }}-url-${{ inputs.CACHE_FILE_ID }}.json + key: ${{ inputs.CONTAINER }}-url diff --git a/.github/workflows/preview_env_closed.yml b/.github/workflows/preview_env_closed.yml new file mode 100644 index 000000000..a328fbada --- /dev/null +++ b/.github/workflows/preview_env_closed.yml @@ -0,0 +1,72 @@ +# This workflow will build and push a new container image to Amazon ECR, +# and then will deploy a new task definition to Amazon ECS, when there is a push to the "staging" branch. + +name: Stop Preview Environment + +# Stop any pending jobs +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +on: + pull_request: + # On PR: label, open, commit push, re-open + types: [ unlabeled, closed ] + +env: + AWS_REGION: us-west-2 + ECR_REPOSITORY: preview-repo + ECS_CLUSTER: preview-env-cluster + + PR_ID_KEY: PullRequestId + PR_ID_VALUE: ${{ github.ref }} + WEBSITE_PR_ID_VALUE: ${{ github.ref }}/website + START_PR_ID_VALUE: ${{ github.ref }}/startpage + +permissions: + contents: read + pull-requests: write + +jobs: + stop_pr: + name: Stop Preview Environment + runs-on: ubuntu-latest + # Only run if we've removed the label "preview environment" or closed the PR + if: | + github.event.label.name == 'preview environment' || + github.event.action == 'closed' + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - uses: actions/github-script@v6 + name: Clean up existing Tasks + with: + script: | + const script = require('${{ github.workspace }}/.github/scripts/delete-tasks.js') + const is_partial = "${{ github.event.label.name }}" === 'preview environment' + let pr_id_value = "${{ env.PR_ID_VALUE }}" + + const options = { + github: github, + context: context, + require: require, + exec: exec, + partial_value: is_partial, + pr_id_key: "${{ env.PR_ID_KEY }}", + pr_id_value: pr_id_value, + ecs_cluster: "${{ env.ECS_CLUSTER }}" + } + await script(options) diff --git a/.github/workflows/preview_env_open.yml b/.github/workflows/preview_env_open.yml new file mode 100644 index 000000000..2be384807 --- /dev/null +++ b/.github/workflows/preview_env_open.yml @@ -0,0 +1,179 @@ +# This workflow will build both website and start page, and then deploy them on AWS +# +name: Deploy Preview Environment + +# Stop any pending jobs +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +on: + pull_request: + # On PR: label, open, commit push, re-open + types: [ labeled, opened, synchronize, reopened ] + +env: + AWS_REGION: us-west-2 + ECR_REPOSITORY: preview-repo + ECS_CLUSTER: preview-env-cluster + + PR_ID_KEY: PullRequestId + + CACHE_FILE_ID: ${{ github.run_id }}${{github.run_attempt}} + + CONTAINER: thunderbird-website + + WEBSITE_CONTAINER: thunderbird-website + WEBSITE_PR_ID_VALUE: ${{ github.ref }}/website + WEBSITE_DOCKER_FILE: 'preview-website.dockerfile' + WEBSITE_TASK_DEFINITION: .aws/website-task-definition.json + + START_CONTAINER: thunderbird-start-page + START_PR_ID_VALUE: ${{ github.ref }}/startpage + START_DOCKER_FILE: 'preview-start-page.dockerfile' + START_TASK_DEFINITION: .aws/start-page-task-definition.json + + +permissions: + contents: read + pull-requests: write + +jobs: + # job..with doesn't support env inputs for some reason... + set_inputs: + runs-on: ubuntu-latest + name: Setting inputs from environment variables... + if: contains(github.event.pull_request.labels.*.name, 'preview environment') + outputs: + AWS_REGION: ${{ steps.step1.outputs.AWS_REGION }} + ECR_REPOSITORY: ${{ steps.step1.outputs.ECR_REPOSITORY }} + ECS_CLUSTER: ${{ steps.step1.outputs.ECS_CLUSTER }} + PR_ID_KEY: ${{ steps.step1.outputs.PR_ID_KEY }} + CACHE_FILE_ID: ${{ steps.step1.outputs.CACHE_FILE_ID }} + WEBSITE_CONTAINER: ${{ steps.step1.outputs.WEBSITE_CONTAINER }} + WEBSITE_PR_ID_VALUE: ${{ steps.step1.outputs.WEBSITE_PR_ID_VALUE }} + WEBSITE_DOCKER_FILE: ${{ steps.step1.outputs.WEBSITE_DOCKER_FILE }} + WEBSITE_TASK_DEFINITION: ${{ steps.step1.outputs.WEBSITE_TASK_DEFINITION }} + START_CONTAINER: ${{ steps.step1.outputs.START_CONTAINER }} + START_PR_ID_VALUE: ${{ steps.step1.outputs.START_PR_ID_VALUE }} + START_DOCKER_FILE: ${{ steps.step1.outputs.START_DOCKER_FILE }} + START_TASK_DEFINITION: ${{ steps.step1.outputs.START_TASK_DEFINITION }} + steps: + - id: step1 + run: | + echo "AWS_REGION=${{ env.AWS_REGION }}" >> "$GITHUB_OUTPUT" + echo "ECR_REPOSITORY=${{ env.ECR_REPOSITORY }}" >> "$GITHUB_OUTPUT" + echo "ECS_CLUSTER=${{ env.ECS_CLUSTER }}" >> "$GITHUB_OUTPUT" + echo "PR_ID_KEY=${{ env.PR_ID_KEY }}" >> "$GITHUB_OUTPUT" + echo "CACHE_FILE_ID=${{ env.CACHE_FILE_ID }}" >> "$GITHUB_OUTPUT" + echo "WEBSITE_CONTAINER=${{ env.WEBSITE_CONTAINER }}" >> "$GITHUB_OUTPUT" + echo "WEBSITE_PR_ID_VALUE=${{ env.WEBSITE_PR_ID_VALUE }}" >> "$GITHUB_OUTPUT" + echo "WEBSITE_DOCKER_FILE=${{ env.WEBSITE_DOCKER_FILE }}" >> "$GITHUB_OUTPUT" + echo "WEBSITE_TASK_DEFINITION=${{ env.WEBSITE_TASK_DEFINITION }}" >> "$GITHUB_OUTPUT" + echo "START_CONTAINER=${{ env.START_CONTAINER }}" >> "$GITHUB_OUTPUT" + echo "START_PR_ID_VALUE=${{ env.START_PR_ID_VALUE }}" >> "$GITHUB_OUTPUT" + echo "START_DOCKER_FILE=${{ env.START_DOCKER_FILE }}" >> "$GITHUB_OUTPUT" + echo "START_TASK_DEFINITION=${{ env.START_TASK_DEFINITION }}" >> "$GITHUB_OUTPUT" + + build_website: + name: Build Thunderbird.net + needs: set_inputs + if: | + contains(github.event.pull_request.labels.*.name, 'preview environment') + uses: ./.github/workflows/build.yml + secrets: inherit + with: + AWS_REGION: ${{ needs.set_inputs.outputs.AWS_REGION }} + ECR_REPOSITORY: ${{ needs.set_inputs.outputs.ECR_REPOSITORY }} + ECS_CLUSTER: ${{ needs.set_inputs.outputs.ECS_CLUSTER }} + ECS_TASK_DEFINITION: ${{ needs.set_inputs.outputs.WEBSITE_TASK_DEFINITION }} + CACHE_FILE_ID: ${{ needs.set_inputs.outputs.CACHE_FILE_ID }} + CONTAINER: ${{ needs.set_inputs.outputs.WEBSITE_CONTAINER }} + DOCKER_FILE: ${{ needs.set_inputs.outputs.WEBSITE_DOCKER_FILE }} + + build_start_page: + name: Build Start.Thunderbird.net + needs: set_inputs + if: | + contains(github.event.pull_request.labels.*.name, 'preview environment') + uses: ./.github/workflows/build.yml + secrets: inherit + with: + AWS_REGION: ${{ needs.set_inputs.outputs.AWS_REGION }} + ECR_REPOSITORY: ${{ needs.set_inputs.outputs.ECR_REPOSITORY }} + ECS_CLUSTER: ${{ needs.set_inputs.outputs.ECS_CLUSTER }} + ECS_TASK_DEFINITION: ${{ needs.set_inputs.outputs.START_TASK_DEFINITION }} + CACHE_FILE_ID: ${{ needs.set_inputs.outputs.CACHE_FILE_ID }} + CONTAINER: ${{ needs.set_inputs.outputs.START_CONTAINER }} + DOCKER_FILE: ${{ needs.set_inputs.outputs.START_DOCKER_FILE }} + + deploy_website: + name: Deploy Thunderbird.net + # Only run if we've labeled the PR as "preview environment" + if: contains(github.event.pull_request.labels.*.name, 'preview environment') + needs: [ set_inputs, build_website ] + uses: ./.github/workflows/deploy.yml + secrets: inherit + with: + AWS_REGION: ${{ needs.set_inputs.outputs.AWS_REGION }} + CACHE_FILE_ID: ${{ needs.set_inputs.outputs.CACHE_FILE_ID }} + CONTAINER: ${{ needs.set_inputs.outputs.WEBSITE_CONTAINER }} + PR_ID_KEY: ${{ needs.set_inputs.outputs.PR_ID_KEY }} + PR_ID_VALUE: ${{ needs.set_inputs.outputs.WEBSITE_PR_ID_VALUE }} + DOCKER_FILE: ${{ needs.set_inputs.outputs.WEBSITE_DOCKER_FILE }} + ECS_CLUSTER: ${{ needs.set_inputs.outputs.ECS_CLUSTER }} + + deploy_start_page: + name: Deploy Start.Thunderbird.net + # Only run if we've labeled the PR as "preview environment" + if: contains(github.event.pull_request.labels.*.name, 'preview environment') + needs: [ set_inputs, build_start_page ] + uses: ./.github/workflows/deploy.yml + secrets: inherit + with: + AWS_REGION: ${{ needs.set_inputs.outputs.AWS_REGION }} + CACHE_FILE_ID: ${{ needs.set_inputs.outputs.CACHE_FILE_ID }} + CONTAINER: ${{ needs.set_inputs.outputs.START_CONTAINER }} + PR_ID_KEY: ${{ needs.set_inputs.outputs.PR_ID_KEY }} + PR_ID_VALUE: ${{ needs.set_inputs.outputs.START_PR_ID_VALUE }} + DOCKER_FILE: ${{ needs.set_inputs.outputs.START_DOCKER_FILE }} + ECS_CLUSTER: ${{ needs.set_inputs.outputs.ECS_CLUSTER }} + + post_comment: + name: Post Links In PR + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'preview environment') + needs: [ deploy_website, deploy_start_page ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get Cached Task Definition ARN + id: cache-task-arn-get-start-page + uses: actions/cache/restore@v3 + with: + path: ./${{ env.START_CONTAINER }}-url-${{ env.CACHE_FILE_ID }}.json + key: ${{ env.START_CONTAINER }}-url + + - name: Get Cached Task Definition ARN + id: cache-task-arn-get-website + uses: actions/cache/restore@v3 + with: + path: ./${{ env.WEBSITE_CONTAINER }}-url-${{ env.CACHE_FILE_ID }}.json + key: ${{ env.WEBSITE_CONTAINER }}-url + + - uses: actions/github-script@v6 + name: Post Comment + with: + script: | + const script = require('${{github.workspace}}/.github/scripts/post-comment-with-link.js') + const options = { + core: core, + github: github, + context: context, + require: require, + website_url_file: "./${{ env.WEBSITE_CONTAINER }}-url-${{ env.CACHE_FILE_ID }}.json", + start_page_url_file: "./${{ env.START_CONTAINER }}-url-${{ env.CACHE_FILE_ID }}.json" + } + await script(options) diff --git a/docker/preview_tb_vhosts.conf b/docker/preview_tb_vhosts.conf new file mode 100644 index 000000000..0ea5468e9 --- /dev/null +++ b/docker/preview_tb_vhosts.conf @@ -0,0 +1,83 @@ +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common +LogFormat "%v: %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" vcombined + +ServerLimit 32 +StartServers 8 +MaxRequestWorkers 1024 +MinSpareThreads 25 +MaxSpareThreads 75 +ThreadsPerChild 32 + +AddOutputFilterByType DEFLATE text/plain +AddOutputFilterByType DEFLATE text/html +AddOutputFilterByType DEFLATE text/xml +AddOutputFilterByType DEFLATE text/css +AddOutputFilterByType DEFLATE application/xml +AddOutputFilterByType DEFLATE application/xhtml+xml +AddOutputFilterByType DEFLATE application/rss+xml +AddOutputFilterByType DEFLATE application/javascript +AddOutputFilterByType DEFLATE application/x-javascript + +# Disabled until we decide what to do with ssl + +#LoadModule ssl_module modules/mod_ssl.so +#Listen 443 +#SSLPassPhraseDialog builtin +#SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000) +#SSLSessionCacheTimeout 300 +#Mutex default +#SSLRandomSeed startup file:/dev/urandom 256 +#SSLRandomSeed connect builtin +#SSLCryptoDevice builtin + + + ServerName localhost + DocumentRoot /var/www/html/start/thunderbird.net + + #SSLEngine on + #SSLCipherSuite DEFAULT + #SSLProtocol TLSv1.3 + #SSLHonorCipherOrder On + #SSLCertificateFile /etc/apache2/ssl/ssl.crt + #SSLCertificateKeyFile /etc/apache2/ssl/ssl.key + + + AllowOverride + Options + Require all granted + + RewriteEngine On +WSGIDaemonProcess thunderbird.net processes=2 threads=15 display-name=%{GROUP} python-home=/var/www/tbservices python-path=/var/www/html/start +WSGIProcessGroup thunderbird.net + +RewriteRule ^/(.*)/thunderbird/60\.0/whatsnew/$ https://support.mozilla.org/$1/kb/new-thunderbird-60 [R=302,L] +# https://github.com/thundernest/thunderbird-website/issues/162 +RewriteRule ^/ja-JP-mac/?(.*)$ /ja/$1 [R=302] +# https://github.com/thundernest/thunderbird.net-l10n/issues/1 +RewriteRule ^/bn-(?:BD|IN)/?(.*)$ /bn/$1 [R=302] +RewriteRule ^/privacy/? https://www.mozilla.org/privacy/thunderbird/ [R=302] +RewriteRule ^/thunderbird/(latest/)?system-requirements/? /system-requirements/ [R=302] +RewriteRule ^/thunderbird/notes/? /notes/ [R=302] +RewriteRule ^/releases/? /thunderbird/releases/ [R=302] +# https://github.com/thundernest/thunderbird-website/issues/160 +RewriteRule ^(.*)/channel/? #channel [R=302,NE] +# URLs with no language code need to be assigned one by wsgi.py +WSGIScriptAliasMatch ^/$ /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/about[/]?$ /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/download /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/donate /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/features[/]?$ /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/newsletter /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/organizations[/]?$ /var/www/html/start/wsgi.py +WSGIScriptAliasMatch ^/thunderbird /var/www/html/start/wsgi.py +RewriteRule ^/calendar/?(.*)$ /en-US/calendar/$1 [R=302] +RewriteRule ^/careers/? /en-US/careers/ [R=302] +RewriteRule ^/contact/? /en-US/contact/ [R=302] +RewriteRule ^/email-providers/? /en-US/email-providers/ [R=302] +RewriteRule ^/get-involved/? /en-US/get-involved/ [R=302] +ExpiresActive On +ExpiresDefault "access plus 2 hours" +Header always set Strict-Transport-Security "max-age=31536000" + + \ No newline at end of file diff --git a/preview-start-page.dockerfile b/preview-start-page.dockerfile new file mode 100644 index 000000000..a5ae1e1ef --- /dev/null +++ b/preview-start-page.dockerfile @@ -0,0 +1,65 @@ +FROM python:3 + +RUN apt-get update && apt-get install -y git apache2 apache2-dev npm libapache2-mod-wsgi-py3 +RUN npm install -g less + +# Copy our Vhost +COPY docker/preview_tb_vhosts.conf /etc/apache2/conf-enabled/tb_vhosts.conf +# Copy our wsgi script +COPY wsgi.py /var/www/html/start/wsgi.py +COPY settings.py /var/www/html/start/settings.py + +COPY . /build +WORKDIR /build + +# Pre-build setup +RUN rm -rf thunderbird_notes +RUN git clone https://github.com/thundernest/thunderbird-notes.git thunderbird_notes + +RUN rm -rf product-details +RUN git clone -b production https://github.com/mozilla-releng/product-details.git + +RUN rm -rf locale +RUN git clone https://github.com/thundernest/thunderbird.net-l10n.git locale +RUN l10n_tools/compile.sh + +# Build the website! +RUN python -m pip install -r ./requirements.txt +RUN python build-site.py --startpage + +RUN cp -R /build/site /var/www/html/start/thunderbird.net + +# Clean up build files +RUN rm -rf /build + +# Create the directory for our wsgi script +RUN mkdir -p /var/www/tbservices/ + +# Log directory +RUN mkdir -p /etc/apache2/logs/ +RUN mkdir -p /var/log/httpd/autoconfig/ +# SSL directory +RUN mkdir -p /etc/apache2/ssl/ + +# Setup our virtualenv +RUN pip install virtualenv +RUN virtualenv -p python /var/www/tbservices/ +# Install some libs into our virtualenv +RUN /var/www/tbservices/bin/pip install requests webob lib + +# Enable some additional mods +RUN a2enmod socache_shmcb +RUN a2enmod rewrite +RUN a2enmod headers +RUN a2enmod expires +#RUN a2enmod ssl + +# Create a symlink so we can get docker logs working +RUN ln -sf /proc/self/fd/1 /var/log/apache2/access.log && \ + ln -sf /proc/self/fd/1 /var/log/apache2/error.log && \ + ln -sf /proc/self/fd/1 /var/log/apache2/other_vhosts_access.log + +# Boot apache +EXPOSE 80 +#EXPOSE 443 +CMD ["apachectl", "-D", "FOREGROUND"] \ No newline at end of file diff --git a/preview-website.dockerfile b/preview-website.dockerfile new file mode 100644 index 000000000..60af63baf --- /dev/null +++ b/preview-website.dockerfile @@ -0,0 +1,65 @@ +FROM python:3 + +RUN apt-get update && apt-get install -y git apache2 apache2-dev npm libapache2-mod-wsgi-py3 +RUN npm install -g less + +# Copy our Vhost +COPY docker/preview_tb_vhosts.conf /etc/apache2/conf-enabled/tb_vhosts.conf +# Copy our wsgi script +COPY wsgi.py /var/www/html/start/wsgi.py +COPY settings.py /var/www/html/start/settings.py + +COPY . /build +WORKDIR /build + +# Pre-build setup +RUN rm -rf thunderbird_notes +RUN git clone https://github.com/thundernest/thunderbird-notes.git thunderbird_notes + +RUN rm -rf product-details +RUN git clone -b production https://github.com/mozilla-releng/product-details.git + +RUN rm -rf locale +RUN git clone https://github.com/thundernest/thunderbird.net-l10n.git locale +RUN l10n_tools/compile.sh + +# Build the website! +RUN python -m pip install -r ./requirements.txt +RUN python build-site.py + +RUN cp -R /build/thunderbird.net /var/www/html/start/thunderbird.net + +# Clean up build files +RUN rm -rf /build + +# Create the directory for our wsgi script +RUN mkdir -p /var/www/tbservices/ + +# Log directory +RUN mkdir -p /etc/apache2/logs/ +RUN mkdir -p /var/log/httpd/autoconfig/ +# SSL directory +RUN mkdir -p /etc/apache2/ssl/ + +# Setup our virtualenv +RUN pip install virtualenv +RUN virtualenv -p python /var/www/tbservices/ +# Install some libs into our virtualenv +RUN /var/www/tbservices/bin/pip install requests webob lib + +# Enable some additional mods +RUN a2enmod socache_shmcb +RUN a2enmod rewrite +RUN a2enmod headers +RUN a2enmod expires +#RUN a2enmod ssl + +# Create a symlink so we can get docker logs working +RUN ln -sf /proc/self/fd/1 /var/log/apache2/access.log && \ + ln -sf /proc/self/fd/1 /var/log/apache2/error.log && \ + ln -sf /proc/self/fd/1 /var/log/apache2/other_vhosts_access.log + +# Boot apache +EXPOSE 80 +#EXPOSE 443 +CMD ["apachectl", "-D", "FOREGROUND"] \ No newline at end of file