From 07099b1c1841a3e11bf894af3426f0734e62690e Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Mon, 14 Oct 2024 16:48:00 +0300 Subject: [PATCH 1/5] wip: adding new domain for api & django admin --- infra/aws/index.ts | 9 +++++---- infra/lib/cloudflare/index.ts | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 infra/lib/cloudflare/index.ts diff --git a/infra/aws/index.ts b/infra/aws/index.ts index b4d0bf68..25be0104 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -24,6 +24,7 @@ import * as op from "@1password/op-js"; import { createVerifierService } from "./verifier"; import { createS3InitiatedECSTask } from "../lib/scorer/s3_initiated_ecs_task"; import { stack, defaultTags, StackType } from "../lib/tags"; +import { createApiDomainRecord } from "../lib/cloudflare"; // The following vars are not allowed to be undefined, hence the `${...}` magic @@ -634,12 +635,10 @@ const serviceTaskRole = new aws.iam.Role("scorer-service-task-role", { ], Resource: "*", }, - // CloudFront + // CloudFront { Effect: "Allow", - Action: [ - "cloudfront:CreateInvalidation", - ], + Action: ["cloudfront:CreateInvalidation"], Resource: "*", }, ], @@ -1915,3 +1914,5 @@ const amplifyAppInfo = coreInfraStack }); export const amplifyAppHookUrl = pulumi.secret(amplifyAppInfo.webHook.url); + +createApiDomainRecord(stack, CLOUDFLARE_ZONE_ID, alb.dnsName); diff --git a/infra/lib/cloudflare/index.ts b/infra/lib/cloudflare/index.ts new file mode 100644 index 00000000..dacdb850 --- /dev/null +++ b/infra/lib/cloudflare/index.ts @@ -0,0 +1,22 @@ +import * as pulumi from "@pulumi/pulumi"; +import * as cloudflare from "@pulumi/cloudflare"; + +export function createApiDomainRecord( + stack: pulumi.Input, + cloudflareZoneId: pulumi.Input, + albDnsName: pulumi.Input +) { + const name = "api-passport-xyz-record"; + const cloudflareApiRecord = + stack === "production" + ? new cloudflare.Record(name, { + name: "api", + zoneId: cloudflareZoneId, + type: "CNAME", + value: albDnsName, + allowOverwrite: true, + comment: "Points to LB handling the backend service requests", + }) + : ""; + return cloudflareApiRecord; +} From c52e356e242bab640f25d13ed1c5ade8428c58d6 Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Tue, 15 Oct 2024 11:25:57 +0300 Subject: [PATCH 2/5] fix: specification of infra-libs version in package.json --- infra/aws/index.ts | 2 +- infra/package.json | 2 +- infra/yarn.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/aws/index.ts b/infra/aws/index.ts index 25be0104..ac87d4bb 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -1915,4 +1915,4 @@ const amplifyAppInfo = coreInfraStack export const amplifyAppHookUrl = pulumi.secret(amplifyAppInfo.webHook.url); -createApiDomainRecord(stack, CLOUDFLARE_ZONE_ID, alb.dnsName); +// createApiDomainRecord(stack, CLOUDFLARE_ZONE_ID, alb.dnsName); diff --git a/infra/package.json b/infra/package.json index 2c39ceec..dc5e345a 100644 --- a/infra/package.json +++ b/infra/package.json @@ -13,6 +13,6 @@ "@pulumi/awsx": "^1.0.5", "@pulumi/command": "^0.10.0", "@pulumi/pulumi": "^3.79.0", - "infra-libs": "passportxyz/infra-libs#semver:^1.2.1" + "infra-libs": "passportxyz/infra-libs#semver:1.2.1" } } diff --git a/infra/yarn.lock b/infra/yarn.lock index fb687b6c..50530744 100644 --- a/infra/yarn.lock +++ b/infra/yarn.lock @@ -1673,7 +1673,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -"infra-libs@passportxyz/infra-libs#semver:^1.2.1": +"infra-libs@passportxyz/infra-libs#semver:1.2.1": version "1.0.0" resolved "https://codeload.github.com/passportxyz/infra-libs/tar.gz/8170b9d4286cad3ad0809c252f65fc0d36776dae" dependencies: From cf7c88ca03efbe6ea82f7b733df0cd47f24c6163 Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Thu, 17 Oct 2024 10:42:45 +0300 Subject: [PATCH 3/5] wip: changes to deploy to new domain --- api/scorer/settings/base.py | 3 +- infra/aws/index.ts | 93 ++++++++++++++++++++++++++++----- infra/lib/cloudflare/index.ts | 7 +++ infra/lib/scorer/new_service.ts | 11 ++-- 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/api/scorer/settings/base.py b/api/scorer/settings/base.py index 7b066b50..55f51cce 100644 --- a/api/scorer/settings/base.py +++ b/api/scorer/settings/base.py @@ -29,7 +29,8 @@ CERAMIC_CACHE_API_KEY = env("CERAMIC_CACHE_API_KEY", default="") -UI_DOMAINS = env("UI_DOMAINS", default=["localhost:3000", "www.localhost:3000"]) +# TODO: geri is this actually used? +# UI_DOMAINS = env("UI_DOMAINS", default=["localhost:3000", "www.localhost:3000"]) SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = env("GOOGLE_OAUTH_CLIENT_ID", default="") SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = env("GOOGLE_CLIENT_SECRET", default="") diff --git a/infra/aws/index.ts b/infra/aws/index.ts index ac87d4bb..51d97448 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -24,9 +24,6 @@ import * as op from "@1password/op-js"; import { createVerifierService } from "./verifier"; import { createS3InitiatedECSTask } from "../lib/scorer/s3_initiated_ecs_task"; import { stack, defaultTags, StackType } from "../lib/tags"; -import { createApiDomainRecord } from "../lib/cloudflare"; - -// The following vars are not allowed to be undefined, hence the `${...}` magic ////////////////////////////////////////////////////////////// // Loading environment variables @@ -45,19 +42,21 @@ const route53ZoneForPublicData = op.read.parse( `op://DevOps/passport-scorer-${stack}-env/ci/ROUTE_53_ZONE_FOR_PUBLIC_DATA` ); -const rootDomain = op.read.parse( +const legacyRootDomain = op.read.parse( `op://DevOps/passport-scorer-${stack}-env/ci/ROOT_DOMAIN` ); -const domain = +const legacyDomain = stack == "production" - ? `api.scorer.${rootDomain}` - : `api.${stack}.scorer.${rootDomain}`; + ? `api.scorer.${legacyRootDomain}` + : `api.${stack}.scorer.${legacyRootDomain}`; +// We only need the publicDataDomain for the legacyRootDomain +// as we hope to be able to drop that before finalizing the mirgation to the new domain const publicDataDomain = stack == "production" - ? `public.scorer.${rootDomain}` - : `public.${stack}.scorer.${rootDomain}`; + ? `public.scorer.${legacyRootDomain}` + : `public.${stack}.scorer.${legacyRootDomain}`; const current = aws.getCallerIdentity({}); const regionData = aws.getRegion({}); @@ -127,6 +126,14 @@ const vpcID = coreInfraStack.getOutput("vpcId"); const vpcPrivateSubnetIds = coreInfraStack.getOutput("privateSubnetIds"); const vpcPublicSubnetIds = coreInfraStack.getOutput("publicSubnetIds"); +const passportXyzDomainName = coreInfraStack.getOutput("passportXyzDomainName"); +const passportXyzHostedZoneId = coreInfraStack.getOutput( + "passportXyzHostedZoneId" +); +const passportXyzCertificateArn = coreInfraStack.getOutput( + "passportXyzCertificateArn" +); + const vpcPublicSubnetId1 = vpcPublicSubnetIds.apply((values) => values[0]); const vpcPublicSubnetId2 = vpcPublicSubnetIds.apply((values) => values[1]); @@ -450,7 +457,7 @@ const httpsListener = HTTPS_ALB_CERT_ARN.apply( // Create a DNS record for the load balancer const www = new aws.route53.Record("scorer", { zoneId: route53Zone, - name: domain, + name: legacyDomain, type: "A", aliases: [ { @@ -666,7 +673,12 @@ const apiEnvironment = [ }, { name: "CSRF_TRUSTED_ORIGINS", - value: JSON.stringify([`https://${domain}`]), + value: passportXyzDomainName.apply((passportXyzDomainNameStr) => + JSON.stringify([ + `https://${legacyDomain}`, + `https://${passportXyzDomainNameStr}`, + ]) + ), }, { name: "CERAMIC_CACHE_CACAO_VALIDATION_URL", @@ -693,12 +705,21 @@ const apiEnvironment = [ value: rescoreQueue.url, }, { + // TODO: geri We can probably remove this name: "UI_DOMAINS", - value: JSON.stringify(["scorer." + rootDomain, "www.scorer." + rootDomain]), + value: passportXyzDomainName.apply((passportXyzDomainNameStr) => + JSON.stringify([ + "scorer." + legacyDomain, + "www.scorer." + legacyDomain, + passportXyzDomainNameStr, + ]) + ), }, { name: "ALLOWED_HOSTS", - value: JSON.stringify([domain, "*"]), + value: passportXyzDomainName.apply((passportXyzDomainNameStr) => + JSON.stringify([legacyDomain, passportXyzDomainNameStr]) + ), }, { name: "VERIFIER_URL", @@ -1267,7 +1288,7 @@ const redashHttpsListener = HTTPS_ALB_CERT_ARN.apply( const redashRecord = new aws.route53.Record("redash", { zoneId: route53Zone, - name: "redash." + domain, + name: "redash." + legacyDomain, type: "A", aliases: [ { @@ -1916,3 +1937,47 @@ const amplifyAppInfo = coreInfraStack export const amplifyAppHookUrl = pulumi.secret(amplifyAppInfo.webHook.url); // createApiDomainRecord(stack, CLOUDFLARE_ZONE_ID, alb.dnsName); + +// import * as cloudflare from "@pulumi/cloudflare"; + +// const name = "api-passport-xyz-record"; +// const cloudflareApiRecord = +// stack === "production" +// ? new cloudflare.Record(name, { +// name: "api1", +// zoneId: CLOUDFLARE_ZONE_ID, +// type: "CNAME", +// value: alb.dnsName, +// allowOverwrite: true, +// comment: "Points to LB handling the backend service requests", +// }) +// : ""; + +// Create a DNS record for the load balancer +// pulumi.all([passportXyzDomainName]).apply((passportXyzDomainNameStr) => { +// const apiDomain = `api.${passportXyzDomainNameStr}`; +// const apiDomainRecord = new aws.route53.Record(apiDomain, { +// zoneId: passportXyzHostedZoneId, +// name: "api", +// type: "CNAME", +// ttl: 300, +// records: [alb.dnsName], +// }); +// }); + +const apiDomainRecord = new aws.route53.Record(`scorer-api`, { + zoneId: passportXyzHostedZoneId, + name: "api", + type: "CNAME", + ttl: 300, + records: [alb.dnsName], +}); + +const coreAlbPassportXyz = new aws.lb.ListenerCertificate( + "core-alb-passport-xyz", + { + listenerArn: httpsListener.arn, + certificateArn: passportXyzCertificateArn, + }, + {} +); diff --git a/infra/lib/cloudflare/index.ts b/infra/lib/cloudflare/index.ts index dacdb850..4ebc13a5 100644 --- a/infra/lib/cloudflare/index.ts +++ b/infra/lib/cloudflare/index.ts @@ -7,6 +7,10 @@ export function createApiDomainRecord( albDnsName: pulumi.Input ) { const name = "api-passport-xyz-record"; + + + // api.review.passport.xyz + const cloudflareApiRecord = stack === "production" ? new cloudflare.Record(name, { @@ -18,5 +22,8 @@ export function createApiDomainRecord( comment: "Points to LB handling the backend service requests", }) : ""; + + + return cloudflareApiRecord; } diff --git a/infra/lib/scorer/new_service.ts b/infra/lib/scorer/new_service.ts index 09378960..101e085c 100644 --- a/infra/lib/scorer/new_service.ts +++ b/infra/lib/scorer/new_service.ts @@ -45,7 +45,7 @@ export function createTargetGroup( vpcId: Input ): TargetGroup { return new TargetGroup(name, { - tags: { + tags: { ...defaultTags, Name: name }, port: 80, @@ -68,7 +68,7 @@ export function createScorerECSService({ environment: secretsManager.EnvironmentVar[]; secrets: pulumi.Output; loadBalancerAlarmThresholds: AlarmConfigurations; -}): awsx.ecs.FargateService { +}): awsx.ecs.FargateService | undefined { ////////////////////////////////////////////////////////////// // Create target group and load balancer rules ////////////////////////////////////////////////////////////// @@ -128,6 +128,7 @@ export function createScorerECSService({ }; const service = new awsx.ecs.FargateService(name, { + name: name, propagateTags: "TASK_DEFINITION", tags: { ...defaultTags, Name: name }, cluster: config.cluster.arn, @@ -144,7 +145,7 @@ export function createScorerECSService({ }, ], taskDefinitionArgs: { - tags: { name: name }, + tags: { ...defaultTags, Name: name }, logGroup: { existing: config.logGroup, }, @@ -914,7 +915,7 @@ export function buildHttpLambdaFn( function: lambdaFunction.name, principal: "elasticloadbalancing.amazonaws.com", sourceArn: lambdaTargetGroup.arn, - + }); const lambdaTargetGroupAttachment = new aws.lb.TargetGroupAttachment( @@ -922,7 +923,7 @@ export function buildHttpLambdaFn( { targetGroupArn: lambdaTargetGroup.arn, targetId: lambdaFunction.arn, - + }, { dependsOn: [withLb], From 6dabe1c19e3f3a02d02db12a38c1407432476dea Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Thu, 17 Oct 2024 18:53:06 +0300 Subject: [PATCH 4/5] feat: changes to host API on passport.xyz domain as well --- api/.env-sample | 1 - api/scorer/settings/base.py | 3 -- infra/aws/index.ts | 71 +++++++++++++------------------------ 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/api/.env-sample b/api/.env-sample index dc2fba15..b59d598b 100644 --- a/api/.env-sample +++ b/api/.env-sample @@ -8,7 +8,6 @@ DATA_MODEL_DATABASE_URL=sqlite:///db_data_model.sqlite3 # DATABASE_URL_FOR_DOCKER=postgres://passport_scorer:passport_scorer_pwd@postgres:5432/passport_scorer ALLOWED_HOSTS='[]' TEST_MNEMONIC=test val is here ... -UI_DOMAINS=[localhost:3000, www.localhost:3000] CELERY_BROKER_URL=redis://localhost:6379/0 diff --git a/api/scorer/settings/base.py b/api/scorer/settings/base.py index 55f51cce..234fc6e5 100644 --- a/api/scorer/settings/base.py +++ b/api/scorer/settings/base.py @@ -29,9 +29,6 @@ CERAMIC_CACHE_API_KEY = env("CERAMIC_CACHE_API_KEY", default="") -# TODO: geri is this actually used? -# UI_DOMAINS = env("UI_DOMAINS", default=["localhost:3000", "www.localhost:3000"]) - SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = env("GOOGLE_OAUTH_CLIENT_ID", default="") SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = env("GOOGLE_CLIENT_SECRET", default="") diff --git a/infra/aws/index.ts b/infra/aws/index.ts index 51d97448..55ca8b32 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -704,21 +704,11 @@ const apiEnvironment = [ name: "RESCORE_QUEUE_URL", value: rescoreQueue.url, }, - { - // TODO: geri We can probably remove this - name: "UI_DOMAINS", - value: passportXyzDomainName.apply((passportXyzDomainNameStr) => - JSON.stringify([ - "scorer." + legacyDomain, - "www.scorer." + legacyDomain, - passportXyzDomainNameStr, - ]) - ), - }, { name: "ALLOWED_HOSTS", value: passportXyzDomainName.apply((passportXyzDomainNameStr) => - JSON.stringify([legacyDomain, passportXyzDomainNameStr]) + // TODO: geri: investigate if using '*' here is a security risk + JSON.stringify([legacyDomain, passportXyzDomainNameStr, "*"]) ), }, { @@ -1936,41 +1926,30 @@ const amplifyAppInfo = coreInfraStack export const amplifyAppHookUrl = pulumi.secret(amplifyAppInfo.webHook.url); -// createApiDomainRecord(stack, CLOUDFLARE_ZONE_ID, alb.dnsName); - -// import * as cloudflare from "@pulumi/cloudflare"; - -// const name = "api-passport-xyz-record"; -// const cloudflareApiRecord = -// stack === "production" -// ? new cloudflare.Record(name, { -// name: "api1", -// zoneId: CLOUDFLARE_ZONE_ID, -// type: "CNAME", -// value: alb.dnsName, -// allowOverwrite: true, -// comment: "Points to LB handling the backend service requests", -// }) -// : ""; - // Create a DNS record for the load balancer -// pulumi.all([passportXyzDomainName]).apply((passportXyzDomainNameStr) => { -// const apiDomain = `api.${passportXyzDomainNameStr}`; -// const apiDomainRecord = new aws.route53.Record(apiDomain, { -// zoneId: passportXyzHostedZoneId, -// name: "api", -// type: "CNAME", -// ttl: 300, -// records: [alb.dnsName], -// }); -// }); - -const apiDomainRecord = new aws.route53.Record(`scorer-api`, { - zoneId: passportXyzHostedZoneId, - name: "api", - type: "CNAME", - ttl: 300, - records: [alb.dnsName], +pulumi.all([passportXyzDomainName]).apply((passportXyzDomainNameStr) => { + const apiDomain = `api.${passportXyzDomainNameStr}`; + const apiDomainRecord = new aws.route53.Record(apiDomain, { + zoneId: passportXyzHostedZoneId, + name: "api", + type: "CNAME", + ttl: 300, + records: [alb.dnsName], + }); + + const redashDomain = `redash.${passportXyzDomainNameStr}`; + const redashRecord = new aws.route53.Record(redashDomain, { + zoneId: passportXyzHostedZoneId, + name: "redash", + type: "A", + aliases: [ + { + name: redashAlb.dnsName, + zoneId: redashAlb.zoneId, + evaluateTargetHealth: true, + }, + ], + }); }); const coreAlbPassportXyz = new aws.lb.ListenerCertificate( From e745efa2d06c693d287c77968676779d0897f332 Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Fri, 18 Oct 2024 15:11:56 +0300 Subject: [PATCH 5/5] fix: removing unused file --- infra/lib/cloudflare/index.ts | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 infra/lib/cloudflare/index.ts diff --git a/infra/lib/cloudflare/index.ts b/infra/lib/cloudflare/index.ts deleted file mode 100644 index 4ebc13a5..00000000 --- a/infra/lib/cloudflare/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -import * as pulumi from "@pulumi/pulumi"; -import * as cloudflare from "@pulumi/cloudflare"; - -export function createApiDomainRecord( - stack: pulumi.Input, - cloudflareZoneId: pulumi.Input, - albDnsName: pulumi.Input -) { - const name = "api-passport-xyz-record"; - - - // api.review.passport.xyz - - const cloudflareApiRecord = - stack === "production" - ? new cloudflare.Record(name, { - name: "api", - zoneId: cloudflareZoneId, - type: "CNAME", - value: albDnsName, - allowOverwrite: true, - comment: "Points to LB handling the backend service requests", - }) - : ""; - - - - return cloudflareApiRecord; -}