From 1e6694ed06e48a118d93f2913ec8a336bcb60e72 Mon Sep 17 00:00:00 2001 From: schultztimothy Date: Mon, 24 Jul 2023 16:35:32 -0600 Subject: [PATCH] feat(api): weekly stamp data dump (#324) * feat(api): create scheduled task definition * chore(api): model updates for data dump * feat(api): data dump export script * fix(infra): fix staging deployment of scheduled task * fix(infra): fix review deployment of scheduled task * chore(api): update pipfile lock after resolving pipfile merge conflict * chore(infra): add aws key values * chore(api): further environment configuration for weekly data dumps * chore(infra): comment out scheduled task until script is merged * refactor(api): paginate stamp query and save count and last_export --------- Co-authored-by: Gerald Iakobinyi-Pich --- api/Pipfile.lock | 90 +++---------------- .../management/commands/__init__.py | 0 .../management/commands/dump_stamp_data.py | 59 ++++++++++++ .../migrations/0009_stampexports.py | 29 ++++++ .../0010_ceramiccache_created_at.py | 18 ++++ .../0011_alter_ceramiccache_created_at.py | 18 ++++ api/ceramic_cache/models.py | 6 ++ api/scorer/settings/s3.py | 1 + infra/prod/index.ts | 12 +++ infra/review/index.ts | 54 +++++++++++ infra/staging/index.ts | 45 ++++++++++ interface/.env.example | 1 + 12 files changed, 254 insertions(+), 79 deletions(-) create mode 100644 api/ceramic_cache/management/commands/__init__.py create mode 100644 api/ceramic_cache/management/commands/dump_stamp_data.py create mode 100644 api/ceramic_cache/migrations/0009_stampexports.py create mode 100644 api/ceramic_cache/migrations/0010_ceramiccache_created_at.py create mode 100644 api/ceramic_cache/migrations/0011_alter_ceramiccache_created_at.py diff --git a/api/Pipfile.lock b/api/Pipfile.lock index 91b1f3822..ce81e8ec9 100644 --- a/api/Pipfile.lock +++ b/api/Pipfile.lock @@ -832,7 +832,7 @@ "sha256:15f9870ca054c09a8e474d2d7e81aff0c32421aebdac896193183fc143e31b50", "sha256:fe738cdb24983adfe89abf727c723c288f8d0029e97fb08160b20bb5290ab475" ], - "markers": "python_version < '4' and python_full_version >= '3.7.2'", + "markers": "python_full_version >= '3.7.2' and python_version < '4'", "version": "==4.1.0" }, "eth-account": { @@ -881,7 +881,7 @@ "sha256:347d50713dd58ab50063b228d8271624ab2de3071bfa32d467b05f0ea31ab4c5", "sha256:7f49610469811ee97ac43eaf6baa294778ce74042d41e61ecf22e5ebe385590f" ], - "markers": "python_version < '4' and python_full_version >= '3.7.2'", + "markers": "python_full_version >= '3.7.2' and python_version < '4'", "version": "==3.4.0" }, "eth-utils": { @@ -892,14 +892,6 @@ "markers": "python_version >= '3.7' and python_version < '4'", "version": "==2.2.0" }, - "exceptiongroup": { - "hashes": [ - "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5", - "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f" - ], - "markers": "python_version < '3.11'", - "version": "==1.1.2" - }, "frozenlist": { "hashes": [ "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6", @@ -1484,9 +1476,6 @@ "version": "==1.10.11" }, "pyjwt": { - "extras": [ - "crypto" - ], "hashes": [ "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de", "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320" @@ -1494,39 +1483,6 @@ "markers": "python_version >= '3.7'", "version": "==2.8.0" }, - "pyrsistent": { - "hashes": [ - "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8", - "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440", - "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a", - "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c", - "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3", - "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393", - "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9", - "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da", - "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf", - "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64", - "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a", - "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3", - "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98", - "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2", - "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8", - "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf", - "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc", - "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7", - "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28", - "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2", - "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b", - "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a", - "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64", - "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19", - "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1", - "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9", - "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c" - ], - "markers": "python_version >= '3.7'", - "version": "==0.19.3" - }, "python-baseconv": { "hashes": [ "sha256:0539f8bd0464013b05ad62e0a1673f0ac9086c76b43ebf9f833053527cd9931b" @@ -1538,7 +1494,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.2" }, "python3-openid": { @@ -1851,14 +1807,6 @@ "markers": "python_version >= '3.7'", "version": "==0.6.1" }, - "setuptools": { - "hashes": [ - "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", - "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" - ], - "markers": "python_version >= '3.7'", - "version": "==68.0.0" - }, "siwe": { "hashes": [ "sha256:2a5f829ebd4d009d75958ae4a4a648dfaedb6081afed0d84d631f1acacb475e4", @@ -1872,7 +1820,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "sniffio": { @@ -2305,7 +2253,7 @@ "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e", "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03" ], - "markers": "python_version < '3.11'", + "markers": "python_version >= '3.11'", "version": "==0.3.7" }, "distlib": { @@ -2339,14 +2287,6 @@ "markers": "python_version >= '3.8'", "version": "==4.2.2" }, - "exceptiongroup": { - "hashes": [ - "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5", - "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f" - ], - "markers": "python_version < '3.11'", - "version": "==1.1.2" - }, "filelock": { "hashes": [ "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", @@ -2561,7 +2501,7 @@ "sha256:06d39a8b70fde873eb2a131141a0e79bb34a432941fb3d66fad247abafc9766c", "sha256:79b1f2497060d0928bc46016793f1fca1057c4aacdf15ef876aa48d75a73a355" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1'", "version": "==0.6.2" }, "pathspec": { @@ -2711,7 +2651,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "sqlparse": { @@ -2722,14 +2662,6 @@ "markers": "python_version >= '3.5'", "version": "==0.4.4" }, - "tomli": { - "hashes": [ - "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" - ], - "markers": "python_version < '3.11'", - "version": "==2.0.1" - }, "tomlkit": { "hashes": [ "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171", @@ -2762,11 +2694,11 @@ }, "virtualenv": { "hashes": [ - "sha256:01aacf8decd346cf9a865ae85c0cdc7f64c8caa07ff0d8b1dfc1733d10677442", - "sha256:2ef6a237c31629da6442b0bcaa3999748108c7166318d1f55cc9f8d7294e97bd" + "sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff", + "sha256:fd8a78f46f6b99a67b7ec5cf73f92357891a7b3a40fd97637c27f854aae3b9e0" ], "markers": "python_version >= '3.7'", - "version": "==20.24.1" + "version": "==20.24.2" }, "wrapt": { "hashes": [ @@ -2846,7 +2778,7 @@ "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559", "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639" ], - "markers": "python_version < '3.11'", + "markers": "python_version >= '3.11'", "version": "==1.15.0" } } diff --git a/api/ceramic_cache/management/commands/__init__.py b/api/ceramic_cache/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/ceramic_cache/management/commands/dump_stamp_data.py b/api/ceramic_cache/management/commands/dump_stamp_data.py new file mode 100644 index 000000000..3ca0197d0 --- /dev/null +++ b/api/ceramic_cache/management/commands/dump_stamp_data.py @@ -0,0 +1,59 @@ +import datetime +import json +import os + +import boto3 +from ceramic_cache.models import CeramicCache, StampExports +from django.conf import settings +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator +from django.utils import timezone + +s3 = boto3.client( + "s3", + aws_access_key_id=settings.S3_DATA_AWS_SECRET_KEY_ID, + aws_secret_access_key=settings.S3_DATA_AWS_SECRET_ACCESS_KEY, +) + + +class Command(BaseCommand): + help = "Weekly data dump of new Stamp data since the last dump." + + def handle(self, *args, **options): + print("Starting dump_stamp_data.py") + + latest_export = StampExports.objects.order_by("-last_export_ts").first() + + if not latest_export: + print("No previous exports found. Exporting all data.") + latest_export = StampExports.objects.create( + last_export_ts=timezone.now() - datetime.timedelta(days=7) + ) + + paginator = Paginator( + CeramicCache.objects.filter( + created_at__gt=latest_export.last_export_ts + ).values_list("stamp", flat=True), + 1000, + ) + + # Generate the dump file name + file_name = f'stamps_{latest_export.last_export_ts.strftime("%Y%m%d_%H%M%S")}_{timezone.now().strftime("%Y%m%d_%H%M%S")}.jsonl' + + # Write serialized data to the file + with open(file_name, "w") as f: + for page in paginator.page_range: + for stamp in paginator.page(page).object_list: + f.write(json.dumps({"stamp": stamp}) + "\n") + + # Upload to S3 bucket + s3.upload_file(file_name, settings.S3_WEEKLY_BACKUP_BUCKET_NAME, file_name) + + # Delete local file after upload + os.remove(file_name) + + StampExports.objects.create( + last_export_ts=timezone.now(), stamp_total=paginator.count + ) + + print(f"Data dump completed and uploaded to S3 as {file_name}") diff --git a/api/ceramic_cache/migrations/0009_stampexports.py b/api/ceramic_cache/migrations/0009_stampexports.py new file mode 100644 index 000000000..21a8bc8ef --- /dev/null +++ b/api/ceramic_cache/migrations/0009_stampexports.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.3 on 2023-07-21 22:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("ceramic_cache", "0008_remove_ceramiccache_deleted_at"), + ] + + operations = [ + migrations.CreateModel( + name="StampExports", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("last_export_ts", models.DateTimeField(auto_now_add=True)), + ("stamp_total", models.IntegerField(default=0)), + ], + ), + ] diff --git a/api/ceramic_cache/migrations/0010_ceramiccache_created_at.py b/api/ceramic_cache/migrations/0010_ceramiccache_created_at.py new file mode 100644 index 000000000..defc67223 --- /dev/null +++ b/api/ceramic_cache/migrations/0010_ceramiccache_created_at.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.3 on 2023-07-21 22:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("ceramic_cache", "0009_stampexports"), + ] + + operations = [ + migrations.AddField( + model_name="ceramiccache", + name="created_at", + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/api/ceramic_cache/migrations/0011_alter_ceramiccache_created_at.py b/api/ceramic_cache/migrations/0011_alter_ceramiccache_created_at.py new file mode 100644 index 000000000..dd518ebaa --- /dev/null +++ b/api/ceramic_cache/migrations/0011_alter_ceramiccache_created_at.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.3 on 2023-07-21 22:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("ceramic_cache", "0010_ceramiccache_created_at"), + ] + + operations = [ + migrations.AlterField( + model_name="ceramiccache", + name="created_at", + field=models.DateTimeField(auto_now_add=True, null=True), + ), + ] diff --git a/api/ceramic_cache/models.py b/api/ceramic_cache/models.py index 5d3836f43..6c3675d45 100644 --- a/api/ceramic_cache/models.py +++ b/api/ceramic_cache/models.py @@ -10,6 +10,12 @@ class CeramicCache(models.Model): null=False, blank=False, default="", max_length=256, db_index=True ) stamp = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) class Meta: unique_together = ["address", "provider"] + + +class StampExports(models.Model): + last_export_ts = models.DateTimeField(auto_now_add=True) + stamp_total = models.IntegerField(default=0) diff --git a/api/scorer/settings/s3.py b/api/scorer/settings/s3.py index 07df276fc..3b2effcdc 100644 --- a/api/scorer/settings/s3.py +++ b/api/scorer/settings/s3.py @@ -5,3 +5,4 @@ # data. Seet the `import_allo_votes` command for an example. S3_DATA_AWS_SECRET_KEY_ID = env("S3_DATA_AWS_SECRET_KEY_ID", default=None) S3_DATA_AWS_SECRET_ACCESS_KEY = env("S3_DATA_AWS_SECRET_ACCESS_KEY", default=None) +S3_WEEKLY_BACKUP_BUCKET_NAME = env("S3_WEEKLY_BACKUP_BUCKET_NAME", default=None) diff --git a/infra/prod/index.ts b/infra/prod/index.ts index 941d42dd5..d68416fe3 100644 --- a/infra/prod/index.ts +++ b/infra/prod/index.ts @@ -384,6 +384,18 @@ const secrets = [ name: "CGRANTS_API_TOKEN", valueFrom: `${SCORER_SERVER_SSM_ARN}:CGRANTS_API_TOKEN::`, }, + { + name: "S3_DATA_AWS_SECRET_KEY_ID", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_KEY_ID::`, + }, + { + name: "S3_DATA_AWS_SECRET_ACCESS_KEY", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_ACCESS_KEY::`, + }, + { + name: "S3_WEEKLY_BACKUP_BUCKET_NAME", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_WEEKLY_BACKUP_BUCKET_NAME::`, + }, ]; const environment = [ { diff --git a/infra/review/index.ts b/infra/review/index.ts index 468714dba..99bc4f188 100644 --- a/infra/review/index.ts +++ b/infra/review/index.ts @@ -312,6 +312,18 @@ const secrets = [ name: "CGRANTS_API_TOKEN", valueFrom: `${SCORER_SERVER_SSM_ARN}:CGRANTS_API_TOKEN::`, }, + { + name: "S3_DATA_AWS_SECRET_KEY_ID", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_KEY_ID::`, + }, + { + name: "S3_DATA_AWS_SECRET_ACCESS_KEY", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_ACCESS_KEY::`, + }, + { + name: "S3_WEEKLY_BACKUP_BUCKET_NAME", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_WEEKLY_BACKUP_BUCKET_NAME::`, + }, ]; const environment = [ { @@ -693,6 +705,48 @@ const secgrp = new aws.ec2.SecurityGroup(`scorer-run-migrations-task`, { export const securityGroupForTaskDefinition = secgrp.id; +////////////////////////////////////////////////////////////// +// ECS Scheduled Task +////////////////////////////////////////////////////////////// +// const weeklyDataDump = new awsx.ecs.FargateTaskDefinition("weekly-data-dump", { +// containers: { +// web: { +// image: dockerGtcPassportScorerImage, +// cpu: 256, +// memory: 2048, +// secrets, +// command: ["python", "manage.py", "dump_stamp_data"], +// }, +// }, +// }); + +// const scheduledEventRule = new aws.cloudwatch.EventRule("scheduledEventRule", { +// // scheduleExpression: "cron(0 12 * * ? *)", // Run the task every day at 12 UTC +// scheduleExpression: "cron(0/5 * ? * * *)", // Run the task every 5 min +// // scheduleExpression: "cron(0 12 ? * FRI *)", // Run the task every friday at 12 UTC +// }); + +// // const serviceLinkRoler = new aws.iam.ServiceLinkedRole("ecs_service_link_roler", { +// // customSuffix: "ecs_scheduled_event", +// // awsServiceName: "ecs.amazonaws.com", +// // }) + +// new aws.cloudwatch.EventTarget("scheduledEventTarget", { +// rule: scheduledEventRule.name, +// arn: cluster.cluster.arn, +// roleArn: dpoppEcsRole.arn, +// ecsTarget: { +// taskCount: 1, +// taskDefinitionArn: weeklyDataDump.taskDefinition.arn, +// launchType: "FARGATE", +// networkConfiguration: { +// assignPublicIp: true, +// subnets: vpcPublicSubnetIds, +// securityGroups: [secgrp.id], +// }, +// }, +// }); + ////////////////////////////////////////////////////////////// // Set up EC2 instance // - it is intended to be used for troubleshooting diff --git a/infra/staging/index.ts b/infra/staging/index.ts index 66fe55fc4..cc50e0af3 100644 --- a/infra/staging/index.ts +++ b/infra/staging/index.ts @@ -316,6 +316,18 @@ const secrets = [ name: "CGRANTS_API_TOKEN", valueFrom: `${SCORER_SERVER_SSM_ARN}:CGRANTS_API_TOKEN::`, }, + { + name: "S3_DATA_AWS_SECRET_KEY_ID", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_KEY_ID::`, + }, + { + name: "S3_DATA_AWS_SECRET_ACCESS_KEY", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_DATA_AWS_SECRET_ACCESS_KEY::`, + }, + { + name: "S3_WEEKLY_BACKUP_BUCKET_NAME", + valueFrom: `${SCORER_SERVER_SSM_ARN}:S3_WEEKLY_BACKUP_BUCKET_NAME::`, + }, ]; const environment = [ { @@ -748,6 +760,39 @@ const flower = new awsx.ecs.FargateService("flower", { }, }); +////////////////////////////////////////////////////////////// +// ECS Scheduled Task +////////////////////////////////////////////////////////////// +// const weeklyDataDump = new awsx.ecs.FargateTaskDefinition("weekly-data-dump", { +// containers: { +// web: { +// image: dockerGtcPassportScorerImage, +// cpu: 256, +// memory: 2048, +// secrets, +// command: ["python", "manage.py", "dump_stamp_data"], +// }, +// }, +// }); + +// const scheduledEventRule = new aws.cloudwatch.EventRule("scheduledEventRule", { +// scheduleExpression: "rate(1 minute)", // Run the task every day at 12pm. +// }); + +// new aws.cloudwatch.EventTarget("scheduledEventTarget", { +// rule: scheduledEventRule.name, +// arn: cluster.cluster.arn, +// ecsTarget: { +// taskCount: 1, +// taskDefinitionArn: weeklyDataDump.taskDefinition.arn, +// launchType: "FARGATE", +// networkConfiguration: { +// subnets: vpc.publicSubnetIds, +// securityGroups: [cluster.securityGroups[0].id], +// }, +// }, +// }); + ////////////////////////////////////////////////////////////// // Set up task to run migrations ////////////////////////////////////////////////////////////// diff --git a/interface/.env.example b/interface/.env.example index 96ab16d80..2671d2c10 100644 --- a/interface/.env.example +++ b/interface/.env.example @@ -19,3 +19,4 @@ NEXT_PUBLIC_WEB3_ONBOARD_EXPLORE_URL=http://localhost:3000/ # See the command `import_allo_votes` for example S3_DATA_AWS_SECRET_KEY_ID= S3_DATA_AWS_SECRET_ACCESS_KEY= +S3_WEEKLY_BACKUP_BUCKET_NAME=