Skip to content

Commit

Permalink
migrate-group: Flag un-migrated objects for manual review
Browse files Browse the repository at this point in the history
I'd been doing this manually while reviewing the previous sync logs, as
before the layout change the sync just migrated everything even though
some objects would be no longer accessible.  Nicer to have it built-in
as part of the rest of the manual steps checklist.
  • Loading branch information
tsibley committed Jun 22, 2022
1 parent 2eca230 commit 7753716
Showing 1 changed file with 20 additions and 0 deletions.
20 changes: 20 additions & 0 deletions scripts/migrate-group
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,17 @@ async function syncData({dryRun = true, group}) {
]
});

// Discover files to consider for manual review
const unsynced = (await s3ListObjects({group})).filter(
key => !key.endsWith(".json")
&& !key.endsWith(".md")
&& key !== "group-overview.md"
&& key !== "group-logo.png"
);

console.groupEnd();

return unsynced.map(key => `Investigate unsynced object s3://${group.bucket}/${key}`);
}


Expand All @@ -148,6 +158,16 @@ async function s3Sync({dryRun = true, group, prefix = "", filters = []}) {
}


async function s3ListObjects({group}) {
const client = new S3.S3Client();

return await collate(
S3.paginateListObjectsV2({client}, {Bucket: group.bucket}),
page => page.Contents.map(object => object.Key),
);
}


async function updateServerPolicies({dryRun = true, oldBucket}) {
const policyFiles = [
"aws/iam/policy/NextstrainDotOrgServerInstance.json",
Expand Down

0 comments on commit 7753716

Please sign in to comment.