From 29bde6516d42364c712863f099501223b81eb05c Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Tue, 25 Feb 2025 14:09:11 -0600 Subject: [PATCH 01/22] adds batch command for quick generation en mass Signed-off-by: Stephen Rugh --- src/commands/commerce/batch.js | 44 ++++++++++++++++++++++++++++++++++ src/commands/commerce/init.js | 1 + src/utils/content.js | 22 +++++++++++++++-- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 src/commands/commerce/batch.js diff --git a/src/commands/commerce/batch.js b/src/commands/commerce/batch.js new file mode 100644 index 0000000..01f3dd1 --- /dev/null +++ b/src/commands/commerce/batch.js @@ -0,0 +1,44 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ +import { Args, Command, Flags } from '@oclif/core' +import { runCommand } from '../../utils/runCommand.js' + +// TODO: allow interactivity, such as to "accept" that you installed the sync bot +// for adobe-summit-l321/322, wont be an issue since the bot is already installed. +// TODO populate with datasources once generated, or allow passing a file/csv +export class BatchCommand extends Command { + async run () { + const { args, flags } = await this.parse(BatchCommand) + const datasources = [ + 'https://na1-sandbox.api.commerce.adobe.com/S8LHdXGTfuMfrg6rb7Q5Mm/graphql' + ] + + for await (const [index, datasource] of datasources.entries()) { + const repo = `${flags.repo}-${(index + 1)}` + await this.config.runCommand('commerce:init', [ + `--template=${flags.template}`, + `--repo=${repo}`, + `--datasource=${datasource}`, + '--skipMesh' + ]) + console.log(`Created storefront: ${repo}`) + } + } +} + +BatchCommand.description = 'Uses the init tool in bulk' + +BatchCommand.flags = { + // datasource: Flags.string({ char: 'd', description: 'either a datasource url or a file with urls separated by newlines' }), + template: Flags.string({ char: 't', description: 'the template to use for the storefronts, ie adobe-commerce/ccdm-demo-store" ' }), + repo: Flags.string({ char: 'r', description: 'the repository name prefix, which will have a count appended to it, ie adobe-commerce-L321/seat' }) +} diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index ac17d6c..fe55273 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -35,6 +35,7 @@ export class InitCommand extends Command { config.set('commerce.template.repo', flags.template.split('/')[1]) config.set('commerce.datasource.paas', flags.datasource) config.set('commerce.datasource.catalog', flags.datasource) + aioLogger.debug(config.get('commerce')) } else { await initialization(args, flags) } diff --git a/src/utils/content.js b/src/utils/content.js index d92b65a..f1c2e78 100644 --- a/src/utils/content.js +++ b/src/utils/content.js @@ -30,8 +30,26 @@ export async function uploadStarterContent () { async function getBulkStatusUrl () { const templateOrg = config.get('commerce.template.org') const templateRepo = config.get('commerce.template.repo') - const { stdout: response } = await runCommand(`curl --data '{ "paths": ["/*"] }' --header "Content-Type: application/json" 'https://admin.hlx.page/status/${templateOrg}/${templateRepo}/main/*'`) - return JSON.parse(response).links.self + '/details' + let res + try { + res = await fetch(`https://admin.hlx.page/status/${templateOrg}/${templateRepo}/main/*`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + paths: ['/*'] + }) + }) + const data = await res.json() + aioLogger.debug(data) + return `${data.links.self}/details` + } catch (e) { + aioLogger.debug(res) + aioLogger.debug(e) + console.log('Failed to fetch status URL!') + throw e + } } /** From 52c9a807c43a6a16ae2d52a5934b87e7e816de54 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Tue, 25 Feb 2025 16:32:26 -0600 Subject: [PATCH 02/22] add notes about rate limits Signed-off-by: Stephen Rugh --- src/commands/commerce/batch.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/commands/commerce/batch.js b/src/commands/commerce/batch.js index 01f3dd1..a52d47f 100644 --- a/src/commands/commerce/batch.js +++ b/src/commands/commerce/batch.js @@ -18,10 +18,20 @@ import { runCommand } from '../../utils/runCommand.js' export class BatchCommand extends Command { async run () { const { args, flags } = await this.parse(BatchCommand) - const datasources = [ - 'https://na1-sandbox.api.commerce.adobe.com/S8LHdXGTfuMfrg6rb7Q5Mm/graphql' - ] + // TODO: fake datasources - 15 endpoints to use... + const datasources = Array(10).fill('https://na1-sandbox.api.commerce.adobe.com/S8LHdXGTfuMfrg6rb7Q5Mm/graphql') + // TODO: rate limit reset for gh user will be ~ same time as bot reset. so we can use this to delay the bot. + // gh api \ 2566ms  Tue Feb 25 16:20:40 2025 + // -H "Accept: application/vnd.github+json" \ + // -H "X-GitHub-Api-Version: 2022-11-28" \ + // /rate_limit | jq .rate + // { + // "limit": 5000, + // "used": 37, + // "remaining": 4963, + // "reset": 1740523308 + // } for await (const [index, datasource] of datasources.entries()) { const repo = `${flags.repo}-${(index + 1)}` await this.config.runCommand('commerce:init', [ From 7b342c259800feaec2bd8742fb082f0b87c05ea7 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Wed, 26 Feb 2025 13:54:19 -0600 Subject: [PATCH 03/22] allow skipping git org creation Signed-off-by: Stephen Rugh --- src/commands/commerce/batch.js | 19 ++++++++++++------- src/commands/commerce/init.js | 32 ++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/commands/commerce/batch.js b/src/commands/commerce/batch.js index a52d47f..b3393f1 100644 --- a/src/commands/commerce/batch.js +++ b/src/commands/commerce/batch.js @@ -34,13 +34,18 @@ export class BatchCommand extends Command { // } for await (const [index, datasource] of datasources.entries()) { const repo = `${flags.repo}-${(index + 1)}` - await this.config.runCommand('commerce:init', [ - `--template=${flags.template}`, - `--repo=${repo}`, - `--datasource=${datasource}`, - '--skipMesh' - ]) - console.log(`Created storefront: ${repo}`) + try { + await this.config.runCommand('commerce:init', [ + `--template=${flags.template}`, + `--repo=${repo}`, + `--datasource=${datasource}`, + '--skipMesh' + ]) + console.log(`Created storefront: ${repo}`) + } catch (e) { + console.error(`! Failed to complete run for ${repo}`) + console.error(e) + } } } } diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index fe55273..70b1bc9 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -62,22 +62,26 @@ export class InitCommand extends Command { console.log('Not creating API Mesh - will use demo environment.') } - await createRepo() - await modifyFstab() - await modifySidekickConfig() - - if (githubOrg === 'adobe-summit-L322' || githubOrg === 'adobe-summit-L321') { - console.log('✅ AEM Code Sync Bot automatically installed :)') + if (flags.skipGit) { + console.log(`Not creating Git Repos - assuming it already exists at https://github.com/${flags.repo}`) } else { - openBrowser('https://github.com/apps/aem-code-sync/installations/select_target') - const res = await promptConfirm('Did you install the AEM Code Sync bot?') - if (!res) { - throw new Error('❌ You must install the AEM Code Sync bot before continuing. Install before running the command again. https://github.com/apps/aem-code-sync/installations/select_target') + await createRepo() + await modifyFstab() + await modifySidekickConfig() + + if (githubOrg === 'adobe-summit-L322' || githubOrg === 'adobe-summit-L321') { + console.log('✅ AEM Code Sync Bot automatically installed :)') + } else { + openBrowser('https://github.com/apps/aem-code-sync/installations/select_target') + const res = await promptConfirm('Did you install the AEM Code Sync bot?') + if (!res) { + throw new Error('❌ You must install the AEM Code Sync bot before continuing. Install before running the command again. https://github.com/apps/aem-code-sync/installations/select_target') + } } + console.log('⏳ Validating code sync...') + await codeSyncComplete() } const filePaths = await uploadStarterContent() - console.log('⏳ Validating code sync...') - await codeSyncComplete() console.log('⏳ Previewing your content files...') await previewContent(filePaths) console.log('⏳ Publishing your content files...') @@ -136,6 +140,10 @@ InitCommand.flags = { char: 'r', description: 'your github repo to create, ie my-git-user/my-site"' }), + skipGit: Flags.boolean({ + default: false, + description: 'Skip creating Git Repo. Assumes you already created at --repo' + }), skipMesh: Flags.boolean({ default: false, description: 'Skip creating API Mesh' From 7456c5e7775466bae9b6b021738c5a4780fcc83d Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 13:10:27 -0600 Subject: [PATCH 04/22] checkSync automation. create-repos script working. --skipGit flag working --- package.json | 3 +- src/commands/commerce/batch.js | 59 --------- src/commands/commerce/create-repos.js | 53 ++++++++ src/commands/commerce/init.js | 2 +- src/scripts/checkSync.js | 183 ++++++++++++++++++++++++++ src/utils/initialization.js | 28 ++-- 6 files changed, 258 insertions(+), 70 deletions(-) delete mode 100644 src/commands/commerce/batch.js create mode 100644 src/commands/commerce/create-repos.js create mode 100644 src/scripts/checkSync.js diff --git a/package.json b/package.json index 8633992..028c07d 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,8 @@ "prepack": "oclif manifest && oclif readme", "postpack": "rm -f oclif.manifest.json", "version": "oclif readme && git add README.md", - "e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --collectCoverage=false --testRegex 'e2e/e2e.js'" + "e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --collectCoverage=false --testRegex 'e2e/e2e.js'", + "checkSync": "node src/scripts/checkSync.js" }, "jest": { "collectCoverage": true, diff --git a/src/commands/commerce/batch.js b/src/commands/commerce/batch.js deleted file mode 100644 index b3393f1..0000000 --- a/src/commands/commerce/batch.js +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2024 Adobe. All rights reserved. -This file is licensed to you under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. You may obtain a copy -of the License at http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under -the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS -OF ANY KIND, either express or implied. See the License for the specific language -governing permissions and limitations under the License. -*/ -import { Args, Command, Flags } from '@oclif/core' -import { runCommand } from '../../utils/runCommand.js' - -// TODO: allow interactivity, such as to "accept" that you installed the sync bot -// for adobe-summit-l321/322, wont be an issue since the bot is already installed. -// TODO populate with datasources once generated, or allow passing a file/csv -export class BatchCommand extends Command { - async run () { - const { args, flags } = await this.parse(BatchCommand) - // TODO: fake datasources - 15 endpoints to use... - const datasources = Array(10).fill('https://na1-sandbox.api.commerce.adobe.com/S8LHdXGTfuMfrg6rb7Q5Mm/graphql') - - // TODO: rate limit reset for gh user will be ~ same time as bot reset. so we can use this to delay the bot. - // gh api \ 2566ms  Tue Feb 25 16:20:40 2025 - // -H "Accept: application/vnd.github+json" \ - // -H "X-GitHub-Api-Version: 2022-11-28" \ - // /rate_limit | jq .rate - // { - // "limit": 5000, - // "used": 37, - // "remaining": 4963, - // "reset": 1740523308 - // } - for await (const [index, datasource] of datasources.entries()) { - const repo = `${flags.repo}-${(index + 1)}` - try { - await this.config.runCommand('commerce:init', [ - `--template=${flags.template}`, - `--repo=${repo}`, - `--datasource=${datasource}`, - '--skipMesh' - ]) - console.log(`Created storefront: ${repo}`) - } catch (e) { - console.error(`! Failed to complete run for ${repo}`) - console.error(e) - } - } - } -} - -BatchCommand.description = 'Uses the init tool in bulk' - -BatchCommand.flags = { - // datasource: Flags.string({ char: 'd', description: 'either a datasource url or a file with urls separated by newlines' }), - template: Flags.string({ char: 't', description: 'the template to use for the storefronts, ie adobe-commerce/ccdm-demo-store" ' }), - repo: Flags.string({ char: 'r', description: 'the repository name prefix, which will have a count appended to it, ie adobe-commerce-L321/seat' }) -} diff --git a/src/commands/commerce/create-repos.js b/src/commands/commerce/create-repos.js new file mode 100644 index 0000000..e2016fd --- /dev/null +++ b/src/commands/commerce/create-repos.js @@ -0,0 +1,53 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ +import { Command } from '@oclif/core' +import { createRepo, modifyFstab, modifySidekickConfig } from '../../utils/github.js' +import config from '@adobe/aio-lib-core-config' +/** + * Clones github repos in preparation of content and mesh provisioning with aio commerce init --skipGit + */ +export class CreateRepos extends Command { + async run () { + // const { args, flags } = await this.parse(CreateRepos) + // const start = 0 // n-1 + // const count = 100 + const start = 16 // n + const end = 100 + const owner = 'adobe-summit-L322' // adobe-summit-L321 + const repoPrefix = 'seat' + const templateOrg = 'adobe-commerce' + const templateRepo = 'adobe-demo-store' + + for (let i = start; i <= end; i++) { + const repo = `${repoPrefix}-${i}` + config.set('commerce.github.org', owner) + config.set('commerce.github.repo', repo) + config.set('commerce.template.org', templateOrg) + config.set('commerce.template.repo', templateRepo) + try { + await createRepo() + await modifyFstab() + await modifySidekickConfig() + } catch (e) { + console.error(`! Failed to complete run for "${repo}". Skipping.`) + console.error(e) + } + } + } +} + +CreateRepos.description = 'Uses the init tool in bulk' + +CreateRepos.flags = { + // template: Flags.string({ char: 't', description: 'the template to use for the storefronts, ie adobe-commerce/ccdm-demo-store" ' }), + // repo: Flags.string({ char: 'r', description: 'the repository name prefix, which will have a count appended to it, ie adobe-commerce-L321/seat' }) +} diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index 70b1bc9..876438b 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -63,7 +63,7 @@ export class InitCommand extends Command { } if (flags.skipGit) { - console.log(`Not creating Git Repos - assuming it already exists at https://github.com/${flags.repo}`) + console.log(`Not creating Git Repos - assuming it already exists at https://github.com/${githubOrg}/${githubRepo}`) } else { await createRepo() await modifyFstab() diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js new file mode 100644 index 0000000..2bff5d1 --- /dev/null +++ b/src/scripts/checkSync.js @@ -0,0 +1,183 @@ +import fs from 'fs' + +const PROGRESS_FILE = 'progress.json' +const BASE_URL = 'https://main--seat-#--adobe-summit-l322.aem.page/scripts/scripts.js' +const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-#/main/*' +const TOTAL_REPOS = 100 +const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour +// the amount of git files we expect the code sync to have to have processed. +// for L322, it seems to be 418. For L321, seems around 411. +const GIT_FILES_TO_CHECK = 418 + +/** + * + */ +function loadProgress () { + try { + if (fs.existsSync(PROGRESS_FILE)) { + const data = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8')) + return { + lastChecked: data.lastChecked || 1, + restartAfter: data.restartAfter || 0 + } + } + } catch (error) { + console.error('Error loading progress:', error) + } + return { lastChecked: 1, restartAfter: 0 } // Default values +} + +// Save progress to file +/** + * + * @param index + * @param restartAfter + */ +function saveProgress (index, restartAfter = 0) { + try { + fs.writeFileSync(PROGRESS_FILE, JSON.stringify({ lastChecked: index, restartAfter })) + } catch (error) { + console.error('Error saving progress:', error) + } +} + +// Function to check a single URL +/** + * + * @param index + */ +async function checkURL (index) { + const url = BASE_URL.replace('#', index) + console.log(`🔎 Checking: ${url}`) + + try { + const response = await fetch(url) + if (response.status === 404) { + console.log(`❌ ${url} not found (404).`) + + try { + await triggerCodeSync(index) + console.log(`✅ Code sync completed for seat-${index}. Proceeding to the next index.`) + return index + 1 // Move to the next repo + } catch (error) { + console.error(`🚨 Error during code sync for seat-${index}:`, error) + return index - 1 // Restart from the previous repo + } + } else { + console.log(`✅ ${url} is available.`) + return index + 1 // Move to the next repo + } + } catch (error) { + console.error(`🚨 Error checking ${url}:`, error) + return index - 1 // Restart from the previous repo + } +} + +/** + * + * @param index + */ +async function saveAndExit (index) { + // Determine restart time in CST + const restartTimestamp = Date.now() + WAIT_TIME_MS + const restartTime = new Date(restartTimestamp).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) + const nextIndex = index - 1 > 0 ? index - 1 : 1 + + console.log(`🚨 Script exiting. Restart at seat-${nextIndex} after ~1 hour at: ${restartTime} CST`) + + saveProgress(nextIndex, restartTimestamp) + process.exit(1) +} +// Function to trigger code sync requests for failing repo (n), n-1, and n+1 +/** + * + * @param index + * @param seat + */ +async function triggerCodeSync (seat) { + const postUrl = POST_URL.replace('#', seat) + console.log(`🔄 Calling Helix Admin Code Sync at: ${postUrl}`) + + try { + const postResponse = await fetch(postUrl, { method: 'POST' }) + + if (postResponse.status !== 202) { + console.error(`No code found for seat-${seat}. Ensure repo exists.`) + saveAndExit(seat) + process.exit(1) + } + const postData = await postResponse.json() + const detailsUrl = `${postData.links.self}/details` + console.log(`🔍 Checking details at: ${detailsUrl}`) + + await checkPhaseCompletion(detailsUrl, seat) + } catch (error) { + console.error(`Error processing seat-${seat}:`, error) + throw error + } +} + +/** + * + * @param detailsUrl + * @param seat + * @param maxRetries + * @param interval + */ +async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval = 1000) { + let attempts = 0 + + while (attempts < maxRetries) { + try { + const detailsResponse = await fetch(detailsUrl) + const detailsData = await detailsResponse.json() + if (detailsData.state === 'stopped') { + if (detailsData.progress && detailsData?.progress.processed >= GIT_FILES_TO_CHECK) { + console.log(`✅ Seat-${seat} processing completed.`) + return + } else if (detailsData.error.includes('rate limit exceeded')) { + console.log(`⚠️ Seat-${seat} processing rate limit exceeded.`) + saveAndExit(seat) + } else { + console.log('Unknown state! Data:', detailsData) + saveAndExit(seat) + } + } else { + console.log(`⏳ Seat-${seat} still processing... retrying (${attempts + 1}/${maxRetries}) in ${(interval / 1000)}s.`) + } + } catch (error) { + console.error(`Error fetching details for seat-${seat}:`, error) + } + + attempts++ + await new Promise(resolve => setTimeout(resolve, interval)) // Wait 1s before retrying + } + + await saveAndExit(seat) +} + +// Main process loop +/** + * + */ +async function runChecks () { + const { lastChecked, restartAfter } = loadProgress() + + // Check if we are running before the allowed restart time + if (Date.now() < restartAfter) { + const restartTime = new Date(restartAfter).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) + console.error(`❌ Cannot start yet. Please wait until: ${restartTime} CST.`) + process.exit(1) + } + + let index = lastChecked + while (index <= TOTAL_REPOS) { + index = await checkURL(index) + if (index < 1) index = 1 // Prevent negative index + saveProgress(index) + } + console.log('✅ All repos checked successfully.') +} + +// Start process +runChecks() diff --git a/src/utils/initialization.js b/src/utils/initialization.js index 69e814e..7d7210c 100644 --- a/src/utils/initialization.js +++ b/src/utils/initialization.js @@ -25,17 +25,27 @@ export async function initialization (args, flags) { 'This tool aims to automate the GitHub repository creation, the content source uploading, and the initial content preview.\nIn just a few minutes, you\'ll have your very own storefront codebase as well as an Edge Delivery Services content space ready to go.\nLet\'s get started!') // GITHUB DESTINATION SELECTION - let { repo, template } = flags - let { stdout: org } = await runCommand('gh api user --jq .login') - if (!org) { - throw new Error('❌ Unable to get github username. Please authenticate first with `gh auth login`".') + let { repo, template, skipGit } = flags + let org + if (!skipGit) { + try { + const { stdout } = await runCommand('gh api user --jq .login') + org = stdout + } catch (e) { + aioLogger.debug(e) + } + if (!org) { + throw new Error('❌ Unable to get github username. Please authenticate first with `gh auth login`".') + } } - const answeredYes = await promptConfirm(`Would you like to create the code repository under your Github username, "${org.trim()}"?`) - if (!answeredYes) { - org = await promptInput('Enter the GitHub organization under which to create the code repository') + let answer + if (org) { + answer = await promptConfirm(`Would you like to create the code and content under your github username, "${org.trim()}"?`) } - - repo = repo?.split('/')[1] || await promptInput('Enter the GitHub storefront repo name to create (must not exist already):') + if (!answer) { + org = await promptInput('Enter the organization under which to create the code and content') + } + repo = repo?.split('/')[1] || await promptInput('Enter the storefront name to create (must not exist already):') if (!org || !repo) { throw new Error('❌ Please provide both the github org/name and repo.') From 883b753e1f383f63add268a191be946b409d4784 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 13:13:29 -0600 Subject: [PATCH 05/22] f Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 2bff5d1..419cb09 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -43,7 +43,8 @@ function saveProgress (index, restartAfter = 0) { // Function to check a single URL /** - * + * Checks that a given seat has scripts/scripts.js file. This is fragile. + * TODO: improve check to, somehow, validate that the expected number of files has been sync'd by Helix Bot * @param index */ async function checkURL (index) { From e87bbba2c1ee7fa83ceb27ecacbe4d49afca515b Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 14:09:58 -0600 Subject: [PATCH 06/22] adds deleterepos script for easy deletion assuming token with permissions. moves fstab and sidekick config into createRepos function Signed-off-by: Stephen Rugh --- src/commands/commerce/create-repos.js | 9 ++-- src/commands/commerce/init.js | 2 - src/scripts/deleteRepos.js | 62 +++++++++++++++++++++++++++ src/utils/github.js | 5 ++- src/utils/initialization.js | 2 +- 5 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 src/scripts/deleteRepos.js diff --git a/src/commands/commerce/create-repos.js b/src/commands/commerce/create-repos.js index e2016fd..ed775b0 100644 --- a/src/commands/commerce/create-repos.js +++ b/src/commands/commerce/create-repos.js @@ -10,17 +10,16 @@ OF ANY KIND, either express or implied. See the License for the specific languag governing permissions and limitations under the License. */ import { Command } from '@oclif/core' -import { createRepo, modifyFstab, modifySidekickConfig } from '../../utils/github.js' +import { createRepo } from '../../utils/github.js' import config from '@adobe/aio-lib-core-config' /** * Clones github repos in preparation of content and mesh provisioning with aio commerce init --skipGit */ export class CreateRepos extends Command { async run () { + console.log('Starting to create repos...') // const { args, flags } = await this.parse(CreateRepos) - // const start = 0 // n-1 - // const count = 100 - const start = 16 // n + const start = 1 const end = 100 const owner = 'adobe-summit-L322' // adobe-summit-L321 const repoPrefix = 'seat' @@ -35,8 +34,6 @@ export class CreateRepos extends Command { config.set('commerce.template.repo', templateRepo) try { await createRepo() - await modifyFstab() - await modifySidekickConfig() } catch (e) { console.error(`! Failed to complete run for "${repo}". Skipping.`) console.error(e) diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index 876438b..2b16da5 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -66,8 +66,6 @@ export class InitCommand extends Command { console.log(`Not creating Git Repos - assuming it already exists at https://github.com/${githubOrg}/${githubRepo}`) } else { await createRepo() - await modifyFstab() - await modifySidekickConfig() if (githubOrg === 'adobe-summit-L322' || githubOrg === 'adobe-summit-L321') { console.log('✅ AEM Code Sync Bot automatically installed :)') diff --git a/src/scripts/deleteRepos.js b/src/scripts/deleteRepos.js new file mode 100644 index 0000000..daa2377 --- /dev/null +++ b/src/scripts/deleteRepos.js @@ -0,0 +1,62 @@ +import childProcess from 'child_process' + +/** + * + * @param cmd + */ +function runCommand (cmd) { + return new Promise((resolve, reject) => { + childProcess.exec(cmd, (error, stdout, stderr) => { + if (error) { + console.error(stderr) + resolve(false) + } else { + console.log(stdout) + resolve(true) + } + }) + }) +} + +/** + * + * @param start + * @param end + */ +function deleteRepos (start, end) { + for (let seat = start; seat <= end; seat++) { + const repoName = `adobe-summit-L322/seat-${seat}` + const command = `gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${repoName}` + + runCommand(command) + .then((success) => { + if (success) { + console.log(`Repo at seat-${seat} deleted successfully`) + } else { + console.error(`Failed to delete repo at seat-${seat}`) + } + }) + .catch((error) => { + console.error(`Error occurred while running command for seat-${seat}: ${error}`) + }) + } +} + +if (process.argv.length !== 4) { + console.log('Usage: node delete-repos.js ') +} else { + const start = parseInt(process.argv[2]) + const end = parseInt(process.argv[3]) + + if (!Number.isInteger(start) || !Number.isInteger(end)) { + console.error('Start and End must be integers') + } else if (start > end) { + console.log('Start cannot be greater than End.') + } else { + deleteRepos(start, end) + } +} diff --git a/src/utils/github.js b/src/utils/github.js index 0af5632..f25006c 100644 --- a/src/utils/github.js +++ b/src/utils/github.js @@ -26,12 +26,13 @@ export async function createRepo () { // console.error(`❌ Exiting. Cannot create repository that already exists: ${githubOrg}/${githubRepo}`) // process.exit(1) // } - console.error(`❌ Exiting. Cannot create repository that already exists: ${githubOrg}/${githubRepo}`) - process.exit(1) + console.error(`❌ Skipping. Cannot create repository that already exists: ${githubOrg}/${githubRepo}`) } else { // If the repository does not exist, proceed with "create" await runCommand(`gh repo create ${githubOrg}/${githubRepo} --template ${templateOrg}/${templateRepo} --public`) console.log(`✅ Created code repository at https://github.com/${githubOrg}/${githubRepo} from template ${templateOrg}/${templateRepo}`) + await modifyFstab() + await modifySidekickConfig() } } diff --git a/src/utils/initialization.js b/src/utils/initialization.js index 7d7210c..df31570 100644 --- a/src/utils/initialization.js +++ b/src/utils/initialization.js @@ -43,7 +43,7 @@ export async function initialization (args, flags) { answer = await promptConfirm(`Would you like to create the code and content under your github username, "${org.trim()}"?`) } if (!answer) { - org = await promptInput('Enter the organization under which to create the code and content') + org = await promptInput('Enter the organization under which to create the code and content:') } repo = repo?.split('/')[1] || await promptInput('Enter the storefront name to create (must not exist already):') From 5ba29184f08102f86a0159fe5a73044c74e2ca42 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 14:14:40 -0600 Subject: [PATCH 07/22] f Signed-off-by: Stephen Rugh --- src/commands/commerce/init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index 2b16da5..2e71133 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -15,7 +15,7 @@ import { uploadStarterContent } from '../../utils/content.js' import { previewContent, publishContent } from '../../utils/preview.js' import { promptConfirm } from '../../utils/prompt.js' import config from '@adobe/aio-lib-core-config' -import { codeSyncComplete, createRepo, modifyFstab, modifySidekickConfig } from '../../utils/github.js' +import { codeSyncComplete, createRepo } from '../../utils/github.js' import { initialization } from '../../utils/initialization.js' import { createMesh, getMeshDetailsPage } from '../../utils/mesh.js' import Logger from '@adobe/aio-lib-core-logging' From 6ca29dc1d57f57e64e8161f318ab2f6a59b82903 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 14:28:44 -0600 Subject: [PATCH 08/22] remove check for specific file and instead just trigger a code sync job, which will inform if rate limit is being hit. Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 419cb09..4d74390 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -41,35 +41,18 @@ function saveProgress (index, restartAfter = 0) { } } -// Function to check a single URL /** - * Checks that a given seat has scripts/scripts.js file. This is fragile. - * TODO: improve check to, somehow, validate that the expected number of files has been sync'd by Helix Bot - * @param index + * Checks that the code sync for a specific seat is complete. + * If not, it triggers the code sync and waits until it's done. + * @param index The index of the seat to check. */ -async function checkURL (index) { - const url = BASE_URL.replace('#', index) - console.log(`🔎 Checking: ${url}`) - +async function checkRepo (index) { try { - const response = await fetch(url) - if (response.status === 404) { - console.log(`❌ ${url} not found (404).`) - - try { - await triggerCodeSync(index) - console.log(`✅ Code sync completed for seat-${index}. Proceeding to the next index.`) - return index + 1 // Move to the next repo - } catch (error) { - console.error(`🚨 Error during code sync for seat-${index}:`, error) - return index - 1 // Restart from the previous repo - } - } else { - console.log(`✅ ${url} is available.`) - return index + 1 // Move to the next repo - } + await triggerCodeSync(index) + console.log(`✅ Code sync completed for seat-${index}. Proceeding to the next repo.`) + return index + 1 // Move to the next repo } catch (error) { - console.error(`🚨 Error checking ${url}:`, error) + console.error(`🚨 Error during code sync for seat-${index}:`, error) return index - 1 // Restart from the previous repo } } @@ -173,7 +156,7 @@ async function runChecks () { let index = lastChecked while (index <= TOTAL_REPOS) { - index = await checkURL(index) + index = await checkRepo(index) if (index < 1) index = 1 // Prevent negative index saveProgress(index) } From f0ea53442793996442cbe6c60c3a643eafa965e4 Mon Sep 17 00:00:00 2001 From: Revanth Kumar Annavarapu <35203638+revanth0212@users.noreply.github.com> Date: Thu, 27 Feb 2025 16:15:23 -0600 Subject: [PATCH 09/22] Converted createRepos from command to script (#17) --- package.json | 3 +- src/commands/commerce/create-repos.js | 50 --------------------------- src/commands/commerce/init.js | 3 +- src/scripts/checkSync.js | 5 +++ src/scripts/createRepos.js | 36 +++++++++++++++++++ src/scripts/deleteRepos.js | 5 +++ src/utils/github.js | 39 +++++++++++---------- 7 files changed, 71 insertions(+), 70 deletions(-) delete mode 100644 src/commands/commerce/create-repos.js create mode 100644 src/scripts/createRepos.js diff --git a/package.json b/package.json index 028c07d..996b001 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,8 @@ "postpack": "rm -f oclif.manifest.json", "version": "oclif readme && git add README.md", "e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --collectCoverage=false --testRegex 'e2e/e2e.js'", - "checkSync": "node src/scripts/checkSync.js" + "checkSync": "node src/scripts/checkSync.js", + "createRepos": "node src/scripts/createRepos.js" }, "jest": { "collectCoverage": true, diff --git a/src/commands/commerce/create-repos.js b/src/commands/commerce/create-repos.js deleted file mode 100644 index ed775b0..0000000 --- a/src/commands/commerce/create-repos.js +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2024 Adobe. All rights reserved. -This file is licensed to you under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. You may obtain a copy -of the License at http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under -the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS -OF ANY KIND, either express or implied. See the License for the specific language -governing permissions and limitations under the License. -*/ -import { Command } from '@oclif/core' -import { createRepo } from '../../utils/github.js' -import config from '@adobe/aio-lib-core-config' -/** - * Clones github repos in preparation of content and mesh provisioning with aio commerce init --skipGit - */ -export class CreateRepos extends Command { - async run () { - console.log('Starting to create repos...') - // const { args, flags } = await this.parse(CreateRepos) - const start = 1 - const end = 100 - const owner = 'adobe-summit-L322' // adobe-summit-L321 - const repoPrefix = 'seat' - const templateOrg = 'adobe-commerce' - const templateRepo = 'adobe-demo-store' - - for (let i = start; i <= end; i++) { - const repo = `${repoPrefix}-${i}` - config.set('commerce.github.org', owner) - config.set('commerce.github.repo', repo) - config.set('commerce.template.org', templateOrg) - config.set('commerce.template.repo', templateRepo) - try { - await createRepo() - } catch (e) { - console.error(`! Failed to complete run for "${repo}". Skipping.`) - console.error(e) - } - } - } -} - -CreateRepos.description = 'Uses the init tool in bulk' - -CreateRepos.flags = { - // template: Flags.string({ char: 't', description: 'the template to use for the storefronts, ie adobe-commerce/ccdm-demo-store" ' }), - // repo: Flags.string({ char: 'r', description: 'the repository name prefix, which will have a count appended to it, ie adobe-commerce-L321/seat' }) -} diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index 2e71133..73290b7 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -40,6 +40,7 @@ export class InitCommand extends Command { await initialization(args, flags) } const { org: githubOrg, repo: githubRepo } = config.get('commerce.github') + const { org: templateOrg, repo: templateRepo } = config.get('commerce.template') const { saas, paas } = config.get('commerce.datasource') const runAIOCommand = async (command, args) => { @@ -65,7 +66,7 @@ export class InitCommand extends Command { if (flags.skipGit) { console.log(`Not creating Git Repos - assuming it already exists at https://github.com/${githubOrg}/${githubRepo}`) } else { - await createRepo() + await createRepo(githubOrg, githubRepo, templateOrg, templateRepo) if (githubOrg === 'adobe-summit-L322' || githubOrg === 'adobe-summit-L321') { console.log('✅ AEM Code Sync Bot automatically installed :)') diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 4d74390..25cf8bd 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -1,3 +1,8 @@ +/** + * This script is designed to check the status of code syncs for a series of repositories. + * Only to be used for housekeeping purposes. Please do not run this script without supervision. + */ + import fs from 'fs' const PROGRESS_FILE = 'progress.json' diff --git a/src/scripts/createRepos.js b/src/scripts/createRepos.js new file mode 100644 index 0000000..ec076af --- /dev/null +++ b/src/scripts/createRepos.js @@ -0,0 +1,36 @@ +/** + * This is a script to create multiple GitHub repositories using a template repository. + * Only to be used for housekeeping purposes. Please do not run this script without supervision. + */ + +import { createRepo } from '../utils/github.js' + +const start = 1 +const end = 100 +const owner = 'adobe-summit-L322' // adobe-summit-L321 +const repoPrefix = 'seat' +const templateOrg = 'adobe-commerce' +const templateRepo = 'adobe-demo-store' + +/** + * + */ +async function createRepos () { + console.log('Starting to create repos...') + + for (let i = start; i <= end; i++) { + const repo = `${repoPrefix}-${i}` + try { + await createRepo(owner, repo, templateOrg, templateRepo) + } catch (e) { + console.error(`! Failed to complete run for "${repo}". Skipping.`) + console.error(e) + } + } +} + +createRepos().then(() => { + console.log('All repos created successfully.') +}).catch((error) => { + console.error('Error creating repos:', error) +}) diff --git a/src/scripts/deleteRepos.js b/src/scripts/deleteRepos.js index daa2377..70ba21a 100644 --- a/src/scripts/deleteRepos.js +++ b/src/scripts/deleteRepos.js @@ -1,3 +1,8 @@ +/** + * This script deletes all the repositories created by the create-repos.js script. + * Only to be used for housekeeping purposes. Please do not run this script without supervision. + */ + import childProcess from 'child_process' /** diff --git a/src/utils/github.js b/src/utils/github.js index f25006c..29491a6 100644 --- a/src/utils/github.js +++ b/src/utils/github.js @@ -6,11 +6,12 @@ const aioLogger = Logger('commerce:github.js') /** * Creates github repo from template + * @param githubOrg + * @param githubRepo + * @param templateOrg + * @param templateRepo */ -export async function createRepo () { - const { org: githubOrg, repo: githubRepo } = config.get('commerce.github') - const { org: templateOrg, repo: templateRepo } = config.get('commerce.template') - +export async function createRepo (githubOrg, githubRepo, templateOrg, templateRepo) { // Check if the repository already exists const cmdResult = await runCommand(`gh api repos/${githubOrg}/${githubRepo}`) if (cmdResult?.stdout) { // if not exist, will throw and return 404. @@ -31,17 +32,18 @@ export async function createRepo () { // If the repository does not exist, proceed with "create" await runCommand(`gh repo create ${githubOrg}/${githubRepo} --template ${templateOrg}/${templateRepo} --public`) console.log(`✅ Created code repository at https://github.com/${githubOrg}/${githubRepo} from template ${templateOrg}/${templateRepo}`) - await modifyFstab() - await modifySidekickConfig() + await modifyFstab(githubOrg, githubRepo, templateRepo) + await modifySidekickConfig(githubOrg, githubRepo) } } /** * fstab must be connected to DA content source + * @param githubOrg + * @param githubRepo + * @param templateRepo */ -export async function modifyFstab () { - const { org, repo } = config.get('commerce.github') - const { repo: templateRepo } = config.get('commerce.template') +export async function modifyFstab (githubOrg, githubRepo, templateRepo) { let repoReady = false let attempts = 0 while (!repoReady && attempts++ <= 10) { @@ -51,7 +53,7 @@ export async function modifyFstab () { // if using config service, or if we can update to only modify the root mountpoint and copy "folders:" in full from source fstab. const standardFstab = `mountpoints: /: - url: https://content.da.live/${org}/${repo}/ + url: https://content.da.live/${githubOrg}/${githubRepo}/ type: markup folders: @@ -59,7 +61,7 @@ folders: ` const adobeStoreFstab = `mountpoints: /: - url: https://content.da.live/${org}/${repo}/ + url: https://content.da.live/${githubOrg}/${githubRepo}/ type: markup folders: @@ -75,8 +77,8 @@ folders: const ENCODED_CONTENT = Buffer.from(fstab, 'utf8').toString('base64') // TODO: this will not work for templates using config-service since this, and other files, are deleted. Need to refactor this a bit to support - const { stdout: FILE_SHA } = await runCommand(`gh api repos/${org}/${repo}/contents/fstab.yaml -q .sha`) - await runCommand(`gh api -X PUT repos/${org}/${repo}/contents/fstab.yaml -f message="update fstab" -f content="${ENCODED_CONTENT.trim()}" -f sha="${FILE_SHA.trim()}"`) + const { stdout: FILE_SHA } = await runCommand(`gh api repos/${githubOrg}/${githubRepo}/contents/fstab.yaml -q .sha`) + await runCommand(`gh api -X PUT repos/${githubOrg}/${githubRepo}/contents/fstab.yaml -f message="update fstab" -f content="${ENCODED_CONTENT.trim()}" -f sha="${FILE_SHA.trim()}"`) repoReady = true aioLogger.debug('fstab mountpoint updated') @@ -90,9 +92,10 @@ folders: /** * Sidekick requires specific config settings in github repo to have "edit" link back to DA + * @param githubOrg + * @param githubRepo */ -export async function modifySidekickConfig () { - const { org, repo } = config.get('commerce.github') +export async function modifySidekickConfig (githubOrg, githubRepo) { let repoReady = false let attempts = 0 while (!repoReady && attempts++ <= 10) { @@ -109,7 +112,7 @@ export async function modifySidekickConfig () { "environments": [ "edit" ], - "url": "https://main--${repo}--${org}.aem.live/tools/picker/dist/index.html", + "url": "https://main--${githubRepo}--${githubOrg}.aem.live/tools/picker/dist/index.html", "isPalette": true, "paletteRect": "top: 54px; left: 5px; bottom: 5px; width: 300px; height: calc(100% - 59px); border-radius: var(--hlx-sk-button-border-radius); overflow: hidden; resize: horizontal;" } @@ -117,8 +120,8 @@ export async function modifySidekickConfig () { } `, 'utf8').toString('base64') - const { stdout: FILE_SHA } = await runCommand(`gh api repos/${org}/${repo}/contents/tools/sidekick/config.json -q .sha`) - await runCommand(`gh api -X PUT repos/${org}/${repo}/contents/tools/sidekick/config.json -f message="update sidekick config" -f content="${ENCODED_CONTENT.trim()}" -f sha="${FILE_SHA.trim()}"`) + const { stdout: FILE_SHA } = await runCommand(`gh api repos/${githubOrg}/${githubRepo}/contents/tools/sidekick/config.json -q .sha`) + await runCommand(`gh api -X PUT repos/${githubOrg}/${githubRepo}/contents/tools/sidekick/config.json -f message="update sidekick config" -f content="${ENCODED_CONTENT.trim()}" -f sha="${FILE_SHA.trim()}"`) repoReady = true aioLogger.debug('sidekick config modified with content source') From d598a877c11b2ba454a084b49dc584c273faf8cf Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Thu, 27 Feb 2025 16:18:24 -0600 Subject: [PATCH 10/22] remove scripts from package Signed-off-by: Stephen Rugh --- package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/package.json b/package.json index 996b001..8633992 100644 --- a/package.json +++ b/package.json @@ -58,9 +58,7 @@ "prepack": "oclif manifest && oclif readme", "postpack": "rm -f oclif.manifest.json", "version": "oclif readme && git add README.md", - "e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --collectCoverage=false --testRegex 'e2e/e2e.js'", - "checkSync": "node src/scripts/checkSync.js", - "createRepos": "node src/scripts/createRepos.js" + "e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --collectCoverage=false --testRegex 'e2e/e2e.js'" }, "jest": { "collectCoverage": true, From 69e31938695cfbcdbb2bd29886c4e4408abecff4 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 07:43:11 -0600 Subject: [PATCH 11/22] pad repo to 2 digits. add args to createRepos Signed-off-by: Stephen Rugh --- src/scripts/createRepos.js | 31 ++++++++++++++++++++++--------- src/scripts/deleteRepos.js | 4 ++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/scripts/createRepos.js b/src/scripts/createRepos.js index ec076af..6d6bb6e 100644 --- a/src/scripts/createRepos.js +++ b/src/scripts/createRepos.js @@ -5,8 +5,6 @@ import { createRepo } from '../utils/github.js' -const start = 1 -const end = 100 const owner = 'adobe-summit-L322' // adobe-summit-L321 const repoPrefix = 'seat' const templateOrg = 'adobe-commerce' @@ -14,12 +12,14 @@ const templateRepo = 'adobe-demo-store' /** * + * @param start + * @param end */ -async function createRepos () { +async function createRepos (start, end) { console.log('Starting to create repos...') for (let i = start; i <= end; i++) { - const repo = `${repoPrefix}-${i}` + const repo = `${repoPrefix}-${i.toString().padStart(2, '0')}` try { await createRepo(owner, repo, templateOrg, templateRepo) } catch (e) { @@ -29,8 +29,21 @@ async function createRepos () { } } -createRepos().then(() => { - console.log('All repos created successfully.') -}).catch((error) => { - console.error('Error creating repos:', error) -}) +if (process.argv.length !== 4) { + console.log('Usage: node createRepos.js ') +} else { + const start = parseInt(process.argv[2]) + const end = parseInt(process.argv[3]) + + if (!Number.isInteger(start) || !Number.isInteger(end)) { + console.error('Start and End must be integers') + } else if (start > end) { + console.log('Start cannot be greater than End.') + } else { + createRepos(start, end).then(() => { + console.log('All repos created successfully.') + }).catch((error) => { + console.error('Error creating repos:', error) + }) + } +} diff --git a/src/scripts/deleteRepos.js b/src/scripts/deleteRepos.js index 70ba21a..d57ee37 100644 --- a/src/scripts/deleteRepos.js +++ b/src/scripts/deleteRepos.js @@ -30,7 +30,7 @@ function runCommand (cmd) { */ function deleteRepos (start, end) { for (let seat = start; seat <= end; seat++) { - const repoName = `adobe-summit-L322/seat-${seat}` + const repoName = `adobe-summit-L322/seat-${seat.toString().padStart(2, '0')}` const command = `gh api \ --method DELETE \ -H "Accept: application/vnd.github+json" \ @@ -52,7 +52,7 @@ function deleteRepos (start, end) { } if (process.argv.length !== 4) { - console.log('Usage: node delete-repos.js ') + console.log('Usage: node deleteRepos.js ') } else { const start = parseInt(process.argv[2]) const end = parseInt(process.argv[3]) From da90bf2a75a8f0acd47fe705f279cb4c3a8125d0 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 08:35:01 -0600 Subject: [PATCH 12/22] padding checkSync repo usage Signed-off-by: Stephen Rugh --- .eslintrc.json | 3 +- src/scripts/checkSync.js | 69 ++++++++++++---------------------------- 2 files changed, 23 insertions(+), 49 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index bf281b3..a092e13 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -8,6 +8,7 @@ } }, "rules": { - "node/no-missing-import": 0 + "node/no-missing-import": 0, + "jsdoc/require-jsdoc": 0 } } diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 25cf8bd..74a9bff 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -6,8 +6,7 @@ import fs from 'fs' const PROGRESS_FILE = 'progress.json' -const BASE_URL = 'https://main--seat-#--adobe-summit-l322.aem.page/scripts/scripts.js' -const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-#/main/*' +const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-##/main/*' const TOTAL_REPOS = 100 const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour // the amount of git files we expect the code sync to have to have processed. @@ -15,14 +14,20 @@ const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour const GIT_FILES_TO_CHECK = 418 /** - * + * Pads the index to two digits (01, 02, ..., 99) + * @param {number} index + * @returns {string} Padded index */ +function padIndex (index) { + return index.toString().padStart(2, '0') +} + function loadProgress () { try { if (fs.existsSync(PROGRESS_FILE)) { const data = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8')) return { - lastChecked: data.lastChecked || 1, + lastChecked: data.lastChecked === 0 ? 0 : data.lastChecked || 1, restartAfter: data.restartAfter || 0 } } @@ -32,12 +37,6 @@ function loadProgress () { return { lastChecked: 1, restartAfter: 0 } // Default values } -// Save progress to file -/** - * - * @param index - * @param restartAfter - */ function saveProgress (index, restartAfter = 0) { try { fs.writeFileSync(PROGRESS_FILE, JSON.stringify({ lastChecked: index, restartAfter })) @@ -46,52 +45,37 @@ function saveProgress (index, restartAfter = 0) { } } -/** - * Checks that the code sync for a specific seat is complete. - * If not, it triggers the code sync and waits until it's done. - * @param index The index of the seat to check. - */ async function checkRepo (index) { try { await triggerCodeSync(index) - console.log(`✅ Code sync completed for seat-${index}. Proceeding to the next repo.`) + console.log(`✅ Code sync completed for seat-${padIndex(index)}. Proceeding to the next repo.`) return index + 1 // Move to the next repo } catch (error) { - console.error(`🚨 Error during code sync for seat-${index}:`, error) + console.error(`🚨 Error during code sync for seat-${padIndex(index)}:`, error) return index - 1 // Restart from the previous repo } } -/** - * - * @param index - */ async function saveAndExit (index) { // Determine restart time in CST const restartTimestamp = Date.now() + WAIT_TIME_MS const restartTime = new Date(restartTimestamp).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) - const nextIndex = index - 1 > 0 ? index - 1 : 1 - console.log(`🚨 Script exiting. Restart at seat-${nextIndex} after ~1 hour at: ${restartTime} CST`) + console.log(`🚨 Script exiting. Restart at seat-${padIndex(index)} after ~1 hour at: ${restartTime} CST`) - saveProgress(nextIndex, restartTimestamp) + saveProgress(index, restartTimestamp) process.exit(1) } -// Function to trigger code sync requests for failing repo (n), n-1, and n+1 -/** - * - * @param index - * @param seat - */ + async function triggerCodeSync (seat) { - const postUrl = POST_URL.replace('#', seat) + const postUrl = POST_URL.replace('##', padIndex(seat)) console.log(`🔄 Calling Helix Admin Code Sync at: ${postUrl}`) try { const postResponse = await fetch(postUrl, { method: 'POST' }) if (postResponse.status !== 202) { - console.error(`No code found for seat-${seat}. Ensure repo exists.`) + console.error(`No code found for seat-${padIndex(seat)}. Ensure repo exists.`) saveAndExit(seat) process.exit(1) } @@ -101,18 +85,11 @@ async function triggerCodeSync (seat) { await checkPhaseCompletion(detailsUrl, seat) } catch (error) { - console.error(`Error processing seat-${seat}:`, error) + console.error(`Error processing seat-${padIndex(seat)}:`, error) throw error } } -/** - * - * @param detailsUrl - * @param seat - * @param maxRetries - * @param interval - */ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval = 1000) { let attempts = 0 @@ -122,20 +99,20 @@ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval const detailsData = await detailsResponse.json() if (detailsData.state === 'stopped') { if (detailsData.progress && detailsData?.progress.processed >= GIT_FILES_TO_CHECK) { - console.log(`✅ Seat-${seat} processing completed.`) + console.log(`✅ Seat-${padIndex(seat)} processing completed.`) return } else if (detailsData.error.includes('rate limit exceeded')) { - console.log(`⚠️ Seat-${seat} processing rate limit exceeded.`) + console.log(`⚠️ Seat-${padIndex(seat)} processing rate limit exceeded.`) saveAndExit(seat) } else { console.log('Unknown state! Data:', detailsData) saveAndExit(seat) } } else { - console.log(`⏳ Seat-${seat} still processing... retrying (${attempts + 1}/${maxRetries}) in ${(interval / 1000)}s.`) + console.log(`⏳ Seat-${padIndex(seat)} still processing... retrying (${attempts + 1}/${maxRetries}) in ${(interval / 1000)}s.`) } } catch (error) { - console.error(`Error fetching details for seat-${seat}:`, error) + console.error(`Error fetching details for seat-${padIndex(seat)}:`, error) } attempts++ @@ -145,10 +122,6 @@ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval await saveAndExit(seat) } -// Main process loop -/** - * - */ async function runChecks () { const { lastChecked, restartAfter } = loadProgress() From 6dcaccbfa14109c5526980567ab4b452980e9271 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 09:35:05 -0600 Subject: [PATCH 13/22] f Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 74a9bff..d1f83e2 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -13,11 +13,6 @@ const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour // for L322, it seems to be 418. For L321, seems around 411. const GIT_FILES_TO_CHECK = 418 -/** - * Pads the index to two digits (01, 02, ..., 99) - * @param {number} index - * @returns {string} Padded index - */ function padIndex (index) { return index.toString().padStart(2, '0') } @@ -60,11 +55,12 @@ async function saveAndExit (index) { // Determine restart time in CST const restartTimestamp = Date.now() + WAIT_TIME_MS const restartTime = new Date(restartTimestamp).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) + const nextIndex = index - 1 > 0 ? index - 1 : 1 - console.log(`🚨 Script exiting. Restart at seat-${padIndex(index)} after ~1 hour at: ${restartTime} CST`) + console.log(`🚨 Script exiting. Restart at seat-${padIndex(nextIndex)} after ~1 hour at: ${restartTime} CST`) - saveProgress(index, restartTimestamp) - process.exit(1) + saveProgress(nextIndex, restartTimestamp) + process.exit(0) } async function triggerCodeSync (seat) { @@ -77,7 +73,6 @@ async function triggerCodeSync (seat) { if (postResponse.status !== 202) { console.error(`No code found for seat-${padIndex(seat)}. Ensure repo exists.`) saveAndExit(seat) - process.exit(1) } const postData = await postResponse.json() const detailsUrl = `${postData.links.self}/details` From b2c42fa40d4dc84965f8fd0641d99aac30da9d3b Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 09:36:33 -0600 Subject: [PATCH 14/22] remove jsdocs Signed-off-by: Stephen Rugh --- src/scripts/createRepos.js | 5 ----- src/scripts/deleteRepos.js | 9 --------- 2 files changed, 14 deletions(-) diff --git a/src/scripts/createRepos.js b/src/scripts/createRepos.js index 6d6bb6e..2999ef8 100644 --- a/src/scripts/createRepos.js +++ b/src/scripts/createRepos.js @@ -10,11 +10,6 @@ const repoPrefix = 'seat' const templateOrg = 'adobe-commerce' const templateRepo = 'adobe-demo-store' -/** - * - * @param start - * @param end - */ async function createRepos (start, end) { console.log('Starting to create repos...') diff --git a/src/scripts/deleteRepos.js b/src/scripts/deleteRepos.js index d57ee37..229d40a 100644 --- a/src/scripts/deleteRepos.js +++ b/src/scripts/deleteRepos.js @@ -5,10 +5,6 @@ import childProcess from 'child_process' -/** - * - * @param cmd - */ function runCommand (cmd) { return new Promise((resolve, reject) => { childProcess.exec(cmd, (error, stdout, stderr) => { @@ -23,11 +19,6 @@ function runCommand (cmd) { }) } -/** - * - * @param start - * @param end - */ function deleteRepos (start, end) { for (let seat = start; seat <= end; seat++) { const repoName = `adobe-summit-L322/seat-${seat.toString().padStart(2, '0')}` From dc00b3274841d72e55a08bfb715367eae182b12f Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 09:58:55 -0600 Subject: [PATCH 15/22] f Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index d1f83e2..f1dd15b 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -55,11 +55,9 @@ async function saveAndExit (index) { // Determine restart time in CST const restartTimestamp = Date.now() + WAIT_TIME_MS const restartTime = new Date(restartTimestamp).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) - const nextIndex = index - 1 > 0 ? index - 1 : 1 + console.log(`🚨 Script exiting. Restart at seat-${padIndex(index)} after ~1 hour at: ${restartTime} CST`) - console.log(`🚨 Script exiting. Restart at seat-${padIndex(nextIndex)} after ~1 hour at: ${restartTime} CST`) - - saveProgress(nextIndex, restartTimestamp) + saveProgress(index, restartTimestamp) process.exit(0) } @@ -98,7 +96,7 @@ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval return } else if (detailsData.error.includes('rate limit exceeded')) { console.log(`⚠️ Seat-${padIndex(seat)} processing rate limit exceeded.`) - saveAndExit(seat) + saveAndExit(seat - 1) // start again on previous seat if rate limit exceeded } else { console.log('Unknown state! Data:', detailsData) saveAndExit(seat) From ad158222e5581a30aad4a0a423b90862bebbcf7d Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 10:00:12 -0600 Subject: [PATCH 16/22] f Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index f1dd15b..ef51537 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -96,7 +96,7 @@ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval return } else if (detailsData.error.includes('rate limit exceeded')) { console.log(`⚠️ Seat-${padIndex(seat)} processing rate limit exceeded.`) - saveAndExit(seat - 1) // start again on previous seat if rate limit exceeded + saveAndExit(seat) } else { console.log('Unknown state! Data:', detailsData) saveAndExit(seat) From cf942e30586412af7eac1a545ad63035e49888a9 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 10:48:14 -0600 Subject: [PATCH 17/22] add auto-sync to restart the node process after the rate limit refresh Signed-off-by: Stephen Rugh --- src/scripts/autoSync.sh | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 src/scripts/autoSync.sh diff --git a/src/scripts/autoSync.sh b/src/scripts/autoSync.sh new file mode 100755 index 0000000..854648d --- /dev/null +++ b/src/scripts/autoSync.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Function to get UTC time from progress.json and display it in CST +get_restart_time() { + local RESTART_AFTER=$(jq -r '.restartAfter' progress.json) + local RESTART_AFTER_SEC=$((RESTART_AFTER / 1000)) + echo "Restart after: $(TZ='America/Chicago' date -r $RESTART_AFTER_SEC "+%Y-%m-%d %H:%M:%S") (CST)" +} + +# Function to check if condition is met +exit_if_all_checked() { + local LAST_CHECKED=$(jq -r '.lastChecked' progress.json) + if [ "$LAST_CHECKED" -gt "100" ]; then + echo "100 repos synced, quitting..." + exit 0 + fi +} + +# Function to format seconds into H:M:S +format_time() { + local SECONDS=$1 + printf "%02d:%02d:%02d" $((SECONDS/3600)) $((SECONDS%3600/60)) $((SECONDS%60)) +} + +# Main loop +while true; do + # Check condition to see if we should quit + exit_if_all_checked + + # Get the current restart time and display it + get_restart_time + + # Wait until restartAfter time has passed + RESTART_AFTER=$(jq -r '.restartAfter' progress.json) + RESTART_AFTER_SEC=$((RESTART_AFTER / 1000)) + CURRENT_TIME=$(date +%s) + SLEEP_TIME=$((RESTART_AFTER_SEC - CURRENT_TIME)) + if [ $SLEEP_TIME -gt 0 ]; then + echo "Sleeping for $(format_time $SLEEP_TIME)" + sleep $SLEEP_TIME + fi + + # Run the script synchronously. When the script ends, it will write to progress.json with new restartAfter time + node src/scripts/checkSync.js +done From 1b09f9c3c0e7125c3c8ff9386b2eb1facca1f28a Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 28 Feb 2025 13:05:58 -0600 Subject: [PATCH 18/22] 5 second interval for processed check Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index ef51537..f75e057 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -83,7 +83,7 @@ async function triggerCodeSync (seat) { } } -async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval = 1000) { +async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval = 5000) { let attempts = 0 while (attempts < maxRetries) { From 279179ba690d62474c3c57e111f8f8e4e9736349 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Fri, 7 Mar 2025 12:42:09 -0600 Subject: [PATCH 19/22] allow empty datasource for empty string Signed-off-by: Stephen Rugh --- src/commands/commerce/init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/commerce/init.js b/src/commands/commerce/init.js index 73290b7..83529b1 100644 --- a/src/commands/commerce/init.js +++ b/src/commands/commerce/init.js @@ -27,7 +27,7 @@ const aioLogger = Logger('commerce:init.js') export class InitCommand extends Command { async run () { const { args, flags } = await this.parse(InitCommand) - if (flags.repo && flags.template && flags.datasource) { + if (flags.repo && flags.template && flags.datasource !== undefined) { // Skip initialization. Instead just set configs here. config.set('commerce.github.org', flags.repo.split('/')[0]) config.set('commerce.github.repo', flags.repo.split('/')[1]) From f1380a377274a62ac18b25b750cdeb0424e2e617 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Mon, 10 Mar 2025 09:38:50 -0500 Subject: [PATCH 20/22] additional Signed-off-by: Stephen Rugh --- README.md | 6 ++++- src/scripts/checkSync.js | 8 +++--- src/scripts/cloneContent.js | 50 +++++++++++++++++++++++++++++++++++++ src/scripts/createRepos.js | 7 ++++-- src/utils/initialization.js | 1 + 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 src/scripts/cloneContent.js diff --git a/README.md b/README.md index 42d2d12..542e7d2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ![image](https://github.com/user-attachments/assets/7cd45e4b-945a-4e13-909d-b1fe2c426fe0) -# Prerequisites +## Prerequisites ❗ You must have Node >= 22. Consider using [nvm](https://formulae.brew.sh/formula/nvm) to manage multiple versions. @@ -25,6 +25,8 @@ brew install gh gh auth login ``` +### Mesh Prerequisites + ❗ You must also have the `api-mesh` plugin installed if you wish to use mesh provisioning. ```sh @@ -42,6 +44,8 @@ aio config set cli.env prod aio console org select ``` +❗ You must have _developer access_ in [Adobe Developer Console](developer.adobe.com/console) to create Meshes. + ## Troubleshooting ### Name Restrictions diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index f75e057..84ee461 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -6,12 +6,14 @@ import fs from 'fs' const PROGRESS_FILE = 'progress.json' -const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-##/main/*' const TOTAL_REPOS = 100 const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour // the amount of git files we expect the code sync to have to have processed. -// for L322, it seems to be 418. For L321, seems around 411. -const GIT_FILES_TO_CHECK = 418 +// for L322, it seems to be 418. For L321, seems to be 412. +const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L321/seat-##/main/*' +const GIT_FILES_TO_CHECK = 412 +// const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-##/main/*' +// const GIT_FILES_TO_CHECK = 418 function padIndex (index) { return index.toString().padStart(2, '0') diff --git a/src/scripts/cloneContent.js b/src/scripts/cloneContent.js new file mode 100644 index 0000000..7f84cd8 --- /dev/null +++ b/src/scripts/cloneContent.js @@ -0,0 +1,50 @@ +/** + * This is a script to clone content automatically. + * Only to be used for housekeeping purposes. Please do not run this script without supervision. + */ +import { execSync } from 'child_process' + +const owner = 'adobe-summit-L321' +const templateRepo = 'ccdm-demo-store' +// const owner = 'adobe-summit-L322' +// const templateRepo = 'adobe-demo-store' +const repoPrefix = 'seat' +const templateOrg = 'adobe-commerce' + +async function cloneContent (start, end) { + console.log('Starting to clone content...') + + for (let i = start; i <= end; i++) { + const repoNumber = i.toString().padStart(2, '0') + const command = `aio commerce:init --template "${templateOrg}/${templateRepo}" --repo "${owner}/${repoPrefix}-${repoNumber}" --datasource "" --skipMesh --skipGit` + console.log(`\nExecuting command ${i} of ${end}:`) + console.log(command) + + try { + execSync(command, { stdio: 'inherit' }) + console.log(`Successfully completed iteration ${i}`) + } catch (error) { + console.error(`Error in iteration ${i}:`, error.message) + throw error + } + } +} + +if (process.argv.length !== 4) { + console.log('Usage: node cloneContent.js ') +} else { + const start = parseInt(process.argv[2]) + const end = parseInt(process.argv[3]) + + if (!Number.isInteger(start) || !Number.isInteger(end)) { + console.error('Start and End must be integers') + } else if (start > end) { + console.log('Start cannot be greater than End.') + } else { + cloneContent(start, end).then(() => { + console.log('All repos created successfully.') + }).catch((error) => { + console.error('Error creating repos:', error) + }) + } +} diff --git a/src/scripts/createRepos.js b/src/scripts/createRepos.js index 2999ef8..b877455 100644 --- a/src/scripts/createRepos.js +++ b/src/scripts/createRepos.js @@ -1,14 +1,17 @@ /** * This is a script to create multiple GitHub repositories using a template repository. * Only to be used for housekeeping purposes. Please do not run this script without supervision. + * */ import { createRepo } from '../utils/github.js' -const owner = 'adobe-summit-L322' // adobe-summit-L321 +const owner = 'adobe-summit-L321' +const templateRepo = 'ccdm-demo-store' +// const owner = 'adobe-summit-L322' +// const templateRepo = 'adobe-demo-store' const repoPrefix = 'seat' const templateOrg = 'adobe-commerce' -const templateRepo = 'adobe-demo-store' async function createRepos (start, end) { console.log('Starting to create repos...') diff --git a/src/utils/initialization.js b/src/utils/initialization.js index df31570..1dbda16 100644 --- a/src/utils/initialization.js +++ b/src/utils/initialization.js @@ -57,6 +57,7 @@ export async function initialization (args, flags) { // TODO: validate template is allowed template = template || await promptSelect('Which template would you like to use?', [ 'adobe-commerce/adobe-demo-store', // ACCS template + 'adobe-commerce/ccdm-demo-store', // CCDM template 'hlxsites/aem-boilerplate-commerce' // PaaS template // 'adobe-rnd/aem-boilerplate-xcom' // UE Template // 'aabsites/citisignal' // TODO: Cannot use citisignal until we resolve how to use templates that use config service as some core files are missing https://magento.slack.com/archives/C085R48U3R7/p1738785011567519 From f889f34b385559087bcbdc18216816c65637ff22 Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Mon, 10 Mar 2025 11:20:47 -0500 Subject: [PATCH 21/22] use args instead of hardcoding source or destination Signed-off-by: Stephen Rugh --- src/scripts/autoSync.sh | 35 +++++++++++++--- src/scripts/checkSync.js | 79 +++++++++++++++++++++++-------------- src/scripts/cloneContent.js | 41 +++++++++++++------ src/scripts/createRepos.js | 39 ++++++++++++------ src/scripts/deleteRepos.js | 35 +++++++++++----- 5 files changed, 158 insertions(+), 71 deletions(-) diff --git a/src/scripts/autoSync.sh b/src/scripts/autoSync.sh index 854648d..b980e8a 100755 --- a/src/scripts/autoSync.sh +++ b/src/scripts/autoSync.sh @@ -3,15 +3,18 @@ # Function to get UTC time from progress.json and display it in CST get_restart_time() { local RESTART_AFTER=$(jq -r '.restartAfter' progress.json) - local RESTART_AFTER_SEC=$((RESTART_AFTER / 1000)) - echo "Restart after: $(TZ='America/Chicago' date -r $RESTART_AFTER_SEC "+%Y-%m-%d %H:%M:%S") (CST)" + if [ "$RESTART_AFTER" != "0" ]; then + local RESTART_AFTER_SEC=$((RESTART_AFTER / 1000)) + echo "Restart after: $(TZ='America/Chicago' date -r $RESTART_AFTER_SEC "+%Y-%m-%d %H:%M:%S") (CST)" + fi } + # Function to check if condition is met exit_if_all_checked() { local LAST_CHECKED=$(jq -r '.lastChecked' progress.json) - if [ "$LAST_CHECKED" -gt "100" ]; then - echo "100 repos synced, quitting..." + if [ "$LAST_CHECKED" -gt "$END" ]; then + echo "All repos synced up to $END, quitting..." exit 0 fi } @@ -22,6 +25,28 @@ format_time() { printf "%02d:%02d:%02d" $((SECONDS/3600)) $((SECONDS%3600/60)) $((SECONDS%60)) } +# Check if destination argument is provided +if [ $# -ne 3 ]; then + echo "Usage: ./autoSync.sh " + echo "Example: ./autoSync.sh adobe-summit-L322/seat 0 100" + exit 1 +fi + +DESTINATION=$1 +START=$2 +END=$3 + +# Validate arguments +if ! [[ "$START" =~ ^[0-9]+$ ]] || ! [[ "$END" =~ ^[0-9]+$ ]]; then + echo "Error: Start and End must be integers" + exit 1 +fi + +if [ "$START" -gt "$END" ]; then + echo "Error: Start cannot be greater than End" + exit 1 +fi + # Main loop while true; do # Check condition to see if we should quit @@ -41,5 +66,5 @@ while true; do fi # Run the script synchronously. When the script ends, it will write to progress.json with new restartAfter time - node src/scripts/checkSync.js + node src/scripts/checkSync.js "$DESTINATION" "$START" "$END" done diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index 84ee461..d82bfcb 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -1,19 +1,23 @@ /** * This script is designed to check the status of code syncs for a series of repositories. * Only to be used for housekeeping purposes. Please do not run this script without supervision. + * + * Usage: node checkSync.js + * Example: node checkSync.js adobe-summit-L322/seat 0 100 */ import fs from 'fs' const PROGRESS_FILE = 'progress.json' -const TOTAL_REPOS = 100 const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour // the amount of git files we expect the code sync to have to have processed. // for L322, it seems to be 418. For L321, seems to be 412. -const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L321/seat-##/main/*' const GIT_FILES_TO_CHECK = 412 -// const POST_URL = 'https://admin.hlx.page/code/adobe-summit-L322/seat-##/main/*' -// const GIT_FILES_TO_CHECK = 418 + +function parseDestinationString (destString) { + const [org, prefix] = destString.split('/') + return { org, prefix } +} function padIndex (index) { return index.toString().padStart(2, '0') @@ -42,50 +46,51 @@ function saveProgress (index, restartAfter = 0) { } } -async function checkRepo (index) { +async function checkRepo (destination, index) { try { - await triggerCodeSync(index) - console.log(`✅ Code sync completed for seat-${padIndex(index)}. Proceeding to the next repo.`) + await triggerCodeSync(destination, index) + console.log(`✅ Code sync completed for ${destination}-${padIndex(index)}. Proceeding to the next repo.`) return index + 1 // Move to the next repo } catch (error) { - console.error(`🚨 Error during code sync for seat-${padIndex(index)}:`, error) + console.error(`🚨 Error during code sync for ${destination}-${padIndex(index)}:`, error) return index - 1 // Restart from the previous repo } } -async function saveAndExit (index) { +async function saveAndExit (destination, index) { // Determine restart time in CST const restartTimestamp = Date.now() + WAIT_TIME_MS const restartTime = new Date(restartTimestamp).toLocaleTimeString('en-US', { timeZone: 'America/Chicago', hour12: true }) - console.log(`🚨 Script exiting. Restart at seat-${padIndex(index)} after ~1 hour at: ${restartTime} CST`) + console.log(`🚨 Script exiting. Restart at ${destination}-${padIndex(index)} after ~1 hour at: ${restartTime} CST`) saveProgress(index, restartTimestamp) process.exit(0) } -async function triggerCodeSync (seat) { - const postUrl = POST_URL.replace('##', padIndex(seat)) +async function triggerCodeSync (destination, seat) { + const { org, prefix } = parseDestinationString(destination) + const postUrl = `https://admin.hlx.page/code/${org}/${prefix}-${padIndex(seat)}/main/*` console.log(`🔄 Calling Helix Admin Code Sync at: ${postUrl}`) try { const postResponse = await fetch(postUrl, { method: 'POST' }) if (postResponse.status !== 202) { - console.error(`No code found for seat-${padIndex(seat)}. Ensure repo exists.`) - saveAndExit(seat) + console.error(`No code found for ${destination}-${padIndex(seat)}. Ensure repo exists.`) + saveAndExit(destination, seat) } const postData = await postResponse.json() const detailsUrl = `${postData.links.self}/details` console.log(`🔍 Checking details at: ${detailsUrl}`) - await checkPhaseCompletion(detailsUrl, seat) + await checkPhaseCompletion(detailsUrl, destination, seat) } catch (error) { - console.error(`Error processing seat-${padIndex(seat)}:`, error) + console.error(`Error processing ${destination}-${padIndex(seat)}:`, error) throw error } } -async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval = 5000) { +async function checkPhaseCompletion (detailsUrl, destination, seat, maxRetries = 60, interval = 5000) { let attempts = 0 while (attempts < maxRetries) { @@ -94,30 +99,30 @@ async function checkPhaseCompletion (detailsUrl, seat, maxRetries = 60, interval const detailsData = await detailsResponse.json() if (detailsData.state === 'stopped') { if (detailsData.progress && detailsData?.progress.processed >= GIT_FILES_TO_CHECK) { - console.log(`✅ Seat-${padIndex(seat)} processing completed.`) + console.log(`✅ ${destination}-${padIndex(seat)} processing completed.`) return } else if (detailsData.error.includes('rate limit exceeded')) { - console.log(`⚠️ Seat-${padIndex(seat)} processing rate limit exceeded.`) - saveAndExit(seat) + console.log(`⚠️ ${destination}-${padIndex(seat)} processing rate limit exceeded.`) + saveAndExit(destination, seat) } else { console.log('Unknown state! Data:', detailsData) - saveAndExit(seat) + saveAndExit(destination, seat) } } else { - console.log(`⏳ Seat-${padIndex(seat)} still processing... retrying (${attempts + 1}/${maxRetries}) in ${(interval / 1000)}s.`) + console.log(`⏳ ${destination}-${padIndex(seat)} still processing... retrying (${attempts + 1}/${maxRetries}) in ${(interval / 1000)}s.`) } } catch (error) { - console.error(`Error fetching details for seat-${padIndex(seat)}:`, error) + console.error(`Error fetching details for ${destination}-${padIndex(seat)}:`, error) } attempts++ await new Promise(resolve => setTimeout(resolve, interval)) // Wait 1s before retrying } - await saveAndExit(seat) + await saveAndExit(destination, seat) } -async function runChecks () { +async function runChecks (destination, start, end) { const { lastChecked, restartAfter } = loadProgress() // Check if we are running before the allowed restart time @@ -128,13 +133,27 @@ async function runChecks () { } let index = lastChecked - while (index <= TOTAL_REPOS) { - index = await checkRepo(index) - if (index < 1) index = 1 // Prevent negative index + while (index <= end) { + index = await checkRepo(destination, index) + if (index < start) index = start // Prevent going below start saveProgress(index) } console.log('✅ All repos checked successfully.') } -// Start process -runChecks() +if (process.argv.length !== 5) { + console.log('Usage: node checkSync.js ') + console.log('Example: node checkSync.js adobe-summit-L322/seat 0 100') +} else { + const destination = process.argv[2] + const start = parseInt(process.argv[3]) + const end = parseInt(process.argv[4]) + + if (!Number.isInteger(start) || !Number.isInteger(end)) { + console.error('Start and End must be integers') + } else if (start > end) { + console.log('Start cannot be greater than End.') + } else { + runChecks(destination, start, end) + } +} diff --git a/src/scripts/cloneContent.js b/src/scripts/cloneContent.js index 7f84cd8..4ac14ee 100644 --- a/src/scripts/cloneContent.js +++ b/src/scripts/cloneContent.js @@ -1,22 +1,34 @@ /** * This is a script to clone content automatically. * Only to be used for housekeeping purposes. Please do not run this script without supervision. + * + * Usage: node cloneContent.js + * Example: node cloneContent.js adobe-commerce/adobe-demo-store adobe-summit-L322/seat 0 100 */ + import { execSync } from 'child_process' -const owner = 'adobe-summit-L321' -const templateRepo = 'ccdm-demo-store' -// const owner = 'adobe-summit-L322' -// const templateRepo = 'adobe-demo-store' -const repoPrefix = 'seat' -const templateOrg = 'adobe-commerce' +function parseRepoString (repoString) { + const [org, repo] = repoString.split('/') + return { org, repo } +} -async function cloneContent (start, end) { +function parseDestinationString (destString) { + const [org, prefix] = destString.split('/') + return { org, prefix } +} + +async function cloneContent (source, destination, start, end) { console.log('Starting to clone content...') + console.log(`Source: ${source}`) + console.log(`Destination: ${destination}`) + + const { org: sourceOrg, repo: sourceRepo } = parseRepoString(source) + const { org: destOrg, prefix: repoPrefix } = parseDestinationString(destination) for (let i = start; i <= end; i++) { const repoNumber = i.toString().padStart(2, '0') - const command = `aio commerce:init --template "${templateOrg}/${templateRepo}" --repo "${owner}/${repoPrefix}-${repoNumber}" --datasource "" --skipMesh --skipGit` + const command = `aio commerce:init --template "${sourceOrg}/${sourceRepo}" --repo "${destOrg}/${repoPrefix}-${repoNumber}" --datasource "" --skipMesh --skipGit` console.log(`\nExecuting command ${i} of ${end}:`) console.log(command) @@ -30,18 +42,21 @@ async function cloneContent (start, end) { } } -if (process.argv.length !== 4) { - console.log('Usage: node cloneContent.js ') +if (process.argv.length !== 6) { + console.log('Usage: node cloneContent.js ') + console.log('Example: node cloneContent.js adobe-commerce/adobe-demo-store adobe-summit-L322/seat 0 100') } else { - const start = parseInt(process.argv[2]) - const end = parseInt(process.argv[3]) + const source = process.argv[2] + const destination = process.argv[3] + const start = parseInt(process.argv[4]) + const end = parseInt(process.argv[5]) if (!Number.isInteger(start) || !Number.isInteger(end)) { console.error('Start and End must be integers') } else if (start > end) { console.log('Start cannot be greater than End.') } else { - cloneContent(start, end).then(() => { + cloneContent(source, destination, start, end).then(() => { console.log('All repos created successfully.') }).catch((error) => { console.error('Error creating repos:', error) diff --git a/src/scripts/createRepos.js b/src/scripts/createRepos.js index b877455..ab7a37d 100644 --- a/src/scripts/createRepos.js +++ b/src/scripts/createRepos.js @@ -2,24 +2,34 @@ * This is a script to create multiple GitHub repositories using a template repository. * Only to be used for housekeeping purposes. Please do not run this script without supervision. * + * Usage: node createRepos.js + * Example: node createRepos.js adobe-commerce/adobe-demo-store adobe-summit-L322/seat 0 100 */ import { createRepo } from '../utils/github.js' -const owner = 'adobe-summit-L321' -const templateRepo = 'ccdm-demo-store' -// const owner = 'adobe-summit-L322' -// const templateRepo = 'adobe-demo-store' -const repoPrefix = 'seat' -const templateOrg = 'adobe-commerce' +function parseRepoString (repoString) { + const [org, repo] = repoString.split('/') + return { org, repo } +} + +function parseDestinationString (destString) { + const [org, prefix] = destString.split('/') + return { org, prefix } +} -async function createRepos (start, end) { +async function createRepos (source, destination, start, end) { console.log('Starting to create repos...') + console.log(`Source: ${source}`) + console.log(`Destination: ${destination}`) + + const { org: sourceOrg, repo: sourceRepo } = parseRepoString(source) + const { org: destOrg, prefix: repoPrefix } = parseDestinationString(destination) for (let i = start; i <= end; i++) { const repo = `${repoPrefix}-${i.toString().padStart(2, '0')}` try { - await createRepo(owner, repo, templateOrg, templateRepo) + await createRepo(destOrg, repo, sourceOrg, sourceRepo) } catch (e) { console.error(`! Failed to complete run for "${repo}". Skipping.`) console.error(e) @@ -27,18 +37,21 @@ async function createRepos (start, end) { } } -if (process.argv.length !== 4) { - console.log('Usage: node createRepos.js ') +if (process.argv.length !== 6) { + console.log('Usage: node createRepos.js ') + console.log('Example: node createRepos.js adobe-commerce/adobe-demo-store adobe-summit-L322/seat 0 100') } else { - const start = parseInt(process.argv[2]) - const end = parseInt(process.argv[3]) + const source = process.argv[2] + const destination = process.argv[3] + const start = parseInt(process.argv[4]) + const end = parseInt(process.argv[5]) if (!Number.isInteger(start) || !Number.isInteger(end)) { console.error('Start and End must be integers') } else if (start > end) { console.log('Start cannot be greater than End.') } else { - createRepos(start, end).then(() => { + createRepos(source, destination, start, end).then(() => { console.log('All repos created successfully.') }).catch((error) => { console.error('Error creating repos:', error) diff --git a/src/scripts/deleteRepos.js b/src/scripts/deleteRepos.js index 229d40a..f384589 100644 --- a/src/scripts/deleteRepos.js +++ b/src/scripts/deleteRepos.js @@ -1,10 +1,18 @@ /** * This script deletes all the repositories created by the create-repos.js script. * Only to be used for housekeeping purposes. Please do not run this script without supervision. + * + * Usage: node deleteRepos.js + * Example: node deleteRepos.js adobe-summit-L322/seat 0 100 */ import childProcess from 'child_process' +function parseDestinationString (destString) { + const [org, prefix] = destString.split('/') + return { org, prefix } +} + function runCommand (cmd) { return new Promise((resolve, reject) => { childProcess.exec(cmd, (error, stdout, stderr) => { @@ -19,9 +27,14 @@ function runCommand (cmd) { }) } -function deleteRepos (start, end) { +function deleteRepos (destination, start, end) { + console.log('Starting to delete repos...') + console.log(`Destination: ${destination}`) + + const { org: destOrg, prefix: repoPrefix } = parseDestinationString(destination) + for (let seat = start; seat <= end; seat++) { - const repoName = `adobe-summit-L322/seat-${seat.toString().padStart(2, '0')}` + const repoName = `${destOrg}/${repoPrefix}-${seat.toString().padStart(2, '0')}` const command = `gh api \ --method DELETE \ -H "Accept: application/vnd.github+json" \ @@ -31,28 +44,30 @@ function deleteRepos (start, end) { runCommand(command) .then((success) => { if (success) { - console.log(`Repo at seat-${seat} deleted successfully`) + console.log(`Repo at ${repoName} deleted successfully`) } else { - console.error(`Failed to delete repo at seat-${seat}`) + console.error(`Failed to delete repo at ${repoName}`) } }) .catch((error) => { - console.error(`Error occurred while running command for seat-${seat}: ${error}`) + console.error(`Error occurred while running command for ${repoName}: ${error}`) }) } } -if (process.argv.length !== 4) { - console.log('Usage: node deleteRepos.js ') +if (process.argv.length !== 5) { + console.log('Usage: node deleteRepos.js ') + console.log('Example: node deleteRepos.js adobe-summit-L322/seat 0 100') } else { - const start = parseInt(process.argv[2]) - const end = parseInt(process.argv[3]) + const destination = process.argv[2] + const start = parseInt(process.argv[3]) + const end = parseInt(process.argv[4]) if (!Number.isInteger(start) || !Number.isInteger(end)) { console.error('Start and End must be integers') } else if (start > end) { console.log('Start cannot be greater than End.') } else { - deleteRepos(start, end) + deleteRepos(destination, start, end) } } From dfc08c0d0988a65a3666b7c657767495474121ab Mon Sep 17 00:00:00 2001 From: Stephen Rugh Date: Mon, 10 Mar 2025 11:22:31 -0500 Subject: [PATCH 22/22] add todo Signed-off-by: Stephen Rugh --- src/scripts/checkSync.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripts/checkSync.js b/src/scripts/checkSync.js index d82bfcb..dcc811e 100644 --- a/src/scripts/checkSync.js +++ b/src/scripts/checkSync.js @@ -12,6 +12,7 @@ const PROGRESS_FILE = 'progress.json' const WAIT_TIME_MS = 1 * 60 * 60 * 1000 // 1 hour // the amount of git files we expect the code sync to have to have processed. // for L322, it seems to be 418. For L321, seems to be 412. +// TODO: this needs to be dynamic, based on the source repo. const GIT_FILES_TO_CHECK = 412 function parseDestinationString (destString) {