diff --git a/build/dockerfiles/Dockerfile b/build/dockerfiles/Dockerfile index 290e94177..12f5daf1a 100644 --- a/build/dockerfiles/Dockerfile +++ b/build/dockerfiles/Dockerfile @@ -42,9 +42,8 @@ RUN yarn build RUN yarn workspace @eclipse-che/dashboard-backend install --production # Prepare air-gapped resources -# ARG GITHUB_TOKEN=$GITHUB_TOKEN -COPY build/dockerfiles/airgap.sh /dashboard/airgap.sh -RUN /dashboard/airgap.sh -d /dashboard/packages/devfile-registry/air-gap +COPY scripts/airgap.sh /dashboard/airgap.sh +RUN /dashboard/airgap.sh -i /dashboard/packages/devfile-registry/air-gap/index.json FROM docker.io/node:18.19.1-alpine3.19 diff --git a/packages/common/src/dto/api/index.ts b/packages/common/src/dto/api/index.ts index 18f823bca..bf3afaaaa 100644 --- a/packages/common/src/dto/api/index.ts +++ b/packages/common/src/dto/api/index.ts @@ -190,6 +190,7 @@ export interface IGettingStartedSample { } export interface IAirGapSample extends IGettingStartedSample { + id: string; project?: { zip?: { filename?: string } }; devfile?: { filename?: string }; } diff --git a/packages/dashboard-backend/src/devworkspaceClient/services/__tests__/fixtures/air-gap/index.json b/packages/dashboard-backend/src/devworkspaceClient/services/__tests__/fixtures/air-gap/index.json index ac083f9b7..ea3f5ab4c 100644 --- a/packages/dashboard-backend/src/devworkspaceClient/services/__tests__/fixtures/air-gap/index.json +++ b/packages/dashboard-backend/src/devworkspaceClient/services/__tests__/fixtures/air-gap/index.json @@ -1,17 +1,21 @@ [ { + "id": "Sample_no_devfile_filename", "displayName": "Sample_no_devfile_filename" }, { + "id": "Sample_no_project_filename", "displayName": "Sample_no_project_filename" }, { + "id": "Sample_devfile_not_exists", "displayName": "Sample_devfile_not_exists", "devfile": { "filename": "not-exists-defile.yaml" } }, { + "id": "Sample_project_not_exists", "displayName": "Sample_project_not_exists", "project": { "zip": { @@ -20,12 +24,14 @@ } }, { + "id": "Sample_devfile", "displayName": "Sample_devfile", "devfile": { "filename": "sample-devfile.yaml" } }, { + "id": "Sample_project", "displayName": "Sample_project", "project": { "zip": { diff --git a/packages/dashboard-backend/src/devworkspaceClient/services/airGapSampleApi.ts b/packages/dashboard-backend/src/devworkspaceClient/services/airGapSampleApi.ts index 74dc72c97..e85d494e0 100644 --- a/packages/dashboard-backend/src/devworkspaceClient/services/airGapSampleApi.ts +++ b/packages/dashboard-backend/src/devworkspaceClient/services/airGapSampleApi.ts @@ -39,23 +39,23 @@ export class AirGapSampleApiService implements IAirGapSampleApi { return this.samples; } - async downloadProject(name: string): Promise { - const sample = this.samples.find(sample => sample.displayName === name); + async downloadProject(id: string): Promise { + const sample = this.samples.find(sample => sample.id === id); if (sample) { return this.download(sample.project?.zip?.filename); } - console.error(`Sample not found: ${name} `); + console.error(`Sample not found: ${id} `); throw new Error(`Sample not found`); } - async downloadDevfile(name: string): Promise { - const sample = this.samples.find(sample => sample.displayName === name); + async downloadDevfile(id: string): Promise { + const sample = this.samples.find(sample => sample.id === id); if (sample) { return this.download(sample.devfile?.filename); } - console.error(`Sample not found: ${name} `); + console.error(`Sample not found: ${id} `); throw new Error(`Sample not found`); } diff --git a/packages/dashboard-backend/src/devworkspaceClient/types/index.ts b/packages/dashboard-backend/src/devworkspaceClient/types/index.ts index 06f048fec..6dcc31c29 100644 --- a/packages/dashboard-backend/src/devworkspaceClient/types/index.ts +++ b/packages/dashboard-backend/src/devworkspaceClient/types/index.ts @@ -455,14 +455,14 @@ export interface IAirGapSampleApi { list(): Promise>; /** - * Downloads the Air Gap sample project by its name. + * Downloads the Air Gap sample project by its id. */ - downloadProject(name: string): Promise; + downloadProject(id: string): Promise; /** - * Reads the devfile content of the Air Gap sample by its name. + * Reads the devfile content of the Air Gap sample by its id. */ - downloadDevfile(name: string): Promise; + downloadDevfile(id: string): Promise; } export interface IEditorsApi { diff --git a/packages/dashboard-backend/src/routes/api/__tests__/airGapSample.spec.ts b/packages/dashboard-backend/src/routes/api/__tests__/airGapSample.spec.ts index 0cc8df794..342331ec4 100644 --- a/packages/dashboard-backend/src/routes/api/__tests__/airGapSample.spec.ts +++ b/packages/dashboard-backend/src/routes/api/__tests__/airGapSample.spec.ts @@ -66,7 +66,7 @@ describe('AirGap Sample Route', () => { } as unknown as DevWorkspaceClient; }); - const res = await app.inject().get(`${baseApiPath}/airgap-sample/devfile/download?name=sample`); + const res = await app.inject().get(`${baseApiPath}/airgap-sample/devfile/download?id=sample`); expect(res.statusCode).toEqual(200); expect(res.headers['content-type']).toEqual('application/octet-stream'); @@ -92,7 +92,7 @@ describe('AirGap Sample Route', () => { } as unknown as DevWorkspaceClient; }); - const res = await app.inject().get(`${baseApiPath}/airgap-sample/project/download?name=sample`); + const res = await app.inject().get(`${baseApiPath}/airgap-sample/project/download?id=sample`); expect(res.statusCode).toEqual(200); expect(res.headers['content-type']).toEqual('application/octet-stream'); diff --git a/packages/dashboard-backend/src/routes/api/airGapSample.ts b/packages/dashboard-backend/src/routes/api/airGapSample.ts index f95401d5e..8f83b70bc 100644 --- a/packages/dashboard-backend/src/routes/api/airGapSample.ts +++ b/packages/dashboard-backend/src/routes/api/airGapSample.ts @@ -43,16 +43,16 @@ export function registerAirGapSampleRoute(instance: FastifyInstance) { `${baseApiPath}/airgap-sample/devfile/download`, Object.assign(rateLimitConfig, getSchema({ tags })), async function (request: FastifyRequest, reply: FastifyReply) { - const name = (request.query as { name: string })['name']; - if (!name) { - return reply.status(400).send('Sample name is required.'); + const sampleId = (request.query as { id: string })['id']; + if (!sampleId) { + return reply.status(400).send('Sample id is required.'); } const token = getServiceAccountToken(); const { airGapSampleApi } = getDevWorkspaceClient(token); try { - const iStreamedFile = await airGapSampleApi.downloadDevfile(name); + const iStreamedFile = await airGapSampleApi.downloadDevfile(sampleId); reply.header('Content-Type', 'application/octet-stream'); reply.header('Content-Length', iStreamedFile.size); return reply.send(iStreamedFile.stream); @@ -67,16 +67,16 @@ export function registerAirGapSampleRoute(instance: FastifyInstance) { `${baseApiPath}/airgap-sample/project/download`, Object.assign(rateLimitConfig, getSchema({ tags })), async function (request: FastifyRequest, reply: FastifyReply) { - const name = (request.query as { name: string })['name']; - if (!name) { - return reply.status(400).send('Sample name is required.'); + const sampleId = (request.query as { id: string })['id']; + if (!sampleId) { + return reply.status(400).send('Sample id is required.'); } const token = getServiceAccountToken(); const { airGapSampleApi } = getDevWorkspaceClient(token); try { - const iStreamedFile = await airGapSampleApi.downloadProject(name); + const iStreamedFile = await airGapSampleApi.downloadProject(sampleId); reply.header('Content-Type', 'application/octet-stream'); reply.header('Content-Length', iStreamedFile.size); return reply.send(iStreamedFile.stream); diff --git a/build/dockerfiles/airgap.sh b/scripts/airgap.sh similarity index 51% rename from build/dockerfiles/airgap.sh rename to scripts/airgap.sh index da3af1e19..5310f58b3 100755 --- a/build/dockerfiles/airgap.sh +++ b/scripts/airgap.sh @@ -9,23 +9,32 @@ # # The script is used to download resources (projects and devfiles) -# for air-gapped (offline) environments. Only https://github.com is supported for now. +# Only https://github.com is supported for now. set -e init() { - unset AIRGAP_RESOURCES_DIR + unset SRC_INDEX_JSON_PATH + unset OUTPUT_DIR while [ "$#" -gt 0 ]; do case $1 in - '--airgap-resources-dir'|'-d') AIRGAP_RESOURCES_DIR=$2; shift 1;; + '--src-index-json-path'|'-i') SRC_INDEX_JSON_PATH=$2; shift 1;; + '--output-dir'|'-o') OUTPUT_DIR=$2; shift 1;; '--help'|'-h') usage; exit;; esac shift 1 done - [ -z "${AIRGAP_RESOURCES_DIR}" ] && { usage; exit; } - SAMPLES_JSON_PATH="${AIRGAP_RESOURCES_DIR}/index.json" + + if [ -z "${SRC_INDEX_JSON_PATH}" ]; then + usage + exit + fi + + if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR=$(dirname "${SRC_INDEX_JSON_PATH}") + fi } usage() { @@ -33,19 +42,24 @@ usage() { Usage: $0 [OPTIONS] Options: - --airgap-resources-dir, -d Directory where airgap resources are stored + --src-index-json-path, -i Path to the JSON file containing the list of samples + --output-dir, -o Directory where the downloaded resources will be stored --help, -h Show this help message EOF } run() { - samplesNum=$(jq -r '. | length' "${SAMPLES_JSON_PATH}") + mkdir -p "${OUTPUT_DIR}" + if [ ! "${SRC_INDEX_JSON_PATH}" = "${OUTPUT_DIR}/index.json" ]; then + cp "${SRC_INDEX_JSON_PATH}" "${OUTPUT_DIR}/index.json" + fi + + samplesNum=$(jq -r '. | length' "${SRC_INDEX_JSON_PATH}") i=0 while [ "${i}" -lt "${samplesNum}" ]; do - url=$(jq -r '.['${i}'].url' "${SAMPLES_JSON_PATH}") - name=$(jq -r '.['${i}'].displayName' "${SAMPLES_JSON_PATH}") - encodedName=$(echo "${name}" | jq -Rr @uri) + url=$(jq -r '.['${i}'].url' "${SRC_INDEX_JSON_PATH}") + sampleId=$(jq -r '.['${i}'].id' "${SRC_INDEX_JSON_PATH}") if [ "${url}" != "null" ]; then strippedURL="${url#https://github.com/}" @@ -71,7 +85,7 @@ run() { "${devfileFileName}" \ "${projectDownloadLink}" \ "${devfileDownloadLink}" \ - "${encodedName}" \ + "${sampleId}" \ "${repository}" fi @@ -84,34 +98,50 @@ processSample() { devfileFileName=$2 projectDownloadLink=$3 devfileDownloadLink=$4 - encodedName=$5 + sampleId=$5 repository=$6 - curl -L \ - -H "Accept: application/vnd.github.raw+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - "${devfileDownloadLink}" \ - -o "${AIRGAP_RESOURCES_DIR}/${devfileFileName}" - - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - "${projectDownloadLink}" \ - -o "${AIRGAP_RESOURCES_DIR}/${archiveFileName}" + if [ -z ${GITHUB_TOKEN} ]; then + curl -L \ + -H "Accept: application/vnd.github.raw+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${devfileDownloadLink}" \ + -o "${OUTPUT_DIR}/${devfileFileName}" + + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${projectDownloadLink}" \ + -o "${OUTPUT_DIR}/${archiveFileName}" + else + curl -L \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.raw+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${devfileDownloadLink}" \ + -o "${OUTPUT_DIR}/${devfileFileName}" + + curl -L \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${projectDownloadLink}" \ + -o "${OUTPUT_DIR}/${archiveFileName}" + fi # CHE_DASHBOARD_INTERNAL_URL is a placeholder that will be replaced # by the actual URL in entrypoint.sh - devfileLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/devfile/download?name=${encodedName}" - projectLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/project/download?name=${encodedName}" + devfileLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/devfile/download?id=${sampleId}" + projectLink="CHE_DASHBOARD_INTERNAL_URL/dashboard/api/airgap-sample/project/download?id=${sampleId}" - echo "$(jq '(.['${i}'].url) = '\"${devfileLink}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}" - echo "$(jq '(.['${i}'].project.zip.filename) = '\"${archiveFileName}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}" - echo "$(jq '(.['${i}'].devfile.filename) = '\"${devfileFileName}\" ${SAMPLES_JSON_PATH})" > "${SAMPLES_JSON_PATH}" + # shellcheck disable=SC2005 + echo "$(cat "${OUTPUT_DIR}/index.json" | \ + jq '(.['${i}'].url) = '\"${devfileLink}\" | \ + jq '(.['${i}'].project.zip.filename) = '\"${archiveFileName}\" | \ + jq '(.['${i}'].devfile.filename) = '\"${devfileFileName}\")" > "${OUTPUT_DIR}/index.json" # Update the devfile with the project link - yq -riY '.projects=[{name: "'${repository}'", zip: {location: "'${projectLink}'"}}]' "${AIRGAP_RESOURCES_DIR}/${devfileFileName}" + yq -riY '.projects=[{name: "'${repository}'", zip: {location: "'${projectLink}'"}}]' "${OUTPUT_DIR}/${devfileFileName}" } init "$@"