diff --git a/.github/ISSUE_TEMPLATE/package-update-addition-to-docker-image.md b/.github/ISSUE_TEMPLATE/package-update-addition-to-docker-image.md index c76c82b2..1b6ff77f 100644 --- a/.github/ISSUE_TEMPLATE/package-update-addition-to-docker-image.md +++ b/.github/ISSUE_TEMPLATE/package-update-addition-to-docker-image.md @@ -1,6 +1,6 @@ --- name: Package update/addition to Docker image -about: This template is used when requesting something be changed in the base or tools +about: This template is used when requesting something be changed in the base image. Dockerfile title: "[Image Update]" labels: image update, triage_needed diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 72eba08d..78589658 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -57,39 +57,19 @@ jobs: scanners: 'vuln,config' severity: 'HIGH,CRITICAL' - # Build tools dockerfile - - name: Build the tools.Dockerfile - run: | - docker build -t tools_cloudshell --build-arg IMAGE_LOCATION=base_cloudshell -f linux/tools.Dockerfile . - - - name: Scan Tools image with Trivy - id: trivy-tools-scan - uses: aquasecurity/trivy-action@0.20.0 - with: - scan-type: 'image' - image-ref: tools_cloudshell - scanners: 'vuln,config' - severity: 'HIGH,CRITICAL' - - # Run the test cases - - name: Run the test cases - run: docker run --volume $(pwd)/tests:/tests tools_cloudshell /bin/bash /tests/test.sh - # Show Docker image size - name: find the pull request id run: echo ISSUEID=$(echo "${{github.ref }}" | sed 's!refs/pull/\([0-9]*\)/merge!\1!') >> $GITHUB_ENV - name: find the base size info run: echo BASE_SIZE=$(docker inspect base_cloudshell:latest --format "{{.Size}}") >> $GITHUB_ENV - - name: find the tools size info - run: echo TOOLS_SIZE=$(docker inspect tools_cloudshell:latest --format "{{.Size}}") >> $GITHUB_ENV - name: update a comment with size run: | - echo "pull id $ISSUEID size $BASE_SIZE $TOOLS_SIZE" && \ + echo "pull id $ISSUEID size $BASE_SIZE" && \ curl --request POST \ --url https://api.github.com/repos/${{ github.repository }}/issues/$ISSUEID/comments \ --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ --header 'content-type: application/json' \ --header 'Accept: application/vnd.github.v3+json' \ --data "{ - \"body\": \"Image size with this change is base: $(($BASE_SIZE / 1048576))MB, tools: $(($TOOLS_SIZE / 1048576))MB. \" + \"body\": \"Image size with this change is base: $(($BASE_SIZE / 1048576))MB\" }" diff --git a/.github/workflows/cloudshell-build-push.yaml b/.github/workflows/cloudshell-build-push.yaml index 602c27e0..26894e51 100644 --- a/.github/workflows/cloudshell-build-push.yaml +++ b/.github/workflows/cloudshell-build-push.yaml @@ -38,12 +38,3 @@ jobs: file: linux/base.Dockerfile push: true tags: ghcr.io/azure/cloudshell/base:${{ steps.tag.outputs.image_tag }},ghcr.io/azure/cloudshell/base:latest - - - name: Tools image - uses: docker/build-push-action@v6 - with: - context: . - file: linux/tools.Dockerfile - push: true - tags: ghcr.io/azure/cloudshell/tools:${{ steps.tag.outputs.image_tag }},ghcr.io/azure/cloudshell/tools:latest - build-args: IMAGE_LOCATION=ghcr.io/azure/cloudshell/base:${{ steps.tag.outputs.image_tag }} diff --git a/README.md b/README.md index 60c2ae19..bdee39bb 100644 --- a/README.md +++ b/README.md @@ -63,32 +63,10 @@ container to connect the shell process to the user interface via a websocket. 1. **Root instead of cloud shell user**. In Azure Cloud Shell you always run as a regular user. When running the image locally, you run as root. -### Understanding the base.Dockerfile and tools.Dockerfile - -The repository contains two Docker configuration files: `base` and `tools`. Normally you just have -one Dockerfile and rely on the container registry to cache the layers that haven't changed. -However, we need to cache the base image explicitly to ensure a fast startup time. Tools is built -on top of the base file and starts from an internal repository where the base image is cached, so -that we know when we need to update the base. - -When building or using the image locally, you don't need to worry about that. Just build using the -instructions below, and be aware that changes to the base layer will take longer to release than -changes to the tools. - -| Layer | Job | -| ---|---| -| Base | Contains large, infrequently changing packages. Changes every 3-4 months. | -| Tools | Contains frequently changing packages. Changes every 2-3 weeks | - ## Building and Testing the image ### Building the images -> [!NOTE] -> Cloud Shell publishes an image on each update to the master branch. If you would like to use the pre-built image, then -> you can skip this step by downloading the latest [base image layer here](ghcr.io/azure/cloudshell/base:latest) -> and the latest [tools image layer here](ghcr.io/azure/cloudshell/tools:latest). You can find all previously built image layers [here](https://github.com/orgs/Azure/packages?repo_name=CloudShell). - Required software: - Docker @@ -100,30 +78,24 @@ Building base.Dockerfile image from the root repository docker build -t base_cloudshell -f linux/base.Dockerfile . ``` -Building tools.Dockerfile image - -```bash -docker build -t tools_cloudshell --build-arg IMAGE_LOCATION=base_cloudshell -f linux/tools.Dockerfile . -``` - ### Testing the images -Running `bash` in the `tools.Dockerfile` based image: +Running `bash` in the `base.Dockerfile` based image: ```bash -docker run -it tools_cloudshell /bin/bash +docker run -it base_cloudshell /bin/bash ``` -Running `pwsh` in the `tools.Dockerfile` based image: +Running `pwsh` in the `base.Dockerfile` based image: ```bash -docker run -it tools_cloudshell /usr/bin/pwsh +docker run -it base_cloudshell /usr/bin/pwsh ``` Testing the Cloud Shell image: ```bash -docker run --volume /path/to/CloudShell/folder/tests:/tests -it tools_cloudshell /tests/test.sh +docker run --volume `pwd`/tests:/tests -it base_cloudshell /tests/test.sh ``` For more information about bind mounts, please see the diff --git a/build.ps1 b/build.ps1 index a1317a09..b0c419fc 100644 --- a/build.ps1 +++ b/build.ps1 @@ -32,9 +32,3 @@ if ($buildbase) { & docker build -t base_cloudshell $args -f linux/base.Dockerfile . Write-Verbose "Finished building base image" } - -if ($image -eq "tools" -or $image -eq "all") { - Write-verbose "Building tools image" - & docker build -t tools_cloudshell $args --build-arg IMAGE_LOCATION=base_cloudshell -f linux/tools.Dockerfile . - Write-Verbose "Finished building tools image" -} \ No newline at end of file diff --git a/linux/base.Dockerfile b/linux/base.Dockerfile index 91872d11..4766c0a7 100644 --- a/linux/base.Dockerfile +++ b/linux/base.Dockerfile @@ -1,20 +1,9 @@ -# base.Dockerfile contains components which are large and change less frequently. -# tools.Dockerfile contains the smaller, more frequently-updated components. - -# Within Azure, the image layers -# built from this file are cached in a number of locations to speed up container startup time. A manual -# step needs to be performed to refresh these locations when the image changes. For this reason, we explicitly -# split the base and the tools docker files into separate files and base the tools file from a version -# of the base docker file stored in a container registry. This avoids accidentally introducing a change in -# the base image - -# CBL-Mariner is an internal Linux distribution for Microsoft’s cloud infrastructure and edge products and services. -# CBL-Mariner is designed to provide a consistent platform for these devices and services and will enhance Microsoft’s -# ability to stay current on Linux updates. -# https://github.com/microsoft/CBL-Mariner FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 LABEL org.opencontainers.image.source="https://github.com/Azure/CloudShell" +RUN mkdir -p /usr/cloudshell +WORKDIR /usr/cloudshell + SHELL ["/bin/bash","-c"] COPY linux/tdnfinstall.sh . @@ -124,16 +113,41 @@ RUN tdnf update -y --refresh && \ redis \ cpio \ gettext && \ + # + # Install latest Azure CLI package. CLI team drops latest (pre-release) + # package here prior to public release We don't support using this location + # elsewhere - it may be removed or updated without notice + wget https://azurecliprod.blob.core.windows.net/cloudshell-release/azure-cli-latest-mariner2.0.rpm && \ + tdnf install -y ./azure-cli-latest-mariner2.0.rpm && \ + rm azure-cli-latest-mariner2.0.rpm && \ + # + # Note: These set of cleanup steps should always be the last ones in this RUN + # statement. tdnf clean all && \ rm -rf /var/cache/tdnf/* && \ rm /var/opt/apache-maven/lib/guava-25.1-android.jar -ENV NPM_CONFIG_LOGLEVEL warn -ENV NODE_ENV production -ENV NODE_OPTIONS=--tls-cipher-list='ECDHE-RSA-AES128-GCM-SHA256:!RC4' +# Install any Azure CLI extensions that should be included by default. +RUN az extension add --system --name ai-examples -y && \ + az extension add --system --name ssh -y && \ + az extension add --system --name ml -y && \ + # + # Install kubectl + az aks install-cli + +ENV NPM_CONFIG_LOGLEVEL=warn \ + NODE_ENV=production \ + NODE_OPTIONS=--tls-cipher-list='ECDHE-RSA-AES128-GCM-SHA256:!RC4' \ + GOROOT="/usr/lib/golang" + +# Add user's home directories to PATH at the front so they can install tools +# which override defaults Add dotnet tools to PATH so users can install a tool +# using dotnet tools and can execute that command from any directory +ENV PATH="~/.local/bin:~/bin:~/.dotnet/tools:$PATH:$GOROOT/bin:/opt/mssql-tools18/bin" \ + AZURE_CLIENTS_SHOW_SECRETS_WARNING=True \ + AZUREPS_HOST_ENVIRONMENT=cloud-shell/1.0 # Get latest version of Terraform. -# Customers require the latest version of Terraform. RUN TF_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M ".current_version") \ && wget -nv -O terraform.zip "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip" \ && wget -nv -O terraform.sha256 "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_SHA256SUMS" \ @@ -147,7 +161,7 @@ RUN TF_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform RUN echo en_US UTF-8 >> /etc/locale.conf && locale-gen.sh ENV LANG="en_US.utf8" -# # BEGIN: Install Ansible in isolated Virtual Environment +# BEGIN: Install Ansible in isolated Virtual Environment COPY ./linux/ansible/ansible* /usr/local/bin/ RUN chmod 755 /usr/local/bin/ansible* \ && cd /opt \ @@ -161,16 +175,12 @@ RUN chmod 755 /usr/local/bin/ansible* \ && wget -nv -q -O /usr/share/ansible/collections/ansible_collections/azure/azcollection/requirements.txt https://raw.githubusercontent.com/ansible-collections/azure/dev/requirements.txt \ && /opt/ansible/bin/python -m pip install -r /usr/share/ansible/collections/ansible_collections/azure/azcollection/requirements.txt - # Install latest version of Istio -ENV ISTIO_ROOT /usr/local/istio-latest +ENV ISTIO_ROOT=/usr/local/istio-latest +ENV PATH=$PATH:$ISTIO_ROOT/bin RUN curl -sSL https://git.io/getLatestIstio | sh - \ && mv $PWD/istio* $ISTIO_ROOT \ && chmod -R 755 $ISTIO_ROOT -ENV PATH $PATH:$ISTIO_ROOT/bin - -ENV GOROOT="/usr/lib/golang" -ENV PATH="$PATH:$GOROOT/bin:/opt/mssql-tools18/bin" RUN gem install bundler --no-document --clear-sources --force \ && bundle config set without 'development test' \ @@ -179,9 +189,9 @@ RUN gem install bundler --no-document --clear-sources --force \ && gem install rspec --no-document --clear-sources --force \ && rm -rf $(gem env gemdir)/cache/*.gem -ENV GEM_HOME=~/bundle -ENV BUNDLE_PATH=~/bundle -ENV PATH=$PATH:$GEM_HOME/bin:$BUNDLE_PATH/gems/bin +ENV GEM_HOME=~/bundle \ + BUNDLE_PATH=~/bundle \ + PATH=$PATH:$GEM_HOME/bin:$BUNDLE_PATH/gems/bin # Install vscode RUN wget -nv -O vscode.tar.gz "https://code.visualstudio.com/sha/download?build=insider&os=cli-alpine-x64" \ @@ -195,17 +205,32 @@ ENV AZD_IN_CLOUDSHELL=1 \ RUN curl -fsSL https://aka.ms/install-azd.sh | bash && \ # # Install Office 365 CLI templates - # npm install -q -g @pnp/cli-microsoft365 && \ # # Install Bicep CLI - # curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64 \ && chmod +x ./bicep \ && mv ./bicep /usr/local/bin/bicep \ && bicep --help && \ # # Add soft links - # ln -s /usr/bin/python3 /usr/bin/python && \ ln -s /usr/bin/node /usr/bin/nodejs + +# Powershell telemetry +ENV POWERSHELL_DISTRIBUTION_CHANNEL=CloudShell \ + # don't tell users to upgrade, they can't + POWERSHELL_UPDATECHECK=Off + +# Copy and run script to install Powershell modules and setup Powershell machine profile +COPY ./linux/powershell/ powershell +RUN /usr/bin/pwsh -File ./powershell/setupPowerShell.ps1 -image Base && \ + cp -r ./powershell/PSCloudShellUtility /usr/local/share/powershell/Modules/PSCloudShellUtility/ && \ + /usr/bin/pwsh -File ./powershell/setupPowerShell.ps1 -image Top && \ + # Install Powershell warmup script + mkdir -p linux/powershell && \ + cp powershell/Invoke-PreparePowerShell.ps1 linux/powershell/Invoke-PreparePowerShell.ps1 && \ + rm -rf ./powershell + +# Remove su so users don't have su access by default. +RUN rm -f ./linux/Dockerfile && rm -f /bin/su diff --git a/linux/tools.Dockerfile b/linux/tools.Dockerfile deleted file mode 100644 index 764c1247..00000000 --- a/linux/tools.Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# IMAGE_LOCATION refers to a Microsoft-internal container registry which stores a cached version -# of the image built from base.Dockerfile. If you are building this file outside Microsoft, you -# won't be able to reach this location, but don't worry! - -# To build yourself locally, override this location with a local image tag. See README.md for more detail - -ARG IMAGE_LOCATION=cdpxb787066ec88f4e20ae65e42a858c42ca00.azurecr.io/official/cloudshell:base.master.11e65d27.20240822.1 -# Copy from base build -FROM ${IMAGE_LOCATION} - -LABEL org.opencontainers.image.source="https://github.com/Azure/CloudShell" - -RUN tdnf clean all && \ - tdnf repolist --refresh && \ - ACCEPT_EULA=Y tdnf update -y && \ - # Install latest Azure CLI package. CLI team drops latest (pre-release) package here prior to public release - # We don't support using this location elsewhere - it may be removed or updated without notice - wget https://azurecliprod.blob.core.windows.net/cloudshell-release/azure-cli-latest-mariner2.0.rpm \ - && tdnf install -y ./azure-cli-latest-mariner2.0.rpm \ - && rm azure-cli-latest-mariner2.0.rpm && \ - tdnf clean all && \ - rm -rf /var/cache/tdnf/* - -# Install any Azure CLI extensions that should be included by default. -RUN az extension add --system --name ai-examples -y \ - && az extension add --system --name ssh -y \ - && az extension add --system --name ml -y - -# Install kubectl -RUN az aks install-cli \ - && chmod +x /usr/local/bin/kubectl \ - && chmod +x /usr/local/bin/kubelogin - -# Install vscode -RUN wget -nv -O vscode.tar.gz "https://code.visualstudio.com/sha/download?build=insider&os=cli-alpine-x64" \ - && tar -xvzf vscode.tar.gz \ - && mv ./code-insiders /bin/vscode \ - && rm vscode.tar.gz - -# Install azure-developer-cli (azd) -ENV AZD_IN_CLOUDSHELL 1 -ENV AZD_SKIP_UPDATE_CHECK 1 -RUN curl -fsSL https://aka.ms/install-azd.sh | bash - -RUN mkdir -p /usr/cloudshell -WORKDIR /usr/cloudshell - -# Install Office 365 CLI templates -RUN npm install -q -g @pnp/cli-microsoft365 - -# Install Bicep CLI -RUN curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64 \ - && chmod +x ./bicep \ - && mv ./bicep /usr/local/bin/bicep \ - && bicep --help - -# Temp: fix ansible modules. Proper fix is to update base layer to use regular python for Ansible. -RUN mkdir -p /usr/share/ansible/collections/ansible_collections/azure/azcollection/ \ - && wget -nv -q -O /usr/share/ansible/collections/ansible_collections/azure/azcollection/requirements.txt https://raw.githubusercontent.com/ansible-collections/azure/dev/requirements.txt \ - && /opt/ansible/bin/python -m pip install -r /usr/share/ansible/collections/ansible_collections/azure/azcollection/requirements.txt - -# Powershell telemetry -ENV POWERSHELL_DISTRIBUTION_CHANNEL=CloudShell \ - # don't tell users to upgrade, they can't - POWERSHELL_UPDATECHECK=Off - -# Copy and run script to install Powershell modules and setup Powershell machine profile -COPY ./linux/powershell/ powershell -RUN /usr/bin/pwsh -File ./powershell/setupPowerShell.ps1 -image Base && \ - cp -r ./powershell/PSCloudShellUtility /usr/local/share/powershell/Modules/PSCloudShellUtility/ && \ - /usr/bin/pwsh -File ./powershell/setupPowerShell.ps1 -image Top && \ - # Install Powershell warmup script - mkdir -p linux/powershell && \ - cp powershell/Invoke-PreparePowerShell.ps1 linux/powershell/Invoke-PreparePowerShell.ps1 && \ - rm -rf ./powershell - -# Remove su so users don't have su access by default. -RUN rm -f ./linux/Dockerfile && rm -f /bin/su - -#Add soft links -RUN ln -sf /usr/bin/python3 /usr/bin/python -RUN ln -sf /usr/bin/node /usr/bin/nodejs - -# Add user's home directories to PATH at the front so they can install tools which -# override defaults -# Add dotnet tools to PATH so users can install a tool using dotnet tools and can execute that command from any directory -ENV PATH ~/.local/bin:~/bin:~/.dotnet/tools:$PATH - -ENV AZURE_CLIENTS_SHOW_SECRETS_WARNING True - -# Set AZUREPS_HOST_ENVIRONMENT -ENV AZUREPS_HOST_ENVIRONMENT cloud-shell/1.0 diff --git a/test.ps1 b/test.ps1 index cda48f48..ce83c6cf 100644 --- a/test.ps1 +++ b/test.ps1 @@ -2,9 +2,9 @@ [CmdletBinding()] param( - [string]$image = "tools_cloudshell" + [string]$image = "base_cloudshell" ) $ErrorActionPreference = "Stop" -& docker run --volume $psscriptroot/tests:/tests $image /bin/bash /tests/test.sh \ No newline at end of file +& docker run --volume $psscriptroot/tests:/tests $image /bin/bash /tests/test.sh