diff --git a/.github/workflows/auto-assign.yaml b/.github/workflows/auto-assign.yaml
index 3da215ecb3fd..75ac0f86f2ea 100644
--- a/.github/workflows/auto-assign.yaml
+++ b/.github/workflows/auto-assign.yaml
@@ -3,8 +3,14 @@ on:
issue_comment:
types: [created, edited]
+permissions:
+ contents: read
+
jobs:
assign:
+ permissions:
+ # write permissions are needed to assign the issue.
+ contents: write
name: Run self assign job
runs-on: ubuntu-latest
steps:
diff --git a/.github/workflows/canary-integration-suite.yml b/.github/workflows/canary-integration-suite.yml
index b13ad4f49037..731d50fcdbbc 100644
--- a/.github/workflows/canary-integration-suite.yml
+++ b/.github/workflows/canary-integration-suite.yml
@@ -24,6 +24,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
canary-tests:
uses: ./.github/workflows/canary-integration-test.yml
diff --git a/.github/workflows/create-tag.yaml b/.github/workflows/create-tag.yaml
deleted file mode 100644
index 22152eac7579..000000000000
--- a/.github/workflows/create-tag.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Tag
-on:
- workflow_dispatch:
- inputs:
- version:
- description: "Release version (e.g. v1.7.0)"
- required: true
-
-defaults:
- run:
- # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
- shell: bash --noprofile --norc -eo pipefail -x {0}
-
-jobs:
- Create-Tag:
- runs-on: ubuntu-22.04
- if: github.repository == 'rook/rook' && contains('travisn,leseb,BlaineEXE,jbw976,galexrt,satoru-takeuchi', github.actor)
- steps:
- - name: checkout
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: set env
- run: |
- echo "FROM_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
- echo "TO_TAG=$(git describe --abbrev=0 --tags)" >> $GITHUB_ENV
- echo "GITHUB_USER=rook" >> $GITHUB_ENV
-
- - name: Create Tag
- uses: negz/create-tag@v1
- with:
- version: ${{ github.event.inputs.version }}
- token: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Get Release Note
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_USER: ${{ env.GITHUB_USER }}
- FROM_BRANCH: ${{ env.FROM_BRANCH }}
- TO_TAG: ${{ env.TO_TAG }}
- run: tests/scripts/gen_release_notes.sh
diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index 4fa73a4edf77..393c58544d99 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -9,6 +9,9 @@ defaults:
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
shell: bash --noprofile --norc -eo pipefail -x {0}
+permissions:
+ contents: read
+
jobs:
canary-arm64:
runs-on: [self-hosted, ubuntu-20.04-arm64, ARM64]
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index 125eb646cc2c..515e0c8525bc 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -18,6 +18,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephHelmSuite:
if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 0fcf2750e5bc..ab53093ea095 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -17,6 +17,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephMgrSuite:
runs-on: ubuntu-22.04
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 0badaee61fec..be6bf74a74e3 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -18,6 +18,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephMultiClusterDeploySuite:
if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index 1754fd5c8af9..ac075715823e 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -18,6 +18,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephObjectSuite:
if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index 67b4808cbcdf..a458ce7b3a6a 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -18,6 +18,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephSmokeSuite:
if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index 07ccb01d2299..679b9e82d88e 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -18,6 +18,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
TestCephUpgradeSuite:
if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml
index 067fd28bd2c9..63b3f9bb180e 100644
--- a/.github/workflows/integration-tests-on-release.yaml
+++ b/.github/workflows/integration-tests-on-release.yaml
@@ -12,6 +12,9 @@ defaults:
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
shell: bash --noprofile --norc -eo pipefail -x {0}
+permissions:
+ contents: read
+
jobs:
TestCephHelmSuite:
runs-on: ubuntu-22.04
diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml
index a24a6289b78b..18955b1f980d 100644
--- a/.github/workflows/multus.yaml
+++ b/.github/workflows/multus.yaml
@@ -22,6 +22,9 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
+permissions:
+ contents: read
+
jobs:
test-validation-tool:
runs-on: ubuntu-latest
diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index aba50fd3122f..f4b067195294 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -26,7 +26,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
* `external`:
* `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](external-cluster/external-cluster.md). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
* `cephVersion`: The version information for launching the ceph daemons.
- * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v18.2.2`. For more details read the [container images section](#ceph-container-images).
+ * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v18.2.4`. For more details read the [container images section](#ceph-container-images).
For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/).
To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v17` will be updated each time a new Quincy build is released.
@@ -80,6 +80,8 @@ For more details on the mons and when to choose a number other than `3`, see the
`useAllNodes` must be set to `false` to use specific nodes and their config.
See [node settings](#node-settings) below.
* `config`: Config settings applied to all OSDs on the node unless overridden by `devices`. See the [config settings](#osd-configuration-settings) below.
+ * `allowDeviceClassUpdate`: Whether to allow changing the device class of an OSD after it is created. The default is false
+ to prevent unintentional data movement or CRUSH changes if the device class is changed accidentally.
* [storage selection settings](#storage-selection-settings)
* [Storage Class Device Sets](#storage-class-device-sets)
* `onlyApplyOSDPlacement`: Whether the placement specific for OSDs is merged with the `all` placement. If `false`, the OSD placement will be merged with the `all` placement. If true, the `OSD placement will be applied` and the `all` placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD:
@@ -114,8 +116,8 @@ These are general purpose Ceph container with all necessary daemons and dependen
| -------------------- | --------------------------------------------------------- |
| vRELNUM | Latest release in this series (e.g., **v17** = Quincy) |
| vRELNUM.Y | Latest stable release in this stable series (e.g., v17.2) |
-| vRELNUM.Y.Z | A specific release (e.g., v18.2.2) |
-| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v18.2.2-20240311) |
+| vRELNUM.Y.Z | A specific release (e.g., v18.2.4) |
+| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v18.2.4-20240724) |
A specific will contain a specific release of Ceph as well as security fixes from the Operating System.
@@ -340,7 +342,7 @@ The following storage selection settings are specific to Ceph and do not apply t
* `metadataDevice`: Name of a device, [partition](#limitations-of-metadata-device) or lvm to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
* `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size.
* `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
-* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](../Block-Storage/ceph-block-pool-crd.md#spec).
+* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](../Block-Storage/ceph-block-pool-crd.md#spec). If updating the device class of an OSD after the OSD is already created, `allowDeviceClassUpdate: true` must be set. Otherwise updates to this `deviceClass` will be ignored.
* `initialWeight`: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity.
* `primaryAffinity`: The [primary-affinity](https://docs.ceph.com/en/latest/rados/operations/crush-map/#primary-affinity) value of an OSD, within range `[0, 1]` (default: `1`).
* `osdsPerDevice`**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device.
@@ -420,7 +422,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
@@ -526,7 +528,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
@@ -654,7 +656,7 @@ kubectl -n rook-ceph get CephCluster -o yaml
deviceClasses:
- name: hdd
version:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
version: 16.2.6-0
conditions:
- lastHeartbeatTime: "2021-03-02T21:22:11Z"
diff --git a/Documentation/CRDs/Cluster/host-cluster.md b/Documentation/CRDs/Cluster/host-cluster.md
index 600cf9a3e873..daa8dd666478 100644
--- a/Documentation/CRDs/Cluster/host-cluster.md
+++ b/Documentation/CRDs/Cluster/host-cluster.md
@@ -22,7 +22,7 @@ metadata:
spec:
cephVersion:
# see the "Cluster Settings" section below for more details on which image of ceph to run
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
@@ -49,7 +49,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
@@ -101,7 +101,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
diff --git a/Documentation/CRDs/Cluster/pvc-cluster.md b/Documentation/CRDs/Cluster/pvc-cluster.md
index f644016ef2b3..231ff7dcaa11 100644
--- a/Documentation/CRDs/Cluster/pvc-cluster.md
+++ b/Documentation/CRDs/Cluster/pvc-cluster.md
@@ -18,7 +18,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
@@ -72,7 +72,7 @@ spec:
requests:
storage: 10Gi
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: false
dashboard:
enabled: true
@@ -128,7 +128,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
diff --git a/Documentation/CRDs/Cluster/stretch-cluster.md b/Documentation/CRDs/Cluster/stretch-cluster.md
index a4f5d483b9eb..00e0adf95c99 100644
--- a/Documentation/CRDs/Cluster/stretch-cluster.md
+++ b/Documentation/CRDs/Cluster/stretch-cluster.md
@@ -34,7 +34,7 @@ spec:
- name: b
- name: c
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: true
# Either storageClassDeviceSets or the storage section can be specified for creating OSDs.
# This example uses all devices for simplicity.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index d79438615518..944a50dd8b00 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -10255,7 +10255,7 @@ the triple using the matching operator
(Optional)
- TopologySpreadConstraint specifies how to spread matching pods among the given topology
+TopologySpreadConstraints specifies how to spread matching pods among the given topology
|
@@ -12218,6 +12218,18 @@ float64
BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
+
+
+allowDeviceClassUpdate
+
+bool
+
+ |
+
+(Optional)
+ Whether to allow updating the device class after the OSD is initially provisioned
+ |
+
StoreType
diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md
index 64abb8050553..291f606fd4b0 100644
--- a/Documentation/Upgrade/ceph-upgrade.md
+++ b/Documentation/Upgrade/ceph-upgrade.md
@@ -50,7 +50,7 @@ Official Ceph container images can be found on [Quay](https://quay.io/repository
These images are tagged in a few ways:
-* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v18.2.2-20240311`).
+* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v18.2.4-20240724`).
These tags are recommended for production clusters, as there is no possibility for the cluster to
be heterogeneous with respect to the version of Ceph running in containers.
* Ceph major version tags (e.g., `v18`) are useful for development and test clusters so that the
@@ -67,7 +67,7 @@ CephCluster CRD (`spec.cephVersion.image`).
```console
ROOK_CLUSTER_NAMESPACE=rook-ceph
-NEW_CEPH_IMAGE='quay.io/ceph/ceph:v18.2.2-20240311'
+NEW_CEPH_IMAGE='quay.io/ceph/ceph:v18.2.4-20240724'
kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $ROOK_CLUSTER_NAMESPACE --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}"
```
@@ -79,7 +79,7 @@ employed by the new Rook operator release. Employing an outdated Ceph version wi
in unexpected behaviour.
```console
-kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v18.2.2-20240311
+kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v18.2.4-20240724
```
#### **3. Wait for the pod updates**
@@ -97,9 +97,9 @@ Confirm the upgrade is completed when the versions are all on the desired Ceph v
kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"ceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' | sort | uniq
This cluster is not yet finished:
ceph-version=v17.2.7-0
- ceph-version=v18.2.2-0
+ ceph-version=v18.2.4-0
This cluster is finished:
- ceph-version=v18.2.2-0
+ ceph-version=v18.2.4-0
```
#### **4. Verify cluster health**
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 556160b3364c..40d5e84f4e20 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -9,3 +9,4 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her
## Features
- Added support for Ceph Squid (v19)
+- Allow updating the device class of OSDs, if `allowDeviceClassUpdate: true` is set
diff --git a/build/csv/ceph/ceph.rook.io_cephclusters.yaml b/build/csv/ceph/ceph.rook.io_cephclusters.yaml
index d5083b7f7d28..e6bd6b6fc665 100644
--- a/build/csv/ceph/ceph.rook.io_cephclusters.yaml
+++ b/build/csv/ceph/ceph.rook.io_cephclusters.yaml
@@ -1551,6 +1551,8 @@ spec:
storage:
nullable: true
properties:
+ allowDeviceClassUpdate:
+ type: boolean
backfillFullRatio:
maximum: 1
minimum: 0
diff --git a/build/csv/ceph/rook-ceph-operator.clusterserviceversion.yaml b/build/csv/ceph/rook-ceph-operator.clusterserviceversion.yaml
index e8e5ecb23897..5fb3f5aa9481 100644
--- a/build/csv/ceph/rook-ceph-operator.clusterserviceversion.yaml
+++ b/build/csv/ceph/rook-ceph-operator.clusterserviceversion.yaml
@@ -1451,446 +1451,442 @@ metadata:
bG9neV9yYmRfcG9vbHM6CiAgICAgICAgICAgIHNlbGYudmFsaWRhdGVfcmJkX3Bvb2wocG9vbCkK
CiAgICBkZWYgaW5pdF90b3BvbG9neV9yYmRfcG9vbHMoc2VsZiwgdG9wb2xvZ3lfcmJkX3Bvb2xz
KToKICAgICAgICBmb3IgcG9vbCBpbiB0b3BvbG9neV9yYmRfcG9vbHM6CiAgICAgICAgICAgIHNl
- bGYuaW5pdF9yYmRfcG9vbChwb29sKQoKICAgIGRlZiBnZXRTY3JpcHRDbGlGbGFncyhzZWxmKToK
- ICAgICAgICByZXR1cm4gIiAiLmpvaW4oc3lzLmFyZ3ZbMTpdKQoKICAgICMgdGhpcyB3aWxsIHJl
- dHVybiB0aGUgZmluYWwgYXJncyB0aGF0IHNjcmlwdCB1c2VzIHRvIHByb2Nlc3MKICAgICMgdGhl
- IHByaW9yaXR5IHRvIHNldCBhIHBhcnRpY3VsYXIgdmFsdWUgaXMsCiAgICAjIGNvbW1hbmQtbGlu
- ZS1hcmdzID4gY29uZmlnLmluaSBmaWxlIHZhbHVlcyA+IGRlZmF1bHQgdmFsdWVzCiAgICBkZWYg
- Z2V0RmluYWxVc2VkQXJncyhzZWxmKToKICAgICAgICBsaXN0ID0gW10KICAgICAgICBmb3IgYXJn
- IGluIHZhcnMoc2VsZi5fYXJnX3BhcnNlcik6CiAgICAgICAgICAgIGlmIHN0cihnZXRhdHRyKHNl
- bGYuX2FyZ19wYXJzZXIsIGFyZykpOgogICAgICAgICAgICAgICAgIyBweXRob24gdHJlYXRzIGZs
- YWctbmFtZSBhcyBmbGFnX25hbWUgaW50ZXJuYWxseSwgc28gY29udmVydGluZyBiYWNrIHRvIGZs
- YWctbmFtZSwKICAgICAgICAgICAgICAgICMgc28gd2UgY2FuIGdldCB0aG9zZSB2YWx1ZXMgZnJv
- bSBjb25maWcgZmlsZQogICAgICAgICAgICAgICAgYXJndW1lbnQgPSAoCiAgICAgICAgICAgICAg
- ICAgICAgYXJnLnJlcGxhY2UoIl8iLCAiLSIpICsgIjogIiArIHN0cihnZXRhdHRyKHNlbGYuX2Fy
- Z19wYXJzZXIsIGFyZykpCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBsaXN0LmFw
- cGVuZChhcmd1bWVudCkKICAgICAgICByZXR1cm4gbGlzdAoKICAgIGRlZiBfZ2VuX291dHB1dF9t
- YXAoc2VsZik6CiAgICAgICAgaWYgc2VsZi5vdXRfbWFwOgogICAgICAgICAgICByZXR1cm4KICAg
- ICAgICAjIHN1cHBvcnQgbGVnYWN5IGZsYWcgd2l0aCB1cGdyYWRlcwogICAgICAgIGlmIHNlbGYu
- X2FyZ19wYXJzZXIuY2x1c3Rlcl9uYW1lOgogICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLms4
- c19jbHVzdGVyX25hbWUgPSBzZWxmLl9hcmdfcGFyc2VyLmNsdXN0ZXJfbmFtZQogICAgICAgIHNl
- bGYuX2FyZ19wYXJzZXIuazhzX2NsdXN0ZXJfbmFtZSA9ICgKICAgICAgICAgICAgc2VsZi5fYXJn
- X3BhcnNlci5rOHNfY2x1c3Rlcl9uYW1lLmxvd2VyKCkKICAgICAgICApICAjIGFsd2F5cyBjb252
- ZXJ0IGNsdXN0ZXIgbmFtZSB0byBsb3dlcmNhc2UgY2hhcmFjdGVycwogICAgICAgIHNlbGYudmFs
- aWRhdGVfcmJkX3Bvb2woc2VsZi5fYXJnX3BhcnNlci5yYmRfZGF0YV9wb29sX25hbWUpCiAgICAg
- ICAgc2VsZi5pbml0X3JiZF9wb29sKHNlbGYuX2FyZ19wYXJzZXIucmJkX2RhdGFfcG9vbF9uYW1l
- KQogICAgICAgIHNlbGYudmFsaWRhdGVfcmFkb3NfbmFtZXNwYWNlKCkKICAgICAgICBzZWxmLl9l
- eGNsdWRlZF9rZXlzLmFkZCgiSzhTX0NMVVNURVJfTkFNRSIpCiAgICAgICAgc2VsZi5nZXRfY2Vw
- aGZzX2RhdGFfcG9vbF9kZXRhaWxzKCkKICAgICAgICAjIGRvdWJsZSBzdHJpbmcgbmVlZGVkIGZv
- ciB1cHN0cmVhbSBleHBvcnRzIG9mIGZsYWdzCiAgICAgICAgc2VsZi5vdXRfbWFwWyJFWFRFUk5B
- TF9DTFVTVEVSX1VTRVJfQ09NTUFORCJdID0gZicie3NlbGYuZ2V0U2NyaXB0Q2xpRmxhZ3MoKX0i
- JwogICAgICAgIHNlbGYub3V0X21hcFsiQVJHUyJdID0gZicie3NlbGYuZ2V0RmluYWxVc2VkQXJn
- cygpfSInCiAgICAgICAgc2VsZi5vdXRfbWFwWyJOQU1FU1BBQ0UiXSA9IHNlbGYuX2FyZ19wYXJz
- ZXIubmFtZXNwYWNlCiAgICAgICAgc2VsZi5vdXRfbWFwWyJLOFNfQ0xVU1RFUl9OQU1FIl0gPSBz
- ZWxmLl9hcmdfcGFyc2VyLms4c19jbHVzdGVyX25hbWUKICAgICAgICBzZWxmLm91dF9tYXBbIlJP
- T0tfRVhURVJOQUxfRlNJRCJdID0gc2VsZi5nZXRfZnNpZCgpCiAgICAgICAgc2VsZi5vdXRfbWFw
- WyJST09LX0VYVEVSTkFMX1VTRVJOQU1FIl0gPSBzZWxmLnJ1bl9hc191c2VyCiAgICAgICAgc2Vs
- Zi5vdXRfbWFwWyJST09LX0VYVEVSTkFMX0NFUEhfTU9OX0RBVEEiXSA9IHNlbGYuZ2V0X2NlcGhf
- ZXh0ZXJuYWxfbW9uX2RhdGEoKQogICAgICAgIHNlbGYub3V0X21hcFsiUk9PS19FWFRFUk5BTF9V
- U0VSX1NFQ1JFVCJdID0gc2VsZi5jcmVhdGVfY2hlY2tlcktleSgKICAgICAgICAgICAgImNsaWVu
- dC5oZWFsdGhjaGVja2VyIgogICAgICAgICkKICAgICAgICBzZWxmLm91dF9tYXBbIlJPT0tfRVhU
- RVJOQUxfREFTSEJPQVJEX0xJTksiXSA9IHNlbGYuZ2V0X2NlcGhfZGFzaGJvYXJkX2xpbmsoKQog
- ICAgICAgICgKICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfUkJEX05PREVfU0VDUkVUIl0s
- CiAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FIl0sCiAg
- ICAgICAgKSA9IHNlbGYuY3JlYXRlX2NlcGhDU0lLZXlyaW5nX3VzZXIoImNsaWVudC5jc2ktcmJk
- LW5vZGUiKQogICAgICAgICgKICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfUkJEX1BST1ZJ
- U0lPTkVSX1NFQ1JFVCJdLAogICAgICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9SQkRfUFJPVklT
- SU9ORVJfU0VDUkVUX05BTUUiXSwKICAgICAgICApID0gc2VsZi5jcmVhdGVfY2VwaENTSUtleXJp
- bmdfdXNlcigiY2xpZW50LmNzaS1yYmQtcHJvdmlzaW9uZXIiKQogICAgICAgIHNlbGYub3V0X21h
- cFsiQ0VQSEZTX1BPT0xfTkFNRSJdID0gc2VsZi5fYXJnX3BhcnNlci5jZXBoZnNfZGF0YV9wb29s
- X25hbWUKICAgICAgICBzZWxmLm91dF9tYXBbIkNFUEhGU19NRVRBREFUQV9QT09MX05BTUUiXSA9
- ICgKICAgICAgICAgICAgc2VsZi5fYXJnX3BhcnNlci5jZXBoZnNfbWV0YWRhdGFfcG9vbF9uYW1l
- CiAgICAgICAgKQogICAgICAgIHNlbGYub3V0X21hcFsiQ0VQSEZTX0ZTX05BTUUiXSA9IHNlbGYu
- X2FyZ19wYXJzZXIuY2VwaGZzX2ZpbGVzeXN0ZW1fbmFtZQogICAgICAgIHNlbGYub3V0X21hcFsi
- UkVTVFJJQ1RFRF9BVVRIX1BFUk1JU1NJT04iXSA9ICgKICAgICAgICAgICAgc2VsZi5fYXJnX3Bh
- cnNlci5yZXN0cmljdGVkX2F1dGhfcGVybWlzc2lvbgogICAgICAgICkKICAgICAgICBzZWxmLm91
- dF9tYXBbIlJBRE9TX05BTUVTUEFDRSJdID0gc2VsZi5fYXJnX3BhcnNlci5yYWRvc19uYW1lc3Bh
- Y2UKICAgICAgICBzZWxmLm91dF9tYXBbIlNVQlZPTFVNRV9HUk9VUCJdID0gc2VsZi5fYXJnX3Bh
- cnNlci5zdWJ2b2x1bWVfZ3JvdXAKICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfTk9E
- RV9TRUNSRVQiXSA9ICIiCiAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX1BST1ZJU0lP
- TkVSX1NFQ1JFVCJdID0gIiIKICAgICAgICAjIGNyZWF0ZSBDZXBoRlMgbm9kZSBhbmQgcHJvdmlz
- aW9uZXIga2V5cmluZyBvbmx5IHdoZW4gTURTIGV4aXN0cwogICAgICAgIGlmIHNlbGYub3V0X21h
- cFsiQ0VQSEZTX0ZTX05BTUUiXSBhbmQgc2VsZi5vdXRfbWFwWyJDRVBIRlNfUE9PTF9OQU1FIl06
- CiAgICAgICAgICAgICgKICAgICAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19O
- T0RFX1NFQ1JFVCJdLAogICAgICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX05P
- REVfU0VDUkVUX05BTUUiXSwKICAgICAgICAgICAgKSA9IHNlbGYuY3JlYXRlX2NlcGhDU0lLZXly
- aW5nX3VzZXIoImNsaWVudC5jc2ktY2VwaGZzLW5vZGUiKQogICAgICAgICAgICAoCiAgICAgICAg
- ICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVUIl0sCiAg
- ICAgICAgICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVU
- X05BTUUiXSwKICAgICAgICAgICAgKSA9IHNlbGYuY3JlYXRlX2NlcGhDU0lLZXlyaW5nX3VzZXIo
- ImNsaWVudC5jc2ktY2VwaGZzLXByb3Zpc2lvbmVyIikKICAgICAgICAgICAgIyBjcmVhdGUgdGhl
- IGRlZmF1bHQgImNzaSIgc3Vidm9sdW1lZ3JvdXAKICAgICAgICAgICAgc2VsZi5nZXRfb3JfY3Jl
- YXRlX3N1YnZvbHVtZV9ncm91cCgKICAgICAgICAgICAgICAgICJjc2kiLCBzZWxmLl9hcmdfcGFy
- c2VyLmNlcGhmc19maWxlc3lzdGVtX25hbWUKICAgICAgICAgICAgKQogICAgICAgICAgICAjIHBp
- biB0aGUgZGVmYXVsdCAiY3NpIiBzdWJ2b2x1bWVncm91cAogICAgICAgICAgICBzZWxmLnBpbl9z
- dWJ2b2x1bWUoCiAgICAgICAgICAgICAgICAiY3NpIiwgc2VsZi5fYXJnX3BhcnNlci5jZXBoZnNf
- ZmlsZXN5c3RlbV9uYW1lLCAiZGlzdHJpYnV0ZWQiLCAiMSIKICAgICAgICAgICAgKQogICAgICAg
- ICAgICBpZiBzZWxmLm91dF9tYXBbIlNVQlZPTFVNRV9HUk9VUCJdOgogICAgICAgICAgICAgICAg
- c2VsZi5nZXRfb3JfY3JlYXRlX3N1YnZvbHVtZV9ncm91cCgKICAgICAgICAgICAgICAgICAgICBz
- ZWxmLl9hcmdfcGFyc2VyLnN1YnZvbHVtZV9ncm91cCwKICAgICAgICAgICAgICAgICAgICBzZWxm
- Ll9hcmdfcGFyc2VyLmNlcGhmc19maWxlc3lzdGVtX25hbWUsCiAgICAgICAgICAgICAgICApCiAg
- ICAgICAgICAgICAgICBzZWxmLnBpbl9zdWJ2b2x1bWUoCiAgICAgICAgICAgICAgICAgICAgc2Vs
- Zi5fYXJnX3BhcnNlci5zdWJ2b2x1bWVfZ3JvdXAsCiAgICAgICAgICAgICAgICAgICAgc2VsZi5f
- YXJnX3BhcnNlci5jZXBoZnNfZmlsZXN5c3RlbV9uYW1lLAogICAgICAgICAgICAgICAgICAgICJk
- aXN0cmlidXRlZCIsCiAgICAgICAgICAgICAgICAgICAgIjEiLAogICAgICAgICAgICAgICAgKQog
- ICAgICAgIHNlbGYub3V0X21hcFsiUkdXX1RMU19DRVJUIl0gPSAiIgogICAgICAgIHNlbGYub3V0
- X21hcFsiTU9OSVRPUklOR19FTkRQT0lOVCJdID0gIiIKICAgICAgICBzZWxmLm91dF9tYXBbIk1P
- TklUT1JJTkdfRU5EUE9JTlRfUE9SVCJdID0gIiIKICAgICAgICBpZiBub3Qgc2VsZi5fYXJnX3Bh
- cnNlci5za2lwX21vbml0b3JpbmdfZW5kcG9pbnQ6CiAgICAgICAgICAgICgKICAgICAgICAgICAg
- ICAgIHNlbGYub3V0X21hcFsiTU9OSVRPUklOR19FTkRQT0lOVCJdLAogICAgICAgICAgICAgICAg
- c2VsZi5vdXRfbWFwWyJNT05JVE9SSU5HX0VORFBPSU5UX1BPUlQiXSwKICAgICAgICAgICAgKSA9
- IHNlbGYuZ2V0X2FjdGl2ZV9hbmRfc3RhbmRieV9tZ3JzKCkKICAgICAgICBzZWxmLm91dF9tYXBb
- IlJCRF9QT09MX05BTUUiXSA9IHNlbGYuX2FyZ19wYXJzZXIucmJkX2RhdGFfcG9vbF9uYW1lCiAg
- ICAgICAgc2VsZi5vdXRfbWFwWyJSQkRfTUVUQURBVEFfRUNfUE9PTF9OQU1FIl0gPSAoCiAgICAg
- ICAgICAgIHNlbGYudmFsaWRhdGVfcmJkX21ldGFkYXRhX2VjX3Bvb2xfbmFtZSgpCiAgICAgICAg
- KQogICAgICAgIHNlbGYub3V0X21hcFsiVE9QT0xPR1lfUE9PTFMiXSA9IHNlbGYuX2FyZ19wYXJz
- ZXIudG9wb2xvZ3lfcG9vbHMKICAgICAgICBzZWxmLm91dF9tYXBbIlRPUE9MT0dZX0ZBSUxVUkVf
- RE9NQUlOX0xBQkVMIl0gPSAoCiAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIudG9wb2xvZ3lf
- ZmFpbHVyZV9kb21haW5fbGFiZWwKICAgICAgICApCiAgICAgICAgc2VsZi5vdXRfbWFwWyJUT1BP
- TE9HWV9GQUlMVVJFX0RPTUFJTl9WQUxVRVMiXSA9ICgKICAgICAgICAgICAgc2VsZi5fYXJnX3Bh
- cnNlci50b3BvbG9neV9mYWlsdXJlX2RvbWFpbl92YWx1ZXMKICAgICAgICApCiAgICAgICAgaWYg
- KAogICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLnRvcG9sb2d5X3Bvb2xzICE9ICIiCiAgICAg
- ICAgICAgIGFuZCBzZWxmLl9hcmdfcGFyc2VyLnRvcG9sb2d5X2ZhaWx1cmVfZG9tYWluX2xhYmVs
- ICE9ICIiCiAgICAgICAgICAgIGFuZCBzZWxmLl9hcmdfcGFyc2VyLnRvcG9sb2d5X2ZhaWx1cmVf
- ZG9tYWluX3ZhbHVlcyAhPSAiIgogICAgICAgICk6CiAgICAgICAgICAgIHNlbGYudmFsaWRhdGVf
- dG9wb2xvZ3lfdmFsdWVzKAogICAgICAgICAgICAgICAgc2VsZi5jb252ZXJ0X2NvbW1hX3NlcGFy
- YXRlZF90b19hcnJheShzZWxmLm91dF9tYXBbIlRPUE9MT0dZX1BPT0xTIl0pLAogICAgICAgICAg
- ICAgICAgc2VsZi5jb252ZXJ0X2NvbW1hX3NlcGFyYXRlZF90b19hcnJheSgKICAgICAgICAgICAg
- ICAgICAgICBzZWxmLm91dF9tYXBbIlRPUE9MT0dZX0ZBSUxVUkVfRE9NQUlOX1ZBTFVFUyJdCiAg
- ICAgICAgICAgICAgICApLAogICAgICAgICAgICApCiAgICAgICAgICAgIHNlbGYudmFsaWRhdGVf
- dG9wb2xvZ3lfcmJkX3Bvb2xzKAogICAgICAgICAgICAgICAgc2VsZi5jb252ZXJ0X2NvbW1hX3Nl
- cGFyYXRlZF90b19hcnJheShzZWxmLm91dF9tYXBbIlRPUE9MT0dZX1BPT0xTIl0pCiAgICAgICAg
- ICAgICkKICAgICAgICAgICAgc2VsZi5pbml0X3RvcG9sb2d5X3JiZF9wb29scygKICAgICAgICAg
- ICAgICAgIHNlbGYuY29udmVydF9jb21tYV9zZXBhcmF0ZWRfdG9fYXJyYXkoc2VsZi5vdXRfbWFw
- WyJUT1BPTE9HWV9QT09MUyJdKQogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAg
- ICAgc2VsZi5yYWlzZV9leGNlcHRpb25faWZfYW55X3RvcG9sb2d5X2ZsYWdfaXNfbWlzc2luZygp
- CgogICAgICAgIHNlbGYub3V0X21hcFsiUkdXX1BPT0xfUFJFRklYIl0gPSBzZWxmLl9hcmdfcGFy
- c2VyLnJnd19wb29sX3ByZWZpeAogICAgICAgIHNlbGYub3V0X21hcFsiUkdXX0VORFBPSU5UIl0g
- PSAiIgogICAgICAgIGlmIHNlbGYuX2FyZ19wYXJzZXIucmd3X2VuZHBvaW50OgogICAgICAgICAg
- ICBpZiBzZWxmLl9hcmdfcGFyc2VyLmRyeV9ydW46CiAgICAgICAgICAgICAgICBzZWxmLmNyZWF0
- ZV9yZ3dfYWRtaW5fb3BzX3VzZXIoKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAg
- aWYgKAogICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmd3X3JlYWxtX25hbWUg
- IT0gIiIKICAgICAgICAgICAgICAgICAgICBhbmQgc2VsZi5fYXJnX3BhcnNlci5yZ3dfem9uZWdy
- b3VwX25hbWUgIT0gIiIKICAgICAgICAgICAgICAgICAgICBhbmQgc2VsZi5fYXJnX3BhcnNlci5y
- Z3dfem9uZV9uYW1lICE9ICIiCiAgICAgICAgICAgICAgICApOgogICAgICAgICAgICAgICAgICAg
- IGVyciA9IHNlbGYudmFsaWRhdGVfcmd3X211bHRpc2l0ZSgKICAgICAgICAgICAgICAgICAgICAg
- ICAgc2VsZi5fYXJnX3BhcnNlci5yZ3dfcmVhbG1fbmFtZSwgInJlYWxtIgogICAgICAgICAgICAg
- ICAgICAgICkKICAgICAgICAgICAgICAgICAgICBlcnIgPSBzZWxmLnZhbGlkYXRlX3Jnd19tdWx0
- aXNpdGUoCiAgICAgICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVn
- cm91cF9uYW1lLCAiem9uZWdyb3VwIgogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAg
- ICAgICAgICBlcnIgPSBzZWxmLnZhbGlkYXRlX3Jnd19tdWx0aXNpdGUoCiAgICAgICAgICAgICAg
- ICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVfbmFtZSwgInpvbmUiCiAgICAgICAg
- ICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgIGlmICgKICAgICAgICAgICAgICAgICAgICBz
- ZWxmLl9hcmdfcGFyc2VyLnJnd19yZWFsbV9uYW1lID09ICIiCiAgICAgICAgICAgICAgICAgICAg
- YW5kIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVncm91cF9uYW1lID09ICIiCiAgICAgICAgICAg
- ICAgICAgICAgYW5kIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVfbmFtZSA9PSAiIgogICAgICAg
- ICAgICAgICAgKSBvciAoCiAgICAgICAgICAgICAgICAgICAgc2VsZi5fYXJnX3BhcnNlci5yZ3df
- cmVhbG1fbmFtZSAhPSAiIgogICAgICAgICAgICAgICAgICAgIGFuZCBzZWxmLl9hcmdfcGFyc2Vy
- LnJnd196b25lZ3JvdXBfbmFtZSAhPSAiIgogICAgICAgICAgICAgICAgICAgIGFuZCBzZWxmLl9h
- cmdfcGFyc2VyLnJnd196b25lX25hbWUgIT0gIiIKICAgICAgICAgICAgICAgICk6CiAgICAgICAg
- ICAgICAgICAgICAgKAogICAgICAgICAgICAgICAgICAgICAgICBzZWxmLm91dF9tYXBbIlJHV19B
- RE1JTl9PUFNfVVNFUl9BQ0NFU1NfS0VZIl0sCiAgICAgICAgICAgICAgICAgICAgICAgIHNlbGYu
- b3V0X21hcFsiUkdXX0FETUlOX09QU19VU0VSX1NFQ1JFVF9LRVkiXSwKICAgICAgICAgICAgICAg
- ICAgICAgICAgaW5mb19jYXBfc3VwcG9ydGVkLAogICAgICAgICAgICAgICAgICAgICAgICBlcnIs
- CiAgICAgICAgICAgICAgICAgICAgKSA9IHNlbGYuY3JlYXRlX3Jnd19hZG1pbl9vcHNfdXNlcigp
- CiAgICAgICAgICAgICAgICAgICAgZXJyID0gc2VsZi52YWxpZGF0ZV9yZ3dfZW5kcG9pbnQoaW5m
- b19jYXBfc3VwcG9ydGVkKQogICAgICAgICAgICAgICAgICAgIGlmIHNlbGYuX2FyZ19wYXJzZXIu
- cmd3X3Rsc19jZXJ0X3BhdGg6CiAgICAgICAgICAgICAgICAgICAgICAgIHNlbGYub3V0X21hcFsi
- UkdXX1RMU19DRVJUIl0gPSAoCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZWxmLnZhbGlk
- YXRlX3Jnd19lbmRwb2ludF90bHNfY2VydCgpCiAgICAgICAgICAgICAgICAgICAgICAgICkKICAg
- ICAgICAgICAgICAgICAgICAjIGlmIHRoZXJlIGlzIG5vIGVycm9yLCBzZXQgdGhlIFJHV19FTkRQ
- T0lOVAogICAgICAgICAgICAgICAgICAgIGlmIGVyciAhPSAiLTEiOgogICAgICAgICAgICAgICAg
- ICAgICAgICBzZWxmLm91dF9tYXBbIlJHV19FTkRQT0lOVCJdID0gc2VsZi5fYXJnX3BhcnNlci5y
- Z3dfZW5kcG9pbnQKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgZXJy
- ID0gIlBsZWFzZSBwcm92aWRlIGFsbCB0aGUgUkdXIG11bHRpc2l0ZSBwYXJhbWV0ZXJzIG9yIG5v
- bmUgb2YgdGhlbSIKICAgICAgICAgICAgICAgICAgICBzeXMuc3RkZXJyLndyaXRlKGVycikKCiAg
- ICBkZWYgZ2VuX3NoZWxsX291dChzZWxmKToKICAgICAgICBzZWxmLl9nZW5fb3V0cHV0X21hcCgp
- CiAgICAgICAgc2hPdXRJTyA9IFN0cmluZ0lPKCkKICAgICAgICBmb3IgaywgdiBpbiBzZWxmLm91
- dF9tYXAuaXRlbXMoKToKICAgICAgICAgICAgaWYgdiBhbmQgayBub3QgaW4gc2VsZi5fZXhjbHVk
- ZWRfa2V5czoKICAgICAgICAgICAgICAgIHNoT3V0SU8ud3JpdGUoZiJleHBvcnQge2t9PXt2fXtM
- SU5FU0VQfSIpCiAgICAgICAgc2hPdXQgPSBzaE91dElPLmdldHZhbHVlKCkKICAgICAgICBzaE91
- dElPLmNsb3NlKCkKICAgICAgICByZXR1cm4gc2hPdXQKCiAgICBkZWYgZ2VuX2pzb25fb3V0KHNl
- bGYpOgogICAgICAgIHNlbGYuX2dlbl9vdXRwdXRfbWFwKCkKICAgICAgICBpZiBzZWxmLl9hcmdf
- cGFyc2VyLmRyeV9ydW46CiAgICAgICAgICAgIHJldHVybiAiIgogICAgICAgIGpzb25fb3V0ID0g
- WwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAibmFtZSI6ICJleHRlcm5hbC1jbHVzdGVy
- LXVzZXItY29tbWFuZCIsCiAgICAgICAgICAgICAgICAia2luZCI6ICJDb25maWdNYXAiLAogICAg
- ICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAgICAgImNvbW1hbmQiOiBzZWxm
- Lm91dF9tYXBbIkVYVEVSTkFMX0NMVVNURVJfVVNFUl9DT01NQU5EIl0sCiAgICAgICAgICAgICAg
- ICAgICAgImFyZ3MiOiBzZWxmLm91dF9tYXBbIkFSR1MiXSwKICAgICAgICAgICAgICAgIH0sCiAg
- ICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICJuYW1lIjogInJvb2st
- Y2VwaC1tb24tZW5kcG9pbnRzIiwKICAgICAgICAgICAgICAgICJraW5kIjogIkNvbmZpZ01hcCIs
- CiAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHNl
- bGYub3V0X21hcFsiUk9PS19FWFRFUk5BTF9DRVBIX01PTl9EQVRBIl0sCiAgICAgICAgICAgICAg
- ICAgICAgIm1heE1vbklkIjogIjAiLAogICAgICAgICAgICAgICAgICAgICJtYXBwaW5nIjogInt9
- IiwKICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAg
- ICAgICAgICAgICJuYW1lIjogInJvb2stY2VwaC1tb24iLAogICAgICAgICAgICAgICAgImtpbmQi
- OiAiU2VjcmV0IiwKICAgICAgICAgICAgICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAg
- ICJhZG1pbi1zZWNyZXQiOiAiYWRtaW4tc2VjcmV0IiwKICAgICAgICAgICAgICAgICAgICAiZnNp
- ZCI6IHNlbGYub3V0X21hcFsiUk9PS19FWFRFUk5BTF9GU0lEIl0sCiAgICAgICAgICAgICAgICAg
- ICAgIm1vbi1zZWNyZXQiOiAibW9uLXNlY3JldCIsCiAgICAgICAgICAgICAgICB9LAogICAgICAg
+ bGYuaW5pdF9yYmRfcG9vbChwb29sKQoKICAgICMgdGhpcyB3aWxsIHJldHVybiB0aGUgZmluYWwg
+ YXJncyB0aGF0IHNjcmlwdCB1c2VzIHRvIHByb2Nlc3MKICAgICMgdGhlIHByaW9yaXR5IHRvIHNl
+ dCBhIHBhcnRpY3VsYXIgdmFsdWUgaXMsCiAgICAjIGNvbW1hbmQtbGluZS1hcmdzID4gY29uZmln
+ LmluaSBmaWxlIHZhbHVlcyA+IGRlZmF1bHQgdmFsdWVzCiAgICBkZWYgZ2V0RmluYWxVc2VkQXJn
+ cyhzZWxmKToKICAgICAgICBhcmd1bWVudCA9IGYiW0NvbmZpZ3VyYXRpb25zXVxuIgogICAgICAg
+ IGZvciBhcmcgaW4gdmFycyhzZWxmLl9hcmdfcGFyc2VyKToKICAgICAgICAgICAgaWYgc3RyKGdl
+ dGF0dHIoc2VsZi5fYXJnX3BhcnNlciwgYXJnKSk6CiAgICAgICAgICAgICAgICAjIHB5dGhvbiB0
+ cmVhdHMgZmxhZy1uYW1lIGFzIGZsYWdfbmFtZSBpbnRlcm5hbGx5LCBzbyBjb252ZXJ0aW5nIGJh
+ Y2sgdG8gZmxhZy1uYW1lLAogICAgICAgICAgICAgICAgIyBzbyB3ZSBjYW4gZ2V0IHRob3NlIHZh
+ bHVlcyBmcm9tIGNvbmZpZyBmaWxlCiAgICAgICAgICAgICAgICBhcmdWYWx1ZSA9IGFyZy5yZXBs
+ YWNlKCJfIiwgIi0iKQogICAgICAgICAgICAgICAgaWYgYXJnVmFsdWUgIT0gImNvbmZpZy1maWxl
+ IjoKICAgICAgICAgICAgICAgICAgICBhcmd1bWVudCArPSBmInthcmdWYWx1ZX0gPSB7c3RyKGdl
+ dGF0dHIoc2VsZi5fYXJnX3BhcnNlciwgYXJnKSl9XG4iCiAgICAgICAgcmV0dXJuIGFyZ3VtZW50
+ CgogICAgZGVmIF9nZW5fb3V0cHV0X21hcChzZWxmKToKICAgICAgICBpZiBzZWxmLm91dF9tYXA6
+ CiAgICAgICAgICAgIHJldHVybgogICAgICAgICMgc3VwcG9ydCBsZWdhY3kgZmxhZyB3aXRoIHVw
+ Z3JhZGVzCiAgICAgICAgaWYgc2VsZi5fYXJnX3BhcnNlci5jbHVzdGVyX25hbWU6CiAgICAgICAg
+ ICAgIHNlbGYuX2FyZ19wYXJzZXIuazhzX2NsdXN0ZXJfbmFtZSA9IHNlbGYuX2FyZ19wYXJzZXIu
+ Y2x1c3Rlcl9uYW1lCiAgICAgICAgc2VsZi5fYXJnX3BhcnNlci5rOHNfY2x1c3Rlcl9uYW1lID0g
+ KAogICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLms4c19jbHVzdGVyX25hbWUubG93ZXIoKQog
+ ICAgICAgICkgICMgYWx3YXlzIGNvbnZlcnQgY2x1c3RlciBuYW1lIHRvIGxvd2VyY2FzZSBjaGFy
+ YWN0ZXJzCiAgICAgICAgc2VsZi52YWxpZGF0ZV9yYmRfcG9vbChzZWxmLl9hcmdfcGFyc2VyLnJi
+ ZF9kYXRhX3Bvb2xfbmFtZSkKICAgICAgICBzZWxmLmluaXRfcmJkX3Bvb2woc2VsZi5fYXJnX3Bh
+ cnNlci5yYmRfZGF0YV9wb29sX25hbWUpCiAgICAgICAgc2VsZi52YWxpZGF0ZV9yYWRvc19uYW1l
+ c3BhY2UoKQogICAgICAgIHNlbGYuX2V4Y2x1ZGVkX2tleXMuYWRkKCJLOFNfQ0xVU1RFUl9OQU1F
+ IikKICAgICAgICBzZWxmLmdldF9jZXBoZnNfZGF0YV9wb29sX2RldGFpbHMoKQogICAgICAgICMg
+ ZG91YmxlIHN0cmluZyBuZWVkZWQgZm9yIHVwc3RyZWFtIGV4cG9ydHMgb2YgZmxhZ3MKICAgICAg
+ ICBzZWxmLm91dF9tYXBbIkFSR1MiXSA9IGYnIntzZWxmLmdldEZpbmFsVXNlZEFyZ3MoKX0iJwog
+ ICAgICAgIHNlbGYub3V0X21hcFsiTkFNRVNQQUNFIl0gPSBzZWxmLl9hcmdfcGFyc2VyLm5hbWVz
+ cGFjZQogICAgICAgIHNlbGYub3V0X21hcFsiSzhTX0NMVVNURVJfTkFNRSJdID0gc2VsZi5fYXJn
+ X3BhcnNlci5rOHNfY2x1c3Rlcl9uYW1lCiAgICAgICAgc2VsZi5vdXRfbWFwWyJST09LX0VYVEVS
+ TkFMX0ZTSUQiXSA9IHNlbGYuZ2V0X2ZzaWQoKQogICAgICAgIHNlbGYub3V0X21hcFsiUk9PS19F
+ WFRFUk5BTF9VU0VSTkFNRSJdID0gc2VsZi5ydW5fYXNfdXNlcgogICAgICAgIHNlbGYub3V0X21h
+ cFsiUk9PS19FWFRFUk5BTF9DRVBIX01PTl9EQVRBIl0gPSBzZWxmLmdldF9jZXBoX2V4dGVybmFs
+ X21vbl9kYXRhKCkKICAgICAgICBzZWxmLm91dF9tYXBbIlJPT0tfRVhURVJOQUxfVVNFUl9TRUNS
+ RVQiXSA9IHNlbGYuY3JlYXRlX2NoZWNrZXJLZXkoCiAgICAgICAgICAgICJjbGllbnQuaGVhbHRo
+ Y2hlY2tlciIKICAgICAgICApCiAgICAgICAgc2VsZi5vdXRfbWFwWyJST09LX0VYVEVSTkFMX0RB
+ U0hCT0FSRF9MSU5LIl0gPSBzZWxmLmdldF9jZXBoX2Rhc2hib2FyZF9saW5rKCkKICAgICAgICAo
+ CiAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX1JCRF9OT0RFX1NFQ1JFVCJdLAogICAgICAg
+ ICAgICBzZWxmLm91dF9tYXBbIkNTSV9SQkRfTk9ERV9TRUNSRVRfTkFNRSJdLAogICAgICAgICkg
+ PSBzZWxmLmNyZWF0ZV9jZXBoQ1NJS2V5cmluZ191c2VyKCJjbGllbnQuY3NpLXJiZC1ub2RlIikK
+ ICAgICAgICAoCiAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX1JCRF9QUk9WSVNJT05FUl9T
+ RUNSRVQiXSwKICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfUkJEX1BST1ZJU0lPTkVSX1NF
+ Q1JFVF9OQU1FIl0sCiAgICAgICAgKSA9IHNlbGYuY3JlYXRlX2NlcGhDU0lLZXlyaW5nX3VzZXIo
+ ImNsaWVudC5jc2ktcmJkLXByb3Zpc2lvbmVyIikKICAgICAgICBzZWxmLm91dF9tYXBbIkNFUEhG
+ U19QT09MX05BTUUiXSA9IHNlbGYuX2FyZ19wYXJzZXIuY2VwaGZzX2RhdGFfcG9vbF9uYW1lCiAg
+ ICAgICAgc2VsZi5vdXRfbWFwWyJDRVBIRlNfTUVUQURBVEFfUE9PTF9OQU1FIl0gPSAoCiAgICAg
+ ICAgICAgIHNlbGYuX2FyZ19wYXJzZXIuY2VwaGZzX21ldGFkYXRhX3Bvb2xfbmFtZQogICAgICAg
+ ICkKICAgICAgICBzZWxmLm91dF9tYXBbIkNFUEhGU19GU19OQU1FIl0gPSBzZWxmLl9hcmdfcGFy
+ c2VyLmNlcGhmc19maWxlc3lzdGVtX25hbWUKICAgICAgICBzZWxmLm91dF9tYXBbIlJFU1RSSUNU
+ RURfQVVUSF9QRVJNSVNTSU9OIl0gPSAoCiAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmVz
+ dHJpY3RlZF9hdXRoX3Blcm1pc3Npb24KICAgICAgICApCiAgICAgICAgc2VsZi5vdXRfbWFwWyJS
+ QURPU19OQU1FU1BBQ0UiXSA9IHNlbGYuX2FyZ19wYXJzZXIucmFkb3NfbmFtZXNwYWNlCiAgICAg
+ ICAgc2VsZi5vdXRfbWFwWyJTVUJWT0xVTUVfR1JPVVAiXSA9IHNlbGYuX2FyZ19wYXJzZXIuc3Vi
+ dm9sdW1lX2dyb3VwCiAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX05PREVfU0VDUkVU
+ Il0gPSAiIgogICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19QUk9WSVNJT05FUl9TRUNS
+ RVQiXSA9ICIiCiAgICAgICAgIyBjcmVhdGUgQ2VwaEZTIG5vZGUgYW5kIHByb3Zpc2lvbmVyIGtl
+ eXJpbmcgb25seSB3aGVuIE1EUyBleGlzdHMKICAgICAgICBpZiBzZWxmLm91dF9tYXBbIkNFUEhG
+ U19GU19OQU1FIl0gYW5kIHNlbGYub3V0X21hcFsiQ0VQSEZTX1BPT0xfTkFNRSJdOgogICAgICAg
+ ICAgICAoCiAgICAgICAgICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfTk9ERV9TRUNS
+ RVQiXSwKICAgICAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19OT0RFX1NFQ1JF
+ VF9OQU1FIl0sCiAgICAgICAgICAgICkgPSBzZWxmLmNyZWF0ZV9jZXBoQ1NJS2V5cmluZ191c2Vy
+ KCJjbGllbnQuY3NpLWNlcGhmcy1ub2RlIikKICAgICAgICAgICAgKAogICAgICAgICAgICAgICAg
+ c2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVCJdLAogICAgICAgICAg
+ ICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FIl0s
+ CiAgICAgICAgICAgICkgPSBzZWxmLmNyZWF0ZV9jZXBoQ1NJS2V5cmluZ191c2VyKCJjbGllbnQu
+ Y3NpLWNlcGhmcy1wcm92aXNpb25lciIpCiAgICAgICAgICAgICMgY3JlYXRlIHRoZSBkZWZhdWx0
+ ICJjc2kiIHN1YnZvbHVtZWdyb3VwCiAgICAgICAgICAgIHNlbGYuZ2V0X29yX2NyZWF0ZV9zdWJ2
+ b2x1bWVfZ3JvdXAoCiAgICAgICAgICAgICAgICAiY3NpIiwgc2VsZi5fYXJnX3BhcnNlci5jZXBo
+ ZnNfZmlsZXN5c3RlbV9uYW1lCiAgICAgICAgICAgICkKICAgICAgICAgICAgIyBwaW4gdGhlIGRl
+ ZmF1bHQgImNzaSIgc3Vidm9sdW1lZ3JvdXAKICAgICAgICAgICAgc2VsZi5waW5fc3Vidm9sdW1l
+ KAogICAgICAgICAgICAgICAgImNzaSIsIHNlbGYuX2FyZ19wYXJzZXIuY2VwaGZzX2ZpbGVzeXN0
+ ZW1fbmFtZSwgImRpc3RyaWJ1dGVkIiwgIjEiCiAgICAgICAgICAgICkKICAgICAgICAgICAgaWYg
+ c2VsZi5vdXRfbWFwWyJTVUJWT0xVTUVfR1JPVVAiXToKICAgICAgICAgICAgICAgIHNlbGYuZ2V0
+ X29yX2NyZWF0ZV9zdWJ2b2x1bWVfZ3JvdXAoCiAgICAgICAgICAgICAgICAgICAgc2VsZi5fYXJn
+ X3BhcnNlci5zdWJ2b2x1bWVfZ3JvdXAsCiAgICAgICAgICAgICAgICAgICAgc2VsZi5fYXJnX3Bh
+ cnNlci5jZXBoZnNfZmlsZXN5c3RlbV9uYW1lLAogICAgICAgICAgICAgICAgKQogICAgICAgICAg
+ ICAgICAgc2VsZi5waW5fc3Vidm9sdW1lKAogICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19w
+ YXJzZXIuc3Vidm9sdW1lX2dyb3VwLAogICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJz
+ ZXIuY2VwaGZzX2ZpbGVzeXN0ZW1fbmFtZSwKICAgICAgICAgICAgICAgICAgICAiZGlzdHJpYnV0
+ ZWQiLAogICAgICAgICAgICAgICAgICAgICIxIiwKICAgICAgICAgICAgICAgICkKICAgICAgICBz
+ ZWxmLm91dF9tYXBbIlJHV19UTFNfQ0VSVCJdID0gIiIKICAgICAgICBzZWxmLm91dF9tYXBbIk1P
+ TklUT1JJTkdfRU5EUE9JTlQiXSA9ICIiCiAgICAgICAgc2VsZi5vdXRfbWFwWyJNT05JVE9SSU5H
+ X0VORFBPSU5UX1BPUlQiXSA9ICIiCiAgICAgICAgaWYgbm90IHNlbGYuX2FyZ19wYXJzZXIuc2tp
+ cF9tb25pdG9yaW5nX2VuZHBvaW50OgogICAgICAgICAgICAoCiAgICAgICAgICAgICAgICBzZWxm
+ Lm91dF9tYXBbIk1PTklUT1JJTkdfRU5EUE9JTlQiXSwKICAgICAgICAgICAgICAgIHNlbGYub3V0
+ X21hcFsiTU9OSVRPUklOR19FTkRQT0lOVF9QT1JUIl0sCiAgICAgICAgICAgICkgPSBzZWxmLmdl
+ dF9hY3RpdmVfYW5kX3N0YW5kYnlfbWdycygpCiAgICAgICAgc2VsZi5vdXRfbWFwWyJSQkRfUE9P
+ TF9OQU1FIl0gPSBzZWxmLl9hcmdfcGFyc2VyLnJiZF9kYXRhX3Bvb2xfbmFtZQogICAgICAgIHNl
+ bGYub3V0X21hcFsiUkJEX01FVEFEQVRBX0VDX1BPT0xfTkFNRSJdID0gKAogICAgICAgICAgICBz
+ ZWxmLnZhbGlkYXRlX3JiZF9tZXRhZGF0YV9lY19wb29sX25hbWUoKQogICAgICAgICkKICAgICAg
+ ICBzZWxmLm91dF9tYXBbIlRPUE9MT0dZX1BPT0xTIl0gPSBzZWxmLl9hcmdfcGFyc2VyLnRvcG9s
+ b2d5X3Bvb2xzCiAgICAgICAgc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9GQUlMVVJFX0RPTUFJTl9M
+ QUJFTCJdID0gKAogICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLnRvcG9sb2d5X2ZhaWx1cmVf
+ ZG9tYWluX2xhYmVsCiAgICAgICAgKQogICAgICAgIHNlbGYub3V0X21hcFsiVE9QT0xPR1lfRkFJ
+ TFVSRV9ET01BSU5fVkFMVUVTIl0gPSAoCiAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIudG9w
+ b2xvZ3lfZmFpbHVyZV9kb21haW5fdmFsdWVzCiAgICAgICAgKQogICAgICAgIGlmICgKICAgICAg
+ ICAgICAgc2VsZi5fYXJnX3BhcnNlci50b3BvbG9neV9wb29scyAhPSAiIgogICAgICAgICAgICBh
+ bmQgc2VsZi5fYXJnX3BhcnNlci50b3BvbG9neV9mYWlsdXJlX2RvbWFpbl9sYWJlbCAhPSAiIgog
+ ICAgICAgICAgICBhbmQgc2VsZi5fYXJnX3BhcnNlci50b3BvbG9neV9mYWlsdXJlX2RvbWFpbl92
+ YWx1ZXMgIT0gIiIKICAgICAgICApOgogICAgICAgICAgICBzZWxmLnZhbGlkYXRlX3RvcG9sb2d5
+ X3ZhbHVlcygKICAgICAgICAgICAgICAgIHNlbGYuY29udmVydF9jb21tYV9zZXBhcmF0ZWRfdG9f
+ YXJyYXkoc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9QT09MUyJdKSwKICAgICAgICAgICAgICAgIHNl
+ bGYuY29udmVydF9jb21tYV9zZXBhcmF0ZWRfdG9fYXJyYXkoCiAgICAgICAgICAgICAgICAgICAg
+ c2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9GQUlMVVJFX0RPTUFJTl9WQUxVRVMiXQogICAgICAgICAg
+ ICAgICAgKSwKICAgICAgICAgICAgKQogICAgICAgICAgICBzZWxmLnZhbGlkYXRlX3RvcG9sb2d5
+ X3JiZF9wb29scygKICAgICAgICAgICAgICAgIHNlbGYuY29udmVydF9jb21tYV9zZXBhcmF0ZWRf
+ dG9fYXJyYXkoc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9QT09MUyJdKQogICAgICAgICAgICApCiAg
+ ICAgICAgICAgIHNlbGYuaW5pdF90b3BvbG9neV9yYmRfcG9vbHMoCiAgICAgICAgICAgICAgICBz
+ ZWxmLmNvbnZlcnRfY29tbWFfc2VwYXJhdGVkX3RvX2FycmF5KHNlbGYub3V0X21hcFsiVE9QT0xP
+ R1lfUE9PTFMiXSkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHNlbGYu
+ cmFpc2VfZXhjZXB0aW9uX2lmX2FueV90b3BvbG9neV9mbGFnX2lzX21pc3NpbmcoKQoKICAgICAg
+ ICBzZWxmLm91dF9tYXBbIlJHV19QT09MX1BSRUZJWCJdID0gc2VsZi5fYXJnX3BhcnNlci5yZ3df
+ cG9vbF9wcmVmaXgKICAgICAgICBzZWxmLm91dF9tYXBbIlJHV19FTkRQT0lOVCJdID0gIiIKICAg
+ ICAgICBpZiBzZWxmLl9hcmdfcGFyc2VyLnJnd19lbmRwb2ludDoKICAgICAgICAgICAgaWYgc2Vs
+ Zi5fYXJnX3BhcnNlci5kcnlfcnVuOgogICAgICAgICAgICAgICAgc2VsZi5jcmVhdGVfcmd3X2Fk
+ bWluX29wc191c2VyKCkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGlmICgKICAg
+ ICAgICAgICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLnJnd19yZWFsbV9uYW1lICE9ICIiCiAg
+ ICAgICAgICAgICAgICAgICAgYW5kIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVncm91cF9uYW1l
+ ICE9ICIiCiAgICAgICAgICAgICAgICAgICAgYW5kIHNlbGYuX2FyZ19wYXJzZXIucmd3X3pvbmVf
+ bmFtZSAhPSAiIgogICAgICAgICAgICAgICAgKToKICAgICAgICAgICAgICAgICAgICBlcnIgPSBz
+ ZWxmLnZhbGlkYXRlX3Jnd19tdWx0aXNpdGUoCiAgICAgICAgICAgICAgICAgICAgICAgIHNlbGYu
+ X2FyZ19wYXJzZXIucmd3X3JlYWxtX25hbWUsICJyZWFsbSIKICAgICAgICAgICAgICAgICAgICAp
+ CiAgICAgICAgICAgICAgICAgICAgZXJyID0gc2VsZi52YWxpZGF0ZV9yZ3dfbXVsdGlzaXRlKAog
+ ICAgICAgICAgICAgICAgICAgICAgICBzZWxmLl9hcmdfcGFyc2VyLnJnd196b25lZ3JvdXBfbmFt
+ ZSwgInpvbmVncm91cCIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAgICAg
+ ZXJyID0gc2VsZi52YWxpZGF0ZV9yZ3dfbXVsdGlzaXRlKAogICAgICAgICAgICAgICAgICAgICAg
+ ICBzZWxmLl9hcmdfcGFyc2VyLnJnd196b25lX25hbWUsICJ6b25lIgogICAgICAgICAgICAgICAg
+ ICAgICkKCiAgICAgICAgICAgICAgICBpZiAoCiAgICAgICAgICAgICAgICAgICAgc2VsZi5fYXJn
+ X3BhcnNlci5yZ3dfcmVhbG1fbmFtZSA9PSAiIgogICAgICAgICAgICAgICAgICAgIGFuZCBzZWxm
+ Ll9hcmdfcGFyc2VyLnJnd196b25lZ3JvdXBfbmFtZSA9PSAiIgogICAgICAgICAgICAgICAgICAg
+ IGFuZCBzZWxmLl9hcmdfcGFyc2VyLnJnd196b25lX25hbWUgPT0gIiIKICAgICAgICAgICAgICAg
+ ICkgb3IgKAogICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmd3X3JlYWxtX25h
+ bWUgIT0gIiIKICAgICAgICAgICAgICAgICAgICBhbmQgc2VsZi5fYXJnX3BhcnNlci5yZ3dfem9u
+ ZWdyb3VwX25hbWUgIT0gIiIKICAgICAgICAgICAgICAgICAgICBhbmQgc2VsZi5fYXJnX3BhcnNl
+ ci5yZ3dfem9uZV9uYW1lICE9ICIiCiAgICAgICAgICAgICAgICApOgogICAgICAgICAgICAgICAg
+ ICAgICgKICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJSR1dfQURNSU5fT1BT
+ X1VTRVJfQUNDRVNTX0tFWSJdLAogICAgICAgICAgICAgICAgICAgICAgICBzZWxmLm91dF9tYXBb
+ IlJHV19BRE1JTl9PUFNfVVNFUl9TRUNSRVRfS0VZIl0sCiAgICAgICAgICAgICAgICAgICAgICAg
+ IGluZm9fY2FwX3N1cHBvcnRlZCwKICAgICAgICAgICAgICAgICAgICAgICAgZXJyLAogICAgICAg
+ ICAgICAgICAgICAgICkgPSBzZWxmLmNyZWF0ZV9yZ3dfYWRtaW5fb3BzX3VzZXIoKQogICAgICAg
+ ICAgICAgICAgICAgIGVyciA9IHNlbGYudmFsaWRhdGVfcmd3X2VuZHBvaW50KGluZm9fY2FwX3N1
+ cHBvcnRlZCkKICAgICAgICAgICAgICAgICAgICBpZiBzZWxmLl9hcmdfcGFyc2VyLnJnd190bHNf
+ Y2VydF9wYXRoOgogICAgICAgICAgICAgICAgICAgICAgICBzZWxmLm91dF9tYXBbIlJHV19UTFNf
+ Q0VSVCJdID0gKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi52YWxpZGF0ZV9yZ3df
+ ZW5kcG9pbnRfdGxzX2NlcnQoKQogICAgICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAg
+ ICAgICAgICAgIyBpZiB0aGVyZSBpcyBubyBlcnJvciwgc2V0IHRoZSBSR1dfRU5EUE9JTlQKICAg
+ ICAgICAgICAgICAgICAgICBpZiBlcnIgIT0gIi0xIjoKICAgICAgICAgICAgICAgICAgICAgICAg
+ c2VsZi5vdXRfbWFwWyJSR1dfRU5EUE9JTlQiXSA9IHNlbGYuX2FyZ19wYXJzZXIucmd3X2VuZHBv
+ aW50CiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGVyciA9ICJQbGVh
+ c2UgcHJvdmlkZSBhbGwgdGhlIFJHVyBtdWx0aXNpdGUgcGFyYW1ldGVycyBvciBub25lIG9mIHRo
+ ZW0iCiAgICAgICAgICAgICAgICAgICAgc3lzLnN0ZGVyci53cml0ZShlcnIpCgogICAgZGVmIGdl
+ bl9zaGVsbF9vdXQoc2VsZik6CiAgICAgICAgc2VsZi5fZ2VuX291dHB1dF9tYXAoKQogICAgICAg
+ IHNoT3V0SU8gPSBTdHJpbmdJTygpCiAgICAgICAgZm9yIGssIHYgaW4gc2VsZi5vdXRfbWFwLml0
+ ZW1zKCk6CiAgICAgICAgICAgIGlmIHYgYW5kIGsgbm90IGluIHNlbGYuX2V4Y2x1ZGVkX2tleXM6
+ CiAgICAgICAgICAgICAgICBzaE91dElPLndyaXRlKGYiZXhwb3J0IHtrfT17dn17TElORVNFUH0i
+ KQogICAgICAgIHNoT3V0ID0gc2hPdXRJTy5nZXR2YWx1ZSgpCiAgICAgICAgc2hPdXRJTy5jbG9z
+ ZSgpCiAgICAgICAgcmV0dXJuIHNoT3V0CgogICAgZGVmIGdlbl9qc29uX291dChzZWxmKToKICAg
+ ICAgICBzZWxmLl9nZW5fb3V0cHV0X21hcCgpCiAgICAgICAgaWYgc2VsZi5fYXJnX3BhcnNlci5k
+ cnlfcnVuOgogICAgICAgICAgICByZXR1cm4gIiIKICAgICAgICBqc29uX291dCA9IFsKICAgICAg
+ ICAgICAgewogICAgICAgICAgICAgICAgIm5hbWUiOiAiZXh0ZXJuYWwtY2x1c3Rlci11c2VyLWNv
+ bW1hbmQiLAogICAgICAgICAgICAgICAgImtpbmQiOiAiQ29uZmlnTWFwIiwKICAgICAgICAgICAg
+ ICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICJhcmdzIjogc2VsZi5vdXRfbWFwWyJB
+ UkdTIl0sCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAg
+ ICAgICAgICAgICAgICAibmFtZSI6ICJyb29rLWNlcGgtbW9uLWVuZHBvaW50cyIsCiAgICAgICAg
+ ICAgICAgICAia2luZCI6ICJDb25maWdNYXAiLAogICAgICAgICAgICAgICAgImRhdGEiOiB7CiAg
+ ICAgICAgICAgICAgICAgICAgImRhdGEiOiBzZWxmLm91dF9tYXBbIlJPT0tfRVhURVJOQUxfQ0VQ
+ SF9NT05fREFUQSJdLAogICAgICAgICAgICAgICAgICAgICJtYXhNb25JZCI6ICIwIiwKICAgICAg
+ ICAgICAgICAgICAgICAibWFwcGluZyI6ICJ7fSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAg
ICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAibmFtZSI6ICJyb29rLWNlcGgt
- b3BlcmF0b3ItY3JlZHMiLAogICAgICAgICAgICAgICAgImtpbmQiOiAiU2VjcmV0IiwKICAgICAg
- ICAgICAgICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICJ1c2VySUQiOiBzZWxmLm91
- dF9tYXBbIlJPT0tfRVhURVJOQUxfVVNFUk5BTUUiXSwKICAgICAgICAgICAgICAgICAgICAidXNl
- cktleSI6IHNlbGYub3V0X21hcFsiUk9PS19FWFRFUk5BTF9VU0VSX1NFQ1JFVCJdLAogICAgICAg
- ICAgICAgICAgfSwKICAgICAgICAgICAgfSwKICAgICAgICBdCgogICAgICAgICMgaWYgJ01PTklU
- T1JJTkdfRU5EUE9JTlQnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAnbW9uaXRvcmluZy1lbmRwb2lu
- dCcgdG8gQ2x1c3RlcgogICAgICAgIGlmICgKICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJNT05J
- VE9SSU5HX0VORFBPSU5UIl0KICAgICAgICAgICAgYW5kIHNlbGYub3V0X21hcFsiTU9OSVRPUklO
- R19FTkRQT0lOVF9QT1JUIl0KICAgICAgICApOgogICAgICAgICAgICBqc29uX291dC5hcHBlbmQo
- CiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAibW9uaXRvcmlu
- Zy1lbmRwb2ludCIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiQ2VwaENsdXN0ZXIiLAog
- ICAgICAgICAgICAgICAgICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAiTW9u
- aXRvcmluZ0VuZHBvaW50Ijogc2VsZi5vdXRfbWFwWyJNT05JVE9SSU5HX0VORFBPSU5UIl0sCiAg
- ICAgICAgICAgICAgICAgICAgICAgICJNb25pdG9yaW5nUG9ydCI6IHNlbGYub3V0X21hcFsiTU9O
- SVRPUklOR19FTkRQT0lOVF9QT1JUIl0sCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAg
- ICAgICAgIH0KICAgICAgICAgICAgKQoKICAgICAgICAjIGlmICdDU0lfUkJEX05PREVfU0VDUkVU
- JyBleGlzdHMsIHRoZW4gb25seSBhZGQgJ3Jvb2stY3NpLXJiZC1wcm92aXNpb25lcicgU2VjcmV0
- CiAgICAgICAgaWYgKAogICAgICAgICAgICBzZWxmLm91dF9tYXBbIkNTSV9SQkRfTk9ERV9TRUNS
- RVQiXQogICAgICAgICAgICBhbmQgc2VsZi5vdXRfbWFwWyJDU0lfUkJEX05PREVfU0VDUkVUX05B
- TUUiXQogICAgICAgICk6CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgKICAgICAgICAgICAg
- ICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydD
- U0lfUkJEX05PREVfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAgICAgICAgICAgICJraW5kIjog
- IlNlY3JldCIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAg
- ICAgICAgICJ1c2VySUQiOiBzZWxmLm91dF9tYXBbIkNTSV9SQkRfTk9ERV9TRUNSRVRfTkFNRSJd
- LAogICAgICAgICAgICAgICAgICAgICAgICAidXNlcktleSI6IHNlbGYub3V0X21hcFsiQ1NJX1JC
- RF9OT0RFX1NFQ1JFVCJdLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9
- CiAgICAgICAgICAgICkKICAgICAgICAjIGlmICdDU0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVCcg
- ZXhpc3RzLCB0aGVuIG9ubHkgYWRkICdyb29rLWNzaS1yYmQtcHJvdmlzaW9uZXInIFNlY3JldAog
- ICAgICAgIGlmICgKICAgICAgICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfUkJEX1BST1ZJU0lPTkVS
- X1NFQ1JFVCJdCiAgICAgICAgICAgIGFuZCBzZWxmLm91dF9tYXBbIkNTSV9SQkRfUFJPVklTSU9O
- RVJfU0VDUkVUX05BTUUiXQogICAgICAgICk6CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgK
- ICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6IGYicm9vay17c2Vs
- Zi5vdXRfbWFwWydDU0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAg
- ICAgICAgICAgICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAgICAgICJkYXRhIjog
- ewogICAgICAgICAgICAgICAgICAgICAgICAidXNlcklEIjogc2VsZi5vdXRfbWFwWyJDU0lfUkJE
- X1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICJ1c2Vy
- S2V5Ijogc2VsZi5vdXRfbWFwWyJDU0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVCJdLAogICAgICAg
- ICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICkKICAgICAgICAj
- IGlmICdDU0lfQ0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVCcgZXhpc3RzLCB0aGVuIG9ubHkgYWRk
- ICdyb29rLWNzaS1jZXBoZnMtcHJvdmlzaW9uZXInIFNlY3JldAogICAgICAgIGlmICgKICAgICAg
- ICAgICAgc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVCJdCiAgICAg
- ICAgICAgIGFuZCBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVUX05B
- TUUiXQogICAgICAgICk6CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgKICAgICAgICAgICAg
- ICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydD
- U0lfQ0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAg
- ICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAgICAgICJkYXRhIjogewogICAgICAg
- ICAgICAgICAgICAgICAgICAiYWRtaW5JRCI6IHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19QUk9W
- SVNJT05FUl9TRUNSRVRfTkFNRSJdLAogICAgICAgICAgICAgICAgICAgICAgICAiYWRtaW5LZXki
- OiBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVUIl0sCiAgICAgICAg
- ICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgKQogICAgICAgICMg
- aWYgJ0NTSV9DRVBIRlNfTk9ERV9TRUNSRVQnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAncm9vay1j
- c2ktY2VwaGZzLW5vZGUnIFNlY3JldAogICAgICAgIGlmICgKICAgICAgICAgICAgc2VsZi5vdXRf
- bWFwWyJDU0lfQ0VQSEZTX05PREVfU0VDUkVUIl0KICAgICAgICAgICAgYW5kIHNlbGYub3V0X21h
- cFsiQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVF9OQU1FIl0KICAgICAgICApOgogICAgICAgICAgICBq
+ bW9uIiwKICAgICAgICAgICAgICAgICJraW5kIjogIlNlY3JldCIsCiAgICAgICAgICAgICAgICAi
+ ZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAiYWRtaW4tc2VjcmV0IjogImFkbWluLXNlY3Jl
+ dCIsCiAgICAgICAgICAgICAgICAgICAgImZzaWQiOiBzZWxmLm91dF9tYXBbIlJPT0tfRVhURVJO
+ QUxfRlNJRCJdLAogICAgICAgICAgICAgICAgICAgICJtb24tc2VjcmV0IjogIm1vbi1zZWNyZXQi
+ LAogICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAg
+ ICAgICAgICAgIm5hbWUiOiAicm9vay1jZXBoLW9wZXJhdG9yLWNyZWRzIiwKICAgICAgICAgICAg
+ ICAgICJraW5kIjogIlNlY3JldCIsCiAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAg
+ ICAgICAgICAgICAidXNlcklEIjogc2VsZi5vdXRfbWFwWyJST09LX0VYVEVSTkFMX1VTRVJOQU1F
+ Il0sCiAgICAgICAgICAgICAgICAgICAgInVzZXJLZXkiOiBzZWxmLm91dF9tYXBbIlJPT0tfRVhU
+ RVJOQUxfVVNFUl9TRUNSRVQiXSwKICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgIH0sCiAg
+ ICAgICAgXQoKICAgICAgICAjIGlmICdNT05JVE9SSU5HX0VORFBPSU5UJyBleGlzdHMsIHRoZW4g
+ b25seSBhZGQgJ21vbml0b3JpbmctZW5kcG9pbnQnIHRvIENsdXN0ZXIKICAgICAgICBpZiAoCiAg
+ ICAgICAgICAgIHNlbGYub3V0X21hcFsiTU9OSVRPUklOR19FTkRQT0lOVCJdCiAgICAgICAgICAg
+ IGFuZCBzZWxmLm91dF9tYXBbIk1PTklUT1JJTkdfRU5EUE9JTlRfUE9SVCJdCiAgICAgICAgKToK
+ ICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAgICAgICAgewogICAgICAgICAg
+ ICAgICAgICAgICJuYW1lIjogIm1vbml0b3JpbmctZW5kcG9pbnQiLAogICAgICAgICAgICAgICAg
+ ICAgICJraW5kIjogIkNlcGhDbHVzdGVyIiwKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsK
+ ICAgICAgICAgICAgICAgICAgICAgICAgIk1vbml0b3JpbmdFbmRwb2ludCI6IHNlbGYub3V0X21h
+ cFsiTU9OSVRPUklOR19FTkRQT0lOVCJdLAogICAgICAgICAgICAgICAgICAgICAgICAiTW9uaXRv
+ cmluZ1BvcnQiOiBzZWxmLm91dF9tYXBbIk1PTklUT1JJTkdfRU5EUE9JTlRfUE9SVCJdLAogICAg
+ ICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICkKCiAgICAg
+ ICAgIyBpZiAnQ1NJX1JCRF9OT0RFX1NFQ1JFVCcgZXhpc3RzLCB0aGVuIG9ubHkgYWRkICdyb29r
+ LWNzaS1yYmQtcHJvdmlzaW9uZXInIFNlY3JldAogICAgICAgIGlmICgKICAgICAgICAgICAgc2Vs
+ Zi5vdXRfbWFwWyJDU0lfUkJEX05PREVfU0VDUkVUIl0KICAgICAgICAgICAgYW5kIHNlbGYub3V0
+ X21hcFsiQ1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FIl0KICAgICAgICApOgogICAgICAgICAgICBq
c29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5h
- bWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVF9OQU1FJ119
- IiwKICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAg
- ICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAiYWRtaW5JRCI6IHNlbGYub3V0
- X21hcFsiQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVF9OQU1FIl0sCiAgICAgICAgICAgICAgICAgICAg
- ICAgICJhZG1pbktleSI6IHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVCJdLAog
- ICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICkKICAg
- ICAgICAjIGlmICdST09LX0VYVEVSTkFMX0RBU0hCT0FSRF9MSU5LJyBleGlzdHMsIHRoZW4gb25s
- eSBhZGQgJ3Jvb2stY2VwaC1kYXNoYm9hcmQtbGluaycgU2VjcmV0CiAgICAgICAgaWYgc2VsZi5v
- dXRfbWFwWyJST09LX0VYVEVSTkFMX0RBU0hCT0FSRF9MSU5LIl06CiAgICAgICAgICAgIGpzb25f
- b3V0LmFwcGVuZCgKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6
- ICJyb29rLWNlcGgtZGFzaGJvYXJkLWxpbmsiLAogICAgICAgICAgICAgICAgICAgICJraW5kIjog
- IlNlY3JldCIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAg
- ICAgICAgICJ1c2VySUQiOiAiY2VwaC1kYXNoYm9hcmQtbGluayIsCiAgICAgICAgICAgICAgICAg
- ICAgICAgICJ1c2VyS2V5Ijogc2VsZi5vdXRfbWFwWyJST09LX0VYVEVSTkFMX0RBU0hCT0FSRF9M
- SU5LIl0sCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgIH0KICAgICAgICAg
- ICAgKQogICAgICAgICMgaWYgJ1JBRE9TX05BTUVTUEFDRScgZXhpc3RzLCB0aGVuIG9ubHkgYWRk
- IHRoZSAiUkFET1NfTkFNRVNQQUNFIiBuYW1lc3BhY2UKICAgICAgICBpZiAoCiAgICAgICAgICAg
- IHNlbGYub3V0X21hcFsiUkFET1NfTkFNRVNQQUNFIl0KICAgICAgICAgICAgYW5kIHNlbGYub3V0
- X21hcFsiUkVTVFJJQ1RFRF9BVVRIX1BFUk1JU1NJT04iXQogICAgICAgICAgICBhbmQgbm90IHNl
- bGYub3V0X21hcFsiUkJEX01FVEFEQVRBX0VDX1BPT0xfTkFNRSJdCiAgICAgICAgKToKICAgICAg
- ICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAg
- ICAgICJuYW1lIjogInJhZG9zLW5hbWVzcGFjZSIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQi
- OiAiQ2VwaEJsb2NrUG9vbFJhZG9zTmFtZXNwYWNlIiwKICAgICAgICAgICAgICAgICAgICAiZGF0
- YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgInJhZG9zTmFtZXNwYWNlTmFtZSI6IHNlbGYu
- b3V0X21hcFsiUkFET1NfTkFNRVNQQUNFIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICJwb29s
- Ijogc2VsZi5vdXRfbWFwWyJSQkRfUE9PTF9OQU1FIl0sCiAgICAgICAgICAgICAgICAgICAgfSwK
- ICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgKQogICAgICAgICAgICBqc29uX291dC5hcHBl
- bmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiY2VwaC1y
- YmQtcmFkb3MtbmFtZXNwYWNlIiwKICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTdG9yYWdl
- Q2xhc3MiLAogICAgICAgICAgICAgICAgICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAg
- ICAgICAicG9vbCI6IHNlbGYub3V0X21hcFsiUkJEX1BPT0xfTkFNRSJdLAogICAgICAgICAgICAg
- ICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL3Byb3Zpc2lvbmVyLXNlY3JldC1uYW1lIjog
- ZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJPVklTSU9ORVJfU0VDUkVUX05BTUUnXX0i
- LAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL2NvbnRyb2xsZXIt
- ZXhwYW5kLXNlY3JldC1uYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJPVklT
- SU9ORVJfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3Jh
- Z2UuazhzLmlvL25vZGUtc3RhZ2Utc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsn
- Q1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICB9LAogICAg
- ICAgICAgICAgICAgfQogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgaWYg
- c2VsZi5vdXRfbWFwWyJSQkRfTUVUQURBVEFfRUNfUE9PTF9OQU1FIl06CiAgICAgICAgICAgICAg
- ICBqc29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAg
- ICAgICAgICAibmFtZSI6ICJjZXBoLXJiZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICJraW5k
- IjogIlN0b3JhZ2VDbGFzcyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJkYXRhIjogewogICAg
- ICAgICAgICAgICAgICAgICAgICAgICAgImRhdGFQb29sIjogc2VsZi5vdXRfbWFwWyJSQkRfUE9P
- TF9OQU1FIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAicG9vbCI6IHNlbGYub3V0X21h
- cFsiUkJEX01FVEFEQVRBX0VDX1BPT0xfTkFNRSJdLAogICAgICAgICAgICAgICAgICAgICAgICAg
- ICAgImNzaS5zdG9yYWdlLms4cy5pby9wcm92aXNpb25lci1zZWNyZXQtbmFtZSI6IGYicm9vay17
+ bWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FJ119IiwK
+ ICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAgICAg
+ ICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAidXNlcklEIjogc2VsZi5vdXRfbWFw
+ WyJDU0lfUkJEX05PREVfU0VDUkVUX05BTUUiXSwKICAgICAgICAgICAgICAgICAgICAgICAgInVz
+ ZXJLZXkiOiBzZWxmLm91dF9tYXBbIkNTSV9SQkRfTk9ERV9TRUNSRVQiXSwKICAgICAgICAgICAg
+ ICAgICAgICB9LAogICAgICAgICAgICAgICAgfQogICAgICAgICAgICApCiAgICAgICAgIyBpZiAn
+ Q1NJX1JCRF9QUk9WSVNJT05FUl9TRUNSRVQnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAncm9vay1j
+ c2ktcmJkLXByb3Zpc2lvbmVyJyBTZWNyZXQKICAgICAgICBpZiAoCiAgICAgICAgICAgIHNlbGYu
+ b3V0X21hcFsiQ1NJX1JCRF9QUk9WSVNJT05FUl9TRUNSRVQiXQogICAgICAgICAgICBhbmQgc2Vs
+ Zi5vdXRfbWFwWyJDU0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FIl0KICAgICAgICApOgog
+ ICAgICAgICAgICBqc29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAg
+ ICAgICAgICAgIm5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX1JCRF9QUk9WSVNJT05F
+ Ul9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiU2VjcmV0IiwK
+ ICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgInVz
+ ZXJJRCI6IHNlbGYub3V0X21hcFsiQ1NJX1JCRF9QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSJdLAog
+ ICAgICAgICAgICAgICAgICAgICAgICAidXNlcktleSI6IHNlbGYub3V0X21hcFsiQ1NJX1JCRF9Q
+ Uk9WSVNJT05FUl9TRUNSRVQiXSwKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAg
+ ICAgfQogICAgICAgICAgICApCiAgICAgICAgIyBpZiAnQ1NJX0NFUEhGU19QUk9WSVNJT05FUl9T
+ RUNSRVQnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAncm9vay1jc2ktY2VwaGZzLXByb3Zpc2lvbmVy
+ JyBTZWNyZXQKICAgICAgICBpZiAoCiAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX0NFUEhG
+ U19QUk9WSVNJT05FUl9TRUNSRVQiXQogICAgICAgICAgICBhbmQgc2VsZi5vdXRfbWFwWyJDU0lf
+ Q0VQSEZTX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FIl0KICAgICAgICApOgogICAgICAgICAgICBq
+ c29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5h
+ bWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX0NFUEhGU19QUk9WSVNJT05FUl9TRUNSRVRf
+ TkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiU2VjcmV0IiwKICAgICAgICAg
+ ICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgImFkbWluSUQiOiBz
+ ZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVUX05BTUUiXSwKICAgICAg
+ ICAgICAgICAgICAgICAgICAgImFkbWluS2V5Ijogc2VsZi5vdXRfbWFwWyJDU0lfQ0VQSEZTX1BS
+ T1ZJU0lPTkVSX1NFQ1JFVCJdLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAg
+ ICB9CiAgICAgICAgICAgICkKICAgICAgICAjIGlmICdDU0lfQ0VQSEZTX05PREVfU0VDUkVUJyBl
+ eGlzdHMsIHRoZW4gb25seSBhZGQgJ3Jvb2stY3NpLWNlcGhmcy1ub2RlJyBTZWNyZXQKICAgICAg
+ ICBpZiAoCiAgICAgICAgICAgIHNlbGYub3V0X21hcFsiQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVCJd
+ CiAgICAgICAgICAgIGFuZCBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfTk9ERV9TRUNSRVRfTkFN
+ RSJdCiAgICAgICAgKToKICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAgICAg
+ ICAgewogICAgICAgICAgICAgICAgICAgICJuYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NT
+ SV9DRVBIRlNfTk9ERV9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQi
+ OiAiU2VjcmV0IiwKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAg
+ ICAgICAgICAgImFkbWluSUQiOiBzZWxmLm91dF9tYXBbIkNTSV9DRVBIRlNfTk9ERV9TRUNSRVRf
+ TkFNRSJdLAogICAgICAgICAgICAgICAgICAgICAgICAiYWRtaW5LZXkiOiBzZWxmLm91dF9tYXBb
+ IkNTSV9DRVBIRlNfTk9ERV9TRUNSRVQiXSwKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAg
+ ICAgICAgICAgfQogICAgICAgICAgICApCiAgICAgICAgIyBpZiAnUk9PS19FWFRFUk5BTF9EQVNI
+ Qk9BUkRfTElOSycgZXhpc3RzLCB0aGVuIG9ubHkgYWRkICdyb29rLWNlcGgtZGFzaGJvYXJkLWxp
+ bmsnIFNlY3JldAogICAgICAgIGlmIHNlbGYub3V0X21hcFsiUk9PS19FWFRFUk5BTF9EQVNIQk9B
+ UkRfTElOSyJdOgogICAgICAgICAgICBqc29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICB7
+ CiAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAicm9vay1jZXBoLWRhc2hib2FyZC1saW5rIiwK
+ ICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAgICAg
+ ICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAidXNlcklEIjogImNlcGgtZGFzaGJv
+ YXJkLWxpbmsiLAogICAgICAgICAgICAgICAgICAgICAgICAidXNlcktleSI6IHNlbGYub3V0X21h
+ cFsiUk9PS19FWFRFUk5BTF9EQVNIQk9BUkRfTElOSyJdLAogICAgICAgICAgICAgICAgICAgIH0s
+ CiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICkKICAgICAgICAjIGlmICdSQURPU19OQU1F
+ U1BBQ0UnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCB0aGUgIlJBRE9TX05BTUVTUEFDRSIgbmFtZXNw
+ YWNlCiAgICAgICAgaWYgKAogICAgICAgICAgICBzZWxmLm91dF9tYXBbIlJBRE9TX05BTUVTUEFD
+ RSJdCiAgICAgICAgICAgIGFuZCBzZWxmLm91dF9tYXBbIlJFU1RSSUNURURfQVVUSF9QRVJNSVNT
+ SU9OIl0KICAgICAgICAgICAgYW5kIG5vdCBzZWxmLm91dF9tYXBbIlJCRF9NRVRBREFUQV9FQ19Q
+ T09MX05BTUUiXQogICAgICAgICk6CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgKICAgICAg
+ ICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJyYWRvcy1uYW1lc3BhY2Ui
+ LAogICAgICAgICAgICAgICAgICAgICJraW5kIjogIkNlcGhCbG9ja1Bvb2xSYWRvc05hbWVzcGFj
+ ZSIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAgICAgICAg
+ ICJyYWRvc05hbWVzcGFjZU5hbWUiOiBzZWxmLm91dF9tYXBbIlJBRE9TX05BTUVTUEFDRSJdLAog
+ ICAgICAgICAgICAgICAgICAgICAgICAicG9vbCI6IHNlbGYub3V0X21hcFsiUkJEX1BPT0xfTkFN
+ RSJdLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAg
+ ICkKICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAgICAgICAgewogICAgICAg
+ ICAgICAgICAgICAgICJuYW1lIjogImNlcGgtcmJkLXJhZG9zLW5hbWVzcGFjZSIsCiAgICAgICAg
+ ICAgICAgICAgICAgImtpbmQiOiAiU3RvcmFnZUNsYXNzIiwKICAgICAgICAgICAgICAgICAgICAi
+ ZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgInBvb2wiOiBzZWxmLm91dF9tYXBbIlJC
+ RF9QT09MX05BTUUiXSwKICAgICAgICAgICAgICAgICAgICAgICAgImNzaS5zdG9yYWdlLms4cy5p
+ by9wcm92aXNpb25lci1zZWNyZXQtbmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydDU0lfUkJE
+ X1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICAgICAgImNz
+ aS5zdG9yYWdlLms4cy5pby9jb250cm9sbGVyLWV4cGFuZC1zZWNyZXQtbmFtZSI6IGYicm9vay17
c2VsZi5vdXRfbWFwWydDU0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FJ119IiwKICAgICAg
- ICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8vY29udHJvbGxlci1leHBh
- bmQtc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX1JCRF9QUk9WSVNJT05F
- Ul9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3Jh
- Z2UuazhzLmlvL25vZGUtc3RhZ2Utc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsn
- Q1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICAgICAgfSwK
- ICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVsc2U6
- CiAgICAgICAgICAgICAgICBqc29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICAgICAgewog
- ICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJjZXBoLXJiZCIsCiAgICAgICAgICAgICAg
- ICAgICAgICAgICJraW5kIjogIlN0b3JhZ2VDbGFzcyIsCiAgICAgICAgICAgICAgICAgICAgICAg
- ICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgInBvb2wiOiBzZWxmLm91dF9t
- YXBbIlJCRF9QT09MX05BTUUiXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3Rv
- cmFnZS5rOHMuaW8vcHJvdmlzaW9uZXItc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21h
- cFsnQ1NJX1JCRF9QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAg
- ICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL2NvbnRyb2xsZXItZXhwYW5kLXNlY3JldC1u
- YW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJPVklTSU9ORVJfU0VDUkVUX05B
- TUUnXX0iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgImNzaS5zdG9yYWdlLms4cy5pby9u
- b2RlLXN0YWdlLXNlY3JldC1uYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfTk9E
- RV9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAg
- ICAgICAgICAgfQogICAgICAgICAgICAgICAgKQoKICAgICAgICAjIGlmICdUT1BPTE9HWV9QT09M
- UycsICdUT1BPTE9HWV9GQUlMVVJFX0RPTUFJTl9MQUJFTCcsICdUT1BPTE9HWV9GQUlMVVJFX0RP
- TUFJTl9WQUxVRVMnICBleGlzdHMsCiAgICAgICAgIyB0aGVuIG9ubHkgYWRkICd0b3BvbG9neScg
- U3RvcmFnZUNsYXNzCiAgICAgICAgaWYgKAogICAgICAgICAgICBzZWxmLm91dF9tYXBbIlRPUE9M
- T0dZX1BPT0xTIl0KICAgICAgICAgICAgYW5kIHNlbGYub3V0X21hcFsiVE9QT0xPR1lfRkFJTFVS
- RV9ET01BSU5fTEFCRUwiXQogICAgICAgICAgICBhbmQgc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9G
- QUlMVVJFX0RPTUFJTl9WQUxVRVMiXQogICAgICAgICk6CiAgICAgICAgICAgIGpzb25fb3V0LmFw
- cGVuZCgKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJjZXBo
- LXJiZC10b3BvbG9neSIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiU3RvcmFnZUNsYXNz
- IiwKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAg
- InRvcG9sb2d5RmFpbHVyZURvbWFpbkxhYmVsIjogc2VsZi5vdXRfbWFwWwogICAgICAgICAgICAg
- ICAgICAgICAgICAgICAgIlRPUE9MT0dZX0ZBSUxVUkVfRE9NQUlOX0xBQkVMIgogICAgICAgICAg
- ICAgICAgICAgICAgICBdLAogICAgICAgICAgICAgICAgICAgICAgICAidG9wb2xvZ3lGYWlsdXJl
- RG9tYWluVmFsdWVzIjogc2VsZi5vdXRfbWFwWwogICAgICAgICAgICAgICAgICAgICAgICAgICAg
- IlRPUE9MT0dZX0ZBSUxVUkVfRE9NQUlOX1ZBTFVFUyIKICAgICAgICAgICAgICAgICAgICAgICAg
- XSwKICAgICAgICAgICAgICAgICAgICAgICAgInRvcG9sb2d5UG9vbHMiOiBzZWxmLm91dF9tYXBb
- IlRPUE9MT0dZX1BPT0xTIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5r
- OHMuaW8vcHJvdmlzaW9uZXItc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJ
- X1JCRF9QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgICAg
- ICJjc2kuc3RvcmFnZS5rOHMuaW8vY29udHJvbGxlci1leHBhbmQtc2VjcmV0LW5hbWUiOiBmInJv
- b2ste3NlbGYub3V0X21hcFsnQ1NJX1JCRF9QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSddfSIsCiAg
- ICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8vbm9kZS1zdGFnZS1zZWNy
- ZXQtbmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydDU0lfUkJEX05PREVfU0VDUkVUX05BTUUn
- XX0iLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAg
- ICkKCiAgICAgICAgIyBpZiAnQ0VQSEZTX0ZTX05BTUUnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAn
- Y2VwaGZzJyBTdG9yYWdlQ2xhc3MKICAgICAgICBpZiBzZWxmLm91dF9tYXBbIkNFUEhGU19GU19O
- QU1FIl06CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgKICAgICAgICAgICAgICAgIHsKICAg
- ICAgICAgICAgICAgICAgICAibmFtZSI6ICJjZXBoZnMiLAogICAgICAgICAgICAgICAgICAgICJr
- aW5kIjogIlN0b3JhZ2VDbGFzcyIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAg
- ICAgICAgICAgICAgICAgICAgICJmc05hbWUiOiBzZWxmLm91dF9tYXBbIkNFUEhGU19GU19OQU1F
- Il0sCiAgICAgICAgICAgICAgICAgICAgICAgICJwb29sIjogc2VsZi5vdXRfbWFwWyJDRVBIRlNf
- UE9PTF9OQU1FIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8v
- cHJvdmlzaW9uZXItc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX0NFUEhG
- U19QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgICAgICJj
- c2kuc3RvcmFnZS5rOHMuaW8vY29udHJvbGxlci1leHBhbmQtc2VjcmV0LW5hbWUiOiBmInJvb2st
- e3NlbGYub3V0X21hcFsnQ1NJX0NFUEhGU19QUk9WSVNJT05FUl9TRUNSRVRfTkFNRSddfSIsCiAg
- ICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8vbm9kZS1zdGFnZS1zZWNy
- ZXQtbmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydDU0lfQ0VQSEZTX05PREVfU0VDUkVUX05B
- TUUnXX0iLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAg
- ICAgICkKICAgICAgICAjIGlmICdSR1dfRU5EUE9JTlQnIGV4aXN0cywgdGhlbiBvbmx5IGFkZCAn
- Y2VwaC1yZ3cnIFN0b3JhZ2VDbGFzcwogICAgICAgIGlmIHNlbGYub3V0X21hcFsiUkdXX0VORFBP
- SU5UIl06CiAgICAgICAgICAgIGpzb25fb3V0LmFwcGVuZCgKICAgICAgICAgICAgICAgIHsKICAg
- ICAgICAgICAgICAgICAgICAibmFtZSI6ICJjZXBoLXJndyIsCiAgICAgICAgICAgICAgICAgICAg
- ImtpbmQiOiAiU3RvcmFnZUNsYXNzIiwKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAg
- ICAgICAgICAgICAgICAgICAgICAgImVuZHBvaW50Ijogc2VsZi5vdXRfbWFwWyJSR1dfRU5EUE9J
- TlQiXSwKICAgICAgICAgICAgICAgICAgICAgICAgInBvb2xQcmVmaXgiOiBzZWxmLm91dF9tYXBb
- IlJHV19QT09MX1BSRUZJWCJdLAogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAg
- ICB9CiAgICAgICAgICAgICkKICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAg
- ICAgICAgewogICAgICAgICAgICAgICAgICAgICJuYW1lIjogInJndy1hZG1pbi1vcHMtdXNlciIs
- CiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiU2VjcmV0IiwKICAgICAgICAgICAgICAgICAg
- ICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgImFjY2Vzc0tleSI6IHNlbGYub3V0
- X21hcFsiUkdXX0FETUlOX09QU19VU0VSX0FDQ0VTU19LRVkiXSwKICAgICAgICAgICAgICAgICAg
- ICAgICAgInNlY3JldEtleSI6IHNlbGYub3V0X21hcFsiUkdXX0FETUlOX09QU19VU0VSX1NFQ1JF
- VF9LRVkiXSwKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgfQogICAgICAg
- ICAgICApCiAgICAgICAgIyBpZiAnUkdXX1RMU19DRVJUJyBleGlzdHMsIHRoZW4gb25seSBhZGQg
- dGhlICJjZXBoLXJndy10bHMtY2VydCIgc2VjcmV0CiAgICAgICAgaWYgc2VsZi5vdXRfbWFwWyJS
- R1dfVExTX0NFUlQiXToKICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAgICAg
- ICAgewogICAgICAgICAgICAgICAgICAgICJuYW1lIjogImNlcGgtcmd3LXRscy1jZXJ0IiwKICAg
- ICAgICAgICAgICAgICAgICAia2luZCI6ICJTZWNyZXQiLAogICAgICAgICAgICAgICAgICAgICJk
- YXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAiY2VydCI6IHNlbGYub3V0X21hcFsiUkdX
- X1RMU19DRVJUIl0sCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgIH0KICAg
- ICAgICAgICAgKQoKICAgICAgICByZXR1cm4ganNvbi5kdW1wcyhqc29uX291dCkgKyBMSU5FU0VQ
- CgogICAgZGVmIHVwZ3JhZGVfdXNlcnNfcGVybWlzc2lvbnMoc2VsZik6CiAgICAgICAgdXNlcnMg
- PSBbCiAgICAgICAgICAgICJjbGllbnQuY3NpLWNlcGhmcy1ub2RlIiwKICAgICAgICAgICAgImNs
- aWVudC5jc2ktY2VwaGZzLXByb3Zpc2lvbmVyIiwKICAgICAgICAgICAgImNsaWVudC5jc2ktcmJk
- LW5vZGUiLAogICAgICAgICAgICAiY2xpZW50LmNzaS1yYmQtcHJvdmlzaW9uZXIiLAogICAgICAg
- ICAgICAiY2xpZW50LmhlYWx0aGNoZWNrZXIiLAogICAgICAgIF0KICAgICAgICBpZiBzZWxmLnJ1
- bl9hc191c2VyICE9ICIiIGFuZCBzZWxmLnJ1bl9hc191c2VyIG5vdCBpbiB1c2VyczoKICAgICAg
- ICAgICAgdXNlcnMuYXBwZW5kKHNlbGYucnVuX2FzX3VzZXIpCiAgICAgICAgZm9yIHVzZXIgaW4g
- dXNlcnM6CiAgICAgICAgICAgIHNlbGYudXBncmFkZV91c2VyX3Blcm1pc3Npb25zKHVzZXIpCgog
- ICAgZGVmIGdldF9yZ3dfcG9vbF9uYW1lX2R1cmluZ191cGdyYWRlKHNlbGYsIHVzZXIsIGNhcHMp
- OgogICAgICAgIGlmIHVzZXIgPT0gImNsaWVudC5oZWFsdGhjaGVja2VyIjoKICAgICAgICAgICAg
- IyB3aGVuIGFkbWluIGhhcyBub3QgcHJvdmlkZWQgcmd3IHBvb2wgbmFtZSBkdXJpbmcgdXBncmFk
- ZSwKICAgICAgICAgICAgIyBnZXQgdGhlIHJndyBwb29sIG5hbWUgZnJvbSBjbGllbnQuaGVhbHRo
- Y2hlY2tlciB1c2VyIHdoaWNoIHdhcyB1c2VkIGR1cmluZyBjb25uZWN0aW9uCiAgICAgICAgICAg
- IGlmIG5vdCBzZWxmLl9hcmdfcGFyc2VyLnJnd19wb29sX3ByZWZpeDoKICAgICAgICAgICAgICAg
- ICMgVG8gZ2V0IHZhbHVlICdkZWZhdWx0JyB3aGljaCBpcyByZ3cgcG9vbCBuYW1lIGZyb20gJ2Fs
- bG93IHJ3eCBwb29sPWRlZmF1bHQucmd3Lm1ldGEnCiAgICAgICAgICAgICAgICBwYXR0ZXJuID0g
- ciJwb29sPSguKj8pXC5yZ3dcLm1ldGEiCiAgICAgICAgICAgICAgICBtYXRjaCA9IHJlLnNlYXJj
- aChwYXR0ZXJuLCBjYXBzKQogICAgICAgICAgICAgICAgaWYgbWF0Y2g6CiAgICAgICAgICAgICAg
- ICAgICAgc2VsZi5fYXJnX3BhcnNlci5yZ3dfcG9vbF9wcmVmaXggPSBtYXRjaC5ncm91cCgxKQog
- ICAgICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgICAgICByYWlzZSBFeGVjdXRpb25G
- YWlsdXJlRXhjZXB0aW9uKAogICAgICAgICAgICAgICAgICAgICAgICAiZmFpbGVkIHRvIGdldCBy
- Z3cgcG9vbCBuYW1lIGZvciB1cGdyYWRlIgogICAgICAgICAgICAgICAgICAgICkKCiAgICBkZWYg
- dXBncmFkZV91c2VyX3Blcm1pc3Npb25zKHNlbGYsIHVzZXIpOgogICAgICAgICMgY2hlY2sgd2hl
- dGhlciB0aGUgZ2l2ZW4gdXNlciBleGlzdHMgb3Igbm90CiAgICAgICAgY21kX2pzb24gPSB7InBy
- ZWZpeCI6ICJhdXRoIGdldCIsICJlbnRpdHkiOiBmInt1c2VyfSIsICJmb3JtYXQiOiAianNvbiJ9
- CiAgICAgICAgcmV0X3ZhbCwganNvbl9vdXQsIGVycl9tc2cgPSBzZWxmLl9jb21tb25fY21kX2pz
- b25fZ2VuKGNtZF9qc29uKQogICAgICAgIGlmIHJldF92YWwgIT0gMCBvciBsZW4oanNvbl9vdXQp
- ID09IDA6CiAgICAgICAgICAgIHByaW50KGYidXNlciB7dXNlcn0gbm90IGZvdW5kIGZvciB1cGdy
- YWRpbmcuIikKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgZXhpc3RpbmdfY2FwcyA9IGpzb25f
- b3V0WzBdWyJjYXBzIl0KICAgICAgICBzZWxmLmdldF9yZ3dfcG9vbF9uYW1lX2R1cmluZ191cGdy
- YWRlKHVzZXIsIHN0cihleGlzdGluZ19jYXBzKSkKICAgICAgICBuZXdfY2FwLCBfID0gc2VsZi5n
- ZXRfY2Fwc19hbmRfZW50aXR5KHVzZXIpCiAgICAgICAgY2FwX2tleXMgPSBbIm1vbiIsICJtZ3Ii
- LCAib3NkIiwgIm1kcyJdCiAgICAgICAgY2FwcyA9IFtdCiAgICAgICAgZm9yIGVhY2hDYXAgaW4g
- Y2FwX2tleXM6CiAgICAgICAgICAgIGN1cl9jYXBfdmFsdWVzID0gZXhpc3RpbmdfY2Fwcy5nZXQo
- ZWFjaENhcCwgIiIpCiAgICAgICAgICAgIG5ld19jYXBfdmFsdWVzID0gbmV3X2NhcC5nZXQoZWFj
- aENhcCwgIiIpCiAgICAgICAgICAgIGN1cl9jYXBfcGVybV9saXN0ID0gWwogICAgICAgICAgICAg
- ICAgeC5zdHJpcCgpIGZvciB4IGluIGN1cl9jYXBfdmFsdWVzLnNwbGl0KCIsIikgaWYgeC5zdHJp
- cCgpCiAgICAgICAgICAgIF0KICAgICAgICAgICAgbmV3X2NhcF9wZXJtX2xpc3QgPSBbCiAgICAg
- ICAgICAgICAgICB4LnN0cmlwKCkgZm9yIHggaW4gbmV3X2NhcF92YWx1ZXMuc3BsaXQoIiwiKSBp
- ZiB4LnN0cmlwKCkKICAgICAgICAgICAgXQogICAgICAgICAgICAjIGFwcGVuZCBuZXdfY2FwX2xp
- c3QgdG8gY3VyX2NhcF9saXN0IHRvIG1haW50YWluIHRoZSBvcmRlciBvZiBjYXBzCiAgICAgICAg
- ICAgIGN1cl9jYXBfcGVybV9saXN0LmV4dGVuZChuZXdfY2FwX3Blcm1fbGlzdCkKICAgICAgICAg
- ICAgIyBlbGltaW5hdGUgZHVwbGljYXRlcyB3aXRob3V0IHVzaW5nICdzZXQnCiAgICAgICAgICAg
- ICMgc2V0IHJlLW9yZGVycyBpdGVtcyBpbiB0aGUgbGlzdCBhbmQgd2UgaGF2ZSB0byBrZWVwIHRo
- ZSBvcmRlcgogICAgICAgICAgICBuZXdfY2FwX2xpc3QgPSBbXQogICAgICAgICAgICBbbmV3X2Nh
- cF9saXN0LmFwcGVuZCh4KSBmb3IgeCBpbiBjdXJfY2FwX3Blcm1fbGlzdCBpZiB4IG5vdCBpbiBu
- ZXdfY2FwX2xpc3RdCiAgICAgICAgICAgIGV4aXN0aW5nX2NhcHNbZWFjaENhcF0gPSAiLCAiLmpv
- aW4obmV3X2NhcF9saXN0KQogICAgICAgICAgICBpZiBleGlzdGluZ19jYXBzW2VhY2hDYXBdOgog
- ICAgICAgICAgICAgICAgY2Fwcy5hcHBlbmQoZWFjaENhcCkKICAgICAgICAgICAgICAgIGNhcHMu
- YXBwZW5kKGV4aXN0aW5nX2NhcHNbZWFjaENhcF0pCiAgICAgICAgY21kX2pzb24gPSB7CiAgICAg
- ICAgICAgICJwcmVmaXgiOiAiYXV0aCBjYXBzIiwKICAgICAgICAgICAgImVudGl0eSI6IHVzZXIs
- CiAgICAgICAgICAgICJjYXBzIjogY2FwcywKICAgICAgICAgICAgImZvcm1hdCI6ICJqc29uIiwK
- ICAgICAgICB9CiAgICAgICAgcmV0X3ZhbCwganNvbl9vdXQsIGVycl9tc2cgPSBzZWxmLl9jb21t
- b25fY21kX2pzb25fZ2VuKGNtZF9qc29uKQogICAgICAgIGlmIHJldF92YWwgIT0gMDoKICAgICAg
- ICAgICAgcmFpc2UgRXhlY3V0aW9uRmFpbHVyZUV4Y2VwdGlvbigKICAgICAgICAgICAgICAgIGYi
- J2F1dGggY2FwcyB7dXNlcn0nIGNvbW1hbmQgZmFpbGVkLlxuIEVycm9yOiB7ZXJyX21zZ30iCiAg
- ICAgICAgICAgICkKICAgICAgICBwcmludChmIlVwZGF0ZWQgdXNlciB7dXNlcn0gc3VjY2Vzc2Z1
- bGx5LiIpCgogICAgZGVmIG1haW4oc2VsZik6CiAgICAgICAgZ2VuZXJhdGVkX291dHB1dCA9ICIi
- CiAgICAgICAgaWYgc2VsZi5fYXJnX3BhcnNlci51cGdyYWRlOgogICAgICAgICAgICBzZWxmLnVw
- Z3JhZGVfdXNlcnNfcGVybWlzc2lvbnMoKQogICAgICAgIGVsaWYgc2VsZi5fYXJnX3BhcnNlci5m
- b3JtYXQgPT0gImpzb24iOgogICAgICAgICAgICBnZW5lcmF0ZWRfb3V0cHV0ID0gc2VsZi5nZW5f
- anNvbl9vdXQoKQogICAgICAgIGVsaWYgc2VsZi5fYXJnX3BhcnNlci5mb3JtYXQgPT0gImJhc2gi
- OgogICAgICAgICAgICBnZW5lcmF0ZWRfb3V0cHV0ID0gc2VsZi5nZW5fc2hlbGxfb3V0KCkKICAg
- ICAgICBlbHNlOgogICAgICAgICAgICByYWlzZSBFeGVjdXRpb25GYWlsdXJlRXhjZXB0aW9uKAog
- ICAgICAgICAgICAgICAgZiJVbnN1cHBvcnRlZCBmb3JtYXQ6IHtzZWxmLl9hcmdfcGFyc2VyLmZv
- cm1hdH0iCiAgICAgICAgICAgICkKICAgICAgICBwcmludChnZW5lcmF0ZWRfb3V0cHV0KQogICAg
- ICAgIGlmIHNlbGYub3V0cHV0X2ZpbGUgYW5kIGdlbmVyYXRlZF9vdXRwdXQ6CiAgICAgICAgICAg
- IGZPdXQgPSBvcGVuKHNlbGYub3V0cHV0X2ZpbGUsIG1vZGU9InciLCBlbmNvZGluZz0iVVRGLTgi
- KQogICAgICAgICAgICBmT3V0LndyaXRlKGdlbmVyYXRlZF9vdXRwdXQpCiAgICAgICAgICAgIGZP
- dXQuY2xvc2UoKQoKCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMj
- IyMjIwojIyMjIyMjIyMjIyMjIyMjIyMjIyMgTUFJTiAjIyMjIyMjIyMjIyMjIyMjIyMjIyMKIyMj
- IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjCmlmIF9fbmFtZV9f
- ID09ICJfX21haW5fXyI6CiAgICByak9iaiA9IFJhZG9zSlNPTigpCiAgICB0cnk6CiAgICAgICAg
- cmpPYmoubWFpbigpCiAgICBleGNlcHQgRXhlY3V0aW9uRmFpbHVyZUV4Y2VwdGlvbiBhcyBlcnI6
- CiAgICAgICAgcHJpbnQoZiJFeGVjdXRpb24gRmFpbGVkOiB7ZXJyfSIpCiAgICAgICAgcmFpc2Ug
- ZXJyCiAgICBleGNlcHQgS2V5RXJyb3IgYXMga0VycjoKICAgICAgICBwcmludChmIktleUVycm9y
- OiB7a0Vycn0iKQogICAgZXhjZXB0IE9TRXJyb3IgYXMgb3NFcnI6CiAgICAgICAgcHJpbnQoZiJF
- cnJvciB3aGlsZSB0cnlpbmcgdG8gb3V0cHV0IHRoZSBkYXRhOiB7b3NFcnJ9IikKICAgIGZpbmFs
- bHk6CiAgICAgICAgcmpPYmouc2h1dGRvd24oKQo=
+ ICAgICAgICAgICAgICAgICAgImNzaS5zdG9yYWdlLms4cy5pby9ub2RlLXN0YWdlLXNlY3JldC1u
+ YW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfTk9ERV9TRUNSRVRfTkFNRSddfSIs
+ CiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgKQog
+ ICAgICAgIGVsc2U6CiAgICAgICAgICAgIGlmIHNlbGYub3V0X21hcFsiUkJEX01FVEFEQVRBX0VD
+ X1BPT0xfTkFNRSJdOgogICAgICAgICAgICAgICAganNvbl9vdXQuYXBwZW5kKAogICAgICAgICAg
+ ICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiY2VwaC1yYmQiLAog
+ ICAgICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTdG9yYWdlQ2xhc3MiLAogICAgICAgICAg
+ ICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkYXRh
+ UG9vbCI6IHNlbGYub3V0X21hcFsiUkJEX1BPT0xfTkFNRSJdLAogICAgICAgICAgICAgICAgICAg
+ ICAgICAgICAgInBvb2wiOiBzZWxmLm91dF9tYXBbIlJCRF9NRVRBREFUQV9FQ19QT09MX05BTUUi
+ XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8vcHJvdmlz
+ aW9uZXItc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21hcFsnQ1NJX1JCRF9QUk9WSVNJ
+ T05FUl9TRUNSRVRfTkFNRSddfSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0
+ b3JhZ2UuazhzLmlvL2NvbnRyb2xsZXItZXhwYW5kLXNlY3JldC1uYW1lIjogZiJyb29rLXtzZWxm
+ Lm91dF9tYXBbJ0NTSV9SQkRfUFJPVklTSU9ORVJfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAg
+ ICAgICAgICAgICAgICAgICAgImNzaS5zdG9yYWdlLms4cy5pby9ub2RlLXN0YWdlLXNlY3JldC1u
+ YW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfTk9ERV9TRUNSRVRfTkFNRSddfSIs
+ CiAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgfQogICAgICAg
+ ICAgICAgICAgKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAganNvbl9vdXQuYXBw
+ ZW5kKAogICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUi
+ OiAiY2VwaC1yYmQiLAogICAgICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTdG9yYWdlQ2xh
+ c3MiLAogICAgICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAg
+ ICAgICAgICAgICJwb29sIjogc2VsZi5vdXRfbWFwWyJSQkRfUE9PTF9OQU1FIl0sCiAgICAgICAg
+ ICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL3Byb3Zpc2lvbmVyLXNlY3Jl
+ dC1uYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJPVklTSU9ORVJfU0VDUkVU
+ X05BTUUnXX0iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgImNzaS5zdG9yYWdlLms4cy5p
+ by9jb250cm9sbGVyLWV4cGFuZC1zZWNyZXQtbmFtZSI6IGYicm9vay17c2VsZi5vdXRfbWFwWydD
+ U0lfUkJEX1BST1ZJU0lPTkVSX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICAg
+ ICAgICAgICJjc2kuc3RvcmFnZS5rOHMuaW8vbm9kZS1zdGFnZS1zZWNyZXQtbmFtZSI6IGYicm9v
+ ay17c2VsZi5vdXRfbWFwWydDU0lfUkJEX05PREVfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAg
+ ICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICkK
+ CiAgICAgICAgIyBpZiAnVE9QT0xPR1lfUE9PTFMnLCAnVE9QT0xPR1lfRkFJTFVSRV9ET01BSU5f
+ TEFCRUwnLCAnVE9QT0xPR1lfRkFJTFVSRV9ET01BSU5fVkFMVUVTJyAgZXhpc3RzLAogICAgICAg
+ ICMgdGhlbiBvbmx5IGFkZCAndG9wb2xvZ3knIFN0b3JhZ2VDbGFzcwogICAgICAgIGlmICgKICAg
+ ICAgICAgICAgc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9QT09MUyJdCiAgICAgICAgICAgIGFuZCBz
+ ZWxmLm91dF9tYXBbIlRPUE9MT0dZX0ZBSUxVUkVfRE9NQUlOX0xBQkVMIl0KICAgICAgICAgICAg
+ YW5kIHNlbGYub3V0X21hcFsiVE9QT0xPR1lfRkFJTFVSRV9ET01BSU5fVkFMVUVTIl0KICAgICAg
+ ICApOgogICAgICAgICAgICBqc29uX291dC5hcHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAg
+ ICAgICAgICAgICAgICAgIm5hbWUiOiAiY2VwaC1yYmQtdG9wb2xvZ3kiLAogICAgICAgICAgICAg
+ ICAgICAgICJraW5kIjogIlN0b3JhZ2VDbGFzcyIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEi
+ OiB7CiAgICAgICAgICAgICAgICAgICAgICAgICJ0b3BvbG9neUZhaWx1cmVEb21haW5MYWJlbCI6
+ IHNlbGYub3V0X21hcFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJUT1BPTE9HWV9GQUlM
+ VVJFX0RPTUFJTl9MQUJFTCIKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAg
+ ICAgICAgICAgICAgInRvcG9sb2d5RmFpbHVyZURvbWFpblZhbHVlcyI6IHNlbGYub3V0X21hcFsK
+ ICAgICAgICAgICAgICAgICAgICAgICAgICAgICJUT1BPTE9HWV9GQUlMVVJFX0RPTUFJTl9WQUxV
+ RVMiCiAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgICAgICJ0
+ b3BvbG9neVBvb2xzIjogc2VsZi5vdXRfbWFwWyJUT1BPTE9HWV9QT09MUyJdLAogICAgICAgICAg
+ ICAgICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL3Byb3Zpc2lvbmVyLXNlY3JldC1uYW1l
+ IjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJPVklTSU9ORVJfU0VDUkVUX05BTUUn
+ XX0iLAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL2NvbnRyb2xs
+ ZXItZXhwYW5kLXNlY3JldC1uYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9SQkRfUFJP
+ VklTSU9ORVJfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0
+ b3JhZ2UuazhzLmlvL25vZGUtc3RhZ2Utc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21h
+ cFsnQ1NJX1JCRF9OT0RFX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICB9LAog
+ ICAgICAgICAgICAgICAgfQogICAgICAgICAgICApCgogICAgICAgICMgaWYgJ0NFUEhGU19GU19O
+ QU1FJyBleGlzdHMsIHRoZW4gb25seSBhZGQgJ2NlcGhmcycgU3RvcmFnZUNsYXNzCiAgICAgICAg
+ aWYgc2VsZi5vdXRfbWFwWyJDRVBIRlNfRlNfTkFNRSJdOgogICAgICAgICAgICBqc29uX291dC5h
+ cHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiY2Vw
+ aGZzIiwKICAgICAgICAgICAgICAgICAgICAia2luZCI6ICJTdG9yYWdlQ2xhc3MiLAogICAgICAg
+ ICAgICAgICAgICAgICJkYXRhIjogewogICAgICAgICAgICAgICAgICAgICAgICAiZnNOYW1lIjog
+ c2VsZi5vdXRfbWFwWyJDRVBIRlNfRlNfTkFNRSJdLAogICAgICAgICAgICAgICAgICAgICAgICAi
+ cG9vbCI6IHNlbGYub3V0X21hcFsiQ0VQSEZTX1BPT0xfTkFNRSJdLAogICAgICAgICAgICAgICAg
+ ICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL3Byb3Zpc2lvbmVyLXNlY3JldC1uYW1lIjogZiJy
+ b29rLXtzZWxmLm91dF9tYXBbJ0NTSV9DRVBIRlNfUFJPVklTSU9ORVJfU0VDUkVUX05BTUUnXX0i
+ LAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0b3JhZ2UuazhzLmlvL2NvbnRyb2xsZXIt
+ ZXhwYW5kLXNlY3JldC1uYW1lIjogZiJyb29rLXtzZWxmLm91dF9tYXBbJ0NTSV9DRVBIRlNfUFJP
+ VklTSU9ORVJfU0VDUkVUX05BTUUnXX0iLAogICAgICAgICAgICAgICAgICAgICAgICAiY3NpLnN0
+ b3JhZ2UuazhzLmlvL25vZGUtc3RhZ2Utc2VjcmV0LW5hbWUiOiBmInJvb2ste3NlbGYub3V0X21h
+ cFsnQ1NJX0NFUEhGU19OT0RFX1NFQ1JFVF9OQU1FJ119IiwKICAgICAgICAgICAgICAgICAgICB9
+ LAogICAgICAgICAgICAgICAgfQogICAgICAgICAgICApCiAgICAgICAgIyBpZiAnUkdXX0VORFBP
+ SU5UJyBleGlzdHMsIHRoZW4gb25seSBhZGQgJ2NlcGgtcmd3JyBTdG9yYWdlQ2xhc3MKICAgICAg
+ ICBpZiBzZWxmLm91dF9tYXBbIlJHV19FTkRQT0lOVCJdOgogICAgICAgICAgICBqc29uX291dC5h
+ cHBlbmQoCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiY2Vw
+ aC1yZ3ciLAogICAgICAgICAgICAgICAgICAgICJraW5kIjogIlN0b3JhZ2VDbGFzcyIsCiAgICAg
+ ICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICJlbmRwb2lu
+ dCI6IHNlbGYub3V0X21hcFsiUkdXX0VORFBPSU5UIl0sCiAgICAgICAgICAgICAgICAgICAgICAg
+ ICJwb29sUHJlZml4Ijogc2VsZi5vdXRfbWFwWyJSR1dfUE9PTF9QUkVGSVgiXSwKICAgICAgICAg
+ ICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgfQogICAgICAgICAgICApCiAgICAgICAgICAg
+ IGpzb25fb3V0LmFwcGVuZCgKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAi
+ bmFtZSI6ICJyZ3ctYWRtaW4tb3BzLXVzZXIiLAogICAgICAgICAgICAgICAgICAgICJraW5kIjog
+ IlNlY3JldCIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiB7CiAgICAgICAgICAgICAgICAg
+ ICAgICAgICJhY2Nlc3NLZXkiOiBzZWxmLm91dF9tYXBbIlJHV19BRE1JTl9PUFNfVVNFUl9BQ0NF
+ U1NfS0VZIl0sCiAgICAgICAgICAgICAgICAgICAgICAgICJzZWNyZXRLZXkiOiBzZWxmLm91dF9t
+ YXBbIlJHV19BRE1JTl9PUFNfVVNFUl9TRUNSRVRfS0VZIl0sCiAgICAgICAgICAgICAgICAgICAg
+ fSwKICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgKQogICAgICAgICMgaWYgJ1JHV19UTFNf
+ Q0VSVCcgZXhpc3RzLCB0aGVuIG9ubHkgYWRkIHRoZSAiY2VwaC1yZ3ctdGxzLWNlcnQiIHNlY3Jl
+ dAogICAgICAgIGlmIHNlbGYub3V0X21hcFsiUkdXX1RMU19DRVJUIl06CiAgICAgICAgICAgIGpz
+ b25fb3V0LmFwcGVuZCgKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAibmFt
+ ZSI6ICJjZXBoLXJndy10bHMtY2VydCIsCiAgICAgICAgICAgICAgICAgICAgImtpbmQiOiAiU2Vj
+ cmV0IiwKICAgICAgICAgICAgICAgICAgICAiZGF0YSI6IHsKICAgICAgICAgICAgICAgICAgICAg
+ ICAgImNlcnQiOiBzZWxmLm91dF9tYXBbIlJHV19UTFNfQ0VSVCJdLAogICAgICAgICAgICAgICAg
+ ICAgIH0sCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICkKCiAgICAgICAgcmV0dXJuIGpz
+ b24uZHVtcHMoanNvbl9vdXQpICsgTElORVNFUAoKICAgIGRlZiB1cGdyYWRlX3VzZXJzX3Blcm1p
+ c3Npb25zKHNlbGYpOgogICAgICAgIHVzZXJzID0gWwogICAgICAgICAgICAiY2xpZW50LmNzaS1j
+ ZXBoZnMtbm9kZSIsCiAgICAgICAgICAgICJjbGllbnQuY3NpLWNlcGhmcy1wcm92aXNpb25lciIs
+ CiAgICAgICAgICAgICJjbGllbnQuY3NpLXJiZC1ub2RlIiwKICAgICAgICAgICAgImNsaWVudC5j
+ c2ktcmJkLXByb3Zpc2lvbmVyIiwKICAgICAgICAgICAgImNsaWVudC5oZWFsdGhjaGVja2VyIiwK
+ ICAgICAgICBdCiAgICAgICAgaWYgc2VsZi5ydW5fYXNfdXNlciAhPSAiIiBhbmQgc2VsZi5ydW5f
+ YXNfdXNlciBub3QgaW4gdXNlcnM6CiAgICAgICAgICAgIHVzZXJzLmFwcGVuZChzZWxmLnJ1bl9h
+ c191c2VyKQogICAgICAgIGZvciB1c2VyIGluIHVzZXJzOgogICAgICAgICAgICBzZWxmLnVwZ3Jh
+ ZGVfdXNlcl9wZXJtaXNzaW9ucyh1c2VyKQoKICAgIGRlZiBnZXRfcmd3X3Bvb2xfbmFtZV9kdXJp
+ bmdfdXBncmFkZShzZWxmLCB1c2VyLCBjYXBzKToKICAgICAgICBpZiB1c2VyID09ICJjbGllbnQu
+ aGVhbHRoY2hlY2tlciI6CiAgICAgICAgICAgICMgd2hlbiBhZG1pbiBoYXMgbm90IHByb3ZpZGVk
+ IHJndyBwb29sIG5hbWUgZHVyaW5nIHVwZ3JhZGUsCiAgICAgICAgICAgICMgZ2V0IHRoZSByZ3cg
+ cG9vbCBuYW1lIGZyb20gY2xpZW50LmhlYWx0aGNoZWNrZXIgdXNlciB3aGljaCB3YXMgdXNlZCBk
+ dXJpbmcgY29ubmVjdGlvbgogICAgICAgICAgICBpZiBub3Qgc2VsZi5fYXJnX3BhcnNlci5yZ3df
+ cG9vbF9wcmVmaXg6CiAgICAgICAgICAgICAgICAjIFRvIGdldCB2YWx1ZSAnZGVmYXVsdCcgd2hp
+ Y2ggaXMgcmd3IHBvb2wgbmFtZSBmcm9tICdhbGxvdyByd3ggcG9vbD1kZWZhdWx0LnJndy5tZXRh
+ JwogICAgICAgICAgICAgICAgcGF0dGVybiA9IHIicG9vbD0oLio/KVwucmd3XC5tZXRhIgogICAg
+ ICAgICAgICAgICAgbWF0Y2ggPSByZS5zZWFyY2gocGF0dGVybiwgY2FwcykKICAgICAgICAgICAg
+ ICAgIGlmIG1hdGNoOgogICAgICAgICAgICAgICAgICAgIHNlbGYuX2FyZ19wYXJzZXIucmd3X3Bv
+ b2xfcHJlZml4ID0gbWF0Y2guZ3JvdXAoMSkKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAg
+ ICAgICAgICAgICAgcmFpc2UgRXhlY3V0aW9uRmFpbHVyZUV4Y2VwdGlvbigKICAgICAgICAgICAg
+ ICAgICAgICAgICAgImZhaWxlZCB0byBnZXQgcmd3IHBvb2wgbmFtZSBmb3IgdXBncmFkZSIKICAg
+ ICAgICAgICAgICAgICAgICApCgogICAgZGVmIHVwZ3JhZGVfdXNlcl9wZXJtaXNzaW9ucyhzZWxm
+ LCB1c2VyKToKICAgICAgICAjIGNoZWNrIHdoZXRoZXIgdGhlIGdpdmVuIHVzZXIgZXhpc3RzIG9y
+ IG5vdAogICAgICAgIGNtZF9qc29uID0geyJwcmVmaXgiOiAiYXV0aCBnZXQiLCAiZW50aXR5Ijog
+ ZiJ7dXNlcn0iLCAiZm9ybWF0IjogImpzb24ifQogICAgICAgIHJldF92YWwsIGpzb25fb3V0LCBl
+ cnJfbXNnID0gc2VsZi5fY29tbW9uX2NtZF9qc29uX2dlbihjbWRfanNvbikKICAgICAgICBpZiBy
+ ZXRfdmFsICE9IDAgb3IgbGVuKGpzb25fb3V0KSA9PSAwOgogICAgICAgICAgICBwcmludChmInVz
+ ZXIge3VzZXJ9IG5vdCBmb3VuZCBmb3IgdXBncmFkaW5nLiIpCiAgICAgICAgICAgIHJldHVybgog
+ ICAgICAgIGV4aXN0aW5nX2NhcHMgPSBqc29uX291dFswXVsiY2FwcyJdCiAgICAgICAgc2VsZi5n
+ ZXRfcmd3X3Bvb2xfbmFtZV9kdXJpbmdfdXBncmFkZSh1c2VyLCBzdHIoZXhpc3RpbmdfY2Fwcykp
+ CiAgICAgICAgbmV3X2NhcCwgXyA9IHNlbGYuZ2V0X2NhcHNfYW5kX2VudGl0eSh1c2VyKQogICAg
+ ICAgIGNhcF9rZXlzID0gWyJtb24iLCAibWdyIiwgIm9zZCIsICJtZHMiXQogICAgICAgIGNhcHMg
+ PSBbXQogICAgICAgIGZvciBlYWNoQ2FwIGluIGNhcF9rZXlzOgogICAgICAgICAgICBjdXJfY2Fw
+ X3ZhbHVlcyA9IGV4aXN0aW5nX2NhcHMuZ2V0KGVhY2hDYXAsICIiKQogICAgICAgICAgICBuZXdf
+ Y2FwX3ZhbHVlcyA9IG5ld19jYXAuZ2V0KGVhY2hDYXAsICIiKQogICAgICAgICAgICBjdXJfY2Fw
+ X3Blcm1fbGlzdCA9IFsKICAgICAgICAgICAgICAgIHguc3RyaXAoKSBmb3IgeCBpbiBjdXJfY2Fw
+ X3ZhbHVlcy5zcGxpdCgiLCIpIGlmIHguc3RyaXAoKQogICAgICAgICAgICBdCiAgICAgICAgICAg
+ IG5ld19jYXBfcGVybV9saXN0ID0gWwogICAgICAgICAgICAgICAgeC5zdHJpcCgpIGZvciB4IGlu
+ IG5ld19jYXBfdmFsdWVzLnNwbGl0KCIsIikgaWYgeC5zdHJpcCgpCiAgICAgICAgICAgIF0KICAg
+ ICAgICAgICAgIyBhcHBlbmQgbmV3X2NhcF9saXN0IHRvIGN1cl9jYXBfbGlzdCB0byBtYWludGFp
+ biB0aGUgb3JkZXIgb2YgY2FwcwogICAgICAgICAgICBjdXJfY2FwX3Blcm1fbGlzdC5leHRlbmQo
+ bmV3X2NhcF9wZXJtX2xpc3QpCiAgICAgICAgICAgICMgZWxpbWluYXRlIGR1cGxpY2F0ZXMgd2l0
+ aG91dCB1c2luZyAnc2V0JwogICAgICAgICAgICAjIHNldCByZS1vcmRlcnMgaXRlbXMgaW4gdGhl
+ IGxpc3QgYW5kIHdlIGhhdmUgdG8ga2VlcCB0aGUgb3JkZXIKICAgICAgICAgICAgbmV3X2NhcF9s
+ aXN0ID0gW10KICAgICAgICAgICAgW25ld19jYXBfbGlzdC5hcHBlbmQoeCkgZm9yIHggaW4gY3Vy
+ X2NhcF9wZXJtX2xpc3QgaWYgeCBub3QgaW4gbmV3X2NhcF9saXN0XQogICAgICAgICAgICBleGlz
+ dGluZ19jYXBzW2VhY2hDYXBdID0gIiwgIi5qb2luKG5ld19jYXBfbGlzdCkKICAgICAgICAgICAg
+ aWYgZXhpc3RpbmdfY2Fwc1tlYWNoQ2FwXToKICAgICAgICAgICAgICAgIGNhcHMuYXBwZW5kKGVh
+ Y2hDYXApCiAgICAgICAgICAgICAgICBjYXBzLmFwcGVuZChleGlzdGluZ19jYXBzW2VhY2hDYXBd
+ KQogICAgICAgIGNtZF9qc29uID0gewogICAgICAgICAgICAicHJlZml4IjogImF1dGggY2FwcyIs
+ CiAgICAgICAgICAgICJlbnRpdHkiOiB1c2VyLAogICAgICAgICAgICAiY2FwcyI6IGNhcHMsCiAg
+ ICAgICAgICAgICJmb3JtYXQiOiAianNvbiIsCiAgICAgICAgfQogICAgICAgIHJldF92YWwsIGpz
+ b25fb3V0LCBlcnJfbXNnID0gc2VsZi5fY29tbW9uX2NtZF9qc29uX2dlbihjbWRfanNvbikKICAg
+ ICAgICBpZiByZXRfdmFsICE9IDA6CiAgICAgICAgICAgIHJhaXNlIEV4ZWN1dGlvbkZhaWx1cmVF
+ eGNlcHRpb24oCiAgICAgICAgICAgICAgICBmIidhdXRoIGNhcHMge3VzZXJ9JyBjb21tYW5kIGZh
+ aWxlZC5cbiBFcnJvcjoge2Vycl9tc2d9IgogICAgICAgICAgICApCiAgICAgICAgcHJpbnQoZiJV
+ cGRhdGVkIHVzZXIge3VzZXJ9IHN1Y2Nlc3NmdWxseS4iKQoKICAgIGRlZiBtYWluKHNlbGYpOgog
+ ICAgICAgIGdlbmVyYXRlZF9vdXRwdXQgPSAiIgogICAgICAgIGlmIHNlbGYuX2FyZ19wYXJzZXIu
+ dXBncmFkZToKICAgICAgICAgICAgc2VsZi51cGdyYWRlX3VzZXJzX3Blcm1pc3Npb25zKCkKICAg
+ ICAgICBlbGlmIHNlbGYuX2FyZ19wYXJzZXIuZm9ybWF0ID09ICJqc29uIjoKICAgICAgICAgICAg
+ Z2VuZXJhdGVkX291dHB1dCA9IHNlbGYuZ2VuX2pzb25fb3V0KCkKICAgICAgICBlbGlmIHNlbGYu
+ X2FyZ19wYXJzZXIuZm9ybWF0ID09ICJiYXNoIjoKICAgICAgICAgICAgZ2VuZXJhdGVkX291dHB1
+ dCA9IHNlbGYuZ2VuX3NoZWxsX291dCgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2Ug
+ RXhlY3V0aW9uRmFpbHVyZUV4Y2VwdGlvbigKICAgICAgICAgICAgICAgIGYiVW5zdXBwb3J0ZWQg
+ Zm9ybWF0OiB7c2VsZi5fYXJnX3BhcnNlci5mb3JtYXR9IgogICAgICAgICAgICApCiAgICAgICAg
+ cHJpbnQoZ2VuZXJhdGVkX291dHB1dCkKICAgICAgICBpZiBzZWxmLm91dHB1dF9maWxlIGFuZCBn
+ ZW5lcmF0ZWRfb3V0cHV0OgogICAgICAgICAgICBmT3V0ID0gb3BlbihzZWxmLm91dHB1dF9maWxl
+ LCBtb2RlPSJ3IiwgZW5jb2Rpbmc9IlVURi04IikKICAgICAgICAgICAgZk91dC53cml0ZShnZW5l
+ cmF0ZWRfb3V0cHV0KQogICAgICAgICAgICBmT3V0LmNsb3NlKCkKCgojIyMjIyMjIyMjIyMjIyMj
+ IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKIyMjIyMjIyMjIyMjIyMjIyMjIyMjIE1B
+ SU4gIyMjIyMjIyMjIyMjIyMjIyMjIyMjCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMj
+ IyMjIyMjIyMjIyMjIyMjIwppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgcmpPYmogPSBS
+ YWRvc0pTT04oKQogICAgdHJ5OgogICAgICAgIHJqT2JqLm1haW4oKQogICAgZXhjZXB0IEV4ZWN1
+ dGlvbkZhaWx1cmVFeGNlcHRpb24gYXMgZXJyOgogICAgICAgIHByaW50KGYiRXhlY3V0aW9uIEZh
+ aWxlZDoge2Vycn0iKQogICAgICAgIHJhaXNlIGVycgogICAgZXhjZXB0IEtleUVycm9yIGFzIGtF
+ cnI6CiAgICAgICAgcHJpbnQoZiJLZXlFcnJvcjoge2tFcnJ9IikKICAgIGV4Y2VwdCBPU0Vycm9y
+ IGFzIG9zRXJyOgogICAgICAgIHByaW50KGYiRXJyb3Igd2hpbGUgdHJ5aW5nIHRvIG91dHB1dCB0
+ aGUgZGF0YToge29zRXJyfSIpCiAgICBmaW5hbGx5OgogICAgICAgIHJqT2JqLnNodXRkb3duKCkK
name: rook-ceph-operator.v4.17.0
namespace: placeholder
spec:
diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go
index 839555088886..372beb449e60 100644
--- a/cmd/rook/ceph/mgr.go
+++ b/cmd/rook/ceph/mgr.go
@@ -17,11 +17,13 @@ limitations under the License.
package ceph
import (
+ "fmt"
"path"
"time"
"github.com/rook/rook/cmd/rook/rook"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ "github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
@@ -30,14 +32,17 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var mgrCmd = &cobra.Command{
Use: "mgr",
}
+
var mgrSidecarCmd = &cobra.Command{
Use: "watch-active",
}
+
var (
updateMgrServicesInterval string
daemonName string
@@ -95,10 +100,9 @@ func runMgrSidecar(cmd *cobra.Command, args []string) error {
}
clusterInfo.CephVersion = *version
- m := mgr.New(context, &clusterInfo, clusterSpec, "")
activeMgr := "unknown"
for {
- activeMgr, err = m.UpdateActiveMgrLabel(daemonName, activeMgr)
+ activeMgr, err = reconcileMgr(context, activeMgr)
if err != nil {
logger.Errorf("failed to reconcile services. %v", err)
} else {
@@ -107,3 +111,35 @@ func runMgrSidecar(cmd *cobra.Command, args []string) error {
time.Sleep(interval)
}
}
+
+// reconcileMgr polls active manager name from Ceph cluster and updates mgr Pod 'mgr_role' label accordingly.
+func reconcileMgr(context *clusterd.Context, prevActiveMgr string) (string, error) {
+ logger.Infof("Checking mgr_role label value of daemon %s (prev active mgr was %s)", daemonName, prevActiveMgr)
+
+ m := mgr.New(context, &clusterInfo, clusterSpec, "")
+ currActiveMgr, err := m.GetActiveMgr()
+ if err != nil {
+ return "", fmt.Errorf("unable to get active mgr: %w", err)
+ }
+
+ if currActiveMgr == prevActiveMgr {
+ logger.Infof("active mgr is still the same (%s). No need to update mgr_role label on daemon %s.", currActiveMgr, daemonName)
+ return currActiveMgr, nil
+ }
+
+ // Active manager has changed
+ // Actualise ceph cluster spec to correctly preserve monitoring labels
+ cephCluster, err := context.RookClientset.CephV1().CephClusters(clusterInfo.Namespace).Get(clusterInfo.Context, clusterName, v1.GetOptions{})
+ if err != nil {
+ return "", fmt.Errorf("unable to get ceph cluster spec: %w", err)
+ }
+ m = mgr.New(context, &clusterInfo, cephCluster.Spec, "")
+
+ isActive := daemonName == currActiveMgr
+ // update labels:
+ err = m.SetMgrRoleLabel(daemonName, isActive)
+ if err != nil {
+ return "", fmt.Errorf("failed to set active mgr labels: %w", err)
+ }
+ return currActiveMgr, nil
+}
diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml
index 6592c42d0142..f154e012628b 100644
--- a/deploy/charts/rook-ceph-cluster/values.yaml
+++ b/deploy/charts/rook-ceph-cluster/values.yaml
@@ -25,7 +25,7 @@ toolbox:
# -- Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md)
enabled: false
# -- Toolbox image, defaults to the image used by the Ceph cluster
- image: #quay.io/ceph/ceph:v18.2.2
+ image: #quay.io/ceph/ceph:v18.2.4
# -- Toolbox tolerations
tolerations: []
# -- Toolbox affinity
@@ -92,9 +92,9 @@ cephClusterSpec:
# v17 is Quincy, v18 is Reef.
# RECOMMENDATION: In production, use a specific version tag instead of the general v18 flag, which pulls the latest release and could result in different
# versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
- # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.2-20240311
+ # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.4-20240724
# This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
# Whether to allow unsupported versions of Ceph. Currently `quincy`, and `reef` are supported.
# Future versions such as `squid` (v19) would require this to be set to `true`.
# Do not set to true in production.
@@ -452,8 +452,8 @@ cephBlockPools:
storageClass:
enabled: true
name: ceph-block
- annotations: { }
- labels: { }
+ annotations: {}
+ labels: {}
isDefault: true
reclaimPolicy: Delete
allowVolumeExpansion: true
@@ -536,8 +536,8 @@ cephFileSystems:
reclaimPolicy: Delete
allowVolumeExpansion: true
volumeBindingMode: "Immediate"
- annotations: { }
- labels: { }
+ annotations: {}
+ labels: {}
mountOptions: []
# see https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md#provision-storage for available configuration
parameters:
@@ -610,8 +610,8 @@ cephObjectStores:
name: ceph-bucket
reclaimPolicy: Delete
volumeBindingMode: "Immediate"
- annotations: { }
- labels: { }
+ annotations: {}
+ labels: {}
# see https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md#storageclass for available configuration
parameters:
# note: objectStoreNamespace and objectStoreName are configured by the chart
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index a9ea8ce8d59f..340af674ff73 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -3204,6 +3204,9 @@ spec:
description: A spec for available storage in the cluster and how it should be used
nullable: true
properties:
+ allowDeviceClassUpdate:
+ description: Whether to allow updating the device class after the OSD is initially provisioned
+ type: boolean
backfillFullRatio:
description: BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
maximum: 1
diff --git a/deploy/examples/cluster-external-management.yaml b/deploy/examples/cluster-external-management.yaml
index e4ad4d2b3513..156e968df17f 100644
--- a/deploy/examples/cluster-external-management.yaml
+++ b/deploy/examples/cluster-external-management.yaml
@@ -19,4 +19,4 @@ spec:
dataDirHostPath: /var/lib/rook
# providing an image is required, if you want to create other CRs (rgw, mds, nfs)
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2 # Should match external cluster version
+ image: quay.io/ceph/ceph:v18.2.4 # Should match external cluster version
diff --git a/deploy/examples/cluster-on-local-pvc.yaml b/deploy/examples/cluster-on-local-pvc.yaml
index 25960b985a81..0d92793f4cc3 100644
--- a/deploy/examples/cluster-on-local-pvc.yaml
+++ b/deploy/examples/cluster-on-local-pvc.yaml
@@ -173,7 +173,7 @@ spec:
requests:
storage: 10Gi
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: false
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-on-pvc.yaml b/deploy/examples/cluster-on-pvc.yaml
index 9efaf587aa4d..2087394aa3cc 100644
--- a/deploy/examples/cluster-on-pvc.yaml
+++ b/deploy/examples/cluster-on-pvc.yaml
@@ -33,7 +33,7 @@ spec:
requests:
storage: 10Gi
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: false
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
@@ -53,6 +53,7 @@ spec:
periodicity: daily # one of: hourly, daily, weekly, monthly
maxLogSize: 500M # SUFFIX may be 'M' or 'G'. Must be at least 1M.
storage:
+ allowDeviceClassUpdate: false # whether to allow changing the device class of an OSD after it is created
storageClassDeviceSets:
- name: set1
# The number of OSDs to create from this device set
@@ -123,7 +124,7 @@ spec:
volumeClaimTemplates:
- metadata:
name: data
- # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph
+ # set a different CRUSH device class on the OSD than the one detected by Ceph
# annotations:
# crushDeviceClass: hybrid
spec:
diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml
index 3a6c773a3d5b..bf82efd38165 100644
--- a/deploy/examples/cluster-stretched-aws.yaml
+++ b/deploy/examples/cluster-stretched-aws.yaml
@@ -44,7 +44,7 @@ spec:
mgr:
count: 2
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: true
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml
index 220656a1fe0b..7f582754ab26 100644
--- a/deploy/examples/cluster-stretched.yaml
+++ b/deploy/examples/cluster-stretched.yaml
@@ -38,7 +38,7 @@ spec:
mgr:
count: 2
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: true
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml
index db4c6d75f0ad..7d33a8e93f3d 100644
--- a/deploy/examples/cluster-test.yaml
+++ b/deploy/examples/cluster-test.yaml
@@ -33,7 +33,10 @@ spec:
storage:
useAllNodes: true
useAllDevices: true
+ allowDeviceClassUpdate: true
#deviceFilter:
+ #config:
+ # deviceClass: testclass
monitoring:
enabled: false
healthCheck:
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 902577817d3e..6e9619774ed9 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -19,9 +19,9 @@ spec:
# v17 is Quincy, v18 is Reef.
# RECOMMENDATION: In production, use a specific version tag instead of the general v17 flag, which pulls the latest release and could result in different
# versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
- # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.2-20240311
+ # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.4-20240724
# This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
# Whether to allow unsupported versions of Ceph. Currently `quincy` and `reef` are supported.
# Future versions such as `squid` (v19) would require this to be set to `true`.
# Do not set to true in production.
@@ -255,6 +255,8 @@ spec:
# databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB
# osdsPerDevice: "1" # this value can be overridden at the node or device level
# encryptedDevice: "true" # the default value for this option is "false"
+ # deviceClass: "myclass" # specify a device class for OSDs in the cluster
+ allowDeviceClassUpdate: false # whether to allow changing the device class of an OSD after it is created
# Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named
# nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label.
# nodes:
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 62e5496c1909..e75eb93ad07a 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -3202,6 +3202,9 @@ spec:
description: A spec for available storage in the cluster and how it should be used
nullable: true
properties:
+ allowDeviceClassUpdate:
+ description: Whether to allow updating the device class after the OSD is initially provisioned
+ type: boolean
backfillFullRatio:
description: BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
maximum: 1
diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py
index bb9a95375f1a..2bc3602002e4 100644
--- a/deploy/examples/create-external-cluster-resources.py
+++ b/deploy/examples/create-external-cluster-resources.py
@@ -1558,23 +1558,19 @@ def init_topology_rbd_pools(self, topology_rbd_pools):
for pool in topology_rbd_pools:
self.init_rbd_pool(pool)
- def getScriptCliFlags(self):
- return " ".join(sys.argv[1:])
-
# this will return the final args that script uses to process
# the priority to set a particular value is,
# command-line-args > config.ini file values > default values
def getFinalUsedArgs(self):
- list = []
+ argument = f"[Configurations]\n"
for arg in vars(self._arg_parser):
if str(getattr(self._arg_parser, arg)):
# python treats flag-name as flag_name internally, so converting back to flag-name,
# so we can get those values from config file
- argument = (
- arg.replace("_", "-") + ": " + str(getattr(self._arg_parser, arg))
- )
- list.append(argument)
- return list
+ argValue = arg.replace("_", "-")
+ if argValue != "config-file":
+ argument += f"{argValue} = {str(getattr(self._arg_parser, arg))}\n"
+ return argument
def _gen_output_map(self):
if self.out_map:
@@ -1591,7 +1587,6 @@ def _gen_output_map(self):
self._excluded_keys.add("K8S_CLUSTER_NAME")
self.get_cephfs_data_pool_details()
# double string needed for upstream exports of flags
- self.out_map["EXTERNAL_CLUSTER_USER_COMMAND"] = f'"{self.getScriptCliFlags()}"'
self.out_map["ARGS"] = f'"{self.getFinalUsedArgs()}"'
self.out_map["NAMESPACE"] = self._arg_parser.namespace
self.out_map["K8S_CLUSTER_NAME"] = self._arg_parser.k8s_cluster_name
@@ -1757,7 +1752,6 @@ def gen_json_out(self):
"name": "external-cluster-user-command",
"kind": "ConfigMap",
"data": {
- "command": self.out_map["EXTERNAL_CLUSTER_USER_COMMAND"],
"args": self.out_map["ARGS"],
},
},
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 2ebc7e1afcb2..fe1d60c72e80 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -1,5 +1,5 @@
gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v20240513-v0.1.0-35-gefb3255
- quay.io/ceph/ceph:v18.2.2
+ quay.io/ceph/ceph:v18.2.4
quay.io/ceph/cosi:v0.1.2
quay.io/cephcsi/cephcsi:v3.11.0
quay.io/csiaddons/k8s-sidecar:v0.8.0
diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh
index 642c2011a5b2..f13c10931011 100644
--- a/deploy/examples/import-external-cluster.sh
+++ b/deploy/examples/import-external-cluster.sh
@@ -174,7 +174,6 @@ function createInputCommadConfigMap() {
create \
configmap \
"$EXTERNAL_COMMAND_CONFIGMAP_NAME" \
- --from-literal=command="$EXTERNAL_CLUSTER_USER_COMMAND" \
--from-literal=args="$ARGS"
else
echo "configmap $EXTERNAL_COMMAND_CONFIGMAP_NAME already exists, updating it"
@@ -182,12 +181,7 @@ function createInputCommadConfigMap() {
patch \
configmap \
"$EXTERNAL_COMMAND_CONFIGMAP_NAME" \
- -p "{\"data\":{\"command\":\"$EXTERNAL_CLUSTER_USER_COMMAND\"}}"
- $KUBECTL -n "$NAMESPACE" \
- patch \
- configmap \
- "$EXTERNAL_COMMAND_CONFIGMAP_NAME" \
- -p "{\"data\":{\"args\":\"$ARGS\"}}"
+ -p "$(jq -n --arg args "$ARGS" '{"data": {"args": $args}}')"
fi
}
diff --git a/deploy/examples/toolbox.yaml b/deploy/examples/toolbox.yaml
index ea62db8f208a..a060b91b51b6 100644
--- a/deploy/examples/toolbox.yaml
+++ b/deploy/examples/toolbox.yaml
@@ -19,7 +19,7 @@ spec:
serviceAccountName: rook-ceph-default
containers:
- name: rook-ceph-tools
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
command:
- /bin/bash
- -c
diff --git a/design/ceph/ceph-cluster-cleanup.md b/design/ceph/ceph-cluster-cleanup.md
index 88901fb4f5a8..1af2fda984ec 100644
--- a/design/ceph/ceph-cluster-cleanup.md
+++ b/design/ceph/ceph-cluster-cleanup.md
@@ -34,7 +34,7 @@ metadata:
namespace: rook-ceph
spec:
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dataDirHostPath: /var/lib/rook
mon:
count: 3
diff --git a/go.mod b/go.mod
index 20767f86a896..d03bf16e5004 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ replace (
require (
github.com/IBM/keyprotect-go-client v0.14.3
- github.com/aws/aws-sdk-go v1.54.19
+ github.com/aws/aws-sdk-go v1.54.20
github.com/banzaicloud/k8s-objectmatcher v1.8.0
github.com/ceph/go-ceph v0.28.0
github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8
@@ -43,12 +43,12 @@ require (
golang.org/x/sync v0.7.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
- k8s.io/api v0.30.2
- k8s.io/apiextensions-apiserver v0.30.2
- k8s.io/apimachinery v0.30.2
- k8s.io/cli-runtime v0.30.2
- k8s.io/client-go v0.30.2
- k8s.io/cloud-provider v0.30.2
+ k8s.io/api v0.30.3
+ k8s.io/apiextensions-apiserver v0.30.3
+ k8s.io/apimachinery v0.30.3
+ k8s.io/cli-runtime v0.30.3
+ k8s.io/client-go v0.30.3
+ k8s.io/cloud-provider v0.30.3
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
sigs.k8s.io/controller-runtime v0.18.4
sigs.k8s.io/mcs-api v0.1.0
diff --git a/go.sum b/go.sum
index 7954adc96846..9ff972827c03 100644
--- a/go.sum
+++ b/go.sum
@@ -144,8 +144,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
-github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo=
+github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A=
github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -1587,15 +1587,15 @@ k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg=
-k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
-k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
+k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ=
+k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04=
k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY=
k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE=
k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio=
k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
-k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE=
-k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw=
+k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U=
+k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4=
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
@@ -1607,14 +1607,14 @@ k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRp
k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=
-k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
+k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw=
k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw=
k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8=
k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
-k8s.io/cli-runtime v0.30.2 h1:ooM40eEJusbgHNEqnHziN9ZpLN5U4WcQGsdLKVxpkKE=
-k8s.io/cli-runtime v0.30.2/go.mod h1:Y4g/2XezFyTATQUbvV5WaChoUGhojv/jZAtdp5Zkm0A=
+k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k=
+k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30=
k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g=
@@ -1623,10 +1623,10 @@ k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA=
k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
-k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50=
-k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs=
-k8s.io/cloud-provider v0.30.2 h1:yov6r02v7sMUNNvzEz51LtL2krn2c1wsC+dy/8BxKQI=
-k8s.io/cloud-provider v0.30.2/go.mod h1:w69t2dSjDtI9BYK6SEqj6HmMKIojEk08fXRoUzjFN2I=
+k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
+k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
+k8s.io/cloud-provider v0.30.3 h1:SNWZmllTymOTzIPJuhtZH6il/qVi75dQARRQAm9k6VY=
+k8s.io/cloud-provider v0.30.3/go.mod h1:Ax0AVdHnM7tMYnJH1Ycy4SMBD98+4zA+tboUR9eYsY8=
k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile
index b5bc029e4dc2..537fe2da6276 100644
--- a/images/ceph/Dockerfile
+++ b/images/ceph/Dockerfile
@@ -19,12 +19,10 @@ FROM BASEIMAGE
ARG S5CMD_VERSION
ARG S5CMD_ARCH
-# 'ip' tool must be installed for Multus. It's present in the base image today, but it will likely
-# be removed in the future. Doing a 'dnf install' sometimes breaks CI when centos repos go down or
-# have other package build errors. In order to make sure Rook CI catches the eventual removal and
-# also limit Rook CI breakage due to centos breakage, simply check that 'ip' is present.
-# Eventually: dnf install -y --repo baseos --setopt=install_weak_deps=False iproute && dnf clean all
-RUN which ip
+# 'ip' tool must be installed for Multus.
+# Doing a 'dnf install' sometimes breaks CI when centos repos go down or have other package build errors.
+RUN dnf install -y --repo baseos --setopt=install_weak_deps=False iproute && dnf clean all
+
# Install the s5cmd package to interact with s3 gateway
RUN curl --fail -sSL -o /s5cmd.tar.gz https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_${S5CMD_ARCH}.tar.gz && \
diff --git a/images/ceph/Makefile b/images/ceph/Makefile
index 00e1510d6c6c..d14f464e13ba 100755
--- a/images/ceph/Makefile
+++ b/images/ceph/Makefile
@@ -18,9 +18,9 @@ include ../image.mk
# Image Build Options
ifeq ($(GOARCH),amd64)
-CEPH_VERSION ?= v18.2.2-20240311
+CEPH_VERSION ?= v18.2.4-20240723
else
-CEPH_VERSION ?= v18.2.2-20240311
+CEPH_VERSION ?= v18.2.4-20240724
endif
REGISTRY_NAME = quay.io
BASEIMAGE = $(REGISTRY_NAME)/ceph/ceph-$(GOARCH):$(CEPH_VERSION)
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 8a376257334e..207f9e65a951 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -2861,6 +2861,9 @@ type StorageScopeSpec struct {
// +optional
// +nullable
BackfillFullRatio *float64 `json:"backfillFullRatio,omitempty"`
+ // Whether to allow updating the device class after the OSD is initially provisioned
+ // +optional
+ AllowDeviceClassUpdate bool `json:"allowDeviceClassUpdate,omitempty"`
}
// OSDStore is the backend storage type used for creating the OSDs
@@ -2942,7 +2945,7 @@ type Placement struct {
// the triple using the matching operator
// +optional
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
- // TopologySpreadConstraint specifies how to spread matching pods among the given topology
+ // TopologySpreadConstraints specifies how to spread matching pods among the given topology
// +optional
TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
}
diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod
index 61b0e1d53728..4c3f000279b2 100644
--- a/pkg/apis/go.mod
+++ b/pkg/apis/go.mod
@@ -21,8 +21,8 @@ require (
github.com/libopenstorage/secrets v0.0.0-20240416031220-a17cf7f72c6c
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.9.0
- k8s.io/api v0.30.2
- k8s.io/apimachinery v0.30.2
+ k8s.io/api v0.30.3
+ k8s.io/apimachinery v0.30.3
)
require (
@@ -31,7 +31,7 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
- k8s.io/client-go v0.30.2 // indirect
+ k8s.io/client-go v0.30.3 // indirect
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum
index edb52e70bf04..4ae9ba9ba0a9 100644
--- a/pkg/apis/go.sum
+++ b/pkg/apis/go.sum
@@ -1403,8 +1403,8 @@ k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg=
-k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
-k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
+k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ=
+k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04=
k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE=
k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
@@ -1417,8 +1417,8 @@ k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRp
k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=
-k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
+k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw=
k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
@@ -1427,8 +1427,8 @@ k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA=
k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
-k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50=
-k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs=
+k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
+k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
diff --git a/pkg/daemon/ceph/client/osd.go b/pkg/daemon/ceph/client/osd.go
index 65c266d2d434..63b9341c2e29 100644
--- a/pkg/daemon/ceph/client/osd.go
+++ b/pkg/daemon/ceph/client/osd.go
@@ -38,6 +38,7 @@ type OSDUsage struct {
type OSDNodeUsage struct {
ID int `json:"id"`
Name string `json:"name"`
+ DeviceClass string `json:"device_class"`
CrushWeight json.Number `json:"crush_weight"`
Depth json.Number `json:"depth"`
Reweight json.Number `json:"reweight"`
@@ -219,6 +220,24 @@ func GetOSDUsage(context *clusterd.Context, clusterInfo *ClusterInfo) (*OSDUsage
return &osdUsage, nil
}
+func SetDeviceClass(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, deviceClass string) error {
+ // First remove the existing device class
+ args := []string{"osd", "crush", "rm-device-class", fmt.Sprintf("osd.%d", osdID)}
+ buf, err := NewCephCommand(context, clusterInfo, args).Run()
+ if err != nil {
+ return errors.Wrap(err, "failed to remove device class. "+string(buf))
+ }
+
+ // Second, apply the desired device class
+ args = []string{"osd", "crush", "set-device-class", deviceClass, fmt.Sprintf("osd.%d", osdID)}
+ buf, err = NewCephCommand(context, clusterInfo, args).Run()
+ if err != nil {
+ return errors.Wrap(err, "failed to set the device class. "+string(buf))
+ }
+
+ return nil
+}
+
func GetOSDPerfStats(context *clusterd.Context, clusterInfo *ClusterInfo) (*OSDPerfStats, error) {
args := []string{"osd", "perf"}
buf, err := NewCephCommand(context, clusterInfo, args).Run()
diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go
index 2b79f1382fd0..f87e2da01043 100644
--- a/pkg/operator/ceph/cluster/mgr/dashboard.go
+++ b/pkg/operator/ceph/cluster/mgr/dashboard.go
@@ -89,16 +89,16 @@ func (c *Cluster) configureDashboardModules() error {
return nil
}
- err := c.initializeSecureDashboard()
+ secureRequiresRestart, err := c.initializeSecureDashboard()
if err != nil {
return errors.Wrap(err, "failed to initialize dashboard")
}
- hasChanged, err := c.configureDashboardModuleSettings()
+ configChanged, err := c.configureDashboardModuleSettings()
if err != nil {
return err
}
- if hasChanged {
+ if secureRequiresRestart || configChanged {
logger.Info("dashboard config has changed. restarting the dashboard module")
return c.restartMgrModule(dashboardModuleName)
}
@@ -199,38 +199,46 @@ func (c *Cluster) configureDashboardModuleSettings() (bool, error) {
return hasChanged, nil
}
-func (c *Cluster) initializeSecureDashboard() error {
+func (c *Cluster) initializeSecureDashboard() (bool, error) {
// we need to wait a short period after enabling the module before we can call the `ceph dashboard` commands.
time.Sleep(dashboardInitWaitTime)
+ restartNeeded := false
password, err := c.getOrGenerateDashboardPassword()
if err != nil {
- return errors.Wrap(err, "failed to generate a password for the ceph dashboard")
+ return restartNeeded, errors.Wrap(err, "failed to generate a password for the ceph dashboard")
}
if c.spec.Dashboard.SSL {
alreadyCreated, err := c.createSelfSignedCert()
if err != nil {
- return errors.Wrap(err, "failed to create a self signed cert for the ceph dashboard")
+ return restartNeeded, errors.Wrap(err, "failed to create a self signed cert for the ceph dashboard")
}
- if alreadyCreated {
- return nil
- }
- if err := c.restartMgrModule(dashboardModuleName); err != nil {
- logger.Warningf("failed to restart dashboard after generating ssl cert. %v", err)
+ if !alreadyCreated {
+ restartNeeded = true
}
}
if err := c.setLoginCredentials(password); err != nil {
- return errors.Wrap(err, "failed to set login credentials for the ceph dashboard")
+ return restartNeeded, errors.Wrap(err, "failed to set login credentials for the ceph dashboard")
}
- return nil
+ return restartNeeded, nil
}
func (c *Cluster) createSelfSignedCert() (bool, error) {
+
+ // Check if the cert already exists
+ args := []string{"config-key", "get", "mgr/dashboard/crt"}
+ output, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout)
+ if err == nil && len(output) > 0 {
+ logger.Info("dashboard is already initialized with a cert")
+ return true, nil
+ }
+ logger.Debugf("dashboard cert does not appear to exist. err=%v", err)
+
// create a self-signed cert for the https connections
- args := []string{"dashboard", "create-self-signed-cert"}
+ args = []string{"dashboard", "create-self-signed-cert"}
// retry a few times in the case that the mgr module is not ready to accept commands
for i := 0; i < 5; i++ {
@@ -242,20 +250,19 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
if err != nil {
exitCode, parsed := c.exitCode(err)
if parsed {
- if exitCode == certAlreadyConfiguredErrorCode {
- logger.Info("dashboard is already initialized with a cert")
- return true, nil
- }
if exitCode == invalidArgErrorCode {
logger.Info("dashboard module is not ready yet. trying again")
time.Sleep(dashboardInitWaitTime)
continue
}
+ } else {
+ return false, errors.Wrap(err, "failed to create self signed cert on mgr")
}
- return false, errors.Wrap(err, "failed to create self signed cert on mgr")
}
- break
+ logger.Info("dashboard cert created")
+ return false, nil
}
+ logger.Info("dashboard cert creation exceeded retries")
return false, nil
}
diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go
index 8b7f14a02339..8a568758f59d 100644
--- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go
+++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go
@@ -136,8 +136,8 @@ func TestStartSecureDashboard(t *testing.T) {
err = c.configureDashboardModules()
assert.NoError(t, err)
// the dashboard is enabled once with the new dashboard and modules
- assert.Equal(t, 3, enables)
- assert.Equal(t, 2, disables)
+ assert.Equal(t, 2, enables)
+ assert.Equal(t, 1, disables)
assert.Equal(t, 2, moduleRetries)
svc, err := c.context.Clientset.CoreV1().Services(clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{})
@@ -152,8 +152,8 @@ func TestStartSecureDashboard(t *testing.T) {
assert.Nil(t, err)
err = c.configureDashboardModules()
assert.NoError(t, err)
- assert.Equal(t, 3, enables)
- assert.Equal(t, 3, disables)
+ assert.Equal(t, 2, enables)
+ assert.Equal(t, 2, disables)
svc, err = c.context.Clientset.CoreV1().Services(clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{})
assert.NotNil(t, err)
diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go
index e75180063241..13cbe397d5f5 100644
--- a/pkg/operator/ceph/cluster/mgr/mgr.go
+++ b/pkg/operator/ceph/cluster/mgr/mgr.go
@@ -248,62 +248,54 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
}
}
-// UpdateActiveMgrLabel updates the mgr_role label value to either
-// active or standby depending on the status of the ceph mgr running
-// in the pod
-func (c *Cluster) UpdateActiveMgrLabel(daemonNameToUpdate string, prevActiveMgr string) (string, error) {
-
- logger.Infof("Checking mgr_role label value of daemon %s (prev active mgr was %s)", daemonNameToUpdate, prevActiveMgr)
- currActiveMgr, err := c.getActiveMgr()
- if err != nil || currActiveMgr == "" {
- logger.Infof("cannot update active mgr, no active mgr found. err=%v", err)
- return "", err
- } else if prevActiveMgr == currActiveMgr {
- logger.Infof("active mgr is still the same (%s). No need to update mgr_role label on daemon %s.", currActiveMgr, daemonNameToUpdate)
- return currActiveMgr, err
- }
-
+// SetMgrRoleLabel sets 'mgr_role: active' label to given manager daemon pods if isActive is true.
+// Otherwise sets 'mgr_role: standby' label to manager pods.
+func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) error {
pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{
- LabelSelector: fmt.Sprintf("mgr=%s", daemonNameToUpdate),
+ LabelSelector: fmt.Sprintf("%s=%s", controller.DaemonIDLabel, daemonNameToUpdate),
})
if err != nil {
logger.Infof("cannot get pod for mgr daemon %s", daemonNameToUpdate)
- return "", err // force mrg_role update in the next call
+ return err // force mrg_role update in the next call
}
+ newMgrRole := standbyMgrStatus
+ if isActive {
+ newMgrRole = activeMgrStatus
+ }
// Normally, there should only be one mgr pod with the specific name daemonNameToUpdate. However,
// during transitions, there might be additional mgr pods shutting down. To handle this, the code
// updates the label mgrRoleLabelName on all mgr pods. If this update fails, the system rolls back
// the currently active manager (currActiveMgr). This way the next call will retry the update.
+ var podLabelUpdErr error
for i, pod := range pods.Items {
-
labels := pod.GetLabels()
- cephDaemonId := labels[controller.DaemonIDLabel]
- newMgrRole := standbyMgrStatus
- if currActiveMgr == cephDaemonId {
- newMgrRole = activeMgrStatus
- }
-
currMgrRole, mgrHasLabel := labels[mgrRoleLabelName]
if !mgrHasLabel || currMgrRole != newMgrRole {
-
- logger.Infof("updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, currActiveMgr)
+ logger.Infof("updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate)
labels[mgrRoleLabelName] = newMgrRole
pod.SetLabels(labels)
_, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &pods.Items[i], metav1.UpdateOptions{})
if err != nil {
- // In case of failure we report as 'active manager' the previous value, this way
- // we force refreshing mgrRoleLabelName label next time this function is called
- currActiveMgr = prevActiveMgr
- logger.Warningf("cannot update the active mgr pod %q. err=%v", pods.Items[i].Name, err)
+ // don't return error from here. First try to update all pods from the list and reconcile services.
+ // return error later to update pods on next reconcile.
+ podLabelUpdErr = fmt.Errorf("cannot update the active mgr pod %q. err=%w", pods.Items[i].Name, err)
}
}
}
- return currActiveMgr, c.reconcileServices()
+ err = c.reconcileServices()
+ if err != nil {
+ return fmt.Errorf("unable to reconcile services: %w", err)
+ }
+ if podLabelUpdErr != nil {
+ return podLabelUpdErr
+ }
+
+ return nil
}
-func (c *Cluster) getActiveMgr() (string, error) {
+func (c *Cluster) GetActiveMgr() (string, error) {
mgrStat, err := cephclient.CephMgrStat(c.context, c.clusterInfo)
if err != nil {
return "", errors.Wrap(err, "failed to get mgr stat for the active mgr")
@@ -313,7 +305,6 @@ func (c *Cluster) getActiveMgr() (string, error) {
// reconcile the services,
func (c *Cluster) reconcileServices() error {
-
if err := c.configureDashboardService(); err != nil {
return errors.Wrap(err, "failed to configure dashboard svc")
}
@@ -461,7 +452,6 @@ func (c *Cluster) restartMgrModule(name string) error {
}
func (c *Cluster) enableBalancerModule() error {
-
// This turns "on" the balancer
err := cephclient.MgrEnableModule(c.context, c.clusterInfo, balancerModuleName, false)
if err != nil {
diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go
index bd17e4135a3a..55bc84f099df 100644
--- a/pkg/operator/ceph/cluster/mgr/mgr_test.go
+++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go
@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"os"
+ "sort"
"testing"
"time"
@@ -189,35 +190,76 @@ func TestActiveMgrLabels(t *testing.T) {
validateStart(t, c)
mgrNames := []string{"a", "b"}
- for i := 0; i < c.spec.Mgr.Count; i++ {
+ for i := 0; i < len(mgrNames); i++ {
// Simulate the Mgr pod having been created
daemonName := mgrNames[i]
mgrPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("rook-ceph-mgr-%s", daemonName),
- Labels: map[string]string{k8sutil.AppAttr: "rook-ceph-mgr",
+ Labels: map[string]string{
+ k8sutil.AppAttr: "rook-ceph-mgr",
"instance": daemonName,
"ceph_daemon_id": daemonName,
"mgr": daemonName,
- "rook_cluster": "ns"}}}
+ "rook_cluster": "ns",
+ },
+ }}
_, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, mgrPod, metav1.CreateOptions{})
assert.NoError(t, err)
}
+ mgrA, mgrB := mgrNames[0], mgrNames[1]
+
+ err = c.SetMgrRoleLabel(mgrA, true)
+ assert.NoError(t, err, "set mgr a to active")
+ err = c.SetMgrRoleLabel(mgrB, false)
+ assert.NoError(t, err, "set mgr b to standby")
+
+ pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
+ assert.NoError(t, err, "lookup all mgr pods")
+ assert.Len(t, pods.Items, 2)
+ sort.Slice(pods.Items, func(i, j int) bool {
+ // sort pods by name, so pod for mgrA will be 0 and for B - 1.
+ return pods.Items[i].GetName() < pods.Items[j].GetName()
+ })
- for i := 0; i < c.spec.Mgr.Count; i++ {
- daemonName := mgrNames[i]
- _, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
- assert.NoError(t, err)
- _, err = c.UpdateActiveMgrLabel(daemonName, "")
- assert.NoError(t, err)
- pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
- assert.NoError(t, err)
- assert.Contains(t, pods.Items[i].Labels, "mgr_role")
- if daemonName == "a" {
- assert.Equal(t, pods.Items[i].Labels["mgr_role"], "active")
- } else if daemonName == "b" {
- assert.Equal(t, pods.Items[i].Labels["mgr_role"], "standby")
- }
- }
+ assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
+ assert.Equal(t, pods.Items[0].Labels["mgr_role"], "active", "mgr a is active")
+
+ assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
+ assert.Equal(t, pods.Items[1].Labels["mgr_role"], "standby", "mgr b is standby")
+
+ err = c.SetMgrRoleLabel(mgrA, false)
+ assert.NoError(t, err, "set mgr a to standby")
+
+ pods, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
+ assert.NoError(t, err, "lookup all mgr pods")
+ assert.Len(t, pods.Items, 2)
+ sort.Slice(pods.Items, func(i, j int) bool {
+ // sort pods by name, so pod for mgrA will be 0 and for B - 1.
+ return pods.Items[i].GetName() < pods.Items[j].GetName()
+ })
+
+ assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
+ assert.Equal(t, pods.Items[0].Labels["mgr_role"], "standby", "mgr a is now standby")
+
+ assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
+ assert.Equal(t, pods.Items[1].Labels["mgr_role"], "standby", "mgr b is still standby")
+
+ err = c.SetMgrRoleLabel(mgrB, true)
+ assert.NoError(t, err, "set mgr b to active")
+
+ pods, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
+ assert.NoError(t, err, "lookup all mgr pods")
+ assert.Len(t, pods.Items, 2)
+ sort.Slice(pods.Items, func(i, j int) bool {
+ // sort pods by name, so pod for mgrA will be 0 and for B - 1.
+ return pods.Items[i].GetName() < pods.Items[j].GetName()
+ })
+
+ assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
+ assert.Equal(t, pods.Items[0].Labels["mgr_role"], "standby", "mgr a is still standby")
+
+ assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
+ assert.Equal(t, pods.Items[1].Labels["mgr_role"], "active", "mgr b is now active")
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
}
diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go
index ea3d5d2d2fdc..31f7eb530975 100644
--- a/pkg/operator/ceph/cluster/osd/create.go
+++ b/pkg/operator/ceph/cluster/osd/create.go
@@ -88,7 +88,7 @@ func (c *createConfig) createNewOSDsFromStatus(
return
}
- for _, osd := range status.OSDs {
+ for i, osd := range status.OSDs {
if c.deployments.Exists(osd.ID) {
// This OSD will be handled by the updater
logger.Debugf("not creating deployment for OSD %d which already exists", osd.ID)
@@ -96,13 +96,13 @@ func (c *createConfig) createNewOSDsFromStatus(
}
if status.PvcBackedOSD {
logger.Infof("creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
- err := createDaemonOnPVCFunc(c.cluster, osd, nodeOrPVCName, c.provisionConfig)
+ err := createDaemonOnPVCFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on PVC %q", osd.ID, nodeOrPVCName))
}
} else {
logger.Infof("creating OSD %d on node %q", osd.ID, nodeOrPVCName)
- err := createDaemonOnNodeFunc(c.cluster, osd, nodeOrPVCName, c.provisionConfig)
+ err := createDaemonOnNodeFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on node %q", osd.ID, nodeOrPVCName))
}
@@ -372,7 +372,7 @@ func (c *Cluster) runPrepareJob(osdProps *osdProperties, config *provisionConfig
return nil
}
-func createDaemonOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) error {
+func createDaemonOnPVC(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) error {
d, err := deploymentOnPVC(c, osd, pvcName, config)
if err != nil {
return err
@@ -402,7 +402,7 @@ func createDaemonOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisio
return nil
}
-func createDaemonOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) error {
+func createDaemonOnNode(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) error {
d, err := deploymentOnNode(c, osd, nodeName, config)
if err != nil {
return err
diff --git a/pkg/operator/ceph/cluster/osd/create_test.go b/pkg/operator/ceph/cluster/osd/create_test.go
index 54722eb53a1c..6a5aea9de09d 100644
--- a/pkg/operator/ceph/cluster/osd/create_test.go
+++ b/pkg/operator/ceph/cluster/osd/create_test.go
@@ -62,7 +62,7 @@ func Test_createNewOSDsFromStatus(t *testing.T) {
}()
createCallsOnNode := []int{}
induceFailureCreatingOSD := -1 // allow causing the create call to fail for a given OSD ID
- createDaemonOnNodeFunc = func(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) error {
+ createDaemonOnNodeFunc = func(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) error {
createCallsOnNode = append(createCallsOnNode, osd.ID)
if induceFailureCreatingOSD == osd.ID {
return errors.Errorf("createOSDDaemonOnNode: induced failure on OSD %d", osd.ID)
@@ -76,7 +76,7 @@ func Test_createNewOSDsFromStatus(t *testing.T) {
}()
createCallsOnPVC := []int{}
// reuse induceFailureCreatingOSD from above
- createDaemonOnPVCFunc = func(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) error {
+ createDaemonOnPVCFunc = func(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) error {
createCallsOnPVC = append(createCallsOnPVC, osd.ID)
if induceFailureCreatingOSD == osd.ID {
return errors.Errorf("createOSDDaemonOnNode: induced failure on OSD %d", osd.ID)
diff --git a/pkg/operator/ceph/cluster/osd/deviceset_test.go b/pkg/operator/ceph/cluster/osd/deviceset_test.go
index cbffcbdc4ece..281b6f18fc14 100644
--- a/pkg/operator/ceph/cluster/osd/deviceset_test.go
+++ b/pkg/operator/ceph/cluster/osd/deviceset_test.go
@@ -294,8 +294,8 @@ func TestPVCName(t *testing.T) {
}
func TestCreateValidImageVersionLabel(t *testing.T) {
- image := "ceph/ceph:v18.2.2"
- assert.Equal(t, "ceph_ceph_v18.2.2", createValidImageVersionLabel(image))
+ image := "ceph/ceph:v18.2.4"
+ assert.Equal(t, "ceph_ceph_v18.2.4", createValidImageVersionLabel(image))
image = "rook/ceph:master"
assert.Equal(t, "rook_ceph_master", createValidImageVersionLabel(image))
image = ".invalid_label"
diff --git a/pkg/operator/ceph/cluster/osd/integration_test.go b/pkg/operator/ceph/cluster/osd/integration_test.go
index 54928ab13654..9ff325f8ae25 100644
--- a/pkg/operator/ceph/cluster/osd/integration_test.go
+++ b/pkg/operator/ceph/cluster/osd/integration_test.go
@@ -585,6 +585,9 @@ func osdIntegrationTestExecutor(t *testing.T, clientset *fake.Clientset, namespa
return `["ssd"]`, nil
}
}
+ if args[1] == "df" {
+ return osdDFResults, nil
+ }
}
if args[0] == "versions" {
// the update deploy code only cares about the mons from the ceph version command results
diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go
index cab46aa5ca9c..24dbc52a1cd3 100644
--- a/pkg/operator/ceph/cluster/osd/osd.go
+++ b/pkg/operator/ceph/cluster/osd/osd.go
@@ -283,6 +283,11 @@ func (c *Cluster) Start() error {
return errors.Wrapf(err, "failed to reconcile key rotation cron jobs")
}
+ err = c.postReconcileUpdateOSDProperties(updateConfig.osdDesiredState)
+ if err != nil {
+ return errors.Wrap(err, "failed post reconcile of osd properties")
+ }
+
err = c.updateCephStorageStatus()
if err != nil {
return errors.Wrapf(err, "failed to update ceph storage status")
@@ -314,6 +319,40 @@ func (c *Cluster) Start() error {
return nil
}
+func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo) error {
+ osdUsage, err := cephclient.GetOSDUsage(c.context, c.clusterInfo)
+ if err != nil {
+ return errors.Wrap(err, "failed to get osd usage")
+ }
+ logger.Debugf("post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
+ for _, actualOSD := range osdUsage.OSDNodes {
+ if desiredOSD, ok := desiredOSDs[actualOSD.ID]; ok {
+ if err := c.updateDeviceClassIfChanged(actualOSD.ID, desiredOSD.DeviceClass, actualOSD.DeviceClass); err != nil {
+ // Log the error and allow other updates to continue
+ logger.Error(err)
+ }
+ }
+ }
+ return nil
+}
+
+func (c *Cluster) updateDeviceClassIfChanged(osdID int, desiredDeviceClass, actualDeviceClass string) error {
+ if !c.spec.Storage.AllowDeviceClassUpdate {
+ // device class updates are not allowed by default
+ return nil
+ }
+ if desiredDeviceClass != "" && desiredDeviceClass != actualDeviceClass {
+ logger.Infof("updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
+ err := cephclient.SetDeviceClass(c.context, c.clusterInfo, osdID, desiredDeviceClass)
+ if err != nil {
+ return errors.Wrapf(err, "failed to set device class on osd %d", osdID)
+ }
+ return nil
+ }
+ logger.Debugf("no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
+ return nil
+}
+
func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.Set[string], error) {
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)}
@@ -332,7 +371,7 @@ func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.Set[string], error) {
return result, nil
}
-func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
+func deploymentOnNode(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdLongName := fmt.Sprintf("OSD %d on node %q", osd.ID, nodeName)
osdProps, err := c.getOSDPropsForNode(nodeName, osd.DeviceClass)
@@ -353,7 +392,7 @@ func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisio
return d, nil
}
-func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
+func deploymentOnPVC(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdLongName := fmt.Sprintf("OSD %d on PVC %q", osd.ID, pvcName)
osdProps, err := c.getOSDPropsForPVC(pvcName, osd.DeviceClass)
@@ -376,7 +415,7 @@ func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionC
// setOSDProperties is used to configure an OSD with parameters which can not be set via explicit
// command-line arguments.
-func setOSDProperties(c *Cluster, osdProps osdProperties, osd OSDInfo) error {
+func setOSDProperties(c *Cluster, osdProps osdProperties, osd *OSDInfo) error {
// OSD's 'primary-affinity' has to be configured via command which goes through mons
if osdProps.storeConfig.PrimaryAffinity != "" {
return cephclient.SetPrimaryAffinity(c.context, c.clusterInfo, osd.ID, osdProps.storeConfig.PrimaryAffinity)
@@ -460,6 +499,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName, osdDeviceClass string) (osdProperti
}
osdProps.storeConfig.InitialWeight = deviceSet.CrushInitialWeight
osdProps.storeConfig.PrimaryAffinity = deviceSet.CrushPrimaryAffinity
+ osdProps.storeConfig.DeviceClass = deviceSet.CrushDeviceClass
// If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized
// or from the osd prepare job from initial creation.
diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go
index 062fe361411b..8d38bcae7c0e 100644
--- a/pkg/operator/ceph/cluster/osd/osd_test.go
+++ b/pkg/operator/ceph/cluster/osd/osd_test.go
@@ -51,6 +51,12 @@ import (
const (
healthyCephStatus = `{"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","health":{"checks":{},"status":"HEALTH_OK"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["a","b","c"],"monmap":{"epoch":3,"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","modified":"2020-11-02 09:58:23.015313","created":"2020-11-02 09:57:37.719235","min_mon_release":14,"min_mon_release_name":"nautilus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.74.42:3300","nonce":0},{"type":"v1","addr":"172.30.74.42:6789","nonce":0}]},"addr":"172.30.74.42:6789/0","public_addr":"172.30.74.42:6789/0"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.101.61:3300","nonce":0},{"type":"v1","addr":"172.30.101.61:6789","nonce":0}]},"addr":"172.30.101.61:6789/0","public_addr":"172.30.101.61:6789/0"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.250.55:3300","nonce":0},{"type":"v1","addr":"172.30.250.55:6789","nonce":0}]},"addr":"172.30.250.55:6789/0","public_addr":"172.30.250.55:6789/0"}]},"osdmap":{"osdmap":{"epoch":19,"num_osds":3,"num_up_osds":3,"num_in_osds":3,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96}],"num_pgs":96,"num_pools":3,"num_objects":79,"data_bytes":81553681,"bytes_used":3255447552,"bytes_avail":1646011994112,"bytes_total":1649267441664,"read_bytes_sec":853,"write_bytes_sec":5118,"read_op_per_sec":1,"write_op_per_sec":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-b","status":"up:active","gid":14161},{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-a","status":"up:standby-replay","gid":24146}],"up:standby":0},"mgrmap":{"epoch":10,"active_gid":14122,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"10.131.0.28:6800","nonce":1},{"type":"v1","addr":"10.131.0.28:6801","nonce":1}]}}}`
unHealthyCephStatus = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
+ osdDFResults = `
+ {"nodes":[
+ {"id":0,"device_class":"hdd","name":"osd.0","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27640,"kb_used_data":432,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915400,"utilization":0.065898895263671875,"var":0.99448308946989694,"pgs":9,"status":"up"},
+ {"id":1,"device_class":"hdd","name":"osd.1","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27960,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915080,"utilization":0.066661834716796875,"var":1.005996641880547,"pgs":15,"status":"up"},
+ {"id":2,"device_class":"hdd","name":"osd.2","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27780,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915260,"utilization":0.066232681274414062,"var":0.99952026864955634,"pgs":8,"status":"up"}],
+ "stray":[],"summary":{"total_kb":125829120,"total_kb_used":83380,"total_kb_used_data":1748,"total_kb_used_omap":3,"total_kb_used_meta":81596,"total_kb_avail":125745740,"average_utilization":0.066264470418294266,"min_var":0.99448308946989694,"max_var":1.005996641880547,"dev":0.00031227879054369131}}`
)
func TestOSDProperties(t *testing.T) {
@@ -80,6 +86,9 @@ func TestStart(t *testing.T) {
// Mock executor for OSD crush class list command, returning ssd as available device class
return `["ssd"]`, nil
}
+ if args[0] == "osd" && args[1] == "df" {
+ return osdDFResults, nil
+ }
return "", nil
},
}
@@ -361,6 +370,68 @@ func TestAddRemoveNode(t *testing.T) {
assert.True(t, k8serrors.IsNotFound(err))
}
+func TestUpdateDeviceClass(t *testing.T) {
+ namespace := "ns"
+ clientset := fake.NewSimpleClientset()
+ removedDeviceClassOSD := ""
+ setDeviceClassOSD := ""
+ setDeviceClass := ""
+ executor := &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ logger.Infof("ExecuteCommandWithOutput: %s %v", command, args)
+ if args[0] == "osd" {
+ if args[1] == "df" {
+ return osdDFResults, nil
+ }
+ if args[1] == "crush" {
+ if args[2] == "rm-device-class" {
+ removedDeviceClassOSD = args[3]
+ } else if args[2] == "set-device-class" {
+ setDeviceClass = args[3]
+ setDeviceClassOSD = args[4]
+ }
+ }
+ }
+ return "", nil
+ },
+ }
+
+ cephCluster := &cephv1.CephCluster{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testing",
+ Namespace: namespace,
+ },
+ Spec: cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{AllowDeviceClassUpdate: true}},
+ }
+ // Objects to track in the fake client.
+ object := []runtime.Object{
+ cephCluster,
+ }
+ s := scheme.Scheme
+ // Create a fake client to mock API calls.
+ client := clientfake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+ clusterInfo := &cephclient.ClusterInfo{
+ Namespace: namespace,
+ CephVersion: cephver.Quincy,
+ Context: context.TODO(),
+ }
+ clusterInfo.SetName("rook-ceph-test")
+ context := &clusterd.Context{Clientset: clientset, Client: client, ConfigDir: "/var/lib/rook", Executor: executor}
+ c := New(context, clusterInfo, cephCluster.Spec, "myversion")
+
+ // Start the first time
+ desiredOSDs := map[int]*OSDInfo{
+ 0: {ID: 0, DeviceClass: "hdd"},
+ 1: {ID: 1, DeviceClass: "hdd"},
+ 2: {ID: 2, DeviceClass: "newclass"},
+ }
+ err := c.postReconcileUpdateOSDProperties(desiredOSDs)
+ assert.Nil(t, err)
+ assert.Equal(t, "newclass", setDeviceClass)
+ assert.Equal(t, "osd.2", setDeviceClassOSD)
+ assert.Equal(t, "osd.2", removedDeviceClassOSD)
+}
+
func TestAddNodeFailure(t *testing.T) {
// create a storage spec with the given nodes/devices/dirs
nodeName := "node1672"
@@ -481,9 +552,9 @@ func TestGetOSDInfo(t *testing.T) {
node := "n1"
location := "root=default host=myhost zone=myzone"
- osd1 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location, TopologyAffinity: "topology.rook.io/rack=rack0"}
- osd2 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true}
- osd3 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "raw"}
+ osd1 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location, TopologyAffinity: "topology.rook.io/rack=rack0"}
+ osd2 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true}
+ osd3 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "raw"}
osdProp := osdProperties{
crushHostname: node,
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc"},
@@ -533,8 +604,8 @@ func TestGetOSDInfo(t *testing.T) {
t.Run("get info from node-based OSDs", func(t *testing.T) {
useAllDevices := true
- osd4 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
- osd5 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm"}
+ osd4 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
+ osd5 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm"}
osdProp = osdProperties{
crushHostname: node,
devices: []cephv1.Device{},
@@ -668,9 +739,9 @@ func TestGetOSDInfoWithCustomRoot(t *testing.T) {
node := "n1"
location := "root=custom-root host=myhost zone=myzone"
- osd1 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location}
- osd2 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true, Location: location}
- osd3 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
+ osd1 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location}
+ osd2 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true, Location: location}
+ osd3 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
osdProp := osdProperties{
crushHostname: node,
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc"},
diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go
index bf8066e042cd..b44aa4ee3225 100644
--- a/pkg/operator/ceph/cluster/osd/spec.go
+++ b/pkg/operator/ceph/cluster/osd/spec.go
@@ -319,7 +319,7 @@ func deploymentName(osdID int) string {
return fmt.Sprintf(osdAppNameFmt, osdID)
}
-func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
+func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
deploymentName := deploymentName(osd.ID)
replicaCount := int32(1)
volumeMounts := controller.CephVolumeMounts(provisionConfig.DataPathMap, false)
@@ -339,6 +339,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
return nil, errors.Errorf("failed to generate deployment for OSD %d. required CVMode is not specified for this OSD", osd.ID)
}
+ if c.spec.Storage.AllowDeviceClassUpdate && osdProps.storeConfig.DeviceClass != "" && osdProps.storeConfig.DeviceClass != osd.DeviceClass {
+ logger.Infof("The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
+ osd.DeviceClass = osdProps.storeConfig.DeviceClass
+ }
+
dataDir := k8sutil.DataDir
// Create volume config for /dev so the pod can access devices on the host
// Only valid when running OSD on device or OSD on LV-backed PVC
@@ -507,10 +512,10 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// needed for luksOpen synchronization when devices are encrypted and the osd is prepared with LVM
hostIPC := osdProps.storeConfig.EncryptedDevice || osdProps.encrypted
- osdLabels := c.getOSDLabels(osd, failureDomainValue, osdProps.portable)
+ osdLabels := c.getOSDLabels(*osd, failureDomainValue, osdProps.portable)
if osd.ExportService {
- osdService, err := c.createOSDService(osd, osdLabels)
+ osdService, err := c.createOSDService(*osd, osdLabels)
if err != nil {
return nil, errors.Wrapf(err, "failed to configure osd service for osd.%d", osd.ID)
}
@@ -598,11 +603,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// so that it can pick the already mounted/activated osd metadata path
// This container will activate the OSD and place the activated files into an empty dir
// The empty dir will be shared by the "activate-osd" pod and the "osd" main pod
- activateOSDVolume, activateOSDContainer := c.getActivateOSDInitContainer(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, osd, osdProps)
+ activateOSDVolume, activateOSDContainer := c.getActivateOSDInitContainer(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd, osdProps)
volumes = append(volumes, activateOSDVolume...)
volumeMounts = append(volumeMounts, activateOSDContainer.VolumeMounts[0])
initContainers = append(initContainers, *activateOSDContainer)
- initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, osd))
+ initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd))
}
// Doing a chown in a post start lifecycle hook does not reliably complete before the OSD
@@ -734,7 +739,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
}
if osdProps.portable {
// portable OSDs must have affinity to the topology where the osd prepare job was executed
- if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, osd); err != nil {
+ if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
return nil, err
}
} else {
diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go
index 6532d7c853fd..b4dbfc2d6507 100644
--- a/pkg/operator/ceph/cluster/osd/spec_test.go
+++ b/pkg/operator/ceph/cluster/osd/spec_test.go
@@ -114,7 +114,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
if len(devices) == 0 && len(dataDir) == 0 {
return
}
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -190,7 +190,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"},
}
// Not needed when running on PVC
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "lvm",
}
@@ -212,7 +212,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 9, len(cont.VolumeMounts), cont.VolumeMounts)
// Test OSD on PVC with RAW
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -251,7 +251,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test OSD on PVC with RAW and metadata device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -275,7 +275,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test encrypted OSD on PVC with RAW and metadata device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -307,7 +307,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test OSD on PVC with RAW / metadata and wal device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -333,7 +333,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test encrypted OSD on PVC with RAW / metadata and wal device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -608,7 +608,7 @@ func TestHostNetwork(t *testing.T) {
c := New(context, clusterInfo, spec, "rook/rook:myversion")
n := c.spec.Storage.ResolveNode(storageSpec.Nodes[0].Name)
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -712,7 +712,7 @@ func TestClusterGetPVCEncryptionInitContainerActivate(t *testing.T) {
// WARNING! modifies c.deviceSets
func getDummyDeploymentOnPVC(clientset *fake.Clientset, c *Cluster, pvcName string, osdID int) *appsv1.Deployment {
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: osdID,
UUID: "some-uuid",
BlockPath: "/some/path",
@@ -736,7 +736,7 @@ func getDummyDeploymentOnPVC(clientset *fake.Clientset, c *Cluster, pvcName stri
// WARNING! modifies c.ValidStorage
func getDummyDeploymentOnNode(clientset *fake.Clientset, c *Cluster, nodeName string, osdID int) *appsv1.Deployment {
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: osdID,
UUID: "some-uuid",
BlockPath: "/dev/vda",
@@ -855,7 +855,7 @@ func TestOSDPlacement(t *testing.T) {
}
c := New(context, clusterInfo, spec, "rook/rook:myversion")
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go
index ea4076fbebd4..eb6ac31a1890 100644
--- a/pkg/operator/ceph/cluster/osd/update.go
+++ b/pkg/operator/ceph/cluster/osd/update.go
@@ -49,6 +49,7 @@ type updateConfig struct {
numUpdatesNeeded int // the number of OSDs that needed updating
deployments *existenceList // these OSDs have existing deployments
osdsToSkipReconcile sets.Set[string] // these OSDs should not be updated during reconcile
+ osdDesiredState map[int]*OSDInfo // the desired state of the OSDs determined during the reconcile
}
func (c *Cluster) newUpdateConfig(
@@ -64,6 +65,7 @@ func (c *Cluster) newUpdateConfig(
queue.Len(),
deployments,
osdsToSkipReconcile,
+ map[int]*OSDInfo{},
}
}
@@ -142,6 +144,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
errs.addError("failed to update OSD %d. failed to extract OSD info from existing deployment %q. %v", osdID, depName, err)
continue
}
+ c.osdDesiredState[osdID] = &osdInfo
if c.osdsToSkipReconcile.Has(strconv.Itoa(osdID)) {
logger.Warningf("Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
@@ -174,7 +177,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
if osdIsOnPVC(dep) {
logger.Infof("updating OSD %d on PVC %q", osdID, nodeOrPVCName)
- updatedDep, err = deploymentOnPVCFunc(c.cluster, osdInfo, nodeOrPVCName, c.provisionConfig)
+ updatedDep, err = deploymentOnPVCFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on PVC %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
} else {
@@ -189,7 +192,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
}
logger.Infof("updating OSD %d on node %q", osdID, nodeOrPVCName)
- updatedDep, err = deploymentOnNodeFunc(c.cluster, osdInfo, nodeOrPVCName, c.provisionConfig)
+ updatedDep, err = deploymentOnNodeFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on node %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
}
diff --git a/pkg/operator/ceph/cluster/osd/update_test.go b/pkg/operator/ceph/cluster/osd/update_test.go
index 4c7bfc786b41..134ba2dba9b7 100644
--- a/pkg/operator/ceph/cluster/osd/update_test.go
+++ b/pkg/operator/ceph/cluster/osd/update_test.go
@@ -175,11 +175,11 @@ func Test_updateExistingOSDs(t *testing.T) {
}
// simple wrappers to allow us to count how many OSDs on nodes/PVCs are identified
- deploymentOnNodeFunc = func(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
+ deploymentOnNodeFunc = func(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdsOnNodes = append(osdsOnNodes, osd.ID)
return deploymentOnNode(c, osd, nodeName, config)
}
- deploymentOnPVCFunc = func(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
+ deploymentOnPVCFunc = func(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdsOnPVCs = append(osdsOnPVCs, osd.ID)
return deploymentOnPVC(c, osd, pvcName, config)
}
diff --git a/pkg/operator/ceph/object/policy.go b/pkg/operator/ceph/object/policy.go
index f1105928f4e4..20b9b81dd2eb 100644
--- a/pkg/operator/ceph/object/policy.go
+++ b/pkg/operator/ceph/object/policy.go
@@ -127,7 +127,7 @@ type PolicyStatement struct {
Sid string `json:"Sid"`
// Effect determines whether the Action(s) are 'Allow'ed or 'Deny'ed.
Effect effect `json:"Effect"`
- // Principle is/are the Ceph user names affected by this PolicyStatement
+ // Principal is/are the Ceph user names affected by this PolicyStatement
// Must be in the format of 'arn:aws:iam:::user/'
Principal map[string][]string `json:"Principal"`
// Action is a list of s3:* actions
diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go
index 06931a4f3bfe..4a7f93f1b493 100644
--- a/pkg/util/sys/device.go
+++ b/pkg/util/sys/device.go
@@ -93,7 +93,7 @@ type LocalDisk struct {
Type string `json:"type"`
// Rotational is the boolean whether the device is rotational: true for hdd, false for ssd and nvme
Rotational bool `json:"rotational"`
- // ReadOnly is the boolean whether the device is readonly
+ // Readonly is the boolean whether the device is readonly
Readonly bool `json:"readOnly"`
// Partitions is a partition slice
Partitions []Partition
diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go
index c66d29d7305e..32288fa1f568 100644
--- a/tests/framework/installer/ceph_manifests.go
+++ b/tests/framework/installer/ceph_manifests.go
@@ -71,7 +71,7 @@ func NewCephManifests(settings *TestCephSettings) CephManifests {
switch settings.RookVersion {
case LocalBuildTag:
return &CephManifestsMaster{settings}
- case Version1_13:
+ case Version1_14:
return &CephManifestsPreviousVersion{settings, &CephManifestsMaster{settings}}
}
panic(fmt.Errorf("unrecognized ceph manifest version: %s", settings.RookVersion))
@@ -237,15 +237,10 @@ spec:
deviceFilter: ` + getDeviceFilter() + `
config:
databaseSizeMB: "1024"
-`
- // Append the storage settings if it's not an upgrade from 1.13 where the settings do not exist
- if m.settings.RookVersion != Version1_13 {
- clusterSpec += `
fullRatio: 0.96
backfillFullRatio: 0.91
nearFullRatio: 0.88
`
- }
}
if m.settings.ConnectionsEncrypted {
diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go
index 3656659cea99..5d9d99b2a5ee 100644
--- a/tests/framework/installer/ceph_manifests_previous.go
+++ b/tests/framework/installer/ceph_manifests_previous.go
@@ -24,7 +24,7 @@ import (
const (
// The version from which the upgrade test will start
- Version1_13 = "v1.13.6"
+ Version1_14 = "v1.14.8"
)
// CephManifestsPreviousVersion wraps rook yaml definitions
diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go
index c94eb4c46e9e..0a8323e7b094 100644
--- a/tests/framework/utils/helm_helper.go
+++ b/tests/framework/utils/helm_helper.go
@@ -21,6 +21,7 @@ import (
"os"
"path"
"path/filepath"
+ "time"
"gopkg.in/yaml.v2"
@@ -99,11 +100,21 @@ func (h *HelmHelper) InstallLocalHelmChart(upgrade bool, namespace, chart string
cmdArgs = append(cmdArgs, "--namespace", namespace)
}
- err = h.installChart(cmdArgs, values)
- if err != nil {
- return fmt.Errorf("failed to install local helm chart %s with in namespace: %v, err=%v", chart, namespace, err)
+ var lastErr error
+ maxRetries := 5
+ for i := 0; i < maxRetries; i++ {
+ err = h.installChart(cmdArgs, values)
+ if err != nil {
+ lastErr = fmt.Errorf("failed to install local helm chart %s with in namespace: %v, err=%v", chart, namespace, err)
+ logger.Error(lastErr)
+ time.Sleep(time.Second)
+ continue
+ }
+ logger.Infof("helm chart %q installed successfully after %d attempt(s)", chart, i+1)
+ return nil
}
- return nil
+ logger.Errorf("failed to install helm chart %s after %d attempts", chart, maxRetries)
+ return lastErr
}
func (h *HelmHelper) InstallVersionedChart(namespace, chart, version string, values map[string]interface{}) error {
diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go
index 9c10a4cdb86e..3500211c4ca0 100644
--- a/tests/integration/ceph_upgrade_test.go
+++ b/tests/integration/ceph_upgrade_test.go
@@ -76,7 +76,7 @@ func (s *UpgradeSuite) TearDownSuite() {
s.installer.UninstallRook()
}
-func (s *UpgradeSuite) baseSetup(useHelm bool, initialCephVersion v1.CephVersionSpec) {
+func (s *UpgradeSuite) baseSetup(useHelm bool, initialRookVersion string, initialCephVersion v1.CephVersionSpec) {
s.namespace = "upgrade"
s.settings = &installer.TestCephSettings{
ClusterName: s.namespace,
@@ -88,7 +88,7 @@ func (s *UpgradeSuite) baseSetup(useHelm bool, initialCephVersion v1.CephVersion
Mons: 1,
EnableDiscovery: true,
SkipClusterCleanup: true,
- RookVersion: installer.Version1_13,
+ RookVersion: initialRookVersion,
CephVersion: initialCephVersion,
}
@@ -105,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeHelm() {
}
func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersionSpec) {
- s.baseSetup(useHelm, initialCephVersion)
+ s.baseSetup(useHelm, installer.Version1_14, initialCephVersion)
objectUserID := "upgraded-user"
preFilename := "pre-upgrade-file"
@@ -129,7 +129,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
//
// Upgrade Rook from v1.13 to master
//
- logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_13)
+ logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_14)
s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade")
s.upgradeToMaster()
@@ -138,7 +138,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
err := s.installer.WaitForToolbox(s.namespace)
assert.NoError(s.T(), err)
- logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_13)
+ logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_14)
newFile := "post-upgrade-previous-to-master-file"
s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead)
rbdFilesToRead = append(rbdFilesToRead, newFile)
@@ -150,7 +150,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
// do not need retry b/c the OBC controller runs parallel to Rook-Ceph orchestration
assert.True(s.T(), s.helper.BucketClient.CheckOBC(obcName, "bound"))
- logger.Infof("Verified upgrade from %s to master", installer.Version1_13)
+ logger.Infof("Verified upgrade from %s to master", installer.Version1_14)
// SKIP the Ceph version upgrades for the helm test
if s.settings.UseHelm {
@@ -183,7 +183,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
}
func (s *UpgradeSuite) TestUpgradeCephToQuincyDevel() {
- s.baseSetup(false, installer.QuincyVersion)
+ s.baseSetup(false, installer.LocalBuildTag, installer.QuincyVersion)
objectUserID := "upgraded-user"
preFilename := "pre-upgrade-file"
@@ -216,7 +216,7 @@ func (s *UpgradeSuite) TestUpgradeCephToQuincyDevel() {
}
func (s *UpgradeSuite) TestUpgradeCephToReefDevel() {
- s.baseSetup(false, installer.ReefVersion)
+ s.baseSetup(false, installer.LocalBuildTag, installer.ReefVersion)
objectUserID := "upgraded-user"
preFilename := "pre-upgrade-file"
@@ -249,7 +249,7 @@ func (s *UpgradeSuite) TestUpgradeCephToReefDevel() {
}
func (s *UpgradeSuite) TestUpgradeCephToSquidDevel() {
- s.baseSetup(false, installer.SquidVersion)
+ s.baseSetup(false, installer.LocalBuildTag, installer.SquidVersion)
objectUserID := "upgraded-user"
preFilename := "pre-upgrade-file"
@@ -330,7 +330,7 @@ func (s *UpgradeSuite) deployClusterforUpgrade(objectUserID, preFilename string)
require.True(s.T(), created)
// verify that we're actually running the right pre-upgrade image
- s.verifyOperatorImage(installer.Version1_13)
+ s.verifyOperatorImage(installer.Version1_14)
assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, simpleTestMessage))
assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, simpleTestMessage))
diff --git a/tests/manifests/test-cluster-on-pvc-encrypted.yaml b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
index 23da35bcf180..7e363abae53b 100644
--- a/tests/manifests/test-cluster-on-pvc-encrypted.yaml
+++ b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -14,7 +14,7 @@ spec:
requests:
storage: 5Gi
cephVersion:
- image: quay.io/ceph/ceph:v18.2.2
+ image: quay.io/ceph/ceph:v18.2.4
dashboard:
enabled: false
network: