Skip to content

Improvement/cldsrv 724 kms tests migration #5951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: improvement/CLDSRV-724-sur-utapi-tests
Choose a base branch
from
Open

Improvement/cldsrv 724 kms tests migration #5951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca351bd
kms tests related code update after sdk migration
benzekrimaha Sep 24, 2025
0bd90df
dependencies update
benzekrimaha Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"dependencies": {
"@aws-sdk/client-s3": "^3.705.0",
"@aws-sdk/credential-providers": "^3.864.0",
"@aws-sdk/middleware-retry": "^3.374.0",
"@azure/storage-blob": "^12.28.0",
"@hapi/joi": "^17.1.1",
"@smithy/node-http-handler": "^3.0.0",
Expand Down
90 changes: 45 additions & 45 deletions tests/functional/sse-kms-migration/arnPrefix.js
View file
Open in desktop

Large diffs are not rendered by default.

62 changes: 31 additions & 31 deletions tests/functional/sse-kms-migration/beforeMigration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,20 @@ describe('SSE KMS before migration', () => {
? bkt.kmsKeyInfo.masterKeyArn
: bkt.kmsKeyInfo.masterKeyId;
}
await helpers.s3.createBucket(({ Bucket: bkt.name })).promise();
await helpers.s3.createBucket(({ Bucket: bkt.vname })).promise();
await helpers.s3.createBucket({ Bucket: bkt.name });
await helpers.s3.createBucket({ Bucket: bkt.vname });
if (bktConf.algo) {
// bucket encryption will be asserted in bucket test
await helpers.s3.putBucketEncryption({
Bucket: bkt.name,
ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({
algo: bktConf.algo, masterKeyId: bkt.kmsKey }),
}).promise();
});
await helpers.s3.putBucketEncryption({
Bucket: bkt.vname,
ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({
algo: bktConf.algo, masterKeyId: bkt.kmsKey }),
}).promise();
});
}

// Put an object for each SSE conf in each bucket
Expand All @@ -79,18 +79,18 @@ describe('SSE KMS before migration', () => {
before(async () => {
console.log('Run before migration',
{ profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId });
const allBuckets = (await helpers.s3.listBuckets().promise()).Buckets.map(b => b.Name);
const allBuckets = ((await helpers.s3.listBuckets({})).Buckets || []).map(b => b.Name);
console.log('List buckets:', allBuckets);
await promisify(metadata.setup.bind(metadata))();

// init copy bucket
await helpers.s3.createBucket(({ Bucket: copyBkt })).promise();
await helpers.s3.createBucket(({ Bucket: mpuCopyBkt })).promise();
await helpers.s3.createBucket({ Bucket: copyBkt });
await helpers.s3.createBucket({ Bucket: mpuCopyBkt });
await helpers.s3.putBucketEncryption({
Bucket: copyBkt,
ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'aws:kms', masterKeyId: copyKmsKey }),
}).promise();
await helpers.s3.putObject({ Bucket: copyBkt, Key: copyObj, Body: 'BODY(copy)' }).promise();
});
await helpers.s3.putObject({ Bucket: copyBkt, Key: copyObj, Body: 'BODY(copy)' });

// Prepare every buckets with 1 object (for copy)
await Promise.all(scenarios.testCases.map(async bktConf => this.initBucket(bktConf)));
Expand Down Expand Up @@ -172,7 +172,7 @@ describe('SSE KMS before migration', () => {
optionalSkip('should encrypt MPU and put 2 encrypted parts', async () => {
const mpuKey = `${obj.name}-mpu`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`];
const newParts = [];
for (const [index, body] of partsBody.entries()) {
Expand Down Expand Up @@ -200,7 +200,7 @@ describe('SSE KMS before migration', () => {
optionalSkip('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => {
const mpuKey = `${obj.name}-mpucopy`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
const part1 = await scenarios.tests.mpuUploadPartCopy({
UploadId: mpu.UploadId,
Bucket: bkt.name,
Expand Down Expand Up @@ -231,7 +231,7 @@ describe('SSE KMS before migration', () => {
optionalSkip('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => {
const mpuKey = `${obj.name}-mpucopyrange`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
// source body is "BODY(copy)"
// [copy, BODY]
const sourceRanges = ['bytes=5-8', 'bytes=0-3'];
Expand Down Expand Up @@ -263,13 +263,13 @@ describe('SSE KMS before migration', () => {
optionalSkip('should prepare empty encrypted MPU without completion', async () => {
const mpuKey = `${obj.name}-migration-mpu-empty`;
await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
});

optionalSkip('should prepare encrypte MPU and put 2 encrypted parts without completion', async () => {
optionalSkip('should prepare encrypted MPU and put 2 encrypted parts without completion', async () => {
const mpuKey = `${obj.name}-migration-mpu`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`];
for (const [index, body] of partsBody.entries()) {
await scenarios.tests.mpuUploadPart({
Expand All @@ -286,7 +286,7 @@ describe('SSE KMS before migration', () => {
'from encrypted bucket without completion', async () => {
const mpuKey = `${obj.name}-migration-mpucopy`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
await scenarios.tests.mpuUploadPartCopy({
UploadId: mpu.UploadId,
Bucket: bkt.name,
Expand All @@ -303,11 +303,11 @@ describe('SSE KMS before migration', () => {
}, mpu, objConf.algo || bktConf.algo, 'before');
});

optionalSkip('should prepare encrypte MPU and copy an encrypted range parts ' +
optionalSkip('should prepare encrypted MPU and copy an encrypted range parts ' +
'from encrypted bucket without completion', async () => {
const mpuKey = `${obj.name}-migration-mpucopyrange`;
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)).promise();
helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey));
// source body is "BODY(copy)"
// [copy, BODY]
const sourceRanges = ['bytes=5-8', 'bytes=0-3'];
Expand All @@ -326,7 +326,7 @@ describe('SSE KMS before migration', () => {
it(`should PutObject versioned with SSE ${obj.name}`, async () => {
// ensure versioned bucket is empty
await helpers.bucketUtil.empty(bkt.vname);
let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise();
let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }) || [];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Versions can then be undefined ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should probably be like this to ensure we do not have an error trying to access Versions from an array that has no fields

Suggested change
let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }) || [];
let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }) || { Versions: [] };

// regularly count versioned objects
assert.strictEqual(Versions.length, 0);

Expand All @@ -336,23 +336,23 @@ describe('SSE KMS before migration', () => {
await scenarios.assertObjectSSE(
{ ...baseAssertion, Body: bodyBase },
{ objConf, obj }, { bktConf, bkt });
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 1);

await helpers.s3.putBucketVersioning({ Bucket: bkt.vname,
VersioningConfiguration: { Status: 'Enabled' },
}).promise();
});

const bodyV1 = `BODY(${obj.name})-v1`;
const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1);
const bodyV2 = `BODY(${obj.name})-v2`;
const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2);
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 3);

const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }).promise();
const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name });
assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 3);

await scenarios.assertObjectSSE(
Expand All @@ -363,12 +363,12 @@ describe('SSE KMS before migration', () => {
{ ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt });
await scenarios.assertObjectSSE(
{ ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt });
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 3);

await helpers.s3.putBucketVersioning({ Bucket: bkt.vname,
VersioningConfiguration: { Status: 'Suspended' },
}).promise();
});

// should be fine after version suspension
await scenarios.assertObjectSSE(
Expand All @@ -379,7 +379,7 @@ describe('SSE KMS before migration', () => {
{ ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt });
await scenarios.assertObjectSSE(
{ ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt });
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 3);

// put a new null version
Expand All @@ -389,7 +389,7 @@ describe('SSE KMS before migration', () => {
{ ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }); // null
await scenarios.assertObjectSSE(
{ ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, 'null');
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }).promise());
({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }));
assert.strictEqual(Versions.length, 3);
});
}));
Expand All @@ -401,10 +401,10 @@ describe('SSE KMS before migration', () => {
Bucket: mpuCopyBkt,
// AES256 because input key is broken for now
ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }),
}).promise();
});
const mpuKey = 'mpucopy';
const mpu = await helpers.s3.createMultipartUpload(
helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)).promise();
helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null));
const copyPartArg = {
UploadId: mpu.UploadId,
Bucket: mpuCopyBkt,
Expand All @@ -422,7 +422,7 @@ describe('SSE KMS before migration', () => {
...copyPartArg,
PartNumber: partNumber,
CopySource: `${bkt.name}/${obj.name}`,
}).promise();
});

return { partNumber, body: obj.body, res: res.CopyPartResult };
}));
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/sse-kms-migration/cleanup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ describe('SSE KMS Cleanup', () => {
it('Empty and delete buckets for SSE KMS Migration', async () => {
console.log('Run cleanup',
{ profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId });
const allBuckets = (await helpers.s3.listBuckets().promise()).Buckets.map(b => b.Name);
const allBuckets = ((await helpers.s3.listBuckets()).Buckets || []).map(b => b.Name);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have same stuff for other tests (for example in beforeMigration). Seems like a debug ? Should we keep it ? Check the length ?

console.log('List buckets:', allBuckets);

await helpers.MD.setup();
Expand Down
Loading
Loading