diff --git a/package-lock.json b/package-lock.json index 2dfdd9efb..13067e970 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,7 +61,7 @@ "husky": "^9.0.11", "mocha": "^10.2.0", "mongodb": "^6.19.0", - "mongodb-runner": "^6.0.0", + "mongodb-runner": "^6.2.0", "node-gyp": "^9.0.0 || ^10.2.0", "nyc": "^15.1.0", "pkg-up": "^3.1.0", @@ -25959,13 +25959,14 @@ "optional": true }, "node_modules/mongodb-runner": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-6.0.0.tgz", - "integrity": "sha512-ijhBVCcTWRlauxp4UdIuktfPjEqlt8yOo0u7XyE99HdaITzL0BjS4x/dOyNzddewC6gw5wxH1uYx+Uo7GfeUnw==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-6.2.0.tgz", + "integrity": "sha512-3RCYNGUJIHkwxmhyfQdY7gp6KqbyqEb2lNw3ZV36XaJFU07bYb9b497ba2YuH3evdNonbqXCUf+yxdAUWNPojg==", "dev": true, "license": "Apache-2.0", "dependencies": { "@mongodb-js/mongodb-downloader": "^1.0.0", + "@mongodb-js/oidc-mock-provider": "^0.12.0", "@mongodb-js/saslprep": "^1.3.2", "debug": "^4.4.0", "mongodb": "^6.9.0", @@ -25991,6 +25992,19 @@ "tar": "^6.1.15" } }, + "node_modules/mongodb-runner/node_modules/@mongodb-js/oidc-mock-provider": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/oidc-mock-provider/-/oidc-mock-provider-0.12.0.tgz", + "integrity": "sha512-OBGqTQ25CJpNhvSy5gYNcTSi+cOExeGliAcuObXX6AswhpW7gt62go9md0jYAs10MPKTPFOvK+RrXX4WIkr/nA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "yargs": "^17.7.2" + }, + "bin": { + "oidc-mock-provider": "bin/oidc-mock-provider.js" + } + }, "node_modules/mongodb-runner/node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", diff --git a/package.json b/package.json index a7b666ea7..d633c8ae3 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "husky": "^9.0.11", "mocha": "^10.2.0", "mongodb": "^6.19.0", - "mongodb-runner": "^6.0.0", + "mongodb-runner": "^6.2.0", "node-gyp": "^9.0.0 || ^10.2.0", "nyc": "^15.1.0", "pkg-up": "^3.1.0", diff --git a/packages/cli-repl/src/cli-repl.spec.ts b/packages/cli-repl/src/cli-repl.spec.ts index 71321df7b..5401535c0 100644 --- a/packages/cli-repl/src/cli-repl.spec.ts +++ b/packages/cli-repl/src/cli-repl.spec.ts @@ -2268,6 +2268,22 @@ describe('CliRepl', function () { context('for server >= 4.1', function () { skipIfServerVersion(testServer, '< 4.1'); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "$where is deprecated" and "operation was interrupted" warnings + unsubscribeAllowWarning = testServer.allowWarning?.((entry) => { + return ( + entry.id === 8996500 || + (entry.id === 23798 && + entry.attr?.error?.codeName === 'ClientDisconnect') + ); + }); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); it('terminates operations on the server side', async function () { if (process.env.MONGOSH_TEST_FORCE_API_STRICT) { diff --git a/packages/e2e-tests/test/e2e-auth.spec.ts b/packages/e2e-tests/test/e2e-auth.spec.ts index 12773ed13..d0f89f6be 100644 --- a/packages/e2e-tests/test/e2e-auth.spec.ts +++ b/packages/e2e-tests/test/e2e-auth.spec.ts @@ -766,6 +766,15 @@ describe('Auth e2e', function () { }); }); describe('logout', function () { + let unsubscribeAllowWarning: (() => void) | undefined; + beforeEach(function () { + // https://jira.mongodb.org/browse/SERVER-56266 + // https://jira.mongodb.org/browse/MONGOSH-2695 + unsubscribeAllowWarning = testServer.allowWarning?.(5626600); + }); + afterEach(function () { + unsubscribeAllowWarning?.(); + }); it('logs out after authenticating', async function () { await shell.executeLine(`use ${dbName}`); expect(await shell.executeLine('db.auth("anna", "pwd")')).to.include( diff --git a/packages/e2e-tests/test/e2e-oidc.spec.ts b/packages/e2e-tests/test/e2e-oidc.spec.ts index 08935926c..5d9994fd2 100644 --- a/packages/e2e-tests/test/e2e-oidc.spec.ts +++ b/packages/e2e-tests/test/e2e-oidc.spec.ts @@ -116,6 +116,15 @@ describe('OIDC auth e2e', function () { ...commonOidcServerArgs, ], }); + // The server will (rightfully) complain about the mock IdP's TLS certificate + // not being trusted - we can ignore that for the purposes of this test. + testServer2.allowWarning( + (entry) => + entry.id === 7938401 && + entry.attr?.error?.includes( + 'SSL peer certificate or SSH remote key was not OK' + ) + ); testServer3 = new MongoRunnerSetup('e2e-oidc-test-idtoken', { args: [ '--setParameter', @@ -154,6 +163,12 @@ describe('OIDC auth e2e', function () { }; }); + afterEach(function () { + testServer?.noServerWarningsCheckpoint(); + testServer2?.noServerWarningsCheckpoint(); + testServer3?.noServerWarningsCheckpoint(); + }); + after(async function () { this.timeout(120_000); await Promise.all([ diff --git a/packages/e2e-tests/test/e2e-proxy.spec.ts b/packages/e2e-tests/test/e2e-proxy.spec.ts index b9afc836d..738d37bea 100644 --- a/packages/e2e-tests/test/e2e-proxy.spec.ts +++ b/packages/e2e-tests/test/e2e-proxy.spec.ts @@ -426,8 +426,18 @@ describe('e2e proxy support', function () { ...commonOidcServerArgs, ], }); + // The server will (rightfully) complain about the mock IdP's TLS certificate + // not being trusted - we can ignore that for the purposes of this test. + oidcTestServer.allowWarning( + (entry) => + entry.id === 7938401 && + entry.attr?.error?.includes( + 'SSL peer certificate or SSH remote key was not OK' + ) + ); await oidcTestServer.start(); }); + after(async function () { this.timeout(120_000); await Promise.all([ @@ -437,6 +447,10 @@ describe('e2e proxy support', function () { ]); }); + afterEach(function () { + oidcTestServer?.noServerWarningsCheckpoint(); + }); + beforeEach(function () { tokenFetches = 0; getTokenPayload = (metadata) => { diff --git a/packages/e2e-tests/test/e2e.spec.ts b/packages/e2e-tests/test/e2e.spec.ts index c6029f248..78b8525b9 100644 --- a/packages/e2e-tests/test/e2e.spec.ts +++ b/packages/e2e-tests/test/e2e.spec.ts @@ -687,6 +687,8 @@ describe('e2e', function () { if (process.env.MONGOSH_TEST_FORCE_API_STRICT) { return this.skip(); // mapReduce is unversioned } + // Allow "map reduce command is deprecated" warning in logs + const unsubscribe = testServer.allowWarning?.(5725801); await shell.executeLine(`use ${dbName}`); await shell.executeLine('db.test.insertMany([{i:1},{i:2},{i:3},{i:4}]);'); const result = await shell.executeLine(`db.test.mapReduce(function() { @@ -696,6 +698,7 @@ describe('e2e', function () { }, { out: { inline: 1 } }).results`); expect(result).to.include('{ _id: 0, value: 6 }'); expect(result).to.include('{ _id: 1, value: 4 }'); + unsubscribe?.(); }); it('rewrites async properly for common libraries', async function () { @@ -800,6 +803,8 @@ describe('e2e', function () { }); it('rewrites async properly for a complex $function', async function () { + // Allow "$function is deprecated" warning in logs + const unsubscribe = testServer.allowWarning?.(8996503); await shell.executeLine(`use ${dbName}`); await shell.executeLine( 'db.test.insertMany([{i:[1,{v:5}]},{i:[2,{v:6}]},{i:[3,{v:7}]},{i:[4,{v:8}]}]);' @@ -819,6 +824,7 @@ describe('e2e', function () { } ])`); expect(result).to.include("{ sum: '12' }"); + unsubscribe?.(); }); }); @@ -2658,6 +2664,7 @@ describe('e2e', function () { context('with 2 shells', function () { let helperShell: TestShell; let currentOpShell: TestShell; + let unsubscribeAllowWarning: undefined | (() => void); const CURRENT_OP_WAIT_TIME = 400; const OPERATION_TIME = CURRENT_OP_WAIT_TIME * 2; @@ -2676,6 +2683,15 @@ describe('e2e', function () { await helperShell.executeLine('db.coll.insertOne({})'); }); + before(function () { + // Allow "$where is deprecated" warnings + unsubscribeAllowWarning = testServer.allowWarning?.(8996500); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); + it('should return the current operation and clear when it is complete', async function () { const currentCommand = helperShell.executeLine( `db.coll.find({$where: function() { sleep(${OPERATION_TIME}) }}).projection({testProjection: 1})` @@ -2706,7 +2722,7 @@ describe('e2e', function () { -1 ); - void helperShell.executeLine( + const currentCommand = helperShell.executeLine( `db.coll.find({$where: function() { sleep(${OPERATION_TIME}) }}).projection({re: BSONRegExp('${stringifiedRegExpString}')})` ); helperShell.assertNoErrors(); @@ -2719,6 +2735,7 @@ describe('e2e', function () { currentOpShell.assertNoErrors(); expect(currentOpCall).to.include(stringifiedRegExpString); + await currentCommand; }); }); }); diff --git a/packages/shell-api/src/integration.spec.ts b/packages/shell-api/src/integration.spec.ts index 2b6e50a1b..dba0a9a14 100644 --- a/packages/shell-api/src/integration.spec.ts +++ b/packages/shell-api/src/integration.spec.ts @@ -743,6 +743,16 @@ describe('Shell API (integration)', function () { describe('#reIndex', function () { skipIfApiStrict(); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "The reIndex command is deprecated" warnings + unsubscribeAllowWarning = testServer.allowWarning?.(6508600); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); beforeEach(async function () { await serviceProvider.createCollection(dbName, collectionName); @@ -1069,6 +1079,16 @@ describe('Shell API (integration)', function () { describe('runCommand', function () { skipIfApiStrict(); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "The collStats command is deprecated" warnings + unsubscribeAllowWarning = testServer.allowWarning?.(7024600); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); beforeEach(async function () { await serviceProvider.createCollection(dbName, collectionName); @@ -1353,6 +1373,21 @@ describe('Shell API (integration)', function () { // https://jira.mongodb.org/browse/SERVER-58076 skipIfServerVersion(testServer, '<= 6.0'); } + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow warning for the failing updateOne() below + unsubscribeAllowWarning = testServer.allowWarning?.( + (entry) => + entry.id === 7267501 && + entry.attr?.error?.codeName === 'DollarPrefixedFieldName' + ); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); + it('can insert, modify and retrieve fields with $-prefixed .-containing names', async function () { await collection.insertOne({ '$x.y': 1, _id: '_id' }); expect(await collection.findOne()).to.deep.equal({ @@ -1991,6 +2026,16 @@ describe('Shell API (integration)', function () { describe('mapReduce', function () { skipIfServerVersion(testServer, '< 4.4'); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "The map reduce command is deprecated" warning + unsubscribeAllowWarning = testServer.allowWarning?.(5725801); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); let mapFn: () => void; let reduceFn: (a: string, b: string[]) => string; @@ -3007,6 +3052,21 @@ describe('Shell API (integration)', function () { describe('maxTimeMS support', function () { skipIfServerVersion(testServer, '< 4.2'); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "$where is deprecated" warning and timeout warnings + unsubscribeAllowWarning = testServer.allowWarning?.( + (entry) => + entry.id === 8996500 || + (entry.id === 23798 && + entry.attr?.error?.codeName === 'MaxTimeMSExpired') + ); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); beforeEach(async function () { await collection.insertMany([...Array(10).keys()].map((i) => ({ i }))); diff --git a/packages/shell-api/src/replica-set.spec.ts b/packages/shell-api/src/replica-set.spec.ts index c50a98f46..f6c5261cc 100644 --- a/packages/shell-api/src/replica-set.spec.ts +++ b/packages/shell-api/src/replica-set.spec.ts @@ -1028,6 +1028,23 @@ describe('ReplicaSet', function () { }); describe('remove member', function () { + let unsubscribeAllowWarnings: (() => void)[]; + + before(function () { + // Allow "Unable to forward progress" warnings + unsubscribeAllowWarnings = [srv0, srv1, srv2, srv3].map((s) => + s.allowWarning?.( + (entry) => + entry.id === 21764 && + entry.attr?.error?.codeName === 'NodeNotFound' + ) + ); + }); + + after(function () { + for (const cb of unsubscribeAllowWarnings) cb(); + }); + it('removes a member of the config', async function () { const removeWithRetry = createRetriableMethod(rs, 'remove'); const version = (await rs.conf()).version; @@ -1061,6 +1078,18 @@ describe('ReplicaSet', function () { }); describe('configureQueryAnalyzer()', function () { skipIfServerVersion(srv0, '< 7.0'); // analyzeShardKey will only be added in 7.0 which is not included in stable yet + let unsubscribeAllowWarnings: (() => void)[]; + + before(function () { + // Allow "Attempted to disable query sampling but query sampling was not active" warnings + unsubscribeAllowWarnings = [srv0, srv1, srv2, srv3].map((s) => + s.allowWarning?.(7724700) + ); + }); + + after(function () { + for (const cb of unsubscribeAllowWarnings) cb(); + }); const docs: any[] = []; for (let i = 0; i < 1000; i++) { @@ -1105,6 +1134,24 @@ describe('ReplicaSet', function () { ); let serviceProvider: NodeDriverServiceProvider; + let unsubscribeAllowWarnings: (() => void)[]; + + before(function () { + // Allow "replSetReconfig" errors + unsubscribeAllowWarnings = [srv0, srv1, srv2].map((s) => + s.allowWarning?.((entry) => { + return ( + entry.id === 21420 && + entry.attr?.error?.codeName === + 'NewReplicaSetConfigurationIncompatible' + ); + }) + ); + }); + + after(function () { + for (const cb of unsubscribeAllowWarnings) cb(); + }); beforeEach(async function () { serviceProvider = await NodeDriverServiceProvider.connect( diff --git a/packages/shell-api/src/session.spec.ts b/packages/shell-api/src/session.spec.ts index d1846dc23..55d85f5f6 100644 --- a/packages/shell-api/src/session.spec.ts +++ b/packages/shell-api/src/session.spec.ts @@ -446,6 +446,17 @@ describe('Session', function () { }); describe('after resetting connection will error with expired session', function () { skipIfApiStrict(); + let unsubscribeAllowWarning: undefined | (() => void); + + before(function () { + // Allow "The logout command has been deprecated" warnings + unsubscribeAllowWarning = srv0.allowWarning?.(5626600); + }); + + after(function () { + unsubscribeAllowWarning?.(); + }); + it('reset connection options', async function () { session = mongo.startSession(); await mongo.setReadConcern('majority'); diff --git a/packages/shell-api/src/shard.spec.ts b/packages/shell-api/src/shard.spec.ts index c52edc91d..cabd7c306 100644 --- a/packages/shell-api/src/shard.spec.ts +++ b/packages/shell-api/src/shard.spec.ts @@ -3035,6 +3035,18 @@ describe('Shard', function () { }); describe('configureQueryAnalyzer()', function () { skipIfServerVersion(mongos, '< 7.0'); // analyzeShardKey will only be added in 7.0 which is not included in stable yet + let unsubscribeAllowWarnings: (() => void)[]; + + before(function () { + // Allow "Attempted to disable query sampling but query sampling was not active" warnings + unsubscribeAllowWarnings = [mongos, rs0, rs1].map((s) => + s.allowWarning?.(7724700) + ); + }); + + after(function () { + for (const cb of unsubscribeAllowWarnings) cb(); + }); let db: Database; const dbName = 'shard-analyze-test'; diff --git a/testing/integration-testing-hooks.ts b/testing/integration-testing-hooks.ts index 7f229cdf1..7c85391c4 100644 --- a/testing/integration-testing-hooks.ts +++ b/testing/integration-testing-hooks.ts @@ -1,12 +1,13 @@ /* eslint-disable mocha/no-exports */ import child_process from 'child_process'; import { promises as fs } from 'fs'; -import { MongoClient, type MongoClientOptions } from 'mongodb'; +import { Log, MongoClient, type MongoClientOptions } from 'mongodb'; import path from 'path'; import semver from 'semver'; import { promisify } from 'util'; import which from 'which'; -import { MongoCluster, type MongoClusterOptions } from 'mongodb-runner'; +import type { LogEntry, MongoClusterOptions } from 'mongodb-runner'; +import { MongoCluster } from 'mongodb-runner'; import { ConnectionString } from 'mongodb-connection-string-url'; import { downloadCryptLibrary } from '@mongosh/build'; @@ -147,6 +148,8 @@ export class MongoRunnerSetup extends MongodSetup { _id: string; _opts: Partial; _cluster: MongoCluster | undefined; + _warnings: LogEntry[] = []; + _warningFilters: ((e: LogEntry) => boolean)[] = []; constructor(id: string, opts: Partial = {}) { super(); @@ -174,12 +177,84 @@ export class MongoRunnerSetup extends MongodSetup { }); this._setConnectionString(this._cluster.connectionString); + + // Setup logging + const ignoreCtx = new Set(); + this._cluster.on('mongoLog', (_server, entry) => { + // Ignore events coming from internal clients (e.g. replset clients) + if (entry.id === 51800 && entry.attr?.doc?.driver?.name === 'MongoDB Internal Client' && entry.context.startsWith('conn')) { + ignoreCtx.add(entry.context); + } + if (ignoreCtx.has(entry.context)) return; + + if (['W', 'E', 'F'].includes(entry.severity) && !this._warningFilters.some(f => f(entry))) { + this._warnings.push(entry); + } + }); + for (const filter of MongoRunnerSetup.defaultAllowedWarnings) + this.allowWarning(filter); } async stop(): Promise { await this._cluster?.close(); this._cluster = undefined; } + + get warnings(): LogEntry[] { + return [...this._warnings]; + } + + noServerWarningsCheckpoint(): void { + const warnings = this.warnings; + this._warnings = []; + if (warnings.length > 0) { + throw new Error(`Unexpected MongoDB server log warnings found: ${JSON.stringify(warnings, null, 2)}`); + } + } + + allowWarning(filter: number | ((entry: LogEntry) => boolean)): () => void { + if (typeof filter === 'number') { + const id = filter; + filter = entry => entry.id === id; + } + this._warningFilters.push(filter); + return () => { + this._warningFilters = this._warningFilters.filter(f => f !== filter); + }; + } + + static defaultAllowedWarnings = [ + 4615610, // "Failed to check socket connectivity", generic disconnect error + 7012500, // "Failed to refresh query analysis configurations", normal sharding behavior + 4906901, // "Arbiters are not supported in quarterly binary versions" + 6100702, // "Failed to get last stable recovery timestamp due to lock acquire timeout. Note this is expected if shutdown is in progress." + 20525, // "Failed to gather storage statistics for slow operation" + 22120, // "Access control is not enabled for the database" + 22140, // "This server is bound to localhost" + 22178, // "transparent_hugepage/enabled is 'always'" + 5123300, // "vm.max_map_count is too low" + 551190, // "Server certificate has no compatible Subject Alternative Name", + 20526, // "Failed to gather storage statistics for slow operation" + (l: LogEntry) => l.component === 'STORAGE', // Outside of mongosh's control + (l: LogEntry) => l.context === 'BackgroundSync', // Outside of mongosh's control + (l: LogEntry) => { + // "Aggregate command executor error", we get this a lot for things like + // $collStats which internally tries to open collections that may or may not exist + return l.id === 23799 && l.attr?.error?.codeName === 'NamespaceNotFound'; + }, + (l: LogEntry) => { + // "getMore command executor error" can happen under normal circumstances + // for client errors + return l.id === 20478 && l.attr?.error?.codeName === 'ClientDisconnect'; + }, + (l: LogEntry) => { + // "$jsonSchema validator does not allow '_id' field" warning + // incorrectly issued for the implicit schema of config.settings + // https://github.com/mongodb/mongo/blob/0c265adbde984c981946f804279693078e0b9f8a/src/mongo/db/global_catalog/ddl/sharding_catalog_manager.cpp#L558-L559 + // https://github.com/mongodb/mongo/blob/0c265adbde984c981946f804279693078e0b9f8a/src/mongo/s/balancer_configuration.cpp#L122-L143 + return l.id === 3216000 && ['ReplWriterWorker', 'OplogApplier'].some(match => l.context.includes(match)); + } + ]; } async function getInstalledMongodVersion(): Promise { @@ -211,11 +286,11 @@ export async function downloadCurrentCryptSharedLibrary( * @export * @returns {MongodSetup} - Object with information about the started server. */ -let sharedSetup: MongodSetup | null = null; +let sharedSetup: MongoRunnerSetup | null = null; export function startTestServer( id: string, args: Partial = {} -): MongodSetup { +): MongoRunnerSetup { const server = new MongoRunnerSetup(id, args); before(async function () { this.timeout(120_000); // Include potential mongod download time. @@ -241,7 +316,7 @@ export function startTestServer( * @export * @returns {MongodSetup} - Object with information about the started server. */ -export function startSharedTestServer(): MongodSetup { +export function startSharedTestServer(): (MongodSetup & Partial) | MongoRunnerSetup { if (process.env.MONGOSH_TEST_SERVER_URL) { return new MongodSetup(process.env.MONGOSH_TEST_SERVER_URL); } @@ -253,6 +328,10 @@ export function startSharedTestServer(): MongodSetup { await server.start(); }); + afterEach(function () { + server.noServerWarningsCheckpoint(); + }); + // NOTE: no after hook here, cause the shared server is only // cleaned up once we're done with everything. return server; @@ -270,7 +349,7 @@ global.after?.(async function () { export function startTestCluster( id: string, ...argLists: Partial[] -): MongodSetup[] { +): MongoRunnerSetup[] { const servers = argLists.map((args) => new MongoRunnerSetup(id, args)); before(async function () { @@ -278,6 +357,12 @@ export function startTestCluster( await Promise.all(servers.map((server: MongodSetup) => server.start())); }); + afterEach(function() { + for (const server of servers) { + server.noServerWarningsCheckpoint(); + } + }); + after(async function () { this.timeout(30_000 * servers.length); await Promise.all(servers.map((server: MongodSetup) => server.stop()));