From b3bdc3b92bfe3102ce171388b7e995fb9675fb73 Mon Sep 17 00:00:00 2001 From: Alex Vasilev Date: Fri, 10 Jan 2025 17:27:07 -0800 Subject: [PATCH 1/5] fix(gateway): skip query rewrite during dry run --- packages/cubejs-api-gateway/src/gateway.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index a00b77c861674..4af6bb3f777ca 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1161,6 +1161,7 @@ class ApiGateway { context: RequestContext, persistent = false, memberExpressions: boolean = false, + skipQueryRewrite: boolean = false, ): Promise<[QueryType, NormalizedQuery[]]> { let query = this.parseQueryParam(inputQuery); @@ -1199,6 +1200,11 @@ class ApiGateway { } const normalizedQuery = normalizeQuery(currentQuery, persistent); + + if (skipQueryRewrite) { + return normalizedQuery; + } + let evaluatedQuery = normalizedQuery; if (hasExpressionsInQuery) { @@ -1454,7 +1460,7 @@ class ApiGateway { try { await this.assertApiScope('data', context.securityContext); - const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context); + const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context, undefined, undefined, true); const sqlQueries = await Promise.all( normalizedQueries.map(async (normalizedQuery) => (await this.getCompilerApi(context)).getSql( From 18e1830ea0e1b2577a48665d91f14ac27922fbd6 Mon Sep 17 00:00:00 2001 From: Alex Vasilev Date: Mon, 13 Jan 2025 11:45:37 -0800 Subject: [PATCH 2/5] pre rewrite queries --- packages/cubejs-api-gateway/src/gateway.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 4af6bb3f777ca..29d1aedc9731e 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1161,8 +1161,7 @@ class ApiGateway { context: RequestContext, persistent = false, memberExpressions: boolean = false, - skipQueryRewrite: boolean = false, - ): Promise<[QueryType, NormalizedQuery[]]> { + ): Promise<[QueryType, NormalizedQuery[], NormalizedQuery[]]> { let query = this.parseQueryParam(inputQuery); let queryType: QueryType = QueryTypeEnum.REGULAR_QUERY; @@ -1185,6 +1184,8 @@ class ApiGateway { const startTime = new Date().getTime(); const compilerApi = await this.getCompilerApi(context); + const normalizedQueriesPreRewrite: NormalizedQuery[] = queries.map((currentQuery) => normalizeQuery(currentQuery, persistent)); + let normalizedQueries: NormalizedQuery[] = await Promise.all( queries.map( async (currentQuery) => { @@ -1201,10 +1202,6 @@ class ApiGateway { const normalizedQuery = normalizeQuery(currentQuery, persistent); - if (skipQueryRewrite) { - return normalizedQuery; - } - let evaluatedQuery = normalizedQuery; if (hasExpressionsInQuery) { @@ -1263,7 +1260,7 @@ class ApiGateway { } } - return [queryType, normalizedQueries]; + return [queryType, normalizedQueries, normalizedQueriesPreRewrite]; } public async sql({ @@ -1460,7 +1457,7 @@ class ApiGateway { try { await this.assertApiScope('data', context.securityContext); - const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context, undefined, undefined, true); + const [queryType, _, normalizedQueries] = await this.getNormalizedQueries(query, context, undefined, undefined); const sqlQueries = await Promise.all( normalizedQueries.map(async (normalizedQuery) => (await this.getCompilerApi(context)).getSql( From 72de5bb085e6fd5eb870a77c2d9fa8830155c60e Mon Sep 17 00:00:00 2001 From: Alex Vasilev Date: Mon, 13 Jan 2025 12:11:50 -0800 Subject: [PATCH 3/5] remap --- packages/cubejs-api-gateway/src/gateway.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 29d1aedc9731e..f106870e44de6 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1184,7 +1184,7 @@ class ApiGateway { const startTime = new Date().getTime(); const compilerApi = await this.getCompilerApi(context); - const normalizedQueriesPreRewrite: NormalizedQuery[] = queries.map((currentQuery) => normalizeQuery(currentQuery, persistent)); + const normalizedQueriesPreRewrite: NormalizedQuery[] = queries.map((currentQuery) => remapToQueryAdapterFormat(normalizeQuery(currentQuery, persistent))); let normalizedQueries: NormalizedQuery[] = await Promise.all( queries.map( From d92209ff4a9fe76ca57170e33aa6e2bea5323fc3 Mon Sep 17 00:00:00 2001 From: Alex Vasilev Date: Mon, 13 Jan 2025 15:15:18 -0800 Subject: [PATCH 4/5] fixes --- packages/cubejs-api-gateway/src/gateway.ts | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index f106870e44de6..c5ccbe1a9813a 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1184,25 +1184,32 @@ class ApiGateway { const startTime = new Date().getTime(); const compilerApi = await this.getCompilerApi(context); - const normalizedQueriesPreRewrite: NormalizedQuery[] = queries.map((currentQuery) => remapToQueryAdapterFormat(normalizeQuery(currentQuery, persistent))); - - let normalizedQueries: NormalizedQuery[] = await Promise.all( - queries.map( - async (currentQuery) => { - const hasExpressionsInQuery = - this.hasExpressionsInQuery(currentQuery); - - if (hasExpressionsInQuery) { - if (!memberExpressions) { - throw new Error('Expressions are not allowed in this context'); - } + const queryNormalizationResult: Array<{ + query: Query, + normalizedQuery: NormalizedQuery, + hasExpressionsInQuery: boolean + }> = queries.map((currentQuery) => { + const hasExpressionsInQuery = this.hasExpressionsInQuery(currentQuery); + + if (hasExpressionsInQuery) { + if (!memberExpressions) { + throw new Error('Expressions are not allowed in this context'); + } - currentQuery = this.parseMemberExpressionsInQuery(currentQuery); - } + currentQuery = this.parseMemberExpressionsInQuery(currentQuery); + } - const normalizedQuery = normalizeQuery(currentQuery, persistent); + return { + query: currentQuery, + normalizedQuery: (normalizeQuery(currentQuery, persistent)), + hasExpressionsInQuery + }; + }); - let evaluatedQuery = normalizedQuery; + let normalizedQueries: NormalizedQuery[] = await Promise.all( + queryNormalizationResult.map( + async ({ query: currentQuery, normalizedQuery, hasExpressionsInQuery }) => { + let evaluatedQuery = currentQuery; if (hasExpressionsInQuery) { // We need to parse/eval all member expressions early as applyRowLevelSecurity @@ -1260,7 +1267,7 @@ class ApiGateway { } } - return [queryType, normalizedQueries, normalizedQueriesPreRewrite]; + return [queryType, normalizedQueries, queryNormalizationResult.map((it) => remapToQueryAdapterFormat(it.normalizedQuery))]; } public async sql({ From c4391b06d06136c372386f292f418d3f2882388e Mon Sep 17 00:00:00 2001 From: Alex Vasilev Date: Mon, 13 Jan 2025 19:06:15 -0800 Subject: [PATCH 5/5] typefix --- packages/cubejs-api-gateway/src/gateway.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index c5ccbe1a9813a..f88b1c675dd59 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1185,7 +1185,6 @@ class ApiGateway { const compilerApi = await this.getCompilerApi(context); const queryNormalizationResult: Array<{ - query: Query, normalizedQuery: NormalizedQuery, hasExpressionsInQuery: boolean }> = queries.map((currentQuery) => { @@ -1200,7 +1199,6 @@ class ApiGateway { } return { - query: currentQuery, normalizedQuery: (normalizeQuery(currentQuery, persistent)), hasExpressionsInQuery }; @@ -1208,14 +1206,13 @@ class ApiGateway { let normalizedQueries: NormalizedQuery[] = await Promise.all( queryNormalizationResult.map( - async ({ query: currentQuery, normalizedQuery, hasExpressionsInQuery }) => { - let evaluatedQuery = currentQuery; + async ({ normalizedQuery, hasExpressionsInQuery }) => { + let evaluatedQuery: Query | NormalizedQuery = normalizedQuery; if (hasExpressionsInQuery) { // We need to parse/eval all member expressions early as applyRowLevelSecurity // needs to access the full SQL query in order to evaluate rules - evaluatedQuery = - this.evalMemberExpressionsInQuery(normalizedQuery); + evaluatedQuery = this.evalMemberExpressionsInQuery(normalizedQuery); } // First apply cube/view level security policies