Skip to content

Commit

Permalink
test(schema-compiler): Replace apla-clickhouse with @clickhouse/client
Browse files Browse the repository at this point in the history
  • Loading branch information
@mcheshkov
mcheshkov committed Nov 21, 2024
1 parent 05984a2 commit 753141c
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 75 deletions.
2 changes: 1 addition & 1 deletion packages/cubejs-schema-compiler/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
"uuid": "^8.3.2"
},
"devDependencies": {
"@cubejs-backend/apla-clickhouse": "^1.7.0",
"@clickhouse/client": "^1.7.0",
"@cubejs-backend/linter": "^1.0.0",
"@cubejs-backend/query-orchestrator": "1.1.7",
"@types/babel__code-frame": "^7.0.6",
Expand Down
153 changes: 84 additions & 69 deletions packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
/* eslint-disable */
// @ts-ignore
import ClickHouse from '@cubejs-backend/apla-clickhouse';
import { createClient } from '@clickhouse/client';
import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client';
import { GenericContainer } from 'testcontainers';
import type { StartedTestContainer } from 'testcontainers';
import { format as formatSql } from 'sqlstring';
import { v4 as uuidv4 } from 'uuid';
import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
import { BaseDbRunner } from "../utils/BaseDbRunner";

// Just a placeholder for now
type ClickHouseClient = any;

process.env.TZ = 'GMT';

export class ClickHouseDbRunner extends BaseDbRunner {
Expand All @@ -35,27 +31,32 @@ export class ClickHouseDbRunner extends BaseDbRunner {
// let engine = 'MergeTree PARTITION BY id ORDER BY (id) SETTINGS index_granularity = 8192'
const engine = 'Memory';

await clickHouse.querying(`
CREATE TEMPORARY TABLE visitors (id UInt64, amount UInt64, created_at DateTime, updated_at DateTime, status UInt64, source Nullable(String), latitude Float64, longitude Float64)
ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
await clickHouse.command({ query: `
CREATE TEMPORARY TABLE visitors (id UInt64, amount UInt64, created_at DateTime, updated_at DateTime, status UInt64, source Nullable(String), latitude Float64, longitude Float64)
ENGINE = ${engine}
` });

await clickHouse.querying(`
CREATE TEMPORARY TABLE visitor_checkins (id UInt64, visitor_id UInt64, created_at DateTime, source Nullable(String))
ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
await clickHouse.command({ query: `
CREATE TEMPORARY TABLE visitor_checkins (id UInt64, visitor_id UInt64, created_at DateTime, source Nullable(String))
ENGINE = ${engine}
` });

await clickHouse.querying(`
CREATE TEMPORARY TABLE cards (id UInt64, visitor_id UInt64, visitor_checkin_id UInt64)
ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
await clickHouse.command({ query: `
CREATE TEMPORARY TABLE cards (id UInt64, visitor_id UInt64, visitor_checkin_id UInt64)
ENGINE = ${engine}
` });

await clickHouse.querying(`
CREATE TEMPORARY TABLE events (id UInt64, type String, name String, started_at DateTime64, ended_at Nullable(DateTime64))
ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
await clickHouse.command({ query: `
CREATE TEMPORARY TABLE events (id UInt64, type String, name String, started_at DateTime64, ended_at Nullable(DateTime64))
ENGINE = ${engine}
` });

await clickHouse.querying(`
CREATE TEMPORARY TABLE numbers (num Int)
ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
await clickHouse.command({ query: `
CREATE TEMPORARY TABLE numbers (num Int)
ENGINE = ${engine}
` });

await clickHouse.querying(`
await clickHouse.command({ query: `
INSERT INTO
visitors
(id, amount, created_at, updated_at, status, source, latitude, longitude) VALUES
Expand All @@ -65,9 +66,9 @@ export class ClickHouseDbRunner extends BaseDbRunner {
(4, 400, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 10.60),
(5, 500, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 58.10),
(6, 500, '2016-09-06 16:00:00', '2016-09-06 16:00:00', 2, null, 120.120, 58.10)
`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
` });

await clickHouse.querying(`
await clickHouse.command({ query: `
INSERT INTO
visitor_checkins
(id, visitor_id, created_at, source) VALUES
Expand All @@ -77,28 +78,28 @@ export class ClickHouseDbRunner extends BaseDbRunner {
(4, 2, '2017-01-04 16:00:00', null),
(5, 2, '2017-01-04 16:00:00', null),
(6, 3, '2017-01-05 16:00:00', null)
`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
` });

await clickHouse.querying(`
await clickHouse.command({ query: `
INSERT INTO
cards
(id, visitor_id, visitor_checkin_id) VALUES
(1, 1, 1),
(2, 1, 2),
(3, 3, 6)
`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
` });

await clickHouse.querying(`
await clickHouse.command({ query: `
INSERT INTO
events
(id, type, name, started_at, ended_at) VALUES
(1, 'moon_missions', 'Apollo 10', '1969-05-18 16:49:00', '1969-05-26 16:52:23'),
(2, 'moon_missions', 'Apollo 11', '1969-07-16 13:32:00', '1969-07-24 16:50:35'),
(3, 'moon_missions', 'Artemis I', '2021-11-16 06:32:00', '2021-12-11 18:50:00'),
(4, 'private_missions', 'Axiom Mission 1', '2022-04-08 15:17:12', '2022-04-25 17:06:00')
`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
` });

await clickHouse.querying(`
await clickHouse.command({ query: `
INSERT INTO
numbers
(num) VALUES
Expand All @@ -108,7 +109,7 @@ export class ClickHouseDbRunner extends BaseDbRunner {
(30), (31), (32), (33), (34), (35), (36), (37), (38), (39),
(40), (41), (42), (43), (44), (45), (46), (47), (48), (49),
(50), (51), (52), (53), (54), (55), (56), (57), (58), (59)
`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
` });
}

public override async testQueries(queries: Array<[string, Array<unknown>]>, prepareDataSet?: ((client: ClickHouseClient) => Promise<void>) | null): Promise<Array<Array<Record<string, unknown>>>> {
Expand All @@ -127,18 +128,17 @@ export class ClickHouseDbRunner extends BaseDbRunner {
port = this.container.getMappedPort(8123);
}

const clickHouse = new ClickHouse({
host,
port,
});
const clickHouse = createClient({
url: `http://${host}:${port}`,

clickHouse.sessionId = uuidv4(); // needed for tests to use temporary tables
// needed for tests to use temporary tables
session_id: uuidv4(),
max_open_connections: 1,
});

prepareDataSet = prepareDataSet || this.gutterDataSet;
await prepareDataSet(clickHouse);

const requests: Array<unknown> = [];

// Controls whether functions return results with extended date and time ranges.
//
// 0 — Functions return Date or DateTime for all arguments (default).
Expand All @@ -150,19 +150,23 @@ export class ClickHouseDbRunner extends BaseDbRunner {
//
// https://clickhouse.com/docs/en/operations/settings/settings#enable-extended-results-for-datetime-functions
const extendedDateTimeResultsOptions = this.supportsExtendedDateTimeResults ? {
enable_extended_results_for_datetime_functions: '1'
} : {};

for (const [query, params] of queries) {
requests.push(clickHouse.querying(formatSql(query, params), {
dataObjects: true,
queryOptions: {
session_id: clickHouse.sessionId,
join_use_nulls: '1',
...extendedDateTimeResultsOptions
}
}));
}
enable_extended_results_for_datetime_functions: 1
} as const : {};

const requests = queries
.map(async ([query, params]) => {
const resultSet = await clickHouse.query({
query: formatSql(query, params),
format: 'JSON',
clickhouse_settings: {
join_use_nulls: 1,
...extendedDateTimeResultsOptions
}
});
// Because we used JSON format we expect each row in result set to be a record of column name => value
const result = await resultSet.json<Record<string, unknown>>();
return result;
});

const results = await Promise.all(requests);

Expand All @@ -189,26 +193,37 @@ export class ClickHouseDbRunner extends BaseDbRunner {
//
// https://github.com/statsbotco/cube.js/pull/98#discussion_r279698399
//
protected static _normaliseResponse(res: any): Array<Record<string, unknown>> {
if (process.env.DEBUG_LOG === 'true') console.log(res);
if (res.data) {
res.data.forEach(row => {
for (const field in row) {
const value = row[field];
if (value !== null) {
const meta = res.meta.find(m => m.name == field);
if (meta.type.includes('DateTime')) {
row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
} else if (meta.type.includes('Date')) {
row[field] = `${value}T00:00:00.000`;
} else if (meta.type.includes('Int') || meta.type.includes('Float')) {
// convert all numbers into strings
row[field] = `${value}`;
protected static _normaliseResponse(res: ResponseJSON<Record<string, unknown>>): Array<Record<string, unknown>> {
if (process.env.DEBUG_LOG === 'true') {
console.log(res);
}

const { meta, data } = res;
if (meta === undefined) {
throw new Error('Unexpected missing meta');
}

data.forEach(row => {
for (const [field, value] of Object.entries(row)) {
if (value !== null) {
const fieldMeta = meta.find(m => m.name === field);
if (fieldMeta === undefined) {
throw new Error(`Missing meta for field ${field}`);
}
if (fieldMeta.type.includes('DateTime')) {
if (typeof value !== 'string') {
throw new Error(`Unexpected value for ${field}`);
}
row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
} else if (fieldMeta.type.includes('Date')) {
row[field] = `${value}T00:00:00.000`;
} else if (fieldMeta.type.includes('Int') || fieldMeta.type.includes('Float')) {
// convert all numbers into strings
row[field] = `${value}`;
}
}
});
}
return res.data;
}
});
return data;
}
}
5 changes: 0 additions & 5 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4365,11 +4365,6 @@
tiny-invariant "^1.3.3"
valid-url "^1.0.9"

"@cubejs-backend/apla-clickhouse@^1.7.0":
version "1.7.0"
resolved "https://registry.yarnpkg.com/@cubejs-backend/apla-clickhouse/-/apla-clickhouse-1.7.0.tgz#6359f46c56492d1704d18be0210c7546fdac5f5e"
integrity sha512-qwXapTC/qosA6RprElRjnl8gmlDQaxtJPtbgcdjyNvkmiyao1HI+w5QkjHWCiVm6aTzE0gjFr6/2y87TZ9fojg==

"@cubejs-backend/dotenv@^9.0.2":
version "9.0.2"
resolved "https://registry.yarnpkg.com/@cubejs-backend/dotenv/-/dotenv-9.0.2.tgz#c3679091b702f0fd38de120c5a63943fcdc0dcbf"
Expand Down

0 comments on commit 753141c

Please sign in to comment.