Skip to content

Commit

Permalink
chore: cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@vpbs2
vpbs2 committed Sep 3, 2024
1 parent 49eae92 commit 548ad4e
Show file tree
Hide file tree
Showing 12 changed files with 230 additions and 313 deletions.
6 changes: 3 additions & 3 deletions benchmarking/src/app/app.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Route, BrowserRouter as Router, Routes } from 'react-router-dom';
import { IndexedDBMProvider } from './dbm-context/indexed-dbm-context';
import { MemoryDBMProvider } from './dbm-context/memory-dbm-context';
import { ParallelParallelDBMProvider } from './dbm-context/parallel-indexed-dbm-context';
import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-context';
import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context';
import { RawDBMProvider } from './dbm-context/raw-dbm-context';
import { FileLoader } from './file-loader/file-loader';
Expand Down Expand Up @@ -69,11 +69,11 @@ export function App() {
element={
<div>
<h1>Parallel Indexed DuckDB</h1>
<ParallelParallelDBMProvider>
<ParallelIndexedDBMProvider>
<FileLoader bufferType="uint8Array">
<QueryBenchmarking />
</FileLoader>
</ParallelParallelDBMProvider>
</ParallelIndexedDBMProvider>
</div>
}
/>
Expand Down
186 changes: 93 additions & 93 deletions benchmarking/src/app/constants.ts
View file
Original file line number Diff line number Diff line change
@@ -1,103 +1,103 @@
export const TEST_QUERIES = [
// `
// WITH group_by_query AS (
// SELECT
// hvfhs_license_num,
// COUNT(*)
// FROM
// taxi.parquet
// GROUP BY
// hvfhs_license_num
// ),
`
WITH group_by_query AS (
SELECT
hvfhs_license_num,
COUNT(*)
FROM
taxi.parquet
GROUP BY
hvfhs_license_num
),
// full_query AS (
// SELECT
// *
// FROM
// taxi.parquet
// )
full_query AS (
SELECT
*
FROM
taxi.parquet
)
// SELECT
// COUNT(*)
// FROM
// group_by_query
// LEFT JOIN
// full_query
// ON
// group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
// LIMIT 1
// `,
SELECT
COUNT(*)
FROM
group_by_query
LEFT JOIN
full_query
ON
group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
LIMIT 1
`,
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
// "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
// 'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
"SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',

// `
// WITH group_by_query AS (
// SELECT
// hvfhs_license_num,
// COUNT(*)
// FROM
// taxi.parquet
// GROUP BY
// hvfhs_license_num
// ),
`
WITH group_by_query AS (
SELECT
hvfhs_license_num,
COUNT(*)
FROM
taxi.parquet
GROUP BY
hvfhs_license_num
),
// full_query AS (
// SELECT
// *
// FROM
// taxi.parquet
// )
full_query AS (
SELECT
*
FROM
taxi.parquet
)
// SELECT
// COUNT(*)
// FROM
// group_by_query
// LEFT JOIN
// full_query
// ON
// group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
// LIMIT 1
// `,
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
// "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
// 'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
// `
// WITH group_by_query AS (
// SELECT
// hvfhs_license_num,
// COUNT(*)
// FROM
// taxi.parquet
// GROUP BY
// hvfhs_license_num
// ),
SELECT
COUNT(*)
FROM
group_by_query
LEFT JOIN
full_query
ON
group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
LIMIT 1
`,
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
"SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
`
WITH group_by_query AS (
SELECT
hvfhs_license_num,
COUNT(*)
FROM
taxi.parquet
GROUP BY
hvfhs_license_num
),
// full_query AS (
// SELECT
// *
// FROM
// taxi.parquet
// )
full_query AS (
SELECT
*
FROM
taxi.parquet
)
// SELECT
// COUNT(*)
// FROM
// group_by_query
// LEFT JOIN
// full_query
// ON
// group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
// LIMIT 1
// `,
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
// "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
// 'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet',
// 'SELECT * FROM taxijson.parquet WHERE price >= 1.0005812645 LIMIT 100',
// 'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet GROUP BY order_count',
// 'SELECT * as total_count FROM taxijson.parquet ORDER BY seconds_in_bucket LIMIT 100',
SELECT
COUNT(*)
FROM
group_by_query
LEFT JOIN
full_query
ON
group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
LIMIT 1
`,
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
"SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet',
'SELECT * FROM taxijson.parquet WHERE price >= 1.0005812645 LIMIT 100',
'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet GROUP BY order_count',
'SELECT * as total_count FROM taxijson.parquet ORDER BY seconds_in_bucket LIMIT 100',
];
2 changes: 1 addition & 1 deletion benchmarking/src/app/dbm-context/parallel-indexed-dbm-context.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { useClassicEffect } from '../hooks/use-classic-effect';
import { InstanceManager } from './instance-manager';
import { useAsyncDuckDB } from './use-async-duckdb';

export const ParallelParallelDBMProvider = ({
export const ParallelIndexedDBMProvider = ({
children,
}: {
children: JSX.Element;
Expand Down
5 changes: 2 additions & 3 deletions meerkat-browser-runner/src/app/app.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
FileManagerType,
getMainAppName,
getRunnerAppName,
ParallelIndexedDBFileManager,
IndexedDBFileManager,
WindowCommunication,
} from '@devrev/meerkat-dbm';

Expand Down Expand Up @@ -72,7 +72,7 @@ export function App() {
}

if (!fileManagerRef.current) {
fileManagerRef.current = new ParallelIndexedDBFileManager({
fileManagerRef.current = new IndexedDBFileManager({
instanceManager: instanceManagerRef.current,
fetchTableFileBuffers: async () => [],
logger: log,
Expand All @@ -82,7 +82,6 @@ export function App() {
payload: event,
});
},
communication: communicationRef.current,
});
}

Expand Down
1 change: 0 additions & 1 deletion meerkat-dbm/src/dbm/dbm-parallel/runner-manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,6 @@ export class IFrameRunnerManager {
if (!manager) {
return;
}

manager.communication.sendResponse(message.uuid, result);
}
);
Expand Down
2 changes: 1 addition & 1 deletion meerkat-dbm/src/file-manager/file-manager-type.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ export interface FileManagerType<BufferType = Uint8Array> {
* @param fileName - The name of the file buffer.
* @returns Uint8Array if found.
*/
getFileBuffer: (fileName: string) => Promise<Uint8Array | undefined>;
getFileBuffer: (fileName: string) => Promise<BufferType | undefined>;

/**
* @description
Expand Down
1 change: 1 addition & 0 deletions meerkat-dbm/src/file-manager/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
export * from './file-manager-type';
export * from './indexed-db/indexed-db-file-manager';
export * from './indexed-db/parallel-indexed-db-file-manager';
export * from './indexed-db/runner-indexed-db-file-manager';
export * from './memory/memory-file-manager';
export * from './memory/parallel-memory-file-manager';
export * from './memory/runner-memory-file-manager';
10 changes: 5 additions & 5 deletions meerkat-dbm/src/file-manager/indexed-db/indexed-db-file-manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,12 +267,12 @@ export class IndexedDBFileManager implements FileManagerType {
async getTableData(table: TableConfig): Promise<Table | undefined> {
const tableData = await this.indexedDB.tablesKey.get(table.name);

if (!tableData) return undefined;
if (!tableData) return undefined;

return {
...tableData,
files: getFilesByPartition(tableData?.files ?? [], table.partitions),
};
return {
...tableData,
files: getFilesByPartition(tableData?.files ?? [], table.partitions),
};
}

async setTableMetadata(tableName: string, metadata: object): Promise<void> {
Expand Down
Loading

0 comments on commit 548ad4e

Please sign in to comment.