Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel Execution of DuckDB Queries using iFrame #88

Merged
merged 43 commits into from
Aug 28, 2024
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
f030e31
WIP Parallel exec
itsTalwar Aug 14, 2024
beabf09
Communication Ref
itsTalwar Aug 21, 2024
2455b28
IG manager
itsTalwar Aug 21, 2024
b8839b4
WIP
itsTalwar Aug 21, 2024
5ec3051
WIP: file register
vpbs2 Aug 21, 2024
da424a4
WIP: parallel memory manager
vpbs2 Aug 21, 2024
9cff6ca
WIP
itsTalwar Aug 21, 2024
81e0c31
Get messages working
itsTalwar Aug 21, 2024
5b1fa7e
Little more fix
itsTalwar Aug 21, 2024
a5932c0
It works idk how
itsTalwar Aug 21, 2024
09c797e
fix: duplicate query execution
vpbs2 Aug 22, 2024
926d74b
Late night work
itsTalwar Aug 22, 2024
21de9ff
Merge back
itsTalwar Aug 22, 2024
266aaaa
some: cleanup
vpbs2 Aug 23, 2024
0fc1bbc
Added comment
itsTalwar Aug 23, 2024
d0c3456
Merge branch 'feat/parallel-exec' of github.com:devrev/meerkat into f…
itsTalwar Aug 23, 2024
e42e80f
update: cache logic
vpbs2 Aug 24, 2024
dcd0853
Feat/parallel exec nik (#83)
itsTalwar Aug 25, 2024
e9d9a22
Remove hardcoding of origin (#85)
itsTalwar Aug 25, 2024
4415ba8
test(parallel-dbm): add tests for parallel dbm (#84)
vpbs2 Aug 25, 2024
afd7e7d
SharedArrayBuffer via Single domain (#86)
itsTalwar Aug 26, 2024
b401c99
fix(parallel-dbm): fix target and app id (#87)
vpbs2 Aug 26, 2024
17e80d4
feat(parallel-dbm): convert json data to shared array (#90)
vpbs2 Aug 26, 2024
8ab7274
feat: Clean types and integrate events with runners (#91)
itsTalwar Aug 26, 2024
a168a1c
Reduce the size of iframe and remove unwanted file (#92)
itsTalwar Aug 26, 2024
b56b0ec
handle: file loader buffer type (#94)
vpbs2 Aug 26, 2024
e231514
chore(dbm): cleanup parallel dbm (#95)
vpbs2 Aug 26, 2024
356614b
fix: types
vpbs2 Aug 26, 2024
75c4bc6
fix: type
vpbs2 Aug 26, 2024
a7074f5
fix: test env
vpbs2 Aug 26, 2024
0a70c8b
fix: test fix attempt 1
vpbs2 Aug 26, 2024
bca126d
update: test
vpbs2 Aug 26, 2024
e5e8941
update: package
vpbs2 Aug 26, 2024
f5bd093
fix: build
vpbs2 Aug 26, 2024
7822865
fix: lint
vpbs2 Aug 26, 2024
914149e
Use node 18
itsTalwar Aug 26, 2024
cc9eb92
Do npm ci instead of install
itsTalwar Aug 26, 2024
4dfb093
Fix types
itsTalwar Aug 26, 2024
3ca8b10
fix: address comments
vpbs2 Aug 27, 2024
69c22cb
fix: types
vpbs2 Aug 27, 2024
3e97c31
update: ref
vpbs2 Aug 28, 2024
d100f94
update: gc
vpbs2 Aug 28, 2024
8fdc30e
address comments
vpbs2 Aug 28, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ jobs:
- uses: actions/checkout@v2
with:
lfs: true
- run: npm install
- name: Use Node.js
uses: actions/setup-node@v2
with:
node-version: '18'
- run: npm ci
- run: npx nx run-many --target=build --all --parallel
- run: npx nx run-many --target=test --all --parallel
- run: npx nx run-many --target=lint --all --parallel
66 changes: 35 additions & 31 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
{
"[javascript]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[javascriptreact]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescript]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescriptreact]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[mdx]": {
"editor.wordWrap": "bounded",
"editor.wordWrapColumn": 120
},
"[markdown]": {
"editor.wordWrap": "bounded",
"editor.wordWrapColumn": 120
},
"editor.codeActionsOnSave": {
"source.fixAll.eslint": true,
"source.organizeImports": true
},
"eslint.validate": ["javascript", "javascriptreact", "typescript", "typescriptreact"],
}

"[javascript]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[javascriptreact]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescript]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescriptreact]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[mdx]": {
"editor.wordWrap": "bounded",
"editor.wordWrapColumn": 120
},
"[markdown]": {
"editor.wordWrap": "bounded",
"editor.wordWrapColumn": 120
},
"editor.codeActionsOnSave": {
"source.fixAll.eslint": "explicit",
"source.organizeImports": "explicit"
},
"eslint.validate": [
"javascript",
"javascriptreact",
"typescript",
"typescriptreact"
]
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"extends": ["plugin:@nx/react", "../../.eslintrc.json"],
"extends": ["plugin:@nx/react", "../.eslintrc.json"],
"ignorePatterns": ["!**/*"],
"overrides": [
{
Expand Down
11 changes: 0 additions & 11 deletions benchmarking/benchmarking-app/.babelrc

This file was deleted.

11 changes: 0 additions & 11 deletions benchmarking/benchmarking-app/jest.config.ts

This file was deleted.

98 changes: 0 additions & 98 deletions benchmarking/benchmarking-app/project.json

This file was deleted.

38 changes: 0 additions & 38 deletions benchmarking/benchmarking-app/src/app/constants.ts

This file was deleted.

40 changes: 0 additions & 40 deletions benchmarking/benchmarking-app/src/app/file-loader/file-loader.tsx

This file was deleted.

9 changes: 0 additions & 9 deletions benchmarking/benchmarking-app/webpack.config.js

This file was deleted.

16 changes: 16 additions & 0 deletions benchmarking/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>BenchmarkingApp</title>
<base href="/" />

<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="stylesheet" href="/src/styles.css" />
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
72 changes: 72 additions & 0 deletions benchmarking/project.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
{
"name": "benchmarking-app",
"$schema": "../node_modules/nx/schemas/project-schema.json",
"sourceRoot": "benchmarking/src",
"projectType": "application",
"targets": {
"build": {
"executor": "@nx/vite:build",
"outputs": ["{options.outputPath}"],
"defaultConfiguration": "production",
"options": {
"outputPath": "dist/benchmarking"
},
"configurations": {
"development": {
"mode": "development"
},
"production": {
"mode": "production"
}
}
},
"serve": {
"executor": "@nx/vite:dev-server",
"defaultConfiguration": "development",
"options": {
"buildTarget": "benchmarking-app:build"
},
"configurations": {
"development": {
"buildTarget": "benchmarking-app:build:development",
"hmr": true
},
"production": {
"buildTarget": "benchmarking-app:build:production",
"hmr": false
}
}
},
"preview": {
"executor": "@nx/vite:preview-server",
"defaultConfiguration": "development",
"options": {
"buildTarget": "benchmarking-app:build"
},
"configurations": {
"development": {
"buildTarget": "benchmarking-app:build:development"
},
"production": {
"buildTarget": "benchmarking-app:build:production"
}
}
},
"test": {
"executor": "@nx/vite:test",
"outputs": ["{options.reportsDirectory}"],
"options": {
"passWithNoTests": true,
"reportsDirectory": "../coverage/benchmarking"
}
},
"lint": {
"executor": "@nx/linter:eslint",
"outputs": ["{options.outputFile}"],
"options": {
"lintFilePatterns": ["benchmarking/**/*.{ts,tsx,js,jsx}"]
}
}
},
"tags": []
}
Binary file added benchmarking/public/favicon.ico
Binary file not shown.
Loading
Loading