add cloudflare testing

.github/workflows/test.yml

@@ -129,3 +129,22 @@ jobs:
               env:
                   CONNECTION_TYPE: turso
               run: npm run test:connection
+
+    test_cloudflare:
+        name: 'Cloudflare D1 Connection'
+        runs-on: ubuntu-latest
+        needs: build
+
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Install modules
+              run: npm install
+
+            - name: Run tests
+              env:
+                  CONNECTION_TYPE: cloudflare
+                  CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
+                  CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+                  CLOUDFLARE_DATABASE_ID: ${{ secrets.CLOUDFLARE_DATABASE_ID }}
+              run: npm run test:connection

src/connections/sqlite/base.ts

@@ -1,137 +1,74 @@
-import {
-    Constraint,
-    ConstraintColumn,
-    TableIndex,
-    TableIndexType,
-    TableColumn,
-    Database,
-    Table,
-} from 'src/models/database';
+import { Database, Table } from 'src/models/database';
 import { Connection, SqlConnection } from '..';
-
 export abstract class SqliteBaseConnection extends SqlConnection {
     public async fetchDatabaseSchema(): Promise<Database> {
-        const exclude_tables = [
-            '_cf_kv',
-            'sqlite_schema',
-            'sqlite_temp_schema',
-        ];
-
-        const schemaMap: Record<string, Record<string, Table>> = {};
-
-        const { data } = await this.query({
-            query: `PRAGMA table_list`,
+        const { data: tableList } = await this.query<{
+            type: string;
+            name: string;
+            tbl_name: string;
+        }>({
+            query: `SELECT * FROM sqlite_master WHERE type = 'table' AND (name NOT LIKE 'sqlite_%' OR name NOT LIKE '_cf_%')`,
         });
 
-        const allTables = (
-            data as {
-                schema: string;
-                name: string;
-                type: string;
-            }[]
-        ).filter(
-            (row) =>
-                !row.name.startsWith('_lite') &&
-                !row.name.startsWith('sqlite_') &&
-                !exclude_tables.includes(row.name?.toLowerCase())
-        );
-
-        for (const table of allTables) {
-            if (exclude_tables.includes(table.name?.toLowerCase())) continue;
-
-            const { data: pragmaData } = await this.query({
-                query: `PRAGMA table_info('${table.name}')`,
-            });
-
-            const tableData = pragmaData as {
-                cid: number;
-                name: string;
-                type: string;
-                notnull: 0 | 1;
-                dflt_value: string | null;
-                pk: 0 | 1;
-            }[];
-
-            const { data: fkConstraintResponse } = await this.query({
-                query: `PRAGMA foreign_key_list('${table.name}')`,
-            });
-
-            const fkConstraintData = (
-                fkConstraintResponse as {
-                    id: number;
-                    seq: number;
-                    table: string;
-                    from: string;
-                    to: string;
-                    on_update: 'NO ACTION' | unknown;
-                    on_delete: 'NO ACTION' | unknown;
-                    match: 'NONE' | unknown;
-                }[]
-            ).filter(
-                (row) =>
-                    !row.table.startsWith('_lite') &&
-                    !row.table.startsWith('sqlite_')
-            );
-
-            const constraints: Constraint[] = [];
+        const { data: columnList } = await this.query<{
+            cid: number;
+            name: string;
+            type: string;
+            notnull: 0 | 1;
+            dflt_value: string | null;
+            pk: 0 | 1;
+            tbl_name: string;
+            ref_table_name: string | null;
+            ref_column_name: string | null;
+        }>({
+            query: `WITH master AS (SELECT tbl_name FROM sqlite_master WHERE type = 'table' AND tbl_name NOT LIKE 'sqlite_%' AND tbl_name NOT LIKE '_cf_%')
+SELECT columns.*, fk."table" AS ref_table_name, fk."to" AS ref_column_name 
+FROM
+  (SELECT fields.*, tbl_name FROM master CROSS JOIN pragma_table_info (master.tbl_name) fields) AS columns LEFT JOIN
+  (SELECT fk.*, tbl_name FROM master CROSS JOIN pragma_foreign_key_list (master.tbl_name) fk) AS fk 
+  ON fk."from" = columns.name AND fk.tbl_name = columns.tbl_name;`,
+        });
 
-            if (fkConstraintData.length > 0) {
-                const fkConstraints: Constraint = {
-                    name: 'FOREIGN KEY',
-                    schema: table.schema,
-                    tableName: table.name,
-                    type: 'FOREIGN KEY',
+        const tableLookup = tableList.reduce(
+            (acc, table) => {
+                acc[table.tbl_name] = {
+                    name: table.name,
                     columns: [],
+                    indexes: [],
+                    constraints: [],
                 };
+                return acc;
+            },
+            {} as Record<string, Table>
+        );
 
-                fkConstraintData.forEach((fkConstraint) => {
-                    const currentConstraint: ConstraintColumn = {
-                        columnName: fkConstraint.from,
-                    };
-                    fkConstraints.columns.push(currentConstraint);
-                });
-                constraints.push(fkConstraints);
-            }
-
-            const indexes: TableIndex[] = [];
-            const columns = tableData.map((column) => {
-                // Primary keys are ALWAYS considered indexes
-                if (column.pk === 1) {
-                    indexes.push({
-                        name: column.name,
-                        type: TableIndexType.PRIMARY,
-                        columns: [column.name],
-                    });
-                }
-
-                const currentColumn: TableColumn = {
-                    name: column.name,
-                    type: column.type,
-                    position: column.cid,
-                    nullable: column.notnull === 0,
-                    default: column.dflt_value,
-                    primary: column.pk === 1,
-                    unique: column.pk === 1,
-                    references: [],
-                };
-
-                return currentColumn;
+        for (const column of columnList) {
+            if (!tableLookup[column.tbl_name]) continue;
+
+            tableLookup[column.tbl_name].columns.push({
+                name: column.name,
+                type: column.type,
+                position: column.cid,
+                nullable: column.notnull === 0,
+                default: column.dflt_value,
+                primary: column.pk === 1,
+                unique: false,
+                references:
+                    column.ref_table_name && column.ref_column_name
+                        ? [
+                              {
+                                  table: column.ref_table_name,
+                                  column: column.ref_column_name,
+                              },
+                          ]
+                        : [],
             });
-
-            const currentTable: Table = {
-                name: table.name,
-                columns: columns,
-                indexes: indexes,
-                constraints: constraints,
-            };
-
-            if (!schemaMap[table.schema]) {
-                schemaMap[table.schema] = {};
-            }
-
-            schemaMap[table.schema][table.name] = currentTable;
         }
 
-        return schemaMap;
+        // Sqlite default schema is "main", since we don't support
+        // ATTACH, we don't need to worry about other schemas
+        return {
+            main: tableLookup,
+        };
     }
 }

src/connections/sqlite/cloudflare.ts

@@ -2,6 +2,15 @@ import { QueryType } from '../../query-params';
 import { Query, constructRawQuery } from '../../query';
 import { DefaultDialect } from '../../query-builder/dialects/default';
 import { SqliteBaseConnection } from './base';
+import {
+    Constraint,
+    ConstraintColumn,
+    Database,
+    Table,
+    TableColumn,
+    TableIndex,
+    TableIndexType,
+} from 'src/models/database';
 
 interface SuccessResponse {
     result: Record<string, any>[]; // Array of objects representing results
@@ -146,4 +155,132 @@ export class CloudflareD1Connection extends SqliteBaseConnection {
             query: rawSQL,
         };
     }
+
+    // For some reason, Cloudflare D1 does not support
+    // cross join with pragma_table_info, so we have to
+    // to expensive loops to get the same data
+    public async fetchDatabaseSchema(): Promise<Database> {
+        const exclude_tables = [
+            '_cf_kv',
+            'sqlite_schema',
+            'sqlite_temp_schema',
+        ];
+
+        const schemaMap: Record<string, Record<string, Table>> = {};
+
+        const { data } = await this.query({
+            query: `PRAGMA table_list`,
+        });
+
+        const allTables = (
+            data as {
+                schema: string;
+                name: string;
+                type: string;
+            }[]
+        ).filter(
+            (row) =>
+                !row.name.startsWith('_lite') &&
+                !row.name.startsWith('sqlite_') &&
+                !exclude_tables.includes(row.name?.toLowerCase())
+        );
+
+        for (const table of allTables) {
+            if (exclude_tables.includes(table.name?.toLowerCase())) continue;
+
+            const { data: pragmaData } = await this.query({
+                query: `PRAGMA table_info('${table.name}')`,
+            });
+
+            const tableData = pragmaData as {
+                cid: number;
+                name: string;
+                type: string;
+                notnull: 0 | 1;
+                dflt_value: string | null;
+                pk: 0 | 1;
+            }[];
+
+            const { data: fkConstraintResponse } = await this.query({
+                query: `PRAGMA foreign_key_list('${table.name}')`,
+            });
+
+            const fkConstraintData = (
+                fkConstraintResponse as {
+                    id: number;
+                    seq: number;
+                    table: string;
+                    from: string;
+                    to: string;
+                    on_update: 'NO ACTION' | unknown;
+                    on_delete: 'NO ACTION' | unknown;
+                    match: 'NONE' | unknown;
+                }[]
+            ).filter(
+                (row) =>
+                    !row.table.startsWith('_lite') &&
+                    !row.table.startsWith('sqlite_')
+            );
+
+            const constraints: Constraint[] = [];
+
+            if (fkConstraintData.length > 0) {
+                const fkConstraints: Constraint = {
+                    name: 'FOREIGN KEY',
+                    schema: table.schema,
+                    tableName: table.name,
+                    type: 'FOREIGN KEY',
+                    columns: [],
+                };
+
+                fkConstraintData.forEach((fkConstraint) => {
+                    const currentConstraint: ConstraintColumn = {
+                        columnName: fkConstraint.from,
+                    };
+                    fkConstraints.columns.push(currentConstraint);
+                });
+                constraints.push(fkConstraints);
+            }
+
+            const indexes: TableIndex[] = [];
+            const columns = tableData.map((column) => {
+                // Primary keys are ALWAYS considered indexes
+                if (column.pk === 1) {
+                    indexes.push({
+                        name: column.name,
+                        type: TableIndexType.PRIMARY,
+                        columns: [column.name],
+                    });
+                }
+
+                const currentColumn: TableColumn = {
+                    name: column.name,
+                    type: column.type,
+                    position: column.cid,
+                    nullable: column.notnull === 0,
+                    default: column.dflt_value,
+                    primary: column.pk === 1,
+                    unique: column.pk === 1,
+                    references: [],
+                };
+
+                return currentColumn;
+            });
+
+            const currentTable: Table = {
+                name: table.name,
+                columns: columns,
+                indexes: indexes,
+                constraints: constraints,
+            };
+
+            if (!schemaMap[table.schema]) {
+                schemaMap[table.schema] = {};
+            }
+
+            schemaMap[table.schema][table.name] = currentTable;
+        }
+
+        return schemaMap;
+    }
 }

tests/connections/create-test-connection.ts

@@ -8,6 +8,7 @@ import {
     MySQLConnection,
     BigQueryConnection,
     TursoConnection,
+    CloudflareD1Connection,
 } from '../../src';
 
 export default function createTestClient(): {
@@ -63,6 +64,13 @@ export default function createTestClient(): {
             createTursoConnection({ url: ':memory:' })
         );
         return { client, defaultSchema: 'main' };
+    } else if (process.env.CONNECTION_TYPE === 'cloudflare') {
+        const client = new CloudflareD1Connection({
+            apiKey: process.env.CLOUDFLARE_API_KEY as string,
+            accountId: process.env.CLOUDFLARE_ACCOUNT_ID as string,
+            databaseId: process.env.CLOUDFLARE_DATABASE_ID as string,
+        });
+        return { client, defaultSchema: 'main' };
     }
 
     throw new Error('Invalid connection type');