From 64ff6ee96560d5da328ae67dfb7864b787b4d0f5 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Sun, 17 Nov 2024 08:56:05 +0000 Subject: [PATCH 1/2] fix(airtable): handle more types correctly --- .changeset/gorgeous-sheep-give.md | 7 + packages/airtable/package.json | 3 +- packages/airtable/src/airtable-loader.ts | 4 +- packages/airtable/src/schema.ts | 174 ++++++++++------- packages/airtable/test/fixtures/fields.ts | 169 ++++++++++++++++ packages/airtable/test/hello.test.ts | 7 - packages/airtable/test/schema.test.ts | 226 ++++++++++++++++++++++ pnpm-lock.yaml | 3 + 8 files changed, 513 insertions(+), 80 deletions(-) create mode 100644 .changeset/gorgeous-sheep-give.md create mode 100644 packages/airtable/test/fixtures/fields.ts delete mode 100644 packages/airtable/test/hello.test.ts create mode 100644 packages/airtable/test/schema.test.ts diff --git a/.changeset/gorgeous-sheep-give.md b/.changeset/gorgeous-sheep-give.md new file mode 100644 index 0000000..a2458be --- /dev/null +++ b/.changeset/gorgeous-sheep-give.md @@ -0,0 +1,7 @@ +--- +"@ascorbic/airtable-loader": patch +--- + +Fixes type mapping for various types. + +Rewrites the type mapping to be more robust and handle more types. This should fix issues with some types not being correctly mapped. diff --git a/packages/airtable/package.json b/packages/airtable/package.json index 176fba8..48ef213 100644 --- a/packages/airtable/package.json +++ b/packages/airtable/package.json @@ -41,6 +41,7 @@ "homepage": "https://github.com/ascorbic/astro-loaders", "dependencies": { "@ascorbic/loader-utils": "workspace:^", - "airtable": "^0.12.2" + "airtable": "^0.12.2", + "ts-pattern": "^5.5.0" } } \ No newline at end of file diff --git a/packages/airtable/src/airtable-loader.ts b/packages/airtable/src/airtable-loader.ts index 80d4244..2bd200b 100644 --- a/packages/airtable/src/airtable-loader.ts +++ b/packages/airtable/src/airtable-loader.ts @@ -1,7 +1,7 @@ import type { Loader } from "astro/loaders"; import { AstroError } from "astro/errors"; import Airtable, { type Query, type FieldSet } from "airtable"; -import { zodSchemaFromAirbaseTable } from "./schema.js"; +import { zodSchemaFromAirtableTable } from "./schema.js"; export interface AirtableLoaderOptions { /** The access token. It must have data.records:read access to the base. Defaults to AIRTABLE_TOKEN env var */ @@ -45,7 +45,7 @@ export function airtableLoader({ logger.info(`Loaded ${records.length} records from "${table}"`); }, schema: () => - zodSchemaFromAirbaseTable({ + zodSchemaFromAirtableTable({ baseID: base, tableIdOrName: table, apiKey: token, diff --git a/packages/airtable/src/schema.ts b/packages/airtable/src/schema.ts index 53b5e29..d93d521 100644 --- a/packages/airtable/src/schema.ts +++ b/packages/airtable/src/schema.ts @@ -1,11 +1,14 @@ -import { AstroError } from "astro/errors"; import { z, type ZodTypeAny } from "astro/zod"; +import { match, P } from "ts-pattern"; -interface AirtableField { +export interface AirtableField { name: string; type: string; options?: { - choices: Array<{ name: string }>; + choices?: Array<{ name: string }>; + result?: { + type: string; + }; }; } @@ -19,12 +22,13 @@ interface AirtableResponse { tables: Array; } +// Define sets for different Airtable field types const STRING_TYPES = new Set([ + "string", "singleLineText", "multilineText", "richText", "phoneNumber", - "formula", "barcode", ]); @@ -50,77 +54,108 @@ const USER_TYPES = new Set([ "lastModifiedBy", ]); +const BOOLEAN_TYPES = new Set(["checkbox", "boolean"]); + +// Define schemas for complex field types const userSchema = z.object({ id: z.string(), email: z.string().email(), name: z.string(), }); -const TYPE_MAP = new Map([ - ["email", z.string().email()], - ["url", z.string().url()], - ["duration", z.string().duration()], - ["singleSelect", z.string()], - ["multipleSelects", z.array(z.string())], - ["multipleCollaborators", z.array(userSchema)], - ["multipleRecordLinks", z.array(z.string())], - ["multipleLookupValues", z.array(z.string())], - [ - "multipleAttachments", - z.array( +const attachmentSchema = z.object({ + id: z.string(), + url: z.string().url(), + filename: z.string(), + size: z.number().optional(), + type: z.string().optional(), +}); + +export const airtableTypeToZodType = (field: AirtableField): ZodTypeAny => { + return match(field) + .with({ type: P.when((t) => STRING_TYPES.has(t)) }, () => z.string()) + .with({ type: P.when((t) => NUMBER_TYPES.has(t)) }, () => z.number()) + .with({ type: P.when((t) => DATE_TYPES.has(t)) }, () => z.coerce.date()) + .with({ type: P.when((t) => USER_TYPES.has(t)) }, () => userSchema) + .with({ type: P.when((t) => BOOLEAN_TYPES.has(t)) }, () => z.boolean()) + .with({ type: "email" }, () => z.string().email()) + .with({ type: "url" }, () => z.string().url()) + .with( + { type: "singleSelect", options: { choices: P.array(P.any) } }, + ({ options }) => { + const choices = options.choices.map(({ name }) => name) as [ + string, + ...string[], + ]; + return z.enum(choices); + }, + ) + .with( + { type: "multipleSelects", options: { choices: P.array(P.any) } }, + ({ options }) => { + const choices = options.choices.map(({ name }) => name) as [ + string, + ...string[], + ]; + return z.array(z.enum(choices)); + }, + ) + .with({ type: "multipleAttachments" }, () => z.array(attachmentSchema)) + .with({ type: "multipleCollaborators" }, () => z.array(userSchema)) + .with({ type: "button" }, () => z.object({ - url: z.string().url(), - filename: z.string(), + label: z.string(), + url: z.string().url().optional(), }), - ), - ], - [ - "button", - z.object({ - label: z.string(), - url: z.string().url(), - }), - ], - ["checkbox", z.boolean()], -]); - -const airtableTypeToZodType = ({ - type, - options, -}: AirtableField): ZodTypeAny => { - if (STRING_TYPES.has(type)) { - return z.string(); - } - if (NUMBER_TYPES.has(type)) { - return z.number(); - } - if (DATE_TYPES.has(type)) { - return z.coerce.date(); - } - - if (USER_TYPES.has(type)) { - return userSchema; - } - - if (options?.choices) { - const choices = options.choices.map(({ name }) => name) as [ - string, - ...string[], - ]; - - if (type === "singleSelect") { - return z.enum(choices); - } - if (type === "multipleSelects") { - return z.array(z.enum(choices)); - } - } - - return TYPE_MAP.get(type) ?? z.any(); + ) + .with( + { type: "formula", options: { result: { type: P.string } } }, + ({ options }) => { + const resultType = options.result.type; + console.log("formula result type:", resultType); + if (resultType === "number") { + return z.number(); + } else if (resultType === "string") { + return z.string(); + } + return z.unknown(); + }, + ) + .with( + { + type: "multipleLookupValues", + options: { result: { type: P.when((t) => STRING_TYPES.has(t)) } }, + }, + () => z.array(z.string()), + ) + .with( + { + type: "multipleLookupValues", + options: { result: { type: P.when((t) => NUMBER_TYPES.has(t)) } }, + }, + () => z.array(z.number()), + ) + .with( + { + type: "multipleLookupValues", + options: { result: { type: P.when((t) => BOOLEAN_TYPES.has(t)) } }, + }, + () => z.array(z.boolean()), + ) + .with( + { type: "multipleLookupValues", options: { result: { type: "array" } } }, + () => z.array(z.array(z.unknown())), + ) + .with( + { type: "multipleLookupValues", options: { result: { type: "object" } } }, + () => z.array(z.object({}).passthrough()), + ) + .with({ type: "duration" }, () => z.number()) + .otherwise(() => z.unknown()); }; -// Generate Zod schema using the Base Schema API -export const zodSchemaFromAirbaseTable = async ({ +// Generate Zod schema from Airtable table +export const zodSchemaFromAirtableTable = async ({ baseID, tableIdOrName, apiKey, @@ -137,7 +172,7 @@ export const zodSchemaFromAirbaseTable = async ({ }); if (!res.ok) { - throw new AstroError(`Failed to fetch Airtable schema: ${res.statusText}`); + throw new Error(`Failed to fetch Airtable schema: ${res.statusText}`); } const response = (await res.json()) as AirtableResponse; @@ -147,16 +182,15 @@ export const zodSchemaFromAirbaseTable = async ({ ); if (!tableSchema) { - throw new AstroError(`Table ${tableIdOrName} not found in base schema.`); + throw new Error(`Table ${tableIdOrName} not found in base schema.`); } - const schemaObject: Record = {}; + const schemaObject: Record = {}; for (const field of tableSchema.fields) { const zodType = airtableTypeToZodType(field).optional(); schemaObject[field.name] = zodType; } - const zodSchema = z.object(schemaObject); - return zodSchema; + return z.object(schemaObject); }; diff --git a/packages/airtable/test/fixtures/fields.ts b/packages/airtable/test/fixtures/fields.ts new file mode 100644 index 0000000..ea2b231 --- /dev/null +++ b/packages/airtable/test/fixtures/fields.ts @@ -0,0 +1,169 @@ +import { AirtableField } from "../../src/schema.js"; + +export const airtableFieldFixtures: Record = { + singleLineText: { + name: "Name", + type: "singleLineText", + }, + number: { + name: "Age", + type: "number", + }, + percent: { + name: "Completion", + type: "percent", + }, + currency: { + name: "Price", + type: "currency", + }, + date: { + name: "Birthdate", + type: "date", + }, + dateTime: { + name: "Appointment", + type: "dateTime", + }, + checkbox: { + name: "IsActive", + type: "checkbox", + }, + singleSelect: { + name: "Status", + type: "singleSelect", + options: { + choices: [{ name: "Active" }, { name: "Inactive" }], + }, + }, + multipleSelects: { + name: "Tags", + type: "multipleSelects", + options: { + choices: [{ name: "Tag1" }, { name: "Tag2" }], + }, + }, + singleCollaborator: { + name: "Owner", + type: "singleCollaborator", + }, + multipleCollaborators: { + name: "Editors", + type: "multipleCollaborators", + }, + multipleAttachments: { + name: "Files", + type: "multipleAttachments", + }, + button: { + name: "Action", + type: "button", + }, + formulaString: { + name: "CalculatedField", + type: "formula", + options: { + result: { + type: "string", + }, + }, + }, + formulaNumber: { + name: "CalculatedNumber", + type: "formula", + options: { + result: { + type: "number", + }, + }, + }, + rollup: { + name: "AggregatedData", + type: "rollup", + }, + lookup: { + name: "RelatedInfo", + type: "lookup", + }, + createdTime: { + name: "CreatedAt", + type: "createdTime", + }, + lastModifiedTime: { + name: "UpdatedAt", + type: "lastModifiedTime", + }, + autoNumber: { + name: "ID", + type: "autoNumber", + }, + rating: { + name: "Rating", + type: "rating", + }, + duration: { + name: "Duration", + type: "duration", + }, + email: { + name: "Email", + type: "email", + }, + url: { + name: "Website", + type: "url", + }, + phoneNumber: { + name: "Phone", + type: "phoneNumber", + }, + barcode: { + name: "Barcode", + type: "barcode", + }, + multipleLookupValuesString: { + name: "Related Names", + type: "multipleLookupValues", + options: { + result: { + type: "string", + }, + }, + }, + multipleLookupValuesNumber: { + name: "Related Scores", + type: "multipleLookupValues", + options: { + result: { + type: "number", + }, + }, + }, + multipleLookupValuesBoolean: { + name: "Related Statuses", + type: "multipleLookupValues", + options: { + result: { + type: "checkbox", + }, + }, + }, + multipleLookupValuesArray: { + name: "Nested Tags", + type: "multipleLookupValues", + options: { + result: { + type: "array", + }, + }, + }, + multipleLookupValuesObject: { + name: "Related Records", + type: "multipleLookupValues", + options: { + result: { + type: "object", + }, + }, + }, +}; diff --git a/packages/airtable/test/hello.test.ts b/packages/airtable/test/hello.test.ts deleted file mode 100644 index 634736b..0000000 --- a/packages/airtable/test/hello.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, it, expect } from "vitest"; - -describe("hello", () => { - it("should say hello", () => { - expect("hello").toBe("hello"); - }); -}) diff --git a/packages/airtable/test/schema.test.ts b/packages/airtable/test/schema.test.ts new file mode 100644 index 0000000..30eadbe --- /dev/null +++ b/packages/airtable/test/schema.test.ts @@ -0,0 +1,226 @@ +import { describe, it, expect } from "vitest"; +import { z } from "astro/zod"; +import { airtableTypeToZodType } from "../src/schema.js"; +import { airtableFieldFixtures } from "./fixtures/fields.js"; + +describe("Airtable to Zod Schema Mapping", () => { + it("should map singleLineText to z.string()", () => { + const field = airtableFieldFixtures.singleLineText; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + }); + + it("should map number to z.number()", () => { + const field = airtableFieldFixtures.number; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map percent to z.number()", () => { + const field = airtableFieldFixtures.percent; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map currency to z.number()", () => { + const field = airtableFieldFixtures.currency; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map date to z.date()", () => { + const field = airtableFieldFixtures.date; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodDate); + }); + + it("should map dateTime to z.date()", () => { + const field = airtableFieldFixtures.dateTime; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodDate); + }); + + it("should map checkbox to z.boolean()", () => { + const field = airtableFieldFixtures.checkbox; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodBoolean); + }); + + it("should map singleSelect to z.enum()", () => { + const field = airtableFieldFixtures.singleSelect; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodEnum); + }); + + it("should map multipleSelects to z.array(z.enum())", () => { + const field = airtableFieldFixtures.multipleSelects; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodEnum); + }); + + it("should map singleCollaborator to userSchema", () => { + const field = airtableFieldFixtures.singleCollaborator; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodObject); + expect((schema as any).shape).toHaveProperty("id"); + expect((schema as any).shape).toHaveProperty("email"); + expect((schema as any).shape).toHaveProperty("name"); + }); + + it("should map multipleCollaborators to z.array(userSchema)", () => { + const field = airtableFieldFixtures.multipleCollaborators; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodObject); + expect(innerType.shape).toHaveProperty("id"); + expect(innerType.shape).toHaveProperty("email"); + expect(innerType.shape).toHaveProperty("name"); + }); + + it("should map multipleAttachments to z.array(attachmentSchema)", () => { + const field = airtableFieldFixtures.multipleAttachments; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodObject); + expect(innerType.shape).toHaveProperty("id"); + expect(innerType.shape).toHaveProperty("url"); + expect(innerType.shape).toHaveProperty("filename"); + }); + + it("should map button to z.object({ label: z.string(), url: z.string().optional() })", () => { + const field = airtableFieldFixtures.button; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodObject); + expect((schema as any).shape).toHaveProperty("label"); + expect((schema as any).shape).toHaveProperty("url"); + }); + + it("should map formula with result type string to z.string()", () => { + const field = airtableFieldFixtures.formulaString; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + }); + + it("should map formula with result type number to z.number()", () => { + const field = airtableFieldFixtures.formulaNumber; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map rollup to z.unknown()", () => { + const field = airtableFieldFixtures.rollup; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodUnknown); + }); + + it("should map lookup to z.unknown()", () => { + const field = airtableFieldFixtures.lookup; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodUnknown); + }); + + it("should map createdTime to z.date()", () => { + const field = airtableFieldFixtures.createdTime; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodDate); + }); + + it("should map lastModifiedTime to z.date()", () => { + const field = airtableFieldFixtures.lastModifiedTime; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodDate); + }); + + it("should map autoNumber to z.number()", () => { + const field = airtableFieldFixtures.autoNumber; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map rating to z.number()", () => { + const field = airtableFieldFixtures.rating; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map duration to z.number()", () => { + const field = airtableFieldFixtures.duration; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodNumber); + }); + + it("should map email to z.string().email()", () => { + const field = airtableFieldFixtures.email; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + expect(() => schema.parse("not-an-email")).toThrow(); + expect(() => schema.parse("test@example.com")).not.toThrow(); + }); + + it("should map url to z.string().url()", () => { + const field = airtableFieldFixtures.url; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + expect(() => schema.parse("not-a-url")).toThrow(); + expect(() => schema.parse("https://example.com")).not.toThrow(); + }); + + it("should map phoneNumber to z.string()", () => { + const field = airtableFieldFixtures.phoneNumber; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + }); + + it("should map barcode to z.string()", () => { + const field = airtableFieldFixtures.barcode; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodString); + }); + + it("should map multipleLookupValues with string result to z.array(z.string())", () => { + const field = airtableFieldFixtures.multipleLookupValuesString; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodString); + }); + + it("should map multipleLookupValues with number result to z.array(z.number())", () => { + const field = airtableFieldFixtures.multipleLookupValuesNumber; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodNumber); + }); + + it("should map multipleLookupValues with boolean result to z.array(z.boolean())", () => { + const field = airtableFieldFixtures.multipleLookupValuesBoolean; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodBoolean); + }); + + it("should map multipleLookupValues with array result to z.array(z.array(z.any()))", () => { + const field = airtableFieldFixtures.multipleLookupValuesArray; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodArray); + const nestedType = (innerType as z.ZodArray).element; + expect(nestedType).toBeInstanceOf(z.ZodUnknown); + }); + + it("should map multipleLookupValues with object result to z.array(z.object({}).passthrough())", () => { + const field = airtableFieldFixtures.multipleLookupValuesObject; + const schema = airtableTypeToZodType(field); + expect(schema).toBeInstanceOf(z.ZodArray); + const innerType = (schema as z.ZodArray).element; + expect(innerType).toBeInstanceOf(z.ZodObject); + expect((innerType as z.ZodObject).shape).toEqual({}); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2953ab..1922f5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,6 +74,9 @@ importers: airtable: specifier: ^0.12.2 version: 0.12.2 + ts-pattern: + specifier: ^5.5.0 + version: 5.5.0 devDependencies: '@arethetypeswrong/cli': specifier: ^0.16.4 From 89420538b186897afdf0b8a0ae8df701b25f87d9 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Sun, 17 Nov 2024 10:37:08 +0000 Subject: [PATCH 2/2] Remove log --- packages/airtable/src/schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/airtable/src/schema.ts b/packages/airtable/src/schema.ts index d93d521..35621f2 100644 --- a/packages/airtable/src/schema.ts +++ b/packages/airtable/src/schema.ts @@ -112,7 +112,6 @@ export const airtableTypeToZodType = (field: AirtableField): ZodTypeAny => { { type: "formula", options: { result: { type: P.string } } }, ({ options }) => { const resultType = options.result.type; - console.log("formula result type:", resultType); if (resultType === "number") { return z.number(); } else if (resultType === "string") {