From 81492c433c3bc11be1729d0a9cf4e64fc867969b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Jan 2026 04:20:35 +0000 Subject: [PATCH 1/3] feat: convert project to CLI-based static site generator - Add CLI tool (`data-explorer build`) using Vite's API - Support --title, --description, --output, --base-path flags - Move models to example/ folder with new Hugging Face examples - Add GitHub Events, IMDB Movies, NYC Taxi datasets - Create landing page in docs/ - Add model validation script (npm run test:models) - Add CLI smoke tests (npm run test:cli) - Update vite.config.ts to support dynamic model paths - Add site-config.ts for runtime title/description config --- .gitignore | 1 + docs/index.html | 441 ++++++++++++++++++ {models => example/models}/.gitkeep | 0 .../models}/E-commerce Orders.malloynb | 0 example/models/GitHub Events.malloynb | 41 ++ example/models/IMDB Movies.malloynb | 49 ++ {models => example/models}/Invoices.malloynb | 0 .../models}/Kids Screen Time.malloynb | 0 example/models/NYC Taxi Trips.malloynb | 57 +++ .../models}/Sales Orders.malloynb | 0 .../models}/Sample Data.malloynb | 0 .../models}/SuperStore.malloynb | 0 .../models}/Users and Products.malloynb | 0 .../models}/business_overview.malloy | 0 {models => example/models}/data/.gitkeep | 0 .../models}/data/Indian_Kids_Screen_Time.csv | 0 {models => example/models}/data/contracts.csv | 0 .../models}/data/invoices.parquet | Bin {models => example/models}/data/orders.csv | 0 .../models}/data/products.jsonl | 0 .../models}/data/sales_orders.xlsx | Bin {models => example/models}/data/users.json | 0 .../models}/ecommerce_orders.malloy | 0 example/models/github_events.malloy | 104 +++++ example/models/imdb_movies.malloy | 140 ++++++ {models => example/models}/invoices.malloy | 0 .../models}/kids_screen_time.malloy | 0 example/models/nyc_taxi.malloy | 164 +++++++ .../models}/sales_orders.malloy | 0 {models => example/models}/sample_data.malloy | 0 {models => example/models}/superstore.malloy | 0 .../models}/users_products.malloy | 0 package-lock.json | 214 ++++++--- package.json | 10 +- scripts/validate-models.ts | 219 +++++++++ src/Home.tsx | 8 +- src/cli/build.ts | 58 +++ src/cli/index.ts | 124 +++++ src/cli/types.ts | 21 + src/index.tsx | 4 + src/site-config.ts | 15 + tests/cli/cli.test.ts | 116 +++++ tsconfig.cli.json | 22 + tsconfig.json | 3 +- vite.config.ts | 48 +- vitest.cli.config.ts | 8 + 46 files changed, 1806 insertions(+), 61 deletions(-) create mode 100644 docs/index.html rename {models => example/models}/.gitkeep (100%) rename {models => example/models}/E-commerce Orders.malloynb (100%) create mode 100644 example/models/GitHub Events.malloynb create mode 100644 example/models/IMDB Movies.malloynb rename {models => example/models}/Invoices.malloynb (100%) rename {models => example/models}/Kids Screen Time.malloynb (100%) create mode 100644 example/models/NYC Taxi Trips.malloynb rename {models => example/models}/Sales Orders.malloynb (100%) rename {models => example/models}/Sample Data.malloynb (100%) rename {models => example/models}/SuperStore.malloynb (100%) rename {models => example/models}/Users and Products.malloynb (100%) rename {models => example/models}/business_overview.malloy (100%) rename {models => example/models}/data/.gitkeep (100%) rename {models => example/models}/data/Indian_Kids_Screen_Time.csv (100%) rename {models => example/models}/data/contracts.csv (100%) rename {models => example/models}/data/invoices.parquet (100%) rename {models => example/models}/data/orders.csv (100%) rename {models => example/models}/data/products.jsonl (100%) rename {models => example/models}/data/sales_orders.xlsx (100%) rename {models => example/models}/data/users.json (100%) rename {models => example/models}/ecommerce_orders.malloy (100%) create mode 100644 example/models/github_events.malloy create mode 100644 example/models/imdb_movies.malloy rename {models => example/models}/invoices.malloy (100%) rename {models => example/models}/kids_screen_time.malloy (100%) create mode 100644 example/models/nyc_taxi.malloy rename {models => example/models}/sales_orders.malloy (100%) rename {models => example/models}/sample_data.malloy (100%) rename {models => example/models}/superstore.malloy (100%) rename {models => example/models}/users_products.malloy (100%) create mode 100644 scripts/validate-models.ts create mode 100644 src/cli/build.ts create mode 100644 src/cli/index.ts create mode 100644 src/cli/types.ts create mode 100644 src/site-config.ts create mode 100644 tests/cli/cli.test.ts create mode 100644 tsconfig.cli.json create mode 100644 vitest.cli.config.ts diff --git a/.gitignore b/.gitignore index 8208276..1b9eb04 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ dist-ssr /playwright-report/ /blob-report/ /playwright/.cache/ +dist-cli/ diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..9499bef --- /dev/null +++ b/docs/index.html @@ -0,0 +1,441 @@ + + + + + + Data Explorer - Static Site Generator for Malloy + + + +
+
+ +
+
+ +
+
+
+

Data Explorer

+

A CLI tool to generate beautiful static websites for your Malloy data models and notebooks. Explore, query, and share your data with ease.

+
+ + View Examples + + + GitHub + +
+
+ npx @aszenz/data-explorer build ./models -o ./dist +
+
+
+ +
+
+

Features

+
+
+

Interactive Schema Explorer

+

Browse your Malloy models with an intuitive schema viewer. See sources, fields, and relationships at a glance.

+
+
+

Query Builder

+

Build and execute queries visually using the Malloy Explorer integration. No code required.

+
+
+

Notebook Support

+

Render Malloy notebooks (.malloynb) with executed results, markdown, and syntax-highlighted code.

+
+
+

DuckDB Powered

+

Runs entirely in the browser using DuckDB WASM. Load CSV, Parquet, JSON, Excel and more.

+
+
+

Hugging Face Datasets

+

Reference public datasets directly from Hugging Face using the hf:// protocol in DuckDB.

+
+
+

Static & Portable

+

Generate static HTML/JS that can be hosted anywhere - GitHub Pages, Netlify, S3, or locally.

+
+
+
+
+ +
+
+

Example Sites

+

See what you can build with Data Explorer

+
+ +

E-commerce Analytics

+

Orders, products, and customer analysis with multi-table joins and visualizations.

+ Local CSV + + +

GitHub Activity

+

Analyze GitHub events data from Hugging Face's public dataset mirror.

+ Hugging Face + + +

NYC Taxi Trips

+

Explore millions of taxi rides with geographic and temporal analysis.

+ Hugging Face + + +

IMDB Movies

+

Movie ratings, genres, and cast analysis from the IMDB dataset.

+ Hugging Face + +
+
+
+ +
+
+

Quick Start

+
+ 
# Build a site from your Malloy models
+npx @aszenz/data-explorer build ./models -o ./dist
+
+# Customize the site title and description
+npx @aszenz/data-explorer build ./models \
+  --title "Sales Analytics" \
+  --description "Explore our sales data" \
+  --output ./dist
+
+# Set base path for GitHub Pages deployment
+npx @aszenz/data-explorer build ./models \
+  --base-path "/my-repo/" \
+  -o ./dist
+
+# View all options
+npx @aszenz/data-explorer --help
+
+
+
+
+ + + + diff --git a/models/.gitkeep b/example/models/.gitkeep similarity index 100% rename from models/.gitkeep rename to example/models/.gitkeep diff --git a/models/E-commerce Orders.malloynb b/example/models/E-commerce Orders.malloynb similarity index 100% rename from models/E-commerce Orders.malloynb rename to example/models/E-commerce Orders.malloynb diff --git a/example/models/GitHub Events.malloynb b/example/models/GitHub Events.malloynb new file mode 100644 index 0000000..a7d437f --- /dev/null +++ b/example/models/GitHub Events.malloynb @@ -0,0 +1,41 @@ +>>>markdown +# GitHub Events Analysis + +Explore GitHub activity data from the Hugging Face datasets. This model analyzes push events, pull requests, issues, and other GitHub activities. + +**Data Source:** [alvarobartt/github-events](https://huggingface.co/datasets/alvarobartt/github-events) on Hugging Face + +>>>malloy +import "github_events.malloy" + +>>>markdown +## Overview Dashboard + +Let's start with an overview of all GitHub events in the dataset: + +>>>malloy +run: github_events -> overview + +>>>markdown +## Event Type Distribution + +Breaking down events by their type: + +>>>malloy +run: github_events -> by_event_type + +>>>markdown +## Top Repositories + +The most active repositories in the dataset: + +>>>malloy +run: github_events -> by_repo + +>>>markdown +## Top Contributors + +Who are the most active contributors? + +>>>malloy +run: github_events -> top_contributors diff --git a/example/models/IMDB Movies.malloynb b/example/models/IMDB Movies.malloynb new file mode 100644 index 0000000..cf553b0 --- /dev/null +++ b/example/models/IMDB Movies.malloynb @@ -0,0 +1,49 @@ +>>>markdown +# IMDB Movies Analysis + +Explore movie ratings, genres, and trends from the IMDB dataset. This data is sourced from Hugging Face and includes movie titles, ratings, vote counts, and more. + +**Data Source:** [Pablinho/imdb-data](https://huggingface.co/datasets/Pablinho/imdb-data) on Hugging Face + +>>>malloy +import "imdb_movies.malloy" + +>>>markdown +## Overview Dashboard + +A comprehensive look at the movie dataset: + +>>>malloy +run: movies -> overview + +>>>markdown +## Top Rated Movies + +The highest-rated movies with significant vote counts (>10,000 votes): + +>>>malloy +run: movies -> top_rated + +>>>markdown +## Most Popular Movies + +Movies sorted by number of votes: + +>>>malloy +run: movies -> most_popular + +>>>markdown +## Genre Analysis + +Deep dive into movie genres with rating trends over time: + +>>>malloy +run: movies -> genre_analysis + +>>>markdown +## Movies by Decade + +How has movie production and quality changed over the decades? + +>>>malloy +run: movies -> by_decade diff --git a/models/Invoices.malloynb b/example/models/Invoices.malloynb similarity index 100% rename from models/Invoices.malloynb rename to example/models/Invoices.malloynb diff --git a/models/Kids Screen Time.malloynb b/example/models/Kids Screen Time.malloynb similarity index 100% rename from models/Kids Screen Time.malloynb rename to example/models/Kids Screen Time.malloynb diff --git a/example/models/NYC Taxi Trips.malloynb b/example/models/NYC Taxi Trips.malloynb new file mode 100644 index 0000000..016c313 --- /dev/null +++ b/example/models/NYC Taxi Trips.malloynb @@ -0,0 +1,57 @@ +>>>markdown +# NYC Taxi Trips Analysis + +Analyze New York City yellow taxi trip data. This dataset contains trip records including pickup/dropoff times, distances, fares, tips, and payment methods. + +**Data Source:** [codelion/nyctaxi](https://huggingface.co/datasets/codelion/nyctaxi) on Hugging Face + +>>>malloy +import "nyc_taxi.malloy" + +>>>markdown +## Overview Dashboard + +A comprehensive look at taxi trip patterns: + +>>>malloy +run: taxi_trips -> overview + +>>>markdown +## Hourly Patterns + +When do New Yorkers take taxi rides? Let's look at trips by hour of day: + +>>>malloy +run: taxi_trips -> by_hour + +>>>markdown +## Daily Patterns + +Trip distribution across days of the week: + +>>>malloy +run: taxi_trips -> by_day + +>>>markdown +## Payment Analysis + +How do people pay for their rides, and how does tipping vary by payment method? + +>>>malloy +run: taxi_trips -> by_payment + +>>>markdown +## Fare Analysis by Hour + +Detailed breakdown of fares throughout the day: + +>>>malloy +run: taxi_trips -> fare_analysis + +>>>markdown +## Long Distance Trips + +The longest trips in the dataset: + +>>>malloy +run: taxi_trips -> long_trips diff --git a/models/Sales Orders.malloynb b/example/models/Sales Orders.malloynb similarity index 100% rename from models/Sales Orders.malloynb rename to example/models/Sales Orders.malloynb diff --git a/models/Sample Data.malloynb b/example/models/Sample Data.malloynb similarity index 100% rename from models/Sample Data.malloynb rename to example/models/Sample Data.malloynb diff --git a/models/SuperStore.malloynb b/example/models/SuperStore.malloynb similarity index 100% rename from models/SuperStore.malloynb rename to example/models/SuperStore.malloynb diff --git a/models/Users and Products.malloynb b/example/models/Users and Products.malloynb similarity index 100% rename from models/Users and Products.malloynb rename to example/models/Users and Products.malloynb diff --git a/models/business_overview.malloy b/example/models/business_overview.malloy similarity index 100% rename from models/business_overview.malloy rename to example/models/business_overview.malloy diff --git a/models/data/.gitkeep b/example/models/data/.gitkeep similarity index 100% rename from models/data/.gitkeep rename to example/models/data/.gitkeep diff --git a/models/data/Indian_Kids_Screen_Time.csv b/example/models/data/Indian_Kids_Screen_Time.csv similarity index 100% rename from models/data/Indian_Kids_Screen_Time.csv rename to example/models/data/Indian_Kids_Screen_Time.csv diff --git a/models/data/contracts.csv b/example/models/data/contracts.csv similarity index 100% rename from models/data/contracts.csv rename to example/models/data/contracts.csv diff --git a/models/data/invoices.parquet b/example/models/data/invoices.parquet similarity index 100% rename from models/data/invoices.parquet rename to example/models/data/invoices.parquet diff --git a/models/data/orders.csv b/example/models/data/orders.csv similarity index 100% rename from models/data/orders.csv rename to example/models/data/orders.csv diff --git a/models/data/products.jsonl b/example/models/data/products.jsonl similarity index 100% rename from models/data/products.jsonl rename to example/models/data/products.jsonl diff --git a/models/data/sales_orders.xlsx b/example/models/data/sales_orders.xlsx similarity index 100% rename from models/data/sales_orders.xlsx rename to example/models/data/sales_orders.xlsx diff --git a/models/data/users.json b/example/models/data/users.json similarity index 100% rename from models/data/users.json rename to example/models/data/users.json diff --git a/models/ecommerce_orders.malloy b/example/models/ecommerce_orders.malloy similarity index 100% rename from models/ecommerce_orders.malloy rename to example/models/ecommerce_orders.malloy diff --git a/example/models/github_events.malloy b/example/models/github_events.malloy new file mode 100644 index 0000000..a4bfd7b --- /dev/null +++ b/example/models/github_events.malloy @@ -0,0 +1,104 @@ +-- GitHub Events Analysis Model +-- Analyzes GitHub activity data from Hugging Face datasets +-- Uses: hf://datasets/alvarobartt/github-events/data/*.parquet + +source: github_events is duckdb.table('hf://datasets/alvarobartt/github-events/data/train-00000-of-00001.parquet') extend { + -- Core measures + measure: event_count is count() + measure: unique_repos is count(distinct repo.name) + measure: unique_actors is count(distinct actor.login) + + -- Event type measures + measure: push_events is count() { where: `type` = 'PushEvent' } + measure: pr_events is count() { where: `type` = 'PullRequestEvent' } + measure: issue_events is count() { where: `type` = 'IssuesEvent' } + measure: watch_events is count() { where: `type` = 'WatchEvent' } + measure: fork_events is count() { where: `type` = 'ForkEvent' } + measure: create_events is count() { where: `type` = 'CreateEvent' } + + -- Dimensions + dimension: event_type is `type` + dimension: repo_name is repo.name + dimension: actor_login is actor.login + dimension: event_date is created_at::date + + -- Views + view: by_event_type is { + group_by: event_type + aggregate: + event_count + unique_repos + unique_actors + order_by: event_count desc + } + + view: by_repo is { + group_by: repo_name + aggregate: + event_count + unique_actors + order_by: event_count desc + limit: 20 + } + + view: by_actor is { + group_by: actor_login + aggregate: + event_count + unique_repos + order_by: event_count desc + limit: 20 + } + + view: activity_timeline is { + group_by: event_date + aggregate: + event_count + unique_repos + unique_actors + order_by: event_date + } + + # dashboard + view: overview is { + aggregate: + event_count + unique_repos + unique_actors + push_events + pr_events + issue_events + watch_events + fork_events + # bar_chart + nest: by_event_type + # bar_chart + nest: top_repos is by_repo + # line_chart + nest: activity_timeline + } + + -- Top contributors view + view: top_contributors is { + group_by: actor_login + aggregate: + event_count + push_events + pr_events + issue_events + # bar_chart + nest: activity_by_type is { + group_by: event_type + aggregate: event_count + } + order_by: event_count desc + limit: 15 + } +} + +-- Named queries +query: github_overview is github_events -> overview +query: event_breakdown is github_events -> by_event_type +query: top_repos is github_events -> by_repo +query: contributors is github_events -> top_contributors +query: timeline is github_events -> activity_timeline diff --git a/example/models/imdb_movies.malloy b/example/models/imdb_movies.malloy new file mode 100644 index 0000000..ef61af9 --- /dev/null +++ b/example/models/imdb_movies.malloy @@ -0,0 +1,140 @@ +-- IMDB Movies Analysis Model +-- Analyzes movie data from the IMDB dataset on Hugging Face +-- Uses: hf://datasets/Pablinho/imdb-data/data/*.parquet + +source: movies is duckdb.table('hf://datasets/Pablinho/imdb-data/data/train-00000-of-00001.parquet') extend { + -- Core measures + measure: movie_count is count() + measure: avg_rating is avg(score) + measure: total_votes is sum(votes) + measure: avg_votes is avg(votes) + measure: avg_runtime is avg(runtime) + + -- Rating distribution + measure: highly_rated is count() { where: score >= 8.0 } + measure: low_rated is count() { where: score < 5.0 } + + -- Dimensions + dimension: decade is floor(year / 10) * 10 + dimension: rating_tier is pick + 'Excellent (8+)' when score >= 8.0 + 'Good (7-8)' when score >= 7.0 + 'Average (5-7)' when score >= 5.0 + else 'Poor (<5)' + + -- Basic views + view: all_movies is { + select: + title + year + score + votes + runtime + genre + order_by: score desc + limit: 100 + } + + view: by_year is { + group_by: year + aggregate: + movie_count + avg_rating + total_votes + order_by: year + } + + view: by_decade is { + group_by: decade + aggregate: + movie_count + avg_rating + total_votes + highly_rated + order_by: decade + } + + view: by_genre is { + group_by: genre + aggregate: + movie_count + avg_rating + avg_votes + order_by: movie_count desc + } + + view: by_rating_tier is { + group_by: rating_tier + aggregate: + movie_count + avg_votes + order_by: movie_count desc + } + + # dashboard + view: overview is { + aggregate: + movie_count + avg_rating + total_votes + highly_rated + low_rated + # bar_chart + nest: by_decade + # bar_chart + nest: by_genre + # bar_chart + nest: by_rating_tier + } + + -- Top rated movies view + view: top_rated is { + select: + title + year + score + votes + genre + runtime + where: votes > 10000 + order_by: score desc + limit: 50 + } + + -- Most voted movies + view: most_popular is { + select: + title + year + score + votes + genre + order_by: votes desc + limit: 50 + } + + -- Genre analysis with nested details + view: genre_analysis is { + group_by: genre + aggregate: + movie_count + avg_rating + total_votes + highly_rated + # line_chart + nest: rating_over_time is { + group_by: decade + aggregate: avg_rating + order_by: decade + } + order_by: movie_count desc + } +} + +-- Named queries +query: movies_overview is movies -> overview +query: top_movies is movies -> top_rated +query: popular_movies is movies -> most_popular +query: movies_by_year is movies -> by_year +query: movies_by_genre is movies -> genre_analysis +query: all_movies_list is movies -> all_movies diff --git a/models/invoices.malloy b/example/models/invoices.malloy similarity index 100% rename from models/invoices.malloy rename to example/models/invoices.malloy diff --git a/models/kids_screen_time.malloy b/example/models/kids_screen_time.malloy similarity index 100% rename from models/kids_screen_time.malloy rename to example/models/kids_screen_time.malloy diff --git a/example/models/nyc_taxi.malloy b/example/models/nyc_taxi.malloy new file mode 100644 index 0000000..e3b3299 --- /dev/null +++ b/example/models/nyc_taxi.malloy @@ -0,0 +1,164 @@ +-- NYC Taxi Trips Analysis Model +-- Analyzes New York City taxi trip data +-- Uses: hf://datasets/codelion/nyctaxi/yellow_tripdata_2023-01.parquet + +source: taxi_trips is duckdb.table('hf://datasets/codelion/nyctaxi/yellow_tripdata_2023-01.parquet') extend { + -- Core measures + measure: trip_count is count() + measure: total_fare is sum(fare_amount) + measure: total_tips is sum(tip_amount) + measure: total_distance is sum(trip_distance) + measure: avg_fare is avg(fare_amount) + measure: avg_tip is avg(tip_amount) + measure: avg_distance is avg(trip_distance) + measure: avg_passengers is avg(passenger_count) + + -- Trip duration (in minutes) + dimension: trip_duration_mins is + (tpep_dropoff_datetime - tpep_pickup_datetime)::bigint / 60000000 + + measure: avg_duration_mins is avg(trip_duration_mins) + + -- Tip percentage + dimension: tip_percentage is + case when fare_amount > 0 then (tip_amount / fare_amount) * 100 else 0 end + measure: avg_tip_percentage is avg(tip_percentage) + + -- Time dimensions + dimension: pickup_hour is hour(tpep_pickup_datetime) + dimension: pickup_day is dayofweek(tpep_pickup_datetime) + dimension: pickup_date is tpep_pickup_datetime::date + + -- Day name + dimension: day_name is pick + 'Sunday' when pickup_day = 0 + 'Monday' when pickup_day = 1 + 'Tuesday' when pickup_day = 2 + 'Wednesday' when pickup_day = 3 + 'Thursday' when pickup_day = 4 + 'Friday' when pickup_day = 5 + 'Saturday' when pickup_day = 6 + else 'Unknown' + + -- Payment type dimension + dimension: payment_type_name is pick + 'Credit Card' when payment_type = 1 + 'Cash' when payment_type = 2 + 'No Charge' when payment_type = 3 + 'Dispute' when payment_type = 4 + 'Unknown' when payment_type = 5 + 'Voided' when payment_type = 6 + else 'Other' + + -- Views + view: by_hour is { + group_by: pickup_hour + aggregate: + trip_count + avg_fare + avg_distance + avg_tip_percentage + order_by: pickup_hour + } + + view: by_day is { + group_by: day_name + aggregate: + trip_count + total_fare + avg_fare + avg_tip + order_by: trip_count desc + } + + view: by_payment is { + group_by: payment_type_name + aggregate: + trip_count + total_fare + avg_tip + avg_tip_percentage + order_by: trip_count desc + } + + view: by_passenger_count is { + group_by: passenger_count + aggregate: + trip_count + avg_fare + avg_distance + order_by: passenger_count + } + + view: daily_trends is { + group_by: pickup_date + aggregate: + trip_count + total_fare + avg_fare + avg_distance + order_by: pickup_date + } + + # dashboard + view: overview is { + aggregate: + trip_count + total_fare + total_tips + total_distance + avg_fare + avg_tip + avg_distance + avg_duration_mins + avg_tip_percentage + # bar_chart + nest: by_hour + # bar_chart + nest: by_day + # bar_chart + nest: by_payment + # line_chart + nest: daily_trends + } + + -- Fare analysis + view: fare_analysis is { + group_by: pickup_hour + aggregate: + trip_count + avg_fare + avg_tip + avg_distance + # bar_chart + nest: by_payment_method is { + group_by: payment_type_name + aggregate: + trip_count + avg_fare + } + order_by: pickup_hour + } + + -- Long trips analysis + view: long_trips is { + select: + tpep_pickup_datetime + tpep_dropoff_datetime + trip_distance + fare_amount + tip_amount + passenger_count + where: trip_distance > 20 + order_by: trip_distance desc + limit: 100 + } +} + +-- Named queries +query: taxi_overview is taxi_trips -> overview +query: hourly_analysis is taxi_trips -> by_hour +query: payment_analysis is taxi_trips -> by_payment +query: fare_breakdown is taxi_trips -> fare_analysis +query: long_distance_trips is taxi_trips -> long_trips +query: daily_summary is taxi_trips -> daily_trends diff --git a/models/sales_orders.malloy b/example/models/sales_orders.malloy similarity index 100% rename from models/sales_orders.malloy rename to example/models/sales_orders.malloy diff --git a/models/sample_data.malloy b/example/models/sample_data.malloy similarity index 100% rename from models/sample_data.malloy rename to example/models/sample_data.malloy diff --git a/models/superstore.malloy b/example/models/superstore.malloy similarity index 100% rename from models/superstore.malloy rename to example/models/superstore.malloy diff --git a/models/users_products.malloy b/example/models/users_products.malloy similarity index 100% rename from models/users_products.malloy rename to example/models/users_products.malloy diff --git a/package-lock.json b/package-lock.json index 1e335ba..984ea34 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@aszenz/data-explorer", - "version": "0.0.0", + "version": "0.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@aszenz/data-explorer", - "version": "0.0.0", + "version": "0.0.1", "dependencies": { "@duckdb/duckdb-wasm": "1.33.1-dev13.0", "@malloydata/db-duckdb": "^0.0.332", @@ -16,11 +16,17 @@ "@malloydata/malloy-query-builder": "^0.0.332", "@malloydata/render": "^0.0.332", "@malloydata/syntax-highlight": "^0.0.335", + "@shikijs/core": "^3.20.0", + "@shikijs/engine-javascript": "^3.20.0", + "@shikijs/themes": "^3.20.0", "react": "^19.2.3", "react-dom": "^19.2.3", "react-markdown": "^10.1.0", "react-router": "^7.11.0" }, + "bin": { + "data-explorer": "dist-cli/index.js" + }, "devDependencies": { "@eslint/js": "^9.39.2", "@malloydata/cli": "^0.0.50", @@ -37,6 +43,7 @@ "globals": "^17.0.0", "playwright": "^1.57.0", "prettier": "^3.7.4", + "tsx": "^4.19.0", "typescript": "^5.9.3", "typescript-eslint": "^8.51.0", "vite": "^7.3.0", @@ -75,7 +82,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -466,7 +472,6 @@ "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz", "integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==", "license": "MIT", - "peer": true, "dependencies": { "@dnd-kit/accessibility": "^3.1.1", "@dnd-kit/utilities": "^3.2.2", @@ -1453,7 +1458,6 @@ "resolved": "https://registry.npmjs.org/@malloydata/malloy-filter/-/malloy-filter-0.0.332.tgz", "integrity": "sha512-I5tVqkwLumE4ODUsuM0dalFfXj1MFfpKJyL+7CivquJKQXyTOxtQ1H5VAbLHdOJ9aui0FhExuXkgpCvHM0pAjQ==", "license": "MIT", - "peer": true, "dependencies": { "jest-diff": "^29.6.2", "luxon": "^3.5.0", @@ -1469,7 +1473,6 @@ "resolved": "https://registry.npmjs.org/@malloydata/malloy-interfaces/-/malloy-interfaces-0.0.332.tgz", "integrity": "sha512-fsZVUEMYQ+GVBoki70UNBXfDRDe7ZG1w4Ryk3v3sbbTXvPYI/ZXNpe4ykiT44W8JLzkRQ5uooNA0nys8TVGM0w==", "license": "MIT", - "peer": true, "dependencies": { "@creditkarma/thrift-server-core": "^1.0.4" }, @@ -1482,7 +1485,6 @@ "resolved": "https://registry.npmjs.org/@malloydata/malloy-query-builder/-/malloy-query-builder-0.0.332.tgz", "integrity": "sha512-lMKH+dRg2M1IL5Etb6iEvAW1hJ6B3VkVEt3ZT1DZCDXXHqbiarEfhW2/asNlPDOdZZjF9ARypj5T0Pl5nXh3hg==", "license": "MIT", - "peer": true, "dependencies": { "@malloydata/malloy-filter": "0.0.332", "@malloydata/malloy-interfaces": "0.0.332", @@ -1497,7 +1499,6 @@ "resolved": "https://registry.npmjs.org/@malloydata/malloy-tag/-/malloy-tag-0.0.332.tgz", "integrity": "sha512-sg8mtxhE4JCkHG+M78ulLLU+lOzdkF6Md9H1VxdaiFdOsIwdrHR1D1JFw1m0pkQqJB7DfbI1GxlUhyecEY3wSw==", "license": "MIT", - "peer": true, "dependencies": { "antlr4ts": "^0.5.0-alpha.4", "assert": "^2.0.0", @@ -1512,7 +1513,6 @@ "resolved": "https://registry.npmjs.org/@malloydata/render/-/render-0.0.332.tgz", "integrity": "sha512-iRKCy6KTJbGRHzXqMoEswz40T/9hyzOnQ5yuSr0ZFiScVcjx6t0BPp3VXT7Sg41LhjTUPU651kWoH45KbliXOg==", "license": "MIT", - "peer": true, "dependencies": { "@malloydata/malloy": "0.0.332", "@malloydata/malloy-interfaces": "0.0.332", @@ -3370,7 +3370,6 @@ "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/core": "^7.21.3", "@svgr/babel-preset": "8.1.0", @@ -3565,7 +3564,8 @@ "version": "7946.0.4", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.4.tgz", "integrity": "sha512-MHmwBtCb7OCv1DSivz2UNJXPGU/1btAWRKlqJ2saEhVJkpkvqHMMaOpKg0v4sAbDWSQekHGvPVMM8nQ+Jen03Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/hast": { "version": "3.0.4", @@ -3610,7 +3610,6 @@ "integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -3620,7 +3619,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz", "integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -3631,7 +3629,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -3641,7 +3638,8 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -3691,7 +3689,6 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.51.0.tgz", "integrity": "sha512-3xP4XzzDNQOIqBMWogftkwxhg5oMKApqY0BAflmLZiFYHqyhSOxv/cd/zPQLTcCXr4AkaKb25joocY0BD1WC6A==", "dev": true, - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.51.0", "@typescript-eslint/types": "8.51.0", @@ -4059,7 +4056,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4129,7 +4125,6 @@ "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-17.0.0.tgz", "integrity": "sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", @@ -4432,7 +4427,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -5255,6 +5249,7 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", + "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -5571,7 +5566,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6115,6 +6109,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", + "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -7232,6 +7239,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -7952,6 +7960,7 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "license": "MIT", + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -8498,7 +8507,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8508,7 +8516,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -8788,6 +8795,16 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", @@ -8809,7 +8826,6 @@ "integrity": "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -8933,7 +8949,6 @@ "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.3.2.tgz", "integrity": "sha512-RbcPH1n5cfwKrru7v7+zrZvjLurgHhGyso3HTyGtRivGWgYjbOmGuivCQaORNELjNONoK35nj28EoWul9sb1zQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=10" } @@ -9126,7 +9141,6 @@ "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.10.tgz", "integrity": "sha512-Coz956cos/EPDlhs6+jsdTxKuJDPT7B5SVIWgABwROyxjY7Xbr8wkzD68Et+NxnV7DLJ3nJdAC2r9InuV/4Jew==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.1.0", "seroval": "~1.3.0", @@ -9484,7 +9498,8 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/trim-lines": { "version": "3.0.1", @@ -9524,6 +9539,41 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -9621,7 +9671,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -9892,6 +9941,7 @@ "resolved": "https://registry.npmjs.org/vega/-/vega-5.33.0.tgz", "integrity": "sha512-jNAGa7TxLojOpMMMrKMXXBos4K6AaLJbCgGDOw1YEkLRjUkh12pcf65J2lMSdEHjcEK47XXjKiOUVZ8L+MniBA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-crossfilter": "~4.1.3", "vega-dataflow": "~5.7.7", @@ -9926,13 +9976,15 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/vega-canvas/-/vega-canvas-1.2.7.tgz", "integrity": "sha512-OkJ9CACVcN9R5Pi9uF6MZBF06pO6qFpDYHWSKBJsdHP5o724KrsgR6UvbnXFH82FdsiTOff/HqjuaG8C7FL+9Q==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-crossfilter": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/vega-crossfilter/-/vega-crossfilter-4.1.3.tgz", "integrity": "sha512-nyPJAXAUABc3EocUXvAL1J/IWotZVsApIcvOeZaUdEQEtZ7bt8VtP2nj3CLbHBA8FZZVV+K6SmdwvCOaAD4wFQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "vega-dataflow": "^5.7.7", @@ -9943,13 +9995,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-dataflow": { "version": "5.7.8", "resolved": "https://registry.npmjs.org/vega-dataflow/-/vega-dataflow-5.7.8.tgz", "integrity": "sha512-jrllcIjSYU5Jh130RDR44o/SbUbJndLuoiM9IsKWW+a7HayKnfmbdHWm7MvCrj/YLupFZVojRaS1tTs53EXTdA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-format": "^1.1.4", "vega-loader": "^4.5.4", @@ -9960,13 +10014,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-encode": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/vega-encode/-/vega-encode-4.10.2.tgz", "integrity": "sha512-fsjEY1VaBAmqwt7Jlpz0dpPtfQFiBdP9igEefvumSpy7XUxOJmDQcRDnT3Qh9ctkv3itfPfI9g8FSnGcv2b4jQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-interpolate": "^3.0.1", @@ -9979,7 +10035,8 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-event-selector": { "version": "3.0.1", @@ -10008,6 +10065,7 @@ "resolved": "https://registry.npmjs.org/vega-force/-/vega-force-4.2.2.tgz", "integrity": "sha512-cHZVaY2VNNIG2RyihhSiWniPd2W9R9kJq0znxzV602CgUVgxEfTKtx/lxnVCn8nNrdKAYrGiqIsBzIeKG1GWHw==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-force": "^3.0.0", "vega-dataflow": "^5.7.7", @@ -10018,13 +10076,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-format": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/vega-format/-/vega-format-1.1.4.tgz", "integrity": "sha512-+oz6UvXjQSbweW9P8q+1o2qFYyBYPFax94j6a9PQMnCIWMovFSss1wEElljOT8CEpnHyS15yiGlmz4qbWTQwnQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-format": "^3.1.0", @@ -10037,13 +10097,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-functions": { "version": "5.18.1", "resolved": "https://registry.npmjs.org/vega-functions/-/vega-functions-5.18.1.tgz", "integrity": "sha512-qEBAbo0jxGGebRvbX1zmxzmjwFz8/UtncRhzwk9/KcI0WudULNmCM1iTu+DGFRnNHdcKi6kUlwJBPIp7zDu3HQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-color": "^3.1.0", @@ -10063,6 +10125,7 @@ "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-5.2.1.tgz", "integrity": "sha512-9KKbI2q9qTI55NSjD/dVWg3aeCtw+gwyWCiLMM47ha6iXrAN9pQ+EKRJfxOHuoDfCTlJJTaUfnnXgbqm0HEszg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@types/estree": "^1.0.0", "vega-util": "^1.17.4" @@ -10072,13 +10135,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-geo": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/vega-geo/-/vega-geo-4.4.3.tgz", "integrity": "sha512-+WnnzEPKIU1/xTFUK3EMu2htN35gp9usNZcC0ZFg2up1/Vqu6JyZsX0PIO51oXSIeXn9bwk6VgzlOmJUcx92tA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-color": "^3.1.0", @@ -10094,13 +10159,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-hierarchy": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/vega-hierarchy/-/vega-hierarchy-4.1.3.tgz", "integrity": "sha512-0Z+TYKRgOEo8XYXnJc2HWg1EGpcbNAhJ9Wpi9ubIbEyEHqIgjCIyFVN8d4nSfsJOcWDzsSmRqohBztxAhOCSaw==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-hierarchy": "^3.1.2", "vega-dataflow": "^5.7.7", @@ -10111,7 +10178,8 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-interpreter": { "version": "2.2.1", @@ -10127,6 +10195,7 @@ "resolved": "https://registry.npmjs.org/vega-label/-/vega-label-1.3.1.tgz", "integrity": "sha512-Emx4b5s7pvuRj3fBkAJ/E2snCoZACfKAwxVId7f/4kYVlAYLb5Swq6W8KZHrH4M9Qds1XJRUYW9/Y3cceqzEFA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-canvas": "^1.2.7", "vega-dataflow": "^5.7.7", @@ -10138,7 +10207,8 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-lite": { "version": "5.23.0", @@ -10177,6 +10247,7 @@ "resolved": "https://registry.npmjs.org/vega-loader/-/vega-loader-4.5.4.tgz", "integrity": "sha512-AOJPsDVz009aTdD9hzigUaO/NFmuN1o83rzvZu/g37TJfhU+3DOvgnO0rnqJbnSOfcBkLWER6XghlKS3j77w4A==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-dsv": "^3.0.1", "node-fetch": "^2.6.7", @@ -10189,13 +10260,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-parser": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/vega-parser/-/vega-parser-6.6.0.tgz", "integrity": "sha512-jltyrwCTtWeidi/6VotLCybhIl+ehwnzvFWYOdWNUP0z/EskdB64YmawNwjCjzTBMemeiQtY6sJPPbewYqe3Vg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-dataflow": "^5.7.7", "vega-event-selector": "^3.0.1", @@ -10208,13 +10281,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-projection": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/vega-projection/-/vega-projection-1.6.2.tgz", "integrity": "sha512-3pcVaQL9R3Zfk6PzopLX6awzrQUeYOXJzlfLGP2Xd93mqUepBa6m/reVrTUoSFXA3v9lfK4W/PS2AcVzD/MIcQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-geo": "^3.1.0", "d3-geo-projection": "^4.0.0", @@ -10226,6 +10301,7 @@ "resolved": "https://registry.npmjs.org/vega-regression/-/vega-regression-1.3.1.tgz", "integrity": "sha512-AmccF++Z9uw4HNZC/gmkQGe6JsRxTG/R4QpbcSepyMvQN1Rj5KtVqMcmVFP1r3ivM4dYGFuPlzMWvuqp0iKMkQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "vega-dataflow": "^5.7.7", @@ -10237,13 +10313,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-runtime": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/vega-runtime/-/vega-runtime-6.2.1.tgz", "integrity": "sha512-b4eot3tWKCk++INWqot+6sLn3wDTj/HE+tRSbiaf8aecuniPMlwJEK7wWuhVGeW2Ae5n8fI/8TeTViaC94bNHA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-dataflow": "^5.7.7", "vega-util": "^1.17.3" @@ -10253,13 +10331,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-scale": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/vega-scale/-/vega-scale-7.4.3.tgz", "integrity": "sha512-f7SSN2YJowtrdkt7nJIR6YYhjDk8oB37q5So2/OxXQv5CBHipFPQSHS1ZVw9vD3V5wLnrZCxC4Ji27gmsTefgA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-interpolate": "^3.0.1", @@ -10273,13 +10353,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-scenegraph": { "version": "4.13.2", "resolved": "https://registry.npmjs.org/vega-scenegraph/-/vega-scenegraph-4.13.2.tgz", "integrity": "sha512-eCutgcLzdUg23HLc6MTZ9pHCdH0hkqSmlbcoznspwT0ajjATk6M09JNyJddiaKR55HuQo03mBWsPeRCd5kOi0g==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-path": "^3.1.0", "d3-shape": "^3.2.0", @@ -10293,13 +10375,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-selections": { "version": "5.6.3", "resolved": "https://registry.npmjs.org/vega-selections/-/vega-selections-5.6.3.tgz", "integrity": "sha512-DXd+XVKcIjBAtSCcgtPx7cXuqG/7L98SWoFh6GKNu26EBUyn3zm0GAlZxNLPoI01Jz9Fb3YpSsewk2aIAbM68g==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "3.2.4", "vega-expression": "^5.2.1", @@ -10311,6 +10395,7 @@ "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-5.2.1.tgz", "integrity": "sha512-9KKbI2q9qTI55NSjD/dVWg3aeCtw+gwyWCiLMM47ha6iXrAN9pQ+EKRJfxOHuoDfCTlJJTaUfnnXgbqm0HEszg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@types/estree": "^1.0.0", "vega-util": "^1.17.4" @@ -10320,13 +10405,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-statistics": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/vega-statistics/-/vega-statistics-1.9.0.tgz", "integrity": "sha512-GAqS7mkatpXcMCQKWtFu1eMUKLUymjInU0O8kXshWaQrVWjPIO2lllZ1VNhdgE0qGj4oOIRRS11kzuijLshGXQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2" } @@ -10336,6 +10423,7 @@ "resolved": "https://registry.npmjs.org/vega-time/-/vega-time-2.1.4.tgz", "integrity": "sha512-DBMRps5myYnSAlvQ+oiX8CycJZjGQNqyGE04xaZrpOgHll7vlvezpET2FnGZC7wS3DsqMcPjnpnI1h7+qJox1Q==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-time": "^3.1.0", @@ -10346,13 +10434,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-transforms": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/vega-transforms/-/vega-transforms-4.12.1.tgz", "integrity": "sha512-Qxo+xeEEftY1jYyKgzOGc9NuW4/MqGm1YPZ5WrL9eXg2G0410Ne+xL/MFIjHF4hRX+3mgFF4Io2hPpfy/thjLg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "vega-dataflow": "^5.7.7", @@ -10365,13 +10455,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-typings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/vega-typings/-/vega-typings-1.5.0.tgz", "integrity": "sha512-tcZ2HwmiQEOXIGyBMP8sdCnoFoVqHn4KQ4H0MQiHwzFU1hb1EXURhfc+Uamthewk4h/9BICtAM3AFQMjBGpjQA==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@types/geojson": "7946.0.4", "vega-event-selector": "^3.0.1", @@ -10384,6 +10476,7 @@ "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-5.2.1.tgz", "integrity": "sha512-9KKbI2q9qTI55NSjD/dVWg3aeCtw+gwyWCiLMM47ha6iXrAN9pQ+EKRJfxOHuoDfCTlJJTaUfnnXgbqm0HEszg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@types/estree": "^1.0.0", "vega-util": "^1.17.4" @@ -10393,7 +10486,8 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-util": { "version": "2.1.0", @@ -10406,6 +10500,7 @@ "resolved": "https://registry.npmjs.org/vega-view/-/vega-view-5.16.0.tgz", "integrity": "sha512-Nxp1MEAY+8bphIm+7BeGFzWPoJnX9+hgvze6wqCAPoM69YiyVR0o0VK8M2EESIL+22+Owr0Fdy94hWHnmon5tQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-array": "^3.2.2", "d3-timer": "^3.0.1", @@ -10422,6 +10517,7 @@ "resolved": "https://registry.npmjs.org/vega-view-transforms/-/vega-view-transforms-4.6.1.tgz", "integrity": "sha512-RYlyMJu5kZV4XXjmyTQKADJWDB25SMHsiF+B1rbE1p+pmdQPlp5tGdPl9r5dUJOp3p8mSt/NGI8GPGucmPMxtw==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-dataflow": "^5.7.7", "vega-scenegraph": "^4.13.1", @@ -10432,19 +10528,22 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-view/node_modules/vega-util": { "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-voronoi": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/vega-voronoi/-/vega-voronoi-4.2.4.tgz", "integrity": "sha512-lWNimgJAXGeRFu2Pz8axOUqVf1moYhD+5yhBzDSmckE9I5jLOyZc/XvgFTXwFnsVkMd1QW1vxJa+y9yfUblzYw==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "d3-delaunay": "^6.0.2", "vega-dataflow": "^5.7.7", @@ -10455,13 +10554,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega-wordcloud": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/vega-wordcloud/-/vega-wordcloud-4.1.6.tgz", "integrity": "sha512-lFmF3u9/ozU0P+WqPjeThQfZm0PigdbXDwpIUCxczrCXKYJLYFmZuZLZR7cxtmpZ0/yuvRvAJ4g123LXbSZF8A==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "vega-canvas": "^1.2.7", "vega-dataflow": "^5.7.7", @@ -10474,13 +10575,15 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vega/node_modules/vega-expression": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-5.2.1.tgz", "integrity": "sha512-9KKbI2q9qTI55NSjD/dVWg3aeCtw+gwyWCiLMM47ha6iXrAN9pQ+EKRJfxOHuoDfCTlJJTaUfnnXgbqm0HEszg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@types/estree": "^1.0.0", "vega-util": "^1.17.4" @@ -10490,7 +10593,8 @@ "version": "1.17.4", "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz", "integrity": "sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/vfile": { "version": "6.0.3", @@ -10526,7 +10630,6 @@ "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -10714,13 +10817,15 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "peer": true }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "license": "MIT", + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -10945,7 +11050,6 @@ "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "dev": true, "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index df727df..58670db 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,21 @@ { "name": "@aszenz/data-explorer", "private": true, - "version": "0.0.0", + "version": "0.0.1", "type": "module", + "bin": { + "data-explorer": "./dist-cli/index.js" + }, "scripts": { "dev": "vite", "build": "tsc -b && vite build", + "build:cli": "tsc -p tsconfig.cli.json", + "build:example": "npm run build:cli && node ./dist-cli/index.js build ./example/models -o ./dist", "lint": "prettier --check . && tsc --noEmit && eslint .", "format": "prettier --write .", "test": "playwright test", + "test:cli": "vitest run --config vitest.cli.config.ts", + "test:models": "tsx scripts/validate-models.ts ./example/models", "vitest": "vitest", "preview": "vite preview --port 3000", "start": "npm run build && npm run preview" @@ -47,6 +54,7 @@ "playwright": "^1.57.0", "prettier": "^3.7.4", "typescript": "^5.9.3", + "tsx": "^4.19.0", "typescript-eslint": "^8.51.0", "vite": "^7.3.0", "vite-plugin-svgr": "^4.5.0", diff --git a/scripts/validate-models.ts b/scripts/validate-models.ts new file mode 100644 index 0000000..b9aa7c4 --- /dev/null +++ b/scripts/validate-models.ts @@ -0,0 +1,219 @@ +#!/usr/bin/env npx tsx + +/** + * Validates that all Malloy models in a directory have valid syntax. + * This performs a basic syntax check - it cannot fully validate models + * without a real database connection to fetch table schemas. + * + * Usage: npx tsx scripts/validate-models.ts [models-directory] + */ + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, resolve, extname } from "node:path"; + +const modelsDir = process.argv[2] ?? "./example/models"; +const resolvedDir = resolve(modelsDir); + +interface ValidationResult { + file: string; + success: boolean; + fileSize: number; + hasSource: boolean; + hasQuery: boolean; + error?: string; +} + +// Basic syntax patterns to check for in Malloy files +const MALLOY_PATTERNS = { + source: /\bsource:\s+\w+\s+is\b/, + query: /\bquery:\s+\w+\s+is\b/, + view: /\bview:\s+\w+\s+is\b/, + measure: /\bmeasure:\s+\w+\s+is\b/, + dimension: /\bdimension:\s+\w+\s+is\b/, + import: /\bimport\s+["']/, +}; + +// Common syntax errors to detect +const SYNTAX_ERROR_PATTERNS = [ + { pattern: /source:\s*$/m, message: "Incomplete source definition" }, + { pattern: /query:\s*$/m, message: "Incomplete query definition" }, +]; + +function validateMalloyFile(filePath: string): ValidationResult { + const fileName = filePath.split("/").pop() ?? filePath; + + try { + const content = readFileSync(filePath, "utf-8"); + const stats = statSync(filePath); + + // Check for basic Malloy constructs + const hasSource = MALLOY_PATTERNS.source.test(content); + const hasQuery = MALLOY_PATTERNS.query.test(content); + const hasView = MALLOY_PATTERNS.view.test(content); + const hasImport = MALLOY_PATTERNS.import.test(content); + + // A valid Malloy file should have at least one of these + const hasValidContent = hasSource || hasQuery || hasView || hasImport; + + if (!hasValidContent && content.trim().length > 0) { + return { + file: fileName, + success: false, + fileSize: stats.size, + hasSource, + hasQuery, + error: "No valid Malloy constructs found (source, query, view, or import)", + }; + } + + // Check for common syntax errors + for (const { pattern, message } of SYNTAX_ERROR_PATTERNS) { + if (pattern.test(content)) { + return { + file: fileName, + success: false, + fileSize: stats.size, + hasSource, + hasQuery, + error: `Potential syntax error: ${message}`, + }; + } + } + + // Check for balanced braces + const openBraces = (content.match(/\{/g) || []).length; + const closeBraces = (content.match(/\}/g) || []).length; + if (openBraces !== closeBraces) { + return { + file: fileName, + success: false, + fileSize: stats.size, + hasSource, + hasQuery, + error: `Unbalanced braces: ${openBraces} opening, ${closeBraces} closing`, + }; + } + + return { + file: fileName, + success: true, + fileSize: stats.size, + hasSource, + hasQuery, + }; + } catch (e) { + return { + file: fileName, + success: false, + fileSize: 0, + hasSource: false, + hasQuery: false, + error: `Failed to read file: ${e instanceof Error ? e.message : String(e)}`, + }; + } +} + +async function validateModels(): Promise { + console.log(`Validating Malloy models in: ${resolvedDir}\n`); + + const files = readdirSync(resolvedDir); + const malloyFiles = files.filter((f) => extname(f) === ".malloy"); + const notebookFiles = files.filter((f) => extname(f) === ".malloynb"); + + if (malloyFiles.length === 0 && notebookFiles.length === 0) { + console.log("No .malloy or .malloynb files found."); + process.exit(0); + } + + console.log(`Found ${malloyFiles.length} .malloy files and ${notebookFiles.length} .malloynb files\n`); + + const results: ValidationResult[] = []; + let hasErrors = false; + + // Validate .malloy files + for (const file of malloyFiles) { + const filePath = join(resolvedDir, file); + const result = validateMalloyFile(filePath); + results.push(result); + + if (result.success) { + const features = []; + if (result.hasSource) features.push("source"); + if (result.hasQuery) features.push("query"); + console.log(` ✓ ${file} (${features.join(", ") || "import only"})`); + } else { + hasErrors = true; + console.log(` ✗ ${file}`); + console.log(` Error: ${result.error}`); + } + } + + // Basic check for .malloynb files (just verify they exist and have content) + for (const file of notebookFiles) { + const filePath = join(resolvedDir, file); + try { + const content = readFileSync(filePath, "utf-8"); + const stats = statSync(filePath); + + // Check for notebook markers + const hasMalloyCell = content.includes(">>>malloy"); + const hasMarkdownCell = content.includes(">>>markdown"); + + if (hasMalloyCell || hasMarkdownCell) { + console.log(` ✓ ${file} (notebook)`); + results.push({ + file, + success: true, + fileSize: stats.size, + hasSource: false, + hasQuery: hasMalloyCell, + }); + } else { + console.log(` ✗ ${file}`); + console.log(` Error: No >>>malloy or >>>markdown cells found`); + hasErrors = true; + results.push({ + file, + success: false, + fileSize: stats.size, + hasSource: false, + hasQuery: false, + error: "No notebook cells found", + }); + } + } catch (e) { + hasErrors = true; + console.log(` ✗ ${file}`); + console.log(` Error: Failed to read file`); + results.push({ + file, + success: false, + fileSize: 0, + hasSource: false, + hasQuery: false, + error: `Failed to read: ${e}`, + }); + } + } + + console.log(""); + console.log("─".repeat(50)); + + const passed = results.filter((r) => r.success).length; + const failed = results.filter((r) => !r.success).length; + + console.log(`Results: ${passed} passed, ${failed} failed`); + + if (hasErrors) { + console.log("\nValidation completed with errors!"); + process.exit(1); + } else { + console.log("\nAll models validated successfully!"); + process.exit(0); + } +} + +validateModels().catch((e) => { + console.error("Validation script error:", e); + process.exit(1); +}); diff --git a/src/Home.tsx b/src/Home.tsx index 62c461e..89764d1 100644 --- a/src/Home.tsx +++ b/src/Home.tsx @@ -5,6 +5,7 @@ import ModelIcon from "../img/model-icon.svg?react"; import NotebookIcon from "../img/notebook-icon.svg?react"; import FaviconLogo from "../img/favicon-logo.svg?react"; import { humanizeName } from "./utils/humanize"; +import { getSiteConfig } from "./site-config"; export default Home; @@ -16,20 +17,21 @@ type HomeProps = { function Home({ models, notebooks }: HomeProps): JSX.Element { const modelCount = Object.keys(models).length; const notebookCount = Object.keys(notebooks).length; + const siteConfig = getSiteConfig(); return (
-

Data Explorer

+

{siteConfig.title}

- Explore and analyze your{" "} + {siteConfig.description}{" "} - Malloy models and notebooks + Powered by Malloy

diff --git a/src/cli/build.ts b/src/cli/build.ts new file mode 100644 index 0000000..93bdeff --- /dev/null +++ b/src/cli/build.ts @@ -0,0 +1,58 @@ +import { build as viteBuild, createLogger, type InlineConfig } from "vite"; +import { writeFileSync, rmSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import type { DataExplorerConfig } from "./types.js"; + +/** + * Build the data-explorer static site using Vite's API + */ +export async function build( + config: DataExplorerConfig, + packageRoot: string +): Promise { + // Write config to a temporary JSON file that vite.config.ts will read + const configPath = join(packageRoot, ".data-explorer-config.json"); + + writeFileSync( + configPath, + JSON.stringify( + { + inputPath: config.inputPath, + outputPath: config.outputPath, + title: config.title, + description: config.description, + basePath: config.basePath, + }, + null, + 2 + ) + ); + + try { + // Set environment variable for vite.config.ts to find the config + process.env["DATA_EXPLORER_CONFIG_PATH"] = configPath; + + const viteConfig: InlineConfig = { + root: packageRoot, + configFile: join(packageRoot, "vite.config.ts"), + build: { + outDir: config.outputPath, + emptyOutDir: true, + }, + logLevel: "info", + customLogger: createLogger("info", { prefix: "[data-explorer]" }), + }; + + await viteBuild(viteConfig); + + console.log(""); + console.log("Build completed successfully!"); + console.log(`Output: ${config.outputPath}`); + } finally { + // Clean up config file and env variable + if (existsSync(configPath)) { + rmSync(configPath); + } + delete process.env["DATA_EXPLORER_CONFIG_PATH"]; + } +} diff --git a/src/cli/index.ts b/src/cli/index.ts new file mode 100644 index 0000000..57b627b --- /dev/null +++ b/src/cli/index.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +import { parseArgs } from "node:util"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { existsSync, statSync } from "node:fs"; +import { build } from "./build.js"; +import { DEFAULT_CONFIG, type DataExplorerConfig } from "./types.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const HELP_TEXT = ` +data-explorer - Static site generator for Malloy data models and notebooks + +Usage: + data-explorer build [options] + +Commands: + build Build the static site from Malloy files + +Options: + -o, --output Output directory for the built site (default: ./dist) + -t, --title Site title (default: "Data Explorer") + -d, --description <desc> Home page description + -b, --base-path <path> Base public path for deployment (default: "/") + -h, --help Show this help message + -v, --version Show version number + +Examples: + data-explorer build ./models + data-explorer build ./models -o ./dist -t "My Data Site" + data-explorer build ./models --title "Sales Analytics" --description "Explore sales data" +`; + +function showHelp(): void { + console.log(HELP_TEXT); + process.exit(0); +} + +function showVersion(): void { + console.log("data-explorer v0.0.1"); + process.exit(0); +} + +function validateInputPath(inputPath: string): string { + const resolved = resolve(inputPath); + if (!existsSync(resolved)) { + console.error(`Error: Input path does not exist: ${resolved}`); + process.exit(1); + } + if (!statSync(resolved).isDirectory()) { + console.error(`Error: Input path is not a directory: ${resolved}`); + process.exit(1); + } + return resolved; +} + +async function main(): Promise<void> { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + output: { type: "string", short: "o" }, + title: { type: "string", short: "t" }, + description: { type: "string", short: "d" }, + "base-path": { type: "string", short: "b" }, + help: { type: "boolean", short: "h" }, + version: { type: "boolean", short: "v" }, + }, + }); + + if (values.help) { + showHelp(); + } + + if (values.version) { + showVersion(); + } + + const command = positionals[0]; + + if (!command) { + console.error("Error: No command specified. Use --help for usage information."); + process.exit(1); + } + + if (command === "build") { + const inputPath = positionals[1]; + if (!inputPath) { + console.error("Error: No input path specified for build command."); + console.error("Usage: data-explorer build <input-path> [options]"); + process.exit(1); + } + + const config: DataExplorerConfig = { + inputPath: validateInputPath(inputPath), + outputPath: values.output ? resolve(values.output) : resolve("./dist"), + title: values.title ?? DEFAULT_CONFIG.title, + description: values.description ?? DEFAULT_CONFIG.description, + basePath: values["base-path"] ?? DEFAULT_CONFIG.basePath, + }; + + // Get the package root (where vite.config.ts lives) + const packageRoot = resolve(__dirname, "../.."); + + console.log("Building data-explorer site..."); + console.log(` Input: ${config.inputPath}`); + console.log(` Output: ${config.outputPath}`); + console.log(` Title: ${config.title}`); + console.log(` Description: ${config.description}`); + console.log(` Base Path: ${config.basePath}`); + console.log(""); + + await build(config, packageRoot); + } else { + console.error(`Error: Unknown command "${command}". Use --help for usage information.`); + process.exit(1); + } +} + +main().catch((error: unknown) => { + console.error("Build failed:", error); + process.exit(1); +}); diff --git a/src/cli/types.ts b/src/cli/types.ts new file mode 100644 index 0000000..c7229d7 --- /dev/null +++ b/src/cli/types.ts @@ -0,0 +1,21 @@ +/** + * Configuration for the data-explorer build + */ +export interface DataExplorerConfig { + /** Path to the directory containing Malloy files (.malloy, .malloynb) */ + inputPath: string; + /** Path to output the built website */ + outputPath: string; + /** Title for the site (shown in browser tab and home page) */ + title: string; + /** Description shown on the home page */ + description: string; + /** Base public path for deployment (e.g., "/my-repo/" for GitHub Pages) */ + basePath: string; +} + +export const DEFAULT_CONFIG: Omit<DataExplorerConfig, "inputPath" | "outputPath"> = { + title: "Data Explorer", + description: "Explore and analyze your Malloy models and notebooks", + basePath: "/", +}; diff --git a/src/index.tsx b/src/index.tsx index 91a3283..beb90cc 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,6 +1,10 @@ import { createRoot } from "react-dom/client"; import App from "./App"; import { StrictMode } from "react"; +import { getSiteConfig } from "./site-config"; + +// Set document title from config +document.title = getSiteConfig().title; const rootElement = document.getElementById("root"); if (null === rootElement) { diff --git a/src/site-config.ts b/src/site-config.ts new file mode 100644 index 0000000..83acdd0 --- /dev/null +++ b/src/site-config.ts @@ -0,0 +1,15 @@ +/** + * Site configuration injected at build time via Vite's define option + */ +export interface SiteConfig { + title: string; + description: string; +} + +declare global { + const __DATA_EXPLORER_CONFIG__: SiteConfig; +} + +export function getSiteConfig(): SiteConfig { + return __DATA_EXPLORER_CONFIG__; +} diff --git a/tests/cli/cli.test.ts b/tests/cli/cli.test.ts new file mode 100644 index 0000000..c2ea26d --- /dev/null +++ b/tests/cli/cli.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { execSync, spawn, type ChildProcess } from "node:child_process"; +import { existsSync, rmSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +const PROJECT_ROOT = resolve(__dirname, "../.."); +const CLI_PATH = join(PROJECT_ROOT, "dist-cli/index.js"); +const EXAMPLE_MODELS = join(PROJECT_ROOT, "example/models"); + +// Build CLI before tests +beforeAll(() => { + console.log("Building CLI..."); + execSync("npm run build:cli", { cwd: PROJECT_ROOT, stdio: "inherit" }); +}, 30000); + +describe("CLI smoke tests", () => { + describe("--help flag", () => { + it("should display help text", () => { + const output = execSync(`node ${CLI_PATH} --help`, { + encoding: "utf-8", + }); + + expect(output).toContain("data-explorer"); + expect(output).toContain("build"); + expect(output).toContain("--output"); + expect(output).toContain("--title"); + expect(output).toContain("--description"); + }); + }); + + describe("--version flag", () => { + it("should display version", () => { + const output = execSync(`node ${CLI_PATH} --version`, { + encoding: "utf-8", + }); + + expect(output).toContain("data-explorer"); + expect(output).toMatch(/v?\d+\.\d+\.\d+/); + }); + }); + + describe("build command validation", () => { + it("should error when no input path provided", () => { + expect(() => { + execSync(`node ${CLI_PATH} build`, { + encoding: "utf-8", + stdio: "pipe", + }); + }).toThrow(); + }); + + it("should error when input path does not exist", () => { + expect(() => { + execSync(`node ${CLI_PATH} build /nonexistent/path`, { + encoding: "utf-8", + stdio: "pipe", + }); + }).toThrow(); + }); + + it("should error when unknown command is given", () => { + expect(() => { + execSync(`node ${CLI_PATH} unknowncommand`, { + encoding: "utf-8", + stdio: "pipe", + }); + }).toThrow(); + }); + }); + + describe("build command execution", () => { + const testOutputDir = join(PROJECT_ROOT, "test-output-cli"); + + afterAll(() => { + // Clean up test output + if (existsSync(testOutputDir)) { + rmSync(testOutputDir, { recursive: true, force: true }); + } + }); + + it("should build example site successfully", () => { + // Clean up any previous test output + if (existsSync(testOutputDir)) { + rmSync(testOutputDir, { recursive: true, force: true }); + } + + const output = execSync( + `node ${CLI_PATH} build ${EXAMPLE_MODELS} -o ${testOutputDir} -t "Test Site" -d "Test description"`, + { + encoding: "utf-8", + cwd: PROJECT_ROOT, + timeout: 120000, + } + ); + + expect(output).toContain("Build completed successfully"); + + // Verify output files exist + expect(existsSync(testOutputDir)).toBe(true); + expect(existsSync(join(testOutputDir, "index.html"))).toBe(true); + expect(existsSync(join(testOutputDir, "assets"))).toBe(true); + }, 120000); + + it("should include custom title in built site", () => { + // The index.html should contain the title (set via JS, but we can check for the bundle) + const indexHtml = readFileSync( + join(testOutputDir, "index.html"), + "utf-8" + ); + + // The HTML should at least have the basic structure + expect(indexHtml).toContain("<!DOCTYPE html>"); + expect(indexHtml).toContain('<div id="root">'); + }); + }); +}); diff --git a/tsconfig.cli.json b/tsconfig.cli.json new file mode 100644 index 0000000..98262c8 --- /dev/null +++ b/tsconfig.cli.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.cli.tsbuildinfo", + "target": "ES2022", + "lib": ["ES2023"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "./dist-cli", + "rootDir": "./src/cli", + "skipLibCheck": true, + "esModuleInterop": true, + "isolatedModules": true, + "incremental": true, + "declaration": true, + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/cli/**/*.ts"] +} diff --git a/tsconfig.json b/tsconfig.json index 9a66b8a..78bdc79 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,7 @@ "files": [], "references": [ { "path": "./tsconfig.app.json" }, - { "path": "./tsconfig.node.json" } + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.cli.json" } ] } diff --git a/vite.config.ts b/vite.config.ts index bf25cdd..594e744 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -2,14 +2,60 @@ import { defineConfig, type UserConfig } from "vite"; import react from "@vitejs/plugin-react"; import svgr from "vite-plugin-svgr"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve, join } from "node:path"; + +interface DataExplorerConfig { + inputPath: string; + outputPath: string; + title: string; + description: string; + basePath: string; +} + +// Load config from file if it exists (when running via CLI) +function loadConfig(): DataExplorerConfig | null { + const configPath = + process.env["DATA_EXPLORER_CONFIG_PATH"] ?? + join(process.cwd(), ".data-explorer-config.json"); + + if (existsSync(configPath)) { + const content = readFileSync(configPath, "utf-8"); + return JSON.parse(content) as DataExplorerConfig; + } + return null; +} + +const cliConfig = loadConfig(); + +// Default values for development (uses example/models) +const siteConfig = { + title: cliConfig?.title ?? "Data Explorer", + description: + cliConfig?.description ?? + "Explore and analyze your Malloy models and notebooks", + basePath: cliConfig?.basePath ?? process.env["BASE_PUBLIC_PATH"] ?? "/", + modelsPath: cliConfig?.inputPath ?? resolve(process.cwd(), "example/models"), +}; // https://vite.dev/config/ const config: UserConfig = defineConfig({ // NOTE: THIS PATH MUST END WITH A TRAILING SLASH - base: process.env["BASE_PUBLIC_PATH"] ?? "/", + base: siteConfig.basePath.endsWith("/") + ? siteConfig.basePath + : siteConfig.basePath + "/", plugins: [react(), svgr()], define: { "process.env": {}, + __DATA_EXPLORER_CONFIG__: JSON.stringify({ + title: siteConfig.title, + description: siteConfig.description, + }), + }, + resolve: { + alias: { + "/models": siteConfig.modelsPath, + }, }, optimizeDeps: { esbuildOptions: { diff --git a/vitest.cli.config.ts b/vitest.cli.config.ts new file mode 100644 index 0000000..7e1eb69 --- /dev/null +++ b/vitest.cli.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/cli/**/*.test.ts"], + testTimeout: 60000, // CLI tests may take longer due to build process + }, +}); From d7675b4691140ea64016ef2c26dc247848fb9ac3 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Mon, 26 Jan 2026 04:34:05 +0000 Subject: [PATCH 2/3] refactor: reorganize examples and improve CLI - Split examples into separate folders (ecommerce, huggingface, sample-data) - Add preview command using Vite's programmatic API - Create minimalist landing page matching app style - Add build-all script to build all examples with landing page - Update validation script to scan all example directories - Fix CLI package root path resolution - Update tests for new example structure --- docs/index.html | 441 ------------------ example/models/.gitkeep | 0 example/models/data/.gitkeep | 0 .../ecommerce}/E-commerce Orders.malloynb | 0 .../ecommerce}/data/contracts.csv | 0 .../ecommerce}/data/orders.csv | 0 .../ecommerce}/ecommerce_orders.malloy | 0 .../huggingface}/GitHub Events.malloynb | 0 .../huggingface}/IMDB Movies.malloynb | 0 .../huggingface}/NYC Taxi Trips.malloynb | 0 .../huggingface}/github_events.malloy | 0 .../huggingface}/imdb_movies.malloy | 0 .../huggingface}/nyc_taxi.malloy | 0 .../sample-data}/Invoices.malloynb | 0 .../sample-data}/Kids Screen Time.malloynb | 0 .../sample-data}/Sales Orders.malloynb | 0 .../sample-data}/Sample Data.malloynb | 0 .../sample-data}/SuperStore.malloynb | 0 .../sample-data}/Users and Products.malloynb | 0 .../sample-data}/business_overview.malloy | 0 .../data/Indian_Kids_Screen_Time.csv | 0 .../sample-data}/data/invoices.parquet | Bin .../sample-data}/data/products.jsonl | 0 .../sample-data}/data/sales_orders.xlsx | Bin .../sample-data}/data/users.json | 0 .../sample-data}/invoices.malloy | 0 .../sample-data}/kids_screen_time.malloy | 0 .../sample-data}/sales_orders.malloy | 0 .../sample-data}/sample_data.malloy | 0 .../sample-data}/superstore.malloy | 0 .../sample-data}/users_products.malloy | 0 landing/index.html | 233 +++++++++ package.json | 4 +- scripts/build-all.ts | 128 +++++ scripts/validate-models.ts | 209 +++++---- src/cli/build.ts | 52 ++- src/cli/index.ts | 58 ++- tests/cli/cli.test.ts | 18 +- vite.config.ts | 5 +- 39 files changed, 572 insertions(+), 576 deletions(-) delete mode 100644 docs/index.html delete mode 100644 example/models/.gitkeep delete mode 100644 example/models/data/.gitkeep rename {example/models => examples/ecommerce}/E-commerce Orders.malloynb (100%) rename {example/models => examples/ecommerce}/data/contracts.csv (100%) rename {example/models => examples/ecommerce}/data/orders.csv (100%) rename {example/models => examples/ecommerce}/ecommerce_orders.malloy (100%) rename {example/models => examples/huggingface}/GitHub Events.malloynb (100%) rename {example/models => examples/huggingface}/IMDB Movies.malloynb (100%) rename {example/models => examples/huggingface}/NYC Taxi Trips.malloynb (100%) rename {example/models => examples/huggingface}/github_events.malloy (100%) rename {example/models => examples/huggingface}/imdb_movies.malloy (100%) rename {example/models => examples/huggingface}/nyc_taxi.malloy (100%) rename {example/models => examples/sample-data}/Invoices.malloynb (100%) rename {example/models => examples/sample-data}/Kids Screen Time.malloynb (100%) rename {example/models => examples/sample-data}/Sales Orders.malloynb (100%) rename {example/models => examples/sample-data}/Sample Data.malloynb (100%) rename {example/models => examples/sample-data}/SuperStore.malloynb (100%) rename {example/models => examples/sample-data}/Users and Products.malloynb (100%) rename {example/models => examples/sample-data}/business_overview.malloy (100%) rename {example/models => examples/sample-data}/data/Indian_Kids_Screen_Time.csv (100%) rename {example/models => examples/sample-data}/data/invoices.parquet (100%) rename {example/models => examples/sample-data}/data/products.jsonl (100%) rename {example/models => examples/sample-data}/data/sales_orders.xlsx (100%) rename {example/models => examples/sample-data}/data/users.json (100%) rename {example/models => examples/sample-data}/invoices.malloy (100%) rename {example/models => examples/sample-data}/kids_screen_time.malloy (100%) rename {example/models => examples/sample-data}/sales_orders.malloy (100%) rename {example/models => examples/sample-data}/sample_data.malloy (100%) rename {example/models => examples/sample-data}/superstore.malloy (100%) rename {example/models => examples/sample-data}/users_products.malloy (100%) create mode 100644 landing/index.html create mode 100644 scripts/build-all.ts diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 9499bef..0000000 --- a/docs/index.html +++ /dev/null @@ -1,441 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="UTF-8"> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <title>Data Explorer - Static Site Generator for Malloy - - - -
-
- -
-
- -
-
-
-

Data Explorer

-

A CLI tool to generate beautiful static websites for your Malloy data models and notebooks. Explore, query, and share your data with ease.

-
- - View Examples - - - GitHub - -
-
- npx @aszenz/data-explorer build ./models -o ./dist -
-
-
- -
-
-

Features

-
-
-

Interactive Schema Explorer

-

Browse your Malloy models with an intuitive schema viewer. See sources, fields, and relationships at a glance.

-
-
-

Query Builder

-

Build and execute queries visually using the Malloy Explorer integration. No code required.

-
-
-

Notebook Support

-

Render Malloy notebooks (.malloynb) with executed results, markdown, and syntax-highlighted code.

-
-
-

DuckDB Powered

-

Runs entirely in the browser using DuckDB WASM. Load CSV, Parquet, JSON, Excel and more.

-
-
-

Hugging Face Datasets

-

Reference public datasets directly from Hugging Face using the hf:// protocol in DuckDB.

-
-
-

Static & Portable

-

Generate static HTML/JS that can be hosted anywhere - GitHub Pages, Netlify, S3, or locally.

-
-
-
-
- -
-
-

Example Sites

-

See what you can build with Data Explorer

-
- -

E-commerce Analytics

-

Orders, products, and customer analysis with multi-table joins and visualizations.

- Local CSV - - -

GitHub Activity

-

Analyze GitHub events data from Hugging Face's public dataset mirror.

- Hugging Face - - -

NYC Taxi Trips

-

Explore millions of taxi rides with geographic and temporal analysis.

- Hugging Face - - -

IMDB Movies

-

Movie ratings, genres, and cast analysis from the IMDB dataset.

- Hugging Face - -
-
-
- -
-
-

Quick Start

-
- 
# Build a site from your Malloy models
-npx @aszenz/data-explorer build ./models -o ./dist
-
-# Customize the site title and description
-npx @aszenz/data-explorer build ./models \
-  --title "Sales Analytics" \
-  --description "Explore our sales data" \
-  --output ./dist
-
-# Set base path for GitHub Pages deployment
-npx @aszenz/data-explorer build ./models \
-  --base-path "/my-repo/" \
-  -o ./dist
-
-# View all options
-npx @aszenz/data-explorer --help
-
-
-
-
- - - - diff --git a/example/models/.gitkeep b/example/models/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/example/models/data/.gitkeep b/example/models/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/example/models/E-commerce Orders.malloynb b/examples/ecommerce/E-commerce Orders.malloynb similarity index 100% rename from example/models/E-commerce Orders.malloynb rename to examples/ecommerce/E-commerce Orders.malloynb diff --git a/example/models/data/contracts.csv b/examples/ecommerce/data/contracts.csv similarity index 100% rename from example/models/data/contracts.csv rename to examples/ecommerce/data/contracts.csv diff --git a/example/models/data/orders.csv b/examples/ecommerce/data/orders.csv similarity index 100% rename from example/models/data/orders.csv rename to examples/ecommerce/data/orders.csv diff --git a/example/models/ecommerce_orders.malloy b/examples/ecommerce/ecommerce_orders.malloy similarity index 100% rename from example/models/ecommerce_orders.malloy rename to examples/ecommerce/ecommerce_orders.malloy diff --git a/example/models/GitHub Events.malloynb b/examples/huggingface/GitHub Events.malloynb similarity index 100% rename from example/models/GitHub Events.malloynb rename to examples/huggingface/GitHub Events.malloynb diff --git a/example/models/IMDB Movies.malloynb b/examples/huggingface/IMDB Movies.malloynb similarity index 100% rename from example/models/IMDB Movies.malloynb rename to examples/huggingface/IMDB Movies.malloynb diff --git a/example/models/NYC Taxi Trips.malloynb b/examples/huggingface/NYC Taxi Trips.malloynb similarity index 100% rename from example/models/NYC Taxi Trips.malloynb rename to examples/huggingface/NYC Taxi Trips.malloynb diff --git a/example/models/github_events.malloy b/examples/huggingface/github_events.malloy similarity index 100% rename from example/models/github_events.malloy rename to examples/huggingface/github_events.malloy diff --git a/example/models/imdb_movies.malloy b/examples/huggingface/imdb_movies.malloy similarity index 100% rename from example/models/imdb_movies.malloy rename to examples/huggingface/imdb_movies.malloy diff --git a/example/models/nyc_taxi.malloy b/examples/huggingface/nyc_taxi.malloy similarity index 100% rename from example/models/nyc_taxi.malloy rename to examples/huggingface/nyc_taxi.malloy diff --git a/example/models/Invoices.malloynb b/examples/sample-data/Invoices.malloynb similarity index 100% rename from example/models/Invoices.malloynb rename to examples/sample-data/Invoices.malloynb diff --git a/example/models/Kids Screen Time.malloynb b/examples/sample-data/Kids Screen Time.malloynb similarity index 100% rename from example/models/Kids Screen Time.malloynb rename to examples/sample-data/Kids Screen Time.malloynb diff --git a/example/models/Sales Orders.malloynb b/examples/sample-data/Sales Orders.malloynb similarity index 100% rename from example/models/Sales Orders.malloynb rename to examples/sample-data/Sales Orders.malloynb diff --git a/example/models/Sample Data.malloynb b/examples/sample-data/Sample Data.malloynb similarity index 100% rename from example/models/Sample Data.malloynb rename to examples/sample-data/Sample Data.malloynb diff --git a/example/models/SuperStore.malloynb b/examples/sample-data/SuperStore.malloynb similarity index 100% rename from example/models/SuperStore.malloynb rename to examples/sample-data/SuperStore.malloynb diff --git a/example/models/Users and Products.malloynb b/examples/sample-data/Users and Products.malloynb similarity index 100% rename from example/models/Users and Products.malloynb rename to examples/sample-data/Users and Products.malloynb diff --git a/example/models/business_overview.malloy b/examples/sample-data/business_overview.malloy similarity index 100% rename from example/models/business_overview.malloy rename to examples/sample-data/business_overview.malloy diff --git a/example/models/data/Indian_Kids_Screen_Time.csv b/examples/sample-data/data/Indian_Kids_Screen_Time.csv similarity index 100% rename from example/models/data/Indian_Kids_Screen_Time.csv rename to examples/sample-data/data/Indian_Kids_Screen_Time.csv diff --git a/example/models/data/invoices.parquet b/examples/sample-data/data/invoices.parquet similarity index 100% rename from example/models/data/invoices.parquet rename to examples/sample-data/data/invoices.parquet diff --git a/example/models/data/products.jsonl b/examples/sample-data/data/products.jsonl similarity index 100% rename from example/models/data/products.jsonl rename to examples/sample-data/data/products.jsonl diff --git a/example/models/data/sales_orders.xlsx b/examples/sample-data/data/sales_orders.xlsx similarity index 100% rename from example/models/data/sales_orders.xlsx rename to examples/sample-data/data/sales_orders.xlsx diff --git a/example/models/data/users.json b/examples/sample-data/data/users.json similarity index 100% rename from example/models/data/users.json rename to examples/sample-data/data/users.json diff --git a/example/models/invoices.malloy b/examples/sample-data/invoices.malloy similarity index 100% rename from example/models/invoices.malloy rename to examples/sample-data/invoices.malloy diff --git a/example/models/kids_screen_time.malloy b/examples/sample-data/kids_screen_time.malloy similarity index 100% rename from example/models/kids_screen_time.malloy rename to examples/sample-data/kids_screen_time.malloy diff --git a/example/models/sales_orders.malloy b/examples/sample-data/sales_orders.malloy similarity index 100% rename from example/models/sales_orders.malloy rename to examples/sample-data/sales_orders.malloy diff --git a/example/models/sample_data.malloy b/examples/sample-data/sample_data.malloy similarity index 100% rename from example/models/sample_data.malloy rename to examples/sample-data/sample_data.malloy diff --git a/example/models/superstore.malloy b/examples/sample-data/superstore.malloy similarity index 100% rename from example/models/superstore.malloy rename to examples/sample-data/superstore.malloy diff --git a/example/models/users_products.malloy b/examples/sample-data/users_products.malloy similarity index 100% rename from example/models/users_products.malloy rename to examples/sample-data/users_products.malloy diff --git a/landing/index.html b/landing/index.html new file mode 100644 index 0000000..c37b3fc --- /dev/null +++ b/landing/index.html @@ -0,0 +1,233 @@ + + + + + + + + + + Data Explorer - Malloy Static Site Generator + + + +
+
+

Data Explorer

+

Static site generator for Malloy data models and notebooks

+
+ +
+

Example Sites

+
+ +

E-commerce Analytics

+

Orders, products, and customer analysis with multi-table joins and visualizations.

+ Local CSV + + +

Hugging Face Datasets

+

GitHub Events, IMDB Movies, and NYC Taxi data from remote Hugging Face datasets.

+ Remote Data + + +

Sample Data Collection

+

Various local data formats including CSV, Parquet, JSON, and Excel files.

+ Mixed Formats + +
+
+ +
+
+

Quick Start

+
+ # Build a site from your Malloy models
+npx @aszenz/data-explorer build ./models -o ./dist

+# Preview the built site
+npx @aszenz/data-explorer preview ./dist

+# Customize title and description
+npx @aszenz/data-explorer build ./models \
+  --title "My Analytics" \
+  --description "Explore data" \
+  --output ./dist +
+
+
+ + +
+ + diff --git a/package.json b/package.json index 58670db..8397fa1 100644 --- a/package.json +++ b/package.json @@ -10,12 +10,12 @@ "dev": "vite", "build": "tsc -b && vite build", "build:cli": "tsc -p tsconfig.cli.json", - "build:example": "npm run build:cli && node ./dist-cli/index.js build ./example/models -o ./dist", + "build:all": "npm run build:cli && tsx scripts/build-all.ts", "lint": "prettier --check . && tsc --noEmit && eslint .", "format": "prettier --write .", "test": "playwright test", "test:cli": "vitest run --config vitest.cli.config.ts", - "test:models": "tsx scripts/validate-models.ts ./example/models", + "test:models": "tsx scripts/validate-models.ts", "vitest": "vitest", "preview": "vite preview --port 3000", "start": "npm run build && npm run preview" diff --git a/scripts/build-all.ts b/scripts/build-all.ts new file mode 100644 index 0000000..97cd007 --- /dev/null +++ b/scripts/build-all.ts @@ -0,0 +1,128 @@ +#!/usr/bin/env npx tsx + +/** + * Builds all example sites and assembles them with the landing page. + * Output structure: + * dist/ + * index.html (landing page) + * favicon.svg + * ecommerce/ (ecommerce example site) + * huggingface/ (huggingface example site) + * sample-data/ (sample-data example site) + */ + +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + existsSync, + mkdirSync, + rmSync, + cpSync, + readdirSync, +} from "node:fs"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const PROJECT_ROOT = resolve(__dirname, ".."); + +interface Example { + name: string; + inputPath: string; + title: string; + description: string; +} + +const EXAMPLES: Example[] = [ + { + name: "ecommerce", + inputPath: "examples/ecommerce", + title: "E-commerce Analytics", + description: "Explore orders, products, and customer data", + }, + { + name: "huggingface", + inputPath: "examples/huggingface", + title: "Hugging Face Datasets", + description: "GitHub Events, IMDB Movies, and NYC Taxi data", + }, + { + name: "sample-data", + inputPath: "examples/sample-data", + title: "Sample Data Collection", + description: "Various local data formats and models", + }, +]; + +async function buildExample(example: Example, outputBase: string): Promise { + const { build } = await import("../src/cli/build.js"); + + const config = { + inputPath: resolve(PROJECT_ROOT, example.inputPath), + outputPath: resolve(outputBase, example.name), + title: example.title, + description: example.description, + basePath: `/${example.name}/`, + }; + + console.log(`\nBuilding: ${example.name}`); + console.log(` Input: ${config.inputPath}`); + console.log(` Output: ${config.outputPath}`); + + await build(config, PROJECT_ROOT); +} + +async function main(): Promise { + const outputDir = resolve(PROJECT_ROOT, "dist"); + + console.log("Building all examples..."); + console.log(`Output directory: ${outputDir}`); + + // Clean output directory + if (existsSync(outputDir)) { + console.log("\nCleaning previous build..."); + rmSync(outputDir, { recursive: true }); + } + mkdirSync(outputDir, { recursive: true }); + + // Build each example + for (const example of EXAMPLES) { + try { + await buildExample(example, outputDir); + } catch (error) { + console.error(`Failed to build ${example.name}:`, error); + process.exit(1); + } + } + + // Copy landing page + console.log("\nCopying landing page..."); + const landingDir = resolve(PROJECT_ROOT, "landing"); + if (existsSync(landingDir)) { + for (const file of readdirSync(landingDir)) { + cpSync(resolve(landingDir, file), resolve(outputDir, file), { + recursive: true, + }); + } + } + + // Copy favicon to root + const faviconSrc = resolve(PROJECT_ROOT, "public/favicon.svg"); + if (existsSync(faviconSrc)) { + cpSync(faviconSrc, resolve(outputDir, "favicon.svg")); + } + + console.log("\n" + "=".repeat(50)); + console.log("Build complete!"); + console.log(`Output: ${outputDir}`); + console.log("\nStructure:"); + console.log(" dist/"); + console.log(" index.html"); + for (const example of EXAMPLES) { + console.log(` ${example.name}/`); + } +} + +main().catch((error) => { + console.error("Build failed:", error); + process.exit(1); +}); diff --git a/scripts/validate-models.ts b/scripts/validate-models.ts index b9aa7c4..ec9a0c8 100644 --- a/scripts/validate-models.ts +++ b/scripts/validate-models.ts @@ -1,21 +1,24 @@ #!/usr/bin/env npx tsx /** - * Validates that all Malloy models in a directory have valid syntax. + * Validates that all Malloy models in examples directories have valid syntax. * This performs a basic syntax check - it cannot fully validate models * without a real database connection to fetch table schemas. * - * Usage: npx tsx scripts/validate-models.ts [models-directory] + * Usage: npx tsx scripts/validate-models.ts [optional-specific-directory] */ -import { readdirSync, readFileSync, statSync } from "node:fs"; -import { join, resolve, extname } from "node:path"; +import { readdirSync, readFileSync, statSync, existsSync } from "node:fs"; +import { join, resolve, extname, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; -const modelsDir = process.argv[2] ?? "./example/models"; -const resolvedDir = resolve(modelsDir); +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const PROJECT_ROOT = resolve(__dirname, ".."); interface ValidationResult { file: string; + dir: string; success: boolean; fileSize: number; hasSource: boolean; @@ -39,63 +42,63 @@ const SYNTAX_ERROR_PATTERNS = [ { pattern: /query:\s*$/m, message: "Incomplete query definition" }, ]; -function validateMalloyFile(filePath: string): ValidationResult { +function validateMalloyFile(filePath: string, dirName: string): ValidationResult { const fileName = filePath.split("/").pop() ?? filePath; try { const content = readFileSync(filePath, "utf-8"); const stats = statSync(filePath); - // Check for basic Malloy constructs const hasSource = MALLOY_PATTERNS.source.test(content); const hasQuery = MALLOY_PATTERNS.query.test(content); const hasView = MALLOY_PATTERNS.view.test(content); const hasImport = MALLOY_PATTERNS.import.test(content); - // A valid Malloy file should have at least one of these const hasValidContent = hasSource || hasQuery || hasView || hasImport; if (!hasValidContent && content.trim().length > 0) { return { file: fileName, + dir: dirName, success: false, fileSize: stats.size, hasSource, hasQuery, - error: "No valid Malloy constructs found (source, query, view, or import)", + error: "No valid Malloy constructs found", }; } - // Check for common syntax errors for (const { pattern, message } of SYNTAX_ERROR_PATTERNS) { if (pattern.test(content)) { return { file: fileName, + dir: dirName, success: false, fileSize: stats.size, hasSource, hasQuery, - error: `Potential syntax error: ${message}`, + error: `Syntax error: ${message}`, }; } } - // Check for balanced braces const openBraces = (content.match(/\{/g) || []).length; const closeBraces = (content.match(/\}/g) || []).length; if (openBraces !== closeBraces) { return { file: fileName, + dir: dirName, success: false, fileSize: stats.size, hasSource, hasQuery, - error: `Unbalanced braces: ${openBraces} opening, ${closeBraces} closing`, + error: `Unbalanced braces: ${openBraces} open, ${closeBraces} close`, }; } return { file: fileName, + dir: dirName, success: true, fileSize: stats.size, hasSource, @@ -104,107 +107,137 @@ function validateMalloyFile(filePath: string): ValidationResult { } catch (e) { return { file: fileName, + dir: dirName, success: false, fileSize: 0, hasSource: false, hasQuery: false, - error: `Failed to read file: ${e instanceof Error ? e.message : String(e)}`, + error: `Read error: ${e instanceof Error ? e.message : String(e)}`, }; } } -async function validateModels(): Promise { - console.log(`Validating Malloy models in: ${resolvedDir}\n`); +function validateNotebook(filePath: string, dirName: string): ValidationResult { + const fileName = filePath.split("/").pop() ?? filePath; - const files = readdirSync(resolvedDir); + try { + const content = readFileSync(filePath, "utf-8"); + const stats = statSync(filePath); + + const hasMalloyCell = content.includes(">>>malloy"); + const hasMarkdownCell = content.includes(">>>markdown"); + + if (hasMalloyCell || hasMarkdownCell) { + return { + file: fileName, + dir: dirName, + success: true, + fileSize: stats.size, + hasSource: false, + hasQuery: hasMalloyCell, + }; + } else { + return { + file: fileName, + dir: dirName, + success: false, + fileSize: stats.size, + hasSource: false, + hasQuery: false, + error: "No notebook cells found", + }; + } + } catch (e) { + return { + file: fileName, + dir: dirName, + success: false, + fileSize: 0, + hasSource: false, + hasQuery: false, + error: `Read error: ${e}`, + }; + } +} + +function validateDirectory(dirPath: string): ValidationResult[] { + const results: ValidationResult[] = []; + const dirName = dirPath.split("/").pop() ?? dirPath; + + if (!existsSync(dirPath)) { + return results; + } + + const files = readdirSync(dirPath); const malloyFiles = files.filter((f) => extname(f) === ".malloy"); const notebookFiles = files.filter((f) => extname(f) === ".malloynb"); - if (malloyFiles.length === 0 && notebookFiles.length === 0) { - console.log("No .malloy or .malloynb files found."); - process.exit(0); + for (const file of malloyFiles) { + results.push(validateMalloyFile(join(dirPath, file), dirName)); } - console.log(`Found ${malloyFiles.length} .malloy files and ${notebookFiles.length} .malloynb files\n`); + for (const file of notebookFiles) { + results.push(validateNotebook(join(dirPath, file), dirName)); + } - const results: ValidationResult[] = []; - let hasErrors = false; + return results; +} - // Validate .malloy files - for (const file of malloyFiles) { - const filePath = join(resolvedDir, file); - const result = validateMalloyFile(filePath); - results.push(result); - - if (result.success) { - const features = []; - if (result.hasSource) features.push("source"); - if (result.hasQuery) features.push("query"); - console.log(` ✓ ${file} (${features.join(", ") || "import only"})`); - } else { - hasErrors = true; - console.log(` ✗ ${file}`); - console.log(` Error: ${result.error}`); +async function main(): Promise { + const specificDir = process.argv[2]; + + let directories: string[]; + + if (specificDir) { + directories = [resolve(specificDir)]; + } else { + // Scan all examples directories + const examplesDir = join(PROJECT_ROOT, "examples"); + if (!existsSync(examplesDir)) { + console.error("No examples directory found"); + process.exit(1); } + directories = readdirSync(examplesDir) + .map((d) => join(examplesDir, d)) + .filter((d) => statSync(d).isDirectory()); } - // Basic check for .malloynb files (just verify they exist and have content) - for (const file of notebookFiles) { - const filePath = join(resolvedDir, file); - try { - const content = readFileSync(filePath, "utf-8"); - const stats = statSync(filePath); - - // Check for notebook markers - const hasMalloyCell = content.includes(">>>malloy"); - const hasMarkdownCell = content.includes(">>>markdown"); - - if (hasMalloyCell || hasMarkdownCell) { - console.log(` ✓ ${file} (notebook)`); - results.push({ - file, - success: true, - fileSize: stats.size, - hasSource: false, - hasQuery: hasMalloyCell, - }); - } else { - console.log(` ✗ ${file}`); - console.log(` Error: No >>>malloy or >>>markdown cells found`); - hasErrors = true; - results.push({ - file, - success: false, - fileSize: stats.size, - hasSource: false, - hasQuery: false, - error: "No notebook cells found", - }); + console.log("Validating Malloy models...\n"); + + const allResults: ValidationResult[] = []; + + for (const dir of directories) { + const dirName = dir.split("/").pop() ?? dir; + const results = validateDirectory(dir); + + if (results.length > 0) { + console.log(`[${dirName}]`); + for (const result of results) { + if (result.success) { + const type = result.file.endsWith(".malloynb") ? "notebook" : ""; + const features = []; + if (result.hasSource) features.push("source"); + if (result.hasQuery) features.push("query"); + const info = type || features.join(", ") || "import"; + console.log(` ✓ ${result.file} (${info})`); + } else { + console.log(` ✗ ${result.file}`); + console.log(` ${result.error}`); + } } - } catch (e) { - hasErrors = true; - console.log(` ✗ ${file}`); - console.log(` Error: Failed to read file`); - results.push({ - file, - success: false, - fileSize: 0, - hasSource: false, - hasQuery: false, - error: `Failed to read: ${e}`, - }); + console.log(""); + allResults.push(...results); } } - console.log(""); console.log("─".repeat(50)); - const passed = results.filter((r) => r.success).length; - const failed = results.filter((r) => !r.success).length; + const passed = allResults.filter((r) => r.success).length; + const failed = allResults.filter((r) => !r.success).length; console.log(`Results: ${passed} passed, ${failed} failed`); - if (hasErrors) { + if (failed > 0) { console.log("\nValidation completed with errors!"); process.exit(1); } else { @@ -213,7 +246,7 @@ async function validateModels(): Promise { } } -validateModels().catch((e) => { +main().catch((e) => { console.error("Validation script error:", e); process.exit(1); }); diff --git a/src/cli/build.ts b/src/cli/build.ts index 93bdeff..2202d6b 100644 --- a/src/cli/build.ts +++ b/src/cli/build.ts @@ -1,4 +1,10 @@ -import { build as viteBuild, createLogger, type InlineConfig } from "vite"; +import { + build as viteBuild, + preview as vitePreview, + createLogger, + type InlineConfig, + type PreviewServer, +} from "vite"; import { writeFileSync, rmSync, existsSync } from "node:fs"; import { join } from "node:path"; import type { DataExplorerConfig } from "./types.js"; @@ -10,26 +16,11 @@ export async function build( config: DataExplorerConfig, packageRoot: string ): Promise { - // Write config to a temporary JSON file that vite.config.ts will read const configPath = join(packageRoot, ".data-explorer-config.json"); - writeFileSync( - configPath, - JSON.stringify( - { - inputPath: config.inputPath, - outputPath: config.outputPath, - title: config.title, - description: config.description, - basePath: config.basePath, - }, - null, - 2 - ) - ); + writeFileSync(configPath, JSON.stringify(config, null, 2)); try { - // Set environment variable for vite.config.ts to find the config process.env["DATA_EXPLORER_CONFIG_PATH"] = configPath; const viteConfig: InlineConfig = { @@ -49,10 +40,35 @@ export async function build( console.log("Build completed successfully!"); console.log(`Output: ${config.outputPath}`); } finally { - // Clean up config file and env variable if (existsSync(configPath)) { rmSync(configPath); } delete process.env["DATA_EXPLORER_CONFIG_PATH"]; } } + +/** + * Preview the built site using Vite's preview server + */ +export async function preview( + outputPath: string, + port: number = 3000 +): Promise { + const viteConfig: InlineConfig = { + preview: { + port, + open: true, + }, + build: { + outDir: outputPath, + }, + }; + + const server = await vitePreview(viteConfig); + + console.log(""); + console.log(`Preview server running at:`); + server.printUrls(); + + return server; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 57b627b..c3699e6 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -4,7 +4,7 @@ import { parseArgs } from "node:util"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { existsSync, statSync } from "node:fs"; -import { build } from "./build.js"; +import { build, preview } from "./build.js"; import { DEFAULT_CONFIG, type DataExplorerConfig } from "./types.js"; const __filename = fileURLToPath(import.meta.url); @@ -15,22 +15,29 @@ data-explorer - Static site generator for Malloy data models and notebooks Usage: data-explorer build [options] + data-explorer preview [options] Commands: build Build the static site from Malloy files + preview Start a preview server for built site -Options: +Build Options: -o, --output Output directory for the built site (default: ./dist) -t, --title Site title (default: "Data Explorer") -d, --description <desc> Home page description -b, --base-path <path> Base public path for deployment (default: "/") + +Preview Options: + -p, --port <port> Port for preview server (default: 3000) + +General Options: -h, --help Show this help message -v, --version Show version number Examples: data-explorer build ./models data-explorer build ./models -o ./dist -t "My Data Site" - data-explorer build ./models --title "Sales Analytics" --description "Explore sales data" + data-explorer preview ./dist -p 8080 `; function showHelp(): void { @@ -43,14 +50,14 @@ function showVersion(): void { process.exit(0); } -function validateInputPath(inputPath: string): string { +function validatePath(inputPath: string, mustBeDir: boolean = true): string { const resolved = resolve(inputPath); if (!existsSync(resolved)) { - console.error(`Error: Input path does not exist: ${resolved}`); + console.error(`Error: Path does not exist: ${resolved}`); process.exit(1); } - if (!statSync(resolved).isDirectory()) { - console.error(`Error: Input path is not a directory: ${resolved}`); + if (mustBeDir && !statSync(resolved).isDirectory()) { + console.error(`Error: Path is not a directory: ${resolved}`); process.exit(1); } return resolved; @@ -64,6 +71,7 @@ async function main(): Promise<void> { title: { type: "string", short: "t" }, description: { type: "string", short: "d" }, "base-path": { type: "string", short: "b" }, + port: { type: "string", short: "p" }, help: { type: "boolean", short: "h" }, version: { type: "boolean", short: "v" }, }, @@ -80,10 +88,15 @@ async function main(): Promise<void> { const command = positionals[0]; if (!command) { - console.error("Error: No command specified. Use --help for usage information."); + console.error( + "Error: No command specified. Use --help for usage information." + ); process.exit(1); } + // __dirname is dist-cli/, so go up one level to get project root + const packageRoot = resolve(__dirname, ".."); + if (command === "build") { const inputPath = positionals[1]; if (!inputPath) { @@ -93,16 +106,13 @@ async function main(): Promise<void> { } const config: DataExplorerConfig = { - inputPath: validateInputPath(inputPath), + inputPath: validatePath(inputPath), outputPath: values.output ? resolve(values.output) : resolve("./dist"), title: values.title ?? DEFAULT_CONFIG.title, description: values.description ?? DEFAULT_CONFIG.description, basePath: values["base-path"] ?? DEFAULT_CONFIG.basePath, }; - // Get the package root (where vite.config.ts lives) - const packageRoot = resolve(__dirname, "../.."); - console.log("Building data-explorer site..."); console.log(` Input: ${config.inputPath}`); console.log(` Output: ${config.outputPath}`); @@ -112,13 +122,33 @@ async function main(): Promise<void> { console.log(""); await build(config, packageRoot); + } else if (command === "preview") { + const distPath = positionals[1]; + if (!distPath) { + console.error("Error: No dist path specified for preview command."); + console.error("Usage: data-explorer preview <dist-path> [options]"); + process.exit(1); + } + + const outputPath = validatePath(distPath); + const port = values.port ? parseInt(values.port, 10) : 3000; + + console.log(`Starting preview server for: ${outputPath}`); + console.log(`Port: ${port}`); + + await preview(outputPath, port); + + // Keep the server running + await new Promise(() => {}); } else { - console.error(`Error: Unknown command "${command}". Use --help for usage information.`); + console.error( + `Error: Unknown command "${command}". Use --help for usage information.` + ); process.exit(1); } } main().catch((error: unknown) => { - console.error("Build failed:", error); + console.error("Error:", error); process.exit(1); }); diff --git a/tests/cli/cli.test.ts b/tests/cli/cli.test.ts index c2ea26d..9d22eee 100644 --- a/tests/cli/cli.test.ts +++ b/tests/cli/cli.test.ts @@ -1,11 +1,11 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { execSync, spawn, type ChildProcess } from "node:child_process"; +import { execSync } from "node:child_process"; import { existsSync, rmSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; const PROJECT_ROOT = resolve(__dirname, "../.."); const CLI_PATH = join(PROJECT_ROOT, "dist-cli/index.js"); -const EXAMPLE_MODELS = join(PROJECT_ROOT, "example/models"); +const EXAMPLE_ECOMMERCE = join(PROJECT_ROOT, "examples/ecommerce"); // Build CLI before tests beforeAll(() => { @@ -22,9 +22,11 @@ describe("CLI smoke tests", () => { expect(output).toContain("data-explorer"); expect(output).toContain("build"); + expect(output).toContain("preview"); expect(output).toContain("--output"); expect(output).toContain("--title"); expect(output).toContain("--description"); + expect(output).toContain("--port"); }); }); @@ -72,20 +74,18 @@ describe("CLI smoke tests", () => { const testOutputDir = join(PROJECT_ROOT, "test-output-cli"); afterAll(() => { - // Clean up test output if (existsSync(testOutputDir)) { rmSync(testOutputDir, { recursive: true, force: true }); } }); it("should build example site successfully", () => { - // Clean up any previous test output if (existsSync(testOutputDir)) { rmSync(testOutputDir, { recursive: true, force: true }); } const output = execSync( - `node ${CLI_PATH} build ${EXAMPLE_MODELS} -o ${testOutputDir} -t "Test Site" -d "Test description"`, + `node ${CLI_PATH} build ${EXAMPLE_ECOMMERCE} -o ${testOutputDir} -t "Test Site" -d "Test description"`, { encoding: "utf-8", cwd: PROJECT_ROOT, @@ -94,22 +94,18 @@ describe("CLI smoke tests", () => { ); expect(output).toContain("Build completed successfully"); - - // Verify output files exist expect(existsSync(testOutputDir)).toBe(true); expect(existsSync(join(testOutputDir, "index.html"))).toBe(true); expect(existsSync(join(testOutputDir, "assets"))).toBe(true); }, 120000); - it("should include custom title in built site", () => { - // The index.html should contain the title (set via JS, but we can check for the bundle) + it("should produce valid HTML structure", () => { const indexHtml = readFileSync( join(testOutputDir, "index.html"), "utf-8" ); - // The HTML should at least have the basic structure - expect(indexHtml).toContain("<!DOCTYPE html>"); + expect(indexHtml.toLowerCase()).toContain("<!doctype html>"); expect(indexHtml).toContain('<div id="root">'); }); }); diff --git a/vite.config.ts b/vite.config.ts index 594e744..4c6c10f 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -28,14 +28,15 @@ function loadConfig(): DataExplorerConfig | null { const cliConfig = loadConfig(); -// Default values for development (uses example/models) +// Default values for development (uses examples/sample-data) const siteConfig = { title: cliConfig?.title ?? "Data Explorer", description: cliConfig?.description ?? "Explore and analyze your Malloy models and notebooks", basePath: cliConfig?.basePath ?? process.env["BASE_PUBLIC_PATH"] ?? "/", - modelsPath: cliConfig?.inputPath ?? resolve(process.cwd(), "example/models"), + modelsPath: + cliConfig?.inputPath ?? resolve(process.cwd(), "examples/sample-data"), }; // https://vite.dev/config/ From 850312a1c1ce924cd6a68b805c3447b87a468c86 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Mon, 26 Jan 2026 04:45:04 +0000 Subject: [PATCH 3/3] feat: update CI to build landing page with examples for GitHub Pages - Updated build-all.ts to accept BASE_PATH environment variable - CI now builds CLI first, then all example sites with landing page - Added CLI smoke tests and model validation to test job - Updated deployment test to use sample-data example (which has invoices model) - Landing page uses relative links, example sites use full base path --- .github/workflows/ci_cd.yml | 21 +++-- scripts/build-all.ts | 159 ++++++++++++++++++++++++++++++++---- 2 files changed, 157 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index da492db..e86959e 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -50,7 +50,13 @@ jobs: npx playwright install fi - - name: Run tests on CI + - name: Run CLI smoke tests + run: npm run test:cli + + - name: Validate Malloy models + run: npm run test:models + + - name: Run E2E tests run: npm test - uses: actions/upload-artifact@v6 @@ -72,11 +78,13 @@ jobs: node-version: lts/* - name: Install dependencies run: npm ci - - name: Build website - run: npm run build - # For gh pages deployment, base path is repository name + - name: Build CLI + run: npm run build:cli + + - name: Build all example sites with landing page + run: npm run build:all env: - BASE_PUBLIC_PATH: /${{ github.event.repository.name }}/ + BASE_PATH: /${{ github.event.repository.name }}/ - name: Upload Build Artifact # only run on pushes to master or workflow_dispatch @@ -123,9 +131,10 @@ jobs: - name: Install Playwright Browsers run: npx playwright install --with-deps - name: Run tests on deployed site + # Tests expect the sample-data example which contains the invoices model run: npm test env: - URL: ${{ needs.deploy.outputs.page_url }} + URL: ${{ needs.deploy.outputs.page_url }}sample-data/ - uses: actions/upload-artifact@v6 if: ${{ !cancelled() }} with: diff --git a/scripts/build-all.ts b/scripts/build-all.ts index 97cd007..490635e 100644 --- a/scripts/build-all.ts +++ b/scripts/build-all.ts @@ -2,6 +2,10 @@ /** * Builds all example sites and assembles them with the landing page. + * + * Environment variables: + * BASE_PATH - Base path for deployment (e.g., "/data-explorer/" for GitHub Pages) + * * Output structure: * dist/ * index.html (landing page) @@ -19,17 +23,25 @@ import { rmSync, cpSync, readdirSync, + readFileSync, + writeFileSync, } from "node:fs"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const PROJECT_ROOT = resolve(__dirname, ".."); +// Get base path from environment, default to "/" +const BASE_PATH = process.env["BASE_PATH"] || "/"; +const normalizedBasePath = BASE_PATH.endsWith("/") ? BASE_PATH : BASE_PATH + "/"; + interface Example { name: string; inputPath: string; title: string; description: string; + tag: string; + tagLabel: string; } const EXAMPLES: Example[] = [ @@ -37,45 +49,159 @@ const EXAMPLES: Example[] = [ name: "ecommerce", inputPath: "examples/ecommerce", title: "E-commerce Analytics", - description: "Explore orders, products, and customer data", + description: "Orders, products, and customer analysis with multi-table joins and visualizations.", + tag: "Local CSV", + tagLabel: "local", }, { name: "huggingface", inputPath: "examples/huggingface", title: "Hugging Face Datasets", - description: "GitHub Events, IMDB Movies, and NYC Taxi data", + description: "GitHub Events, IMDB Movies, and NYC Taxi data from remote Hugging Face datasets.", + tag: "Remote Data", + tagLabel: "remote", }, { name: "sample-data", inputPath: "examples/sample-data", title: "Sample Data Collection", - description: "Various local data formats and models", + description: "Various local data formats including CSV, Parquet, JSON, and Excel files.", + tag: "Mixed Formats", + tagLabel: "mixed", }, ]; async function buildExample(example: Example, outputBase: string): Promise<void> { const { build } = await import("../src/cli/build.js"); + // Combine base path with example name + const exampleBasePath = `${normalizedBasePath}${example.name}/`; + const config = { inputPath: resolve(PROJECT_ROOT, example.inputPath), outputPath: resolve(outputBase, example.name), title: example.title, description: example.description, - basePath: `/${example.name}/`, + basePath: exampleBasePath, }; console.log(`\nBuilding: ${example.name}`); - console.log(` Input: ${config.inputPath}`); - console.log(` Output: ${config.outputPath}`); + console.log(` Input: ${config.inputPath}`); + console.log(` Output: ${config.outputPath}`); + console.log(` Base Path: ${config.basePath}`); await build(config, PROJECT_ROOT); } +function generateLandingPage(): string { + const exampleCards = EXAMPLES.map( + (ex) => ` + <a href="./${ex.name}/" class="example-card"> + <h3>${ex.title}</h3> + <p>${ex.description}</p> + <span class="tag">${ex.tag}</span> + </a>` + ).join(""); + + return `<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="icon" type="image/svg+xml" href="favicon.svg"> + <link rel="preconnect" href="https://fonts.googleapis.com"> + <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> + <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> + <title>Data Explorer - Malloy Static Site Generator + + + +
+
+

Data Explorer

+

Static site generator for Malloy data models and notebooks

+
+
+

Example Sites

+
${exampleCards} +
+
+
+
+

Quick Start

+
+ # Build a site from your Malloy models
+npx @aszenz/data-explorer build ./models -o ./dist

+# Preview the built site
+npx @aszenz/data-explorer preview ./dist

+# Customize title and description
+npx @aszenz/data-explorer build ./models \\
+  --title "My Analytics" \\
+  --description "Explore data" \\
+  --output ./dist +
+
+
+ +
+ +`; +} + async function main(): Promise { const outputDir = resolve(PROJECT_ROOT, "dist"); console.log("Building all examples..."); console.log(`Output directory: ${outputDir}`); + console.log(`Base path: ${normalizedBasePath}`); // Clean output directory if (existsSync(outputDir)) { @@ -94,16 +220,10 @@ async function main(): Promise { } } - // Copy landing page - console.log("\nCopying landing page..."); - const landingDir = resolve(PROJECT_ROOT, "landing"); - if (existsSync(landingDir)) { - for (const file of readdirSync(landingDir)) { - cpSync(resolve(landingDir, file), resolve(outputDir, file), { - recursive: true, - }); - } - } + // Generate and write landing page + console.log("\nGenerating landing page..."); + const landingPageHtml = generateLandingPage(); + writeFileSync(resolve(outputDir, "index.html"), landingPageHtml); // Copy favicon to root const faviconSrc = resolve(PROJECT_ROOT, "public/favicon.svg"); @@ -111,12 +231,17 @@ async function main(): Promise { cpSync(faviconSrc, resolve(outputDir, "favicon.svg")); } + // Create .nojekyll file for GitHub Pages + writeFileSync(resolve(outputDir, ".nojekyll"), ""); + console.log("\n" + "=".repeat(50)); console.log("Build complete!"); console.log(`Output: ${outputDir}`); + console.log(`Base path: ${normalizedBasePath}`); console.log("\nStructure:"); console.log(" dist/"); - console.log(" index.html"); + console.log(" index.html (landing page)"); + console.log(" .nojekyll"); for (const example of EXAMPLES) { console.log(` ${example.name}/`); }