Skip to content

Commit 3bca2f3

Browse files
committed
Revert "Remove Dagger integration and replace with stub"
This reverts commit 1e9ee36.
1 parent df0f5cc commit 3bca2f3

File tree

14 files changed

+1313
-98
lines changed

14 files changed

+1313
-98
lines changed

experimental/apps-mcp/README.md

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,13 @@ extensive validation to ensure high-quality outputs.
1212
1. **Explore your data** - Query Databricks catalogs, schemas, and tables to understand your data
1313
2. **Generate the app** - Scaffold a full-stack TypeScript application (tRPC + React) with proper structure
1414
3. **Customize with AI** - Use workspace tools to read, write, and edit files naturally through conversation
15-
4. **Validate rigorously** - Run builds, type checks, and tests to ensure quality
15+
4. **Validate rigorously** - Run builds, type checks, and tests in isolated containers (Dagger)
1616
5. **Deploy confidently** - Push validated apps directly to Databricks Apps platform
1717

1818
**Why use it:**
1919
- **Speed**: Go from concept to deployed Databricks app in minutes, not hours or days
2020
- **Quality**: Extensive validation ensures your app builds, passes tests, and is production-ready
21+
- **Safety**: Containerized validation prevents breaking changes from reaching production
2122
- **Simplicity**: One natural language conversation handles the entire workflow
2223

2324
Perfect for data engineers and developers who want to build Databricks apps without the manual overhead of project setup, configuration, testing infrastructure, and deployment pipelines.
@@ -103,10 +104,11 @@ Create the application structure:
103104

104105
Ensure production-readiness before deployment:
105106

106-
- **`validate_data_app`** - Comprehensive validation
107+
- **`validate_data_app`** - Comprehensive validation in isolated containers
107108
- Build verification (npm build)
108109
- Type checking (TypeScript compiler)
109110
- Test execution (full test suite)
111+
- Containerized with Dagger (Docker)
110112

111113
*This step guarantees your application is tested and ready for production before deployment.*
112114

@@ -258,12 +260,17 @@ Deploy the app to Databricks as "orders-dashboard"
258260
- Every change is validated before deployment
259261
- No broken builds reach production
260262

261-
**2. Natural Language = Productivity**
263+
**2. Containerized Validation = Safety**
264+
- Dagger containers ensure reproducible builds
265+
- Isolated testing prevents environment issues
266+
- Consistent behavior from development to production
267+
268+
**3. Natural Language = Productivity**
262269
- Describe what you want, not how to build it
263270
- AI handles implementation details
264271
- Focus on requirements, not configuration
265272

266-
**3. End-to-End Workflow = Simplicity**
273+
**4. End-to-End Workflow = Simplicity**
267274
- Single tool for entire lifecycle
268275
- No context switching between tools
269276
- Seamless progression from idea to deployment
@@ -276,6 +283,7 @@ The Databricks MCP server doesn't just generate code—it ensures quality:
276283
-**Build verification** - Ensures code compiles
277284
-**Test suite** - Validates functionality
278285
-**Linting** - Enforces code quality
286+
-**Containerization** - Reproducible environments
279287
-**Databricks integration** - Native SDK usage
280288

281289
---
@@ -293,6 +301,15 @@ databricks experimental apps-mcp --warehouse-id <warehouse-id> --with-workspace-
293301

294302
# Enable deployment
295303
databricks experimental apps-mcp --warehouse-id <warehouse-id> --allow-deployment
304+
305+
# With custom Docker image
306+
databricks experimental apps-mcp --warehouse-id <warehouse-id> --docker-image node:20-alpine
307+
308+
# Without containerized validation
309+
databricks experimental apps-mcp --warehouse-id <warehouse-id> --use-dagger=false
310+
311+
# Check configuration
312+
databricks experimental apps-mcp check
296313
```
297314

298315
### CLI Flags
@@ -302,6 +319,8 @@ databricks experimental apps-mcp --warehouse-id <warehouse-id> --allow-deploymen
302319
| `--warehouse-id` | Databricks SQL Warehouse ID (required) | - |
303320
| `--with-workspace-tools` | Enable workspace file operations | `false` |
304321
| `--allow-deployment` | Enable deployment operations | `false` |
322+
| `--docker-image` | Docker image for validation | `node:20-alpine` |
323+
| `--use-dagger` | Use Dagger for containerized validation | `true` |
305324
| `--help` | Show help | - |
306325

307326
### Environment Variables
@@ -314,6 +333,7 @@ databricks experimental apps-mcp --warehouse-id <warehouse-id> --allow-deploymen
314333
| `DATABRICKS_WAREHOUSE_ID` | Alternative name for warehouse ID | `abc123def456` |
315334
| `ALLOW_DEPLOYMENT` | Enable deployment operations | `true` or `false` |
316335
| `WITH_WORKSPACE_TOOLS` | Enable workspace tools | `true` or `false` |
336+
| `USE_DAGGER` | Enable Dagger sandbox for validation | `true` or `false` |
317337

318338
### Authentication
319339

@@ -328,6 +348,7 @@ For more details, see the [Databricks authentication documentation](https://docs
328348
### Requirements
329349

330350
- **Databricks CLI** (this package)
351+
- **Docker** (optional, for Dagger-based validation)
331352
- **Databricks workspace** with a SQL warehouse
332353
- **MCP-compatible client** (Claude Desktop, Continue, etc.)
333354

experimental/apps-mcp/cmd/apps_mcp.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ func NewMcpCmd() *cobra.Command {
1313
var warehouseID string
1414
var allowDeployment bool
1515
var withWorkspaceTools bool
16+
var dockerImage string
17+
var useDagger bool
1618

1719
cmd := &cobra.Command{
1820
Use: "apps-mcp",
@@ -33,7 +35,13 @@ The server communicates via stdio using the Model Context Protocol.`,
3335
databricks experimental apps-mcp --warehouse-id abc123 --with-workspace-tools
3436
3537
# Start with deployment tools enabled
36-
databricks experimental apps-mcp --warehouse-id abc123 --allow-deployment`,
38+
databricks experimental apps-mcp --warehouse-id abc123 --allow-deployment
39+
40+
# Start with custom Docker image for validation
41+
databricks experimental apps-mcp --warehouse-id abc123 --docker-image node:20-alpine
42+
43+
# Start without containerized validation
44+
databricks experimental apps-mcp --warehouse-id abc123 --use-dagger=false`,
3745
PreRunE: root.MustWorkspaceClient,
3846
RunE: func(cmd *cobra.Command, args []string) error {
3947
ctx := cmd.Context()
@@ -47,7 +55,10 @@ The server communicates via stdio using the Model Context Protocol.`,
4755
WarehouseID: warehouseID,
4856
DatabricksHost: w.Config.Host,
4957
IoConfig: &mcplib.IoConfig{
50-
Validation: &mcplib.ValidationConfig{},
58+
Validation: &mcplib.ValidationConfig{
59+
DockerImage: dockerImage,
60+
UseDagger: useDagger,
61+
},
5162
},
5263
}
5364

@@ -71,6 +82,10 @@ The server communicates via stdio using the Model Context Protocol.`,
7182
cmd.Flags().StringVar(&warehouseID, "warehouse-id", "", "Databricks SQL Warehouse ID")
7283
cmd.Flags().BoolVar(&allowDeployment, "allow-deployment", false, "Enable deployment tools")
7384
cmd.Flags().BoolVar(&withWorkspaceTools, "with-workspace-tools", false, "Enable workspace tools (file operations, bash, grep, glob)")
85+
cmd.Flags().StringVar(&dockerImage, "docker-image", "node:20-alpine", "Docker image for validation")
86+
cmd.Flags().BoolVar(&useDagger, "use-dagger", true, "Use Dagger for containerized validation")
87+
88+
cmd.AddCommand(newCheckCmd())
7489

7590
return cmd
7691
}

experimental/apps-mcp/cmd/check.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package mcp
2+
3+
import (
4+
"github.com/databricks/cli/cmd/root"
5+
"github.com/databricks/cli/libs/cmdctx"
6+
"github.com/databricks/cli/libs/cmdio"
7+
"github.com/databricks/cli/libs/log"
8+
"github.com/spf13/cobra"
9+
)
10+
11+
func newCheckCmd() *cobra.Command {
12+
cmd := &cobra.Command{
13+
Use: "check",
14+
Short: "Check MCP server environment",
15+
Long: `Verify that the environment is correctly configured for running the MCP server.
16+
17+
This command checks:
18+
- Databricks authentication (API token, profile, or other auth methods)
19+
- Workspace connectivity
20+
21+
Use this command to troubleshoot connection issues before starting the MCP server.`,
22+
Example: ` # Check environment configuration
23+
databricks experimental apps-mcp check
24+
25+
# Check with specific profile
26+
databricks experimental apps-mcp check --profile production`,
27+
PreRunE: root.MustWorkspaceClient,
28+
RunE: func(cmd *cobra.Command, args []string) error {
29+
ctx := cmd.Context()
30+
31+
log.Info(ctx, "Checking MCP server environment")
32+
33+
// Check Databricks authentication
34+
w := cmdctx.WorkspaceClient(ctx)
35+
me, err := w.CurrentUser.Me(ctx)
36+
if err != nil {
37+
return err
38+
}
39+
40+
cmdio.LogString(ctx, "✓ Databricks authentication: OK")
41+
cmdio.LogString(ctx, " User: "+me.UserName)
42+
cmdio.LogString(ctx, " Host: "+w.Config.Host)
43+
44+
cmdio.LogString(ctx, "\nEnvironment is ready for MCP server")
45+
46+
return nil
47+
},
48+
}
49+
50+
return cmd
51+
}

experimental/apps-mcp/lib/config.go

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type Config struct {
1616
type IoConfig struct {
1717
Template *TemplateConfig
1818
Validation *ValidationConfig
19+
Dagger *DaggerConfig
1920
}
2021

2122
// TemplateConfig specifies which template to use for scaffolding new projects.
@@ -24,19 +25,30 @@ type TemplateConfig struct {
2425
Path string
2526
}
2627

27-
// ValidationConfig defines custom validation commands for project validation.
28+
// ValidationConfig defines custom validation commands and docker images for project validation.
2829
type ValidationConfig struct {
29-
Command string
30-
Timeout int
30+
Command string
31+
DockerImage string
32+
UseDagger bool
33+
Timeout int
3134
}
3235

3336
// SetDefaults applies default values to ValidationConfig if not explicitly set.
3437
func (v *ValidationConfig) SetDefaults() {
38+
if v.DockerImage == "" {
39+
v.DockerImage = "node:20-alpine"
40+
}
3541
if v.Timeout == 0 {
3642
v.Timeout = 600
3743
}
3844
}
3945

46+
// DaggerConfig configures the Dagger sandbox when use_dagger is enabled.
47+
type DaggerConfig struct {
48+
Image string
49+
ExecuteTimeout int
50+
}
51+
4052
// DefaultConfig returns a Config with sensible default values.
4153
func DefaultConfig() *Config {
4254
validationCfg := &ValidationConfig{}
@@ -51,6 +63,10 @@ func DefaultConfig() *Config {
5163
Path: "",
5264
},
5365
Validation: validationCfg,
66+
Dagger: &DaggerConfig{
67+
Image: "node:20-alpine",
68+
ExecuteTimeout: 600,
69+
},
5470
},
5571
WarehouseID: "",
5672
}

experimental/apps-mcp/lib/providers/io/validate.go

Lines changed: 98 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ import (
77
"os"
88
"path/filepath"
99

10+
mcp "github.com/databricks/cli/experimental/apps-mcp/lib"
11+
"github.com/databricks/cli/experimental/apps-mcp/lib/sandbox"
12+
"github.com/databricks/cli/experimental/apps-mcp/lib/sandbox/dagger"
1013
"github.com/databricks/cli/experimental/apps-mcp/lib/sandbox/local"
1114
"github.com/databricks/cli/libs/log"
1215
)
@@ -44,7 +47,7 @@ func (p *Provider) Validate(ctx context.Context, args *ValidateArgs) (*ValidateR
4447
valConfig := p.config.Validation
4548
if valConfig.Command != "" {
4649
log.Infof(ctx, "using custom validation command: command=%s", valConfig.Command)
47-
validation = NewValidationCmd(valConfig.Command, "")
50+
validation = NewValidationCmd(valConfig.Command, valConfig.DockerImage)
4851
}
4952
}
5053

@@ -53,12 +56,45 @@ func (p *Provider) Validate(ctx context.Context, args *ValidateArgs) (*ValidateR
5356
validation = NewValidationTRPC()
5457
}
5558

56-
log.Info(ctx, "using local sandbox for validation")
57-
sb, err := p.createLocalSandbox(workDir)
58-
if err != nil {
59-
return nil, fmt.Errorf("failed to create local sandbox: %w", err)
59+
validationCfg := p.config.Validation
60+
if validationCfg == nil {
61+
validationCfg = &mcp.ValidationConfig{}
62+
validationCfg.SetDefaults()
63+
} else {
64+
validationCfg.SetDefaults()
65+
}
66+
67+
var sb sandbox.Sandbox
68+
var sandboxType string
69+
if validationCfg.UseDagger {
70+
log.Info(ctx, "attempting to create Dagger sandbox")
71+
daggerSb, err := p.createDaggerSandbox(ctx, workDir, validationCfg)
72+
if err != nil {
73+
log.Warnf(ctx, "failed to create Dagger sandbox, falling back to local: error=%s", err.Error())
74+
sb, err = p.createLocalSandbox(workDir)
75+
if err != nil {
76+
return nil, fmt.Errorf("failed to create local sandbox: %w", err)
77+
}
78+
sandboxType = "local"
79+
} else {
80+
sb = daggerSb
81+
sandboxType = "dagger"
82+
}
83+
} else {
84+
log.Info(ctx, "using local sandbox")
85+
sb, err = p.createLocalSandbox(workDir)
86+
if err != nil {
87+
return nil, fmt.Errorf("failed to create local sandbox: %w", err)
88+
}
89+
sandboxType = "local"
90+
}
91+
92+
// Log which sandbox is being used for transparency
93+
if sandboxType == "dagger" {
94+
log.Info(ctx, "✓ Using Dagger sandbox for validation (containerized, isolated environment)")
95+
} else {
96+
log.Info(ctx, "Using local sandbox for validation (host filesystem)")
6097
}
61-
sandboxType := "local"
6298

6399
defer func() {
64100
if closeErr := sb.Close(); closeErr != nil {
@@ -101,7 +137,62 @@ func (p *Provider) Validate(ctx context.Context, args *ValidateArgs) (*ValidateR
101137
return result, nil
102138
}
103139

104-
func (p *Provider) createLocalSandbox(workDir string) (*local.LocalSandbox, error) {
140+
func (p *Provider) createDaggerSandbox(ctx context.Context, workDir string, cfg *mcp.ValidationConfig) (sandbox.Sandbox, error) {
141+
log.Infof(ctx, "creating Dagger sandbox: image=%s, timeout=%d, workDir=%s",
142+
cfg.DockerImage, cfg.Timeout, workDir)
143+
144+
sb, err := dagger.NewDaggerSandbox(ctx, dagger.Config{
145+
Image: cfg.DockerImage,
146+
ExecuteTimeout: cfg.Timeout,
147+
BaseDir: "/workspace",
148+
})
149+
if err != nil {
150+
log.Errorf(ctx, "failed to create Dagger sandbox: error=%s, image=%s",
151+
err.Error(), cfg.DockerImage)
152+
return nil, err
153+
}
154+
155+
log.Debug(ctx, "propagating environment variables")
156+
if err := p.propagateEnvironment(sb); err != nil {
157+
log.Errorf(ctx, "failed to propagate environment: error=%s", err.Error())
158+
sb.Close()
159+
return nil, fmt.Errorf("failed to set environment: %w", err)
160+
}
161+
162+
log.Debugf(ctx, "syncing files from host to container: workDir=%s", workDir)
163+
if err := sb.RefreshFromHost(ctx, workDir, "/workspace"); err != nil {
164+
log.Errorf(ctx, "failed to sync files: error=%s", err.Error())
165+
sb.Close()
166+
return nil, fmt.Errorf("failed to sync files: %w", err)
167+
}
168+
169+
log.Info(ctx, "Dagger sandbox created successfully")
170+
return sb, nil
171+
}
172+
173+
func (p *Provider) createLocalSandbox(workDir string) (sandbox.Sandbox, error) {
105174
log.Infof(p.ctx, "creating local sandbox: workDir=%s", workDir)
106175
return local.NewLocalSandbox(workDir)
107176
}
177+
178+
func (p *Provider) propagateEnvironment(sb sandbox.Sandbox) error {
179+
daggerSb, ok := sb.(*dagger.DaggerSandbox)
180+
if !ok {
181+
return nil
182+
}
183+
184+
envVars := []string{
185+
"DATABRICKS_HOST",
186+
"DATABRICKS_TOKEN",
187+
"DATABRICKS_WAREHOUSE_ID",
188+
}
189+
190+
for _, key := range envVars {
191+
if value := os.Getenv(key); value != "" {
192+
daggerSb.WithEnv(key, value)
193+
log.Debugf(p.ctx, "propagated environment variable: key=%s", key)
194+
}
195+
}
196+
197+
return nil
198+
}

0 commit comments

Comments
 (0)