diff --git a/docs/workflow/modular-schema-files.mdx b/docs/workflow/modular-schema-files.mdx index 97825960..5e841a8f 100644 --- a/docs/workflow/modular-schema-files.mdx +++ b/docs/workflow/modular-schema-files.mdx @@ -242,6 +242,46 @@ schema/ The key is ensuring your `main.sql` file contains the correct `\i` directives that match your chosen file organization. +### Folder Includes + +The `\i` directive supports including entire folders by adding a trailing slash (`/`). This automatically includes all `.sql` files in the folder in alphabetical order, and recursively processes any subdirectories: + +``` +schema/ +├── main.sql +├── types/ +│ ├── address.sql +│ ├── order_status.sql +│ └── user_status.sql +├── tables/ +│ ├── orders.sql +│ └── users.sql +└── functions/ + ├── auth/ + │ └── validate_user.sql + ├── calculate_total.sql + └── update_timestamp.sql +``` + +Using folder includes in `main.sql`: +```sql +-- Include all types (processed alphabetically: address, order_status, user_status) +\i types/ + +-- Include all tables (processed alphabetically: orders, users) +\i tables/ + +-- Include all functions recursively (auth/validate_user, calculate_total, update_timestamp) +\i functions/ +``` + +Key behaviors: +- Files are processed in **alphabetical order** by filename +- Subdirectories are processed recursively using **depth-first search** +- Only `.sql` files are included; other files are ignored +- Folder paths must end with `/` to be recognized as folders +- Error if folder doesn't exist or if you try to include a file as a folder + ### Nested Includes The `\i` directive can be nested, allowing for hierarchical file organization: @@ -282,4 +322,15 @@ With nested includes: \i functions.sql ``` +You can also combine folder includes with nested approaches: +```sql +-- main.sql - mix folder and file includes +\i core/ +\i modules/auth/auth.sql +\i modules/reporting/ + +-- core/ folder contains individual files that get included alphabetically +-- modules/reporting/ folder gets all files included recursively +``` + This approach allows for modular organization where each subsystem manages its own includes. \ No newline at end of file diff --git a/internal/include/processor.go b/internal/include/processor.go index 79d1e704..d71d1065 100644 --- a/internal/include/processor.go +++ b/internal/include/processor.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" ) @@ -83,19 +84,28 @@ func (p *Processor) processIncludes(content string, currentDir string) (string, if matches != nil { // Found an include directive includePath := matches[1] - + // Resolve the include path - resolvedPath, err := p.resolveIncludePath(includePath, currentDir) + resolvedPath, isFolder, err := p.resolveIncludePath(includePath, currentDir) if err != nil { return "", fmt.Errorf("failed to resolve include path %s: %w", includePath, err) } - - // Process the included file recursively - includedContent, err := p.processFileRecursive(resolvedPath) - if err != nil { - return "", fmt.Errorf("failed to process included file %s: %w", resolvedPath, err) + + var includedContent string + if isFolder { + // Process the folder recursively + includedContent, err = p.processFolderRecursive(resolvedPath) + if err != nil { + return "", fmt.Errorf("failed to process included folder %s: %w", resolvedPath, err) + } + } else { + // Process the included file recursively + includedContent, err = p.processFileRecursive(resolvedPath) + if err != nil { + return "", fmt.Errorf("failed to process included file %s: %w", resolvedPath, err) + } } - + // Split included content into lines and add them includedLines := strings.Split(includedContent, "\n") // Remove the last empty line if the content ends with \n @@ -114,40 +124,108 @@ func (p *Processor) processIncludes(content string, currentDir string) (string, // resolveIncludePath resolves an include path relative to the current directory // Only allows files within the base directory and its subdirectories -func (p *Processor) resolveIncludePath(includePath string, currentDir string) (string, error) { +// Returns the resolved path and a flag indicating if it's a folder +func (p *Processor) resolveIncludePath(includePath string, currentDir string) (string, bool, error) { + // Check if this is a folder path (ends with /) + isFolder := strings.HasSuffix(includePath, "/") + // Clean the path to remove any . or .. components cleanPath := filepath.Clean(includePath) - + // Check for directory traversal attempts if strings.Contains(cleanPath, "..") { - return "", fmt.Errorf("directory traversal not allowed: %s", includePath) + return "", false, fmt.Errorf("directory traversal not allowed: %s", includePath) } - + // Resolve relative to current directory resolvedPath := filepath.Join(currentDir, cleanPath) - + // Get absolute path absPath, err := filepath.Abs(resolvedPath) if err != nil { - return "", fmt.Errorf("failed to get absolute path: %w", err) + return "", false, fmt.Errorf("failed to get absolute path: %w", err) } - + // Ensure the resolved path is within the base directory baseAbs, err := filepath.Abs(p.baseDir) if err != nil { - return "", fmt.Errorf("failed to get absolute base path: %w", err) + return "", false, fmt.Errorf("failed to get absolute base path: %w", err) } - + // Check if the resolved path is within the base directory relPath, err := filepath.Rel(baseAbs, absPath) if err != nil || strings.HasPrefix(relPath, "..") { - return "", fmt.Errorf("include path %s is outside the base directory %s", includePath, p.baseDir) + return "", false, fmt.Errorf("include path %s is outside the base directory %s", includePath, p.baseDir) } - - // Check if file exists - if _, err := os.Stat(absPath); os.IsNotExist(err) { - return "", fmt.Errorf("included file does not exist: %s", absPath) + + // Check if path exists + stat, err := os.Stat(absPath) + if os.IsNotExist(err) { + if isFolder { + return "", false, fmt.Errorf("included folder does not exist: %s", absPath) + } else { + return "", false, fmt.Errorf("included file does not exist: %s", absPath) + } } - - return absPath, nil + if err != nil { + return "", false, fmt.Errorf("failed to stat path %s: %w", absPath, err) + } + + // Validate that the path type matches the expectation + if isFolder && !stat.IsDir() { + return "", false, fmt.Errorf("expected folder but found file: %s", absPath) + } + if !isFolder && stat.IsDir() { + return "", false, fmt.Errorf("expected file but found folder: %s (use %s/ for folder includes)", absPath, includePath) + } + + return absPath, isFolder, nil +} + +// processFolderRecursive processes all .sql files in a folder using DFS +func (p *Processor) processFolderRecursive(folderPath string) (string, error) { + // Read directory contents + entries, err := os.ReadDir(folderPath) + if err != nil { + return "", fmt.Errorf("failed to read directory %s: %w", folderPath, err) + } + + // Sort entries alphabetically (natural filename order) + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() < entries[j].Name() + }) + + var resultParts []string + + // Process each entry in alphabetical order + for _, entry := range entries { + entryPath := filepath.Join(folderPath, entry.Name()) + + if entry.IsDir() { + // Recursively process subdirectory (DFS) + subFolderContent, err := p.processFolderRecursive(entryPath) + if err != nil { + return "", fmt.Errorf("failed to process subdirectory %s: %w", entryPath, err) + } + if subFolderContent != "" { + resultParts = append(resultParts, subFolderContent) + } + } else if strings.HasSuffix(entry.Name(), ".sql") { + // Process .sql file + fileContent, err := p.processFileRecursive(entryPath) + if err != nil { + return "", fmt.Errorf("failed to process file %s: %w", entryPath, err) + } + if fileContent != "" { + // Ensure the file content ends with a newline for proper concatenation + if !strings.HasSuffix(fileContent, "\n") { + fileContent += "\n" + } + resultParts = append(resultParts, fileContent) + } + } + // Ignore non-.sql files + } + + return strings.Join(resultParts, ""), nil } \ No newline at end of file diff --git a/internal/include/processor_test.go b/internal/include/processor_test.go index 0226720a..e44566b0 100644 --- a/internal/include/processor_test.go +++ b/internal/include/processor_test.go @@ -279,6 +279,341 @@ CREATE TABLE users ( } } +func TestProcessFile_FolderInclude(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file with folder include + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i types/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create types directory with multiple files + typesDir := filepath.Join(tempDir, "types") + if err := os.MkdirAll(typesDir, 0755); err != nil { + t.Fatalf("Failed to create types dir: %v", err) + } + + // Create type files (should be processed in alphabetical order) + files := map[string]string{ + "zoo.sql": "CREATE TYPE zoo AS ENUM ('open', 'closed');", + "animal.sql": "CREATE TYPE animal AS ENUM ('cat', 'dog');", + "bird.sql": "CREATE TYPE bird AS ENUM ('eagle', 'robin');", + } + + for filename, content := range files { + filePath := filepath.Join(typesDir, filename) + if err := os.WriteFile(filePath, []byte(content), 0644); err != nil { + t.Fatalf("Failed to write %s: %v", filename, err) + } + } + + // Process the file + processor := NewProcessor(tempDir) + result, err := processor.ProcessFile(mainFile) + if err != nil { + t.Fatalf("ProcessFile failed: %v", err) + } + + // Check that the folder include was processed + if !strings.Contains(result, "CREATE TYPE animal") { + t.Error("animal.sql content not found in result") + } + if !strings.Contains(result, "CREATE TYPE bird") { + t.Error("bird.sql content not found in result") + } + if !strings.Contains(result, "CREATE TYPE zoo") { + t.Error("zoo.sql content not found in result") + } + + // Check that files are processed in alphabetical order + animalIdx := strings.Index(result, "CREATE TYPE animal") + birdIdx := strings.Index(result, "CREATE TYPE bird") + zooIdx := strings.Index(result, "CREATE TYPE zoo") + + if animalIdx == -1 || birdIdx == -1 || zooIdx == -1 { + t.Fatal("Not all type definitions found") + } + + if !(animalIdx < birdIdx && birdIdx < zooIdx) { + t.Error("Files not processed in alphabetical order") + t.Logf("Order found: animal=%d, bird=%d, zoo=%d", animalIdx, birdIdx, zooIdx) + } + + // Check that folder include directive was replaced + if strings.Contains(result, "\\i types/") { + t.Error("Folder include directive should have been replaced") + } +} + +func TestProcessFile_NestedFolderInclude(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i schema/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create nested directory structure + schemaDir := filepath.Join(tempDir, "schema") + typesDir := filepath.Join(schemaDir, "types") + tablesDir := filepath.Join(schemaDir, "tables") + + for _, dir := range []string{schemaDir, typesDir, tablesDir} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("Failed to create dir %s: %v", dir, err) + } + } + + // Create files in different directories + files := map[string]string{ + filepath.Join(schemaDir, "main.sql"): "-- Schema main file", + filepath.Join(typesDir, "user_type.sql"): "CREATE TYPE user_type AS ENUM ('admin', 'user');", + filepath.Join(tablesDir, "users.sql"): "CREATE TABLE users (id SERIAL);", + filepath.Join(typesDir, "status_type.sql"): "CREATE TYPE status_type AS ENUM ('active', 'inactive');", + } + + for filePath, content := range files { + if err := os.WriteFile(filePath, []byte(content), 0644); err != nil { + t.Fatalf("Failed to write %s: %v", filePath, err) + } + } + + // Process the file + processor := NewProcessor(tempDir) + result, err := processor.ProcessFile(mainFile) + if err != nil { + t.Fatalf("ProcessFile failed: %v", err) + } + + // Check that all content is included using DFS order + expected := []string{ + "-- Schema main file", + "CREATE TABLE users", // tables/ comes before types/ alphabetically + "CREATE TYPE status_type", // status_type.sql comes before user_type.sql + "CREATE TYPE user_type", + } + + lastIndex := -1 + for _, expectedContent := range expected { + index := strings.Index(result, expectedContent) + if index == -1 { + t.Errorf("Expected content not found: %s", expectedContent) + continue + } + if index < lastIndex { + t.Errorf("Content out of expected order: %s at position %d, previous was at %d", + expectedContent, index, lastIndex) + } + lastIndex = index + } +} + +func TestProcessFile_FolderNotFound(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file that includes non-existent folder + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i nonexistent/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Process the file - should report folder not found + processor := NewProcessor(tempDir) + _, err := processor.ProcessFile(mainFile) + if err == nil { + t.Fatal("Expected folder not found error") + } + if !strings.Contains(err.Error(), "does not exist") { + t.Errorf("Expected 'does not exist' error, got: %v", err) + } +} + +func TestProcessFile_EmptyFolder(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file with folder include + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i empty/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create empty directory + emptyDir := filepath.Join(tempDir, "empty") + if err := os.MkdirAll(emptyDir, 0755); err != nil { + t.Fatalf("Failed to create empty dir: %v", err) + } + + // Process the file + processor := NewProcessor(tempDir) + result, err := processor.ProcessFile(mainFile) + if err != nil { + t.Fatalf("ProcessFile failed: %v", err) + } + + // Check that only main file content remains + expected := `-- Main file +-- End of main file` + if result != expected { + t.Errorf("Expected:\n%s\nGot:\n%s", expected, result) + } +} + +func TestProcessFile_ExpectedFileButFoundFolder(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file trying to include a folder as file + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i somefolder +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create a directory with the same name + folderPath := filepath.Join(tempDir, "somefolder") + if err := os.MkdirAll(folderPath, 0755); err != nil { + t.Fatalf("Failed to create folder: %v", err) + } + + // Process the file - should report type mismatch + processor := NewProcessor(tempDir) + _, err := processor.ProcessFile(mainFile) + if err == nil { + t.Fatal("Expected type mismatch error") + } + if !strings.Contains(err.Error(), "expected file but found folder") { + t.Errorf("Expected 'expected file but found folder' error, got: %v", err) + } +} + +func TestProcessFile_ExpectedFolderButFoundFile(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file trying to include a file as folder + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i somefile.sql/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create a file with that name + filePath := filepath.Join(tempDir, "somefile.sql") + if err := os.WriteFile(filePath, []byte("CREATE TABLE test (id SERIAL);"), 0644); err != nil { + t.Fatalf("Failed to create file: %v", err) + } + + // Process the file - should report type mismatch + processor := NewProcessor(tempDir) + _, err := processor.ProcessFile(mainFile) + if err == nil { + t.Fatal("Expected type mismatch error") + } + if !strings.Contains(err.Error(), "expected folder but found file") { + t.Errorf("Expected 'expected folder but found file' error, got: %v", err) + } +} + +func TestProcessFile_MixedFilesAndFoldersInFolder(t *testing.T) { + // Create temporary directory + tempDir := t.TempDir() + + // Create main file + mainFile := filepath.Join(tempDir, "main.sql") + mainContent := `-- Main file +\i mixed/ +-- End of main file` + + if err := os.WriteFile(mainFile, []byte(mainContent), 0644); err != nil { + t.Fatalf("Failed to write main file: %v", err) + } + + // Create mixed directory with files and subdirectories + mixedDir := filepath.Join(tempDir, "mixed") + subDir := filepath.Join(mixedDir, "subdir") + + for _, dir := range []string{mixedDir, subDir} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("Failed to create dir %s: %v", dir, err) + } + } + + // Create files and non-SQL files + files := map[string]string{ + filepath.Join(mixedDir, "a_file.sql"): "-- A file content", + filepath.Join(mixedDir, "b_file.txt"): "This should be ignored", + filepath.Join(mixedDir, "z_file.sql"): "-- Z file content", + filepath.Join(subDir, "sub_file.sql"): "-- Sub file content", + } + + for filePath, content := range files { + if err := os.WriteFile(filePath, []byte(content), 0644); err != nil { + t.Fatalf("Failed to write %s: %v", filePath, err) + } + } + + // Process the file + processor := NewProcessor(tempDir) + result, err := processor.ProcessFile(mainFile) + if err != nil { + t.Fatalf("ProcessFile failed: %v", err) + } + + // Check that SQL files are included in proper order + if !strings.Contains(result, "-- A file content") { + t.Error("a_file.sql content not found") + } + if !strings.Contains(result, "-- Sub file content") { + t.Error("sub_file.sql content not found") + } + if !strings.Contains(result, "-- Z file content") { + t.Error("z_file.sql content not found") + } + + // Check that non-SQL files are ignored + if strings.Contains(result, "This should be ignored") { + t.Error("Non-SQL file content should be ignored") + } + + // Check alphabetical order: a_file.sql, subdir/sub_file.sql, z_file.sql + aIdx := strings.Index(result, "-- A file content") + subIdx := strings.Index(result, "-- Sub file content") + zIdx := strings.Index(result, "-- Z file content") + + if !(aIdx < subIdx && subIdx < zIdx) { + t.Error("Files not processed in expected alphabetical order") + t.Logf("Order found: a_file=%d, sub_file=%d, z_file=%d", aIdx, subIdx, zIdx) + } +} + func TestProcessFile_MatchesPreGeneratedSchema(t *testing.T) { // Test that processing main.sql produces the same output as schema.sql // This ensures the include processor works correctly with real test data diff --git a/testdata/include/expected_full_schema.sql b/testdata/include/expected_full_schema.sql index a71879a4..ec3d34cc 100644 --- a/testdata/include/expected_full_schema.sql +++ b/testdata/include/expected_full_schema.sql @@ -2,15 +2,12 @@ -- This represents a modular approach to organizing database schema -- Includes ALL supported PostgreSQL database objects --- Include custom types first (dependencies for tables) +-- Include custom types folder first (dependencies for tables) -- --- Name: user_status; Type: TYPE; Schema: -; Owner: - +-- Name: address; Type: TYPE; Schema: -; Owner: - -- -CREATE TYPE user_status AS ENUM ( - 'active', - 'inactive' -); +CREATE TYPE address AS (street text, city text); -- -- Name: order_status; Type: TYPE; Schema: -; Owner: - -- @@ -20,10 +17,13 @@ CREATE TYPE order_status AS ENUM ( 'completed' ); -- --- Name: address; Type: TYPE; Schema: -; Owner: - +-- Name: user_status; Type: TYPE; Schema: -; Owner: - -- -CREATE TYPE address AS (street text, city text); +CREATE TYPE user_status AS ENUM ( + 'active', + 'inactive' +); -- Include domain types (constrained base types) -- @@ -182,7 +182,7 @@ AS $$ SELECT COUNT(*) FROM orders WHERE user_id = user_id_param; $$; --- Include procedures +-- Include procedures folder -- -- Name: cleanup_orders; Type: PROCEDURE; Schema: -; Owner: - -- diff --git a/testdata/include/main.sql b/testdata/include/main.sql index ef68eb7f..9e85278e 100644 --- a/testdata/include/main.sql +++ b/testdata/include/main.sql @@ -2,10 +2,8 @@ -- This represents a modular approach to organizing database schema -- Includes ALL supported PostgreSQL database objects --- Include custom types first (dependencies for tables) -\i types/user_status.sql -\i types/order_status.sql -\i types/address.sql +-- Include custom types folder first (dependencies for tables) +\i types/ -- Include domain types (constrained base types) \i domains/email_address.sql @@ -26,9 +24,8 @@ \i functions/get_user_count.sql \i functions/get_order_count.sql --- Include procedures -\i procedures/cleanup_orders.sql -\i procedures/update_status.sql +-- Include procedures folder +\i procedures/ -- Include views (depend on tables and functions) \i views/user_summary.sql