diff --git a/README.md b/README.md index acf6f72..59cb065 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ It's intended use case is for preparing content to be provided to AI/LLMs. - Traverse directory structures and generate a tree view - Include/exclude files based on glob patterns -- Generate git diffs and logs +- Parse output directly to Ollama for processing +- Generate and include git diffs and logs - Count approximate tokens for LLM compatibility - Customisable output templates - Copy output to clipboard (when available) @@ -37,7 +38,7 @@ go install github.com/sammcj/ingest@HEAD Basic usage: ```shell -ingest [flags] +ingest [flags] ``` ingest will default the current working directory, if no path is provided, e.g: @@ -61,12 +62,52 @@ Generate a prompt with git diff and copy to clipboard: ingest -d /path/to/project ``` +Generate a prompt for multiple files/directories: + +```shell +ingest /path/to/project /path/to/other/project +``` + Generate a prompt and save to a file: ```shell ingest -o output.md /path/to/project ``` +## Ollama Integration + +Ingest can pass the generated prompt to [Ollama](https://ollama.com) for processing. + +![ingest ollama](ollama-ingest.png) + +```shell +ingest --ollama /path/to/project +``` + +By default this will ask you to enter a prompt: + +```shell +./ingest utils.go --ollama +⠋ Traversing directory and building tree... [0s] +[!] Enter Ollama prompt: +explain this code +This is Go code for a file named `utils.go`. It contains various utility functions for +handling terminal output, clipboard operations, and configuration directories. +... +``` + +## Configuration + +Ingest uses a configuration file located at `~/.config/ingest/config.json`. + +You can make Ollama processing run without prompting setting `"ollama_auto_run": true` in the config file. + +The config file also contains: + +- "ollama_model": The model to use for processing the prompt, e.g. "llama3.1:8b-q5_k_m". +- "ollama_prompt_prefix": An optional prefix to prepend to the prompt, e.g. "This is my application." +- "ollama_prompt_suffix": An optional suffix to append to the prompt, e.g. "explain this code" + ### Flags - `-i, --include`: Patterns to include (can be used multiple times) @@ -76,6 +117,7 @@ ingest -o output.md /path/to/project - `--tokens`: Display the token count of the generated prompt - `-c, --encoding`: Optional tokeniser to use for token count - `-o, --output`: Optional output file path +- `--ollama`: Send the generated prompt to Ollama for processing - `-d, --diff`: Include git diff - `--git-diff-branch`: Generate git diff between two branches - `--git-log-branch`: Retrieve git log between two branches diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..c0d03e2 --- /dev/null +++ b/config/config.go @@ -0,0 +1,74 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/mitchellh/go-homedir" +) + +type OllamaConfig struct { + Model string `json:"ollama_model"` + PromptPrefix string `json:"ollama_prompt_prefix"` + PromptSuffix string `json:"ollama_prompt_suffix"` + AutoRun bool `json:"ollama_auto_run"` +} + +type Config struct { + Ollama []OllamaConfig `json:"ollama"` +} + +func LoadConfig() (*Config, error) { + home, err := homedir.Dir() + if err != nil { + return nil, fmt.Errorf("failed to get home directory: %w", err) + } + + configPath := filepath.Join(home, ".config", "ingest", "ingest.json") + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return createDefaultConfig(configPath) + } + + file, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + var config Config + if err := json.Unmarshal(file, &config); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + return &config, nil +} + +func createDefaultConfig(configPath string) (*Config, error) { + defaultConfig := Config{ + Ollama: []OllamaConfig{ + { + Model: "llama3.1:8b-instruct-q6_K", + PromptPrefix: "Code: ", + PromptSuffix: "", + AutoRun: false, + }, + }, + } + + err := os.MkdirAll(filepath.Dir(configPath), 0755) + if err != nil { + return nil, fmt.Errorf("failed to create config directory: %w", err) + } + + file, err := json.MarshalIndent(defaultConfig, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal default config: %w", err) + } + + if err := os.WriteFile(configPath, file, 0644); err != nil { + return nil, fmt.Errorf("failed to write default config file: %w", err) + } + + return &defaultConfig, nil +} diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index aed9376..487b92e 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -134,37 +134,62 @@ func WalkDirectory(rootPath string, includePatterns, excludePatterns []string, p return "", nil, fmt.Errorf("failed to read .gitignore: %w", err) } - // Generate the tree representation - treeString, err := generateTreeString(rootPath, allExcludePatterns) + // Check if rootPath is a file or directory + fileInfo, err := os.Stat(rootPath) if err != nil { - return "", nil, fmt.Errorf("failed to generate directory tree: %w", err) + return "", nil, fmt.Errorf("failed to get file info: %w", err) } - // Process files - err = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(rootPath, path) - if err != nil { - return err - } + var treeString string - if shouldIncludeFile(relPath, includePatterns, allExcludePatterns, gitignore, includePriority) && !info.IsDir() { + if !fileInfo.IsDir() { + // Handle single file + relPath := filepath.Base(rootPath) + if shouldIncludeFile(relPath, includePatterns, allExcludePatterns, gitignore, includePriority) { wg.Add(1) - go func(path, relPath string, info os.FileInfo) { + go func() { defer wg.Done() - processFile(path, relPath, rootPath, lineNumber, relativePaths, noCodeblock, &mu, &files) - }(path, relPath, info) + processFile(rootPath, relPath, filepath.Dir(rootPath), lineNumber, relativePaths, noCodeblock, &mu, &files) + }() + } + treeString = fmt.Sprintf("File: %s", rootPath) + } else { + // Generate the tree representation for directory + treeString, err = generateTreeString(rootPath, allExcludePatterns) + if err != nil { + return "", nil, fmt.Errorf("failed to generate directory tree: %w", err) } - return nil - }) + // Process files in directory + err = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(rootPath, path) + if err != nil { + return err + } + + if shouldIncludeFile(relPath, includePatterns, allExcludePatterns, gitignore, includePriority) && !info.IsDir() { + wg.Add(1) + go func(path, relPath string, info os.FileInfo) { + defer wg.Done() + processFile(path, relPath, rootPath, lineNumber, relativePaths, noCodeblock, &mu, &files) + }(path, relPath, info) + } + + return nil + }) + } wg.Wait() - return treeString, files, err + if err != nil { + return "", nil, err + } + + return treeString, files, nil } func shouldIncludeFile(path string, includePatterns, excludePatterns []string, gitignore *ignore.GitIgnore, includePriority bool) bool { diff --git a/main.go b/main.go index 9bf0a5e..ec6e81d 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,17 @@ package main import ( + "bufio" "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "sort" "strings" "github.com/fatih/color" + "github.com/sammcj/ingest/config" "github.com/sammcj/ingest/filesystem" "github.com/sammcj/ingest/git" "github.com/sammcj/ingest/template" @@ -42,9 +45,9 @@ var ( func main() { rootCmd := &cobra.Command{ - Use: "ingest [flags] [path]", - Short: "Generate a markdown LLM prompt from a codebase directory", - Long: `ingest is a command-line tool to generate an LLM prompt from a codebase directory.`, + Use: "ingest [flags] [paths...]", + Short: "Generate a markdown LLM prompt from files and directories", + Long: `ingest is a command-line tool to generate an LLM prompt from files and directories.`, RunE: run, } @@ -65,6 +68,7 @@ func main() { rootCmd.Flags().StringVarP(&templatePath, "template", "t", "", "Optional Path to a custom Handlebars template") rootCmd.Flags().BoolVar(&jsonOutput, "json", false, "Print output as JSON") rootCmd.Flags().StringVar(&patternExclude, "pattern-exclude", "", "Path to a specific .glob file for exclude patterns") + rootCmd.Flags().BoolP("ollama", "p", false, "Pass output to Ollama for processing") rootCmd.Flags().BoolVar(&printDefaultExcludes, "print-default-excludes", false, "Print the default exclude patterns") rootCmd.Flags().BoolVar(&printDefaultTemplate, "print-default-template", false, "Print the default template") rootCmd.Flags().BoolP("version", "V", false, "Print the version number") @@ -90,10 +94,20 @@ func run(cmd *cobra.Command, args []string) error { return nil } + cfg, err := config.LoadConfig() + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + if err := utils.EnsureConfigDirectories(); err != nil { return fmt.Errorf("failed to ensure config directories: %w", err) } + paths := args + if len(paths) == 0 { + paths = []string{"."} + } + if printDefaultExcludes { filesystem.PrintDefaultExcludes() return nil @@ -132,11 +146,24 @@ func run(cmd *cobra.Command, args []string) error { printExcludePatterns(activeExcludes) } - // Traverse directory with parallel processing - tree, files, err := filesystem.WalkDirectory(absPath, includePatterns, excludePatterns, patternExclude, includePriority, lineNumber, relativePaths, excludeFromTree, noCodeblock) - if err != nil { - spinner.Finish() - return fmt.Errorf("failed to build directory tree: %w", err) + // Process all provided paths + var allFiles []filesystem.FileInfo + var allTrees []string + + for _, path := range paths { + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("failed to get absolute path for %s: %w", path, err) + } + + // Traverse directory with parallel processing + tree, files, err := filesystem.WalkDirectory(absPath, includePatterns, excludePatterns, patternExclude, includePriority, lineNumber, relativePaths, excludeFromTree, noCodeblock) + if err != nil { + return fmt.Errorf("failed to process %s: %w", path, err) + } + + allFiles = append(allFiles, files...) + allTrees = append(allTrees, fmt.Sprintf("%s:\n%s", absPath, tree)) } // Handle git operations @@ -181,9 +208,9 @@ func run(cmd *cobra.Command, args []string) error { // Prepare data for template data := map[string]interface{}{ + "source_trees": strings.Join(allTrees, "\n\n"), + "files": allFiles, "absolute_code_path": absPath, - "source_tree": tree, - "files": files, "git_diff": gitDiff, "git_diff_branch": gitDiffBranchContent, "git_log_branch": gitLogBranchContent, @@ -192,12 +219,19 @@ func run(cmd *cobra.Command, args []string) error { // Render template rendered, err := template.RenderTemplate(tmpl, data) if err != nil { - return fmt.Errorf("failed to render template: %w", err) + return fmt.Errorf("failed to render template: %w", err) } + useOllama, _ := cmd.Flags().GetBool("ollama") // Handle output - if err := handleOutput(rendered, tokens, encoding, noClipboard, output, jsonOutput, report || verbose, files); err != nil { - return fmt.Errorf("failed to handle output: %w", err) + if useOllama { + if err := handleOllamaOutput(rendered, cfg.Ollama[0]); err != nil { + return fmt.Errorf("failed to handle Ollama output: %w", err) + } + } else { + if err := handleOutput(rendered, tokens, encoding, noClipboard, output, jsonOutput, report || verbose, allFiles); err != nil { + return fmt.Errorf("failed to handle output: %w", err) + } } return nil @@ -262,7 +296,6 @@ func handleOutput(rendered string, countTokens bool, encoding string, noClipboar return nil } - func printExcludePatterns(patterns []string) { utils.PrintColouredMessage("i", "Active exclude patterns:", color.FgCyan) @@ -292,9 +325,9 @@ func printExcludePatterns(patterns []string) { highlighted = strings.ReplaceAll(highlighted, ".", dotColour(".")) // Add padding to align patterns - padding := strings.Repeat(" ", maxWidth - len(pattern) + 2) + padding := strings.Repeat(" ", maxWidth-len(pattern)+2) - if lineWidth + len(pattern) + 2 > w && i > 0 { + if lineWidth+len(pattern)+2 > w && i > 0 { fmt.Println() lineWidth = 0 } @@ -312,3 +345,33 @@ func printExcludePatterns(patterns []string) { } } } + +func handleOllamaOutput(rendered string, ollamaConfig config.OllamaConfig) error { + if ollamaConfig.PromptPrefix != "" { + rendered = ollamaConfig.PromptPrefix + "\n" + rendered + } + if ollamaConfig.PromptSuffix != "" { + rendered += ollamaConfig.PromptSuffix + } + + if ollamaConfig.AutoRun { + return runOllamaCommand(ollamaConfig.Model, rendered) + } + + fmt.Println() + utils.PrintColouredMessage("!", "Enter Ollama prompt:", color.FgRed) + reader := bufio.NewReader(os.Stdin) + prompt, _ := reader.ReadString('\n') + prompt = strings.TrimSpace(prompt) + + return runOllamaCommand(ollamaConfig.Model, rendered+"\n"+prompt) +} + +func runOllamaCommand(model, input string) error { + cmd := exec.Command("ollama", "run", model) + cmd.Stdin = strings.NewReader(input) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} diff --git a/ollama-ingest.png b/ollama-ingest.png new file mode 100644 index 0000000..8459c22 Binary files /dev/null and b/ollama-ingest.png differ diff --git a/template/template.go b/template/template.go index 361b700..7d05bf7 100644 --- a/template/template.go +++ b/template/template.go @@ -59,13 +59,9 @@ func getDefaultTemplate() (string, error) { func readEmbeddedTemplate() (string, error) { return ` -Project Path: {{.absolute_code_path}} +Source Trees: -Source Tree: - -` + "```" + ` -{{.source_tree}} -` + "```" + ` +{{.source_trees}} {{range .files}} {{if .Code}}