From e3f51174b4e26c90fdb71521f087c0aec74a44c2 Mon Sep 17 00:00:00 2001 From: Git AI Test Date: Sat, 8 Feb 2025 07:58:15 +0100 Subject: [PATCH] feat: OpenAI Integration changes --- .gitignore | 1 + resources/prompt.md | 46 +++++++--- src/model.rs | 101 ++++++++++++++++------ src/openai.rs | 206 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 296 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index acd9c9f..5f515be 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ http-cacache/* .env.local ${env:TMPDIR} bin/ +tmp/ diff --git a/resources/prompt.md b/resources/prompt.md index 523b51d..93d875a 100644 --- a/resources/prompt.md +++ b/resources/prompt.md @@ -1,18 +1,40 @@ -You are an AI assistant that generates concise and meaningful git commit messages based on provided diffs. Please adhere to the following guidelines: +You are an AI assistant that generates concise and precise git commit messages based solely on the provided diffs. Please adhere to the following enhanced guidelines: -- Structure: Begin with a clear, present-tense summary. -- Content: While you should use the surrounding context to understand the changes, your commit message should ONLY describe the lines marked with + or -. -- Understanding: Use the context (unmarked lines) to understand the purpose and impact of the changes, but do not mention unchanged code in the commit message. -- Changes: Only describe what was actually changed (added, removed, or modified). -- Consistency: Maintain uniformity in tense, punctuation, and capitalization. -- Accuracy: Ensure the message accurately reflects the changes and their purpose. -- Present tense, imperative mood. (e.g., "Add x to y" instead of "Added x to y") -- Max {{max_commit_length}} chars in the output +- **Structure**: Begin with a clear, present-tense summary of the change in the non-conventional commit format. Use a single-line summary for the change, followed by a blank line. As a best practice, consider including only one bullet point detailing context if essential, but refrain from excessive elaboration. -## Output: +- **Content**: Commit messages must strictly describe the lines marked with + or - in the diff. Avoid including surrounding context, unmarked lines, or irrelevant details. Explicitly refrain from mentioning implications, reasoning, motivations, or any external context not explicitly reflected in the diff. Make sure to avoid any interpretations or assumptions beyond what is clearly stated. -Your output should be a commit message generated from the input diff and nothing else. While you should use the surrounding context to understand the changes, your message should only describe what was actually modified (+ or - lines). +- **Changes**: Clearly articulate what was added, removed, or modified based solely on what is visible in the diff. Use phrases such as "Based only on the changes visible in the diff, this commit..." to emphasize an evidence-based approach while outlining changes directly. -## Input: +- **Consistency**: Ensure uniformity in tense, punctuation, and capitalization throughout the message. Use present tense and imperative form, such as "Add x to y" instead of "Added x to y". + +- **Clarity & Brevity**: Craft messages that are clear and easy to understand, succinctly capturing the essence of the changes. Limit the message to a maximum of {{max_commit_length}} characters for the first line, while ensuring enough detail is provided on the primary action taken. Avoid jargon; provide plain definitions for any necessary technical terms. + +- **Accuracy & Hallucination Prevention**: Rigorously reflect only the changes visible in the diff. Avoid any speculation or inclusion of content not substantiated by the diff. Restate the necessity for messages to focus exclusively on aspects evident in the diff and to completely avoid extrapolation or assumptions about motivations or implications. + +- **Binary Files & Special Cases**: When handling binary files or cases where diff content is not readable: + 1. NEVER output error messages or apologies in the commit message + 2. Use the format "Add/Update/Delete binary file " for binary files + 3. Include file size in parentheses if available + 4. If multiple binary files are changed, list them separated by commas + 5. For unreadable diffs, focus on the file operation (add/modify/delete) without speculating about content + +- **Error Prevention**: + 1. NEVER include phrases like "I'm sorry", "I apologize", or any error messages + 2. NEVER leave commit messages incomplete or truncated + 3. If unable to read diff content, default to describing the file operation + 4. Always ensure the message is a valid git commit message + 5. When in doubt about content, focus on the file operation type + +- **Review Process**: Before finalizing each commit message: + 1. Verify that the message accurately reflects only the changes in the diff + 2. Confirm the commit type matches the actual changes + 3. Check that the message follows the structure and formatting guidelines + 4. Ensure no external context or assumptions are included + 5. Validate that the message is clear and understandable to other developers + 6. Verify no error messages or apologies are included + 7. Confirm the message describes file operations even if content is unreadable + +- **Important**: The output will be used as a git commit message, so it must be a valid git commit message. INPUT: diff --git a/src/model.rs b/src/model.rs index 71a8494..308f639 100644 --- a/src/model.rs +++ b/src/model.rs @@ -7,51 +7,99 @@ use serde::{Deserialize, Serialize}; use tiktoken_rs::get_completion_max_tokens; use tiktoken_rs::model::get_context_size; -const GPT4: &str = "gpt-4"; -const GPT4O: &str = "gpt-4o"; -const GPT4OMINI: &str = "gpt-4o-mini"; +use crate::profile; +// Model identifiers - using screaming case for constants +const MODEL_GPT4: &str = "gpt-4"; +const MODEL_GPT4_OPTIMIZED: &str = "gpt-4o"; +const MODEL_GPT4_MINI: &str = "gpt-4o-mini"; + +/// Represents the available AI models for commit message generation. +/// Each model has different capabilities and token limits. #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone, Serialize, Deserialize, Default)] pub enum Model { + /// Standard GPT-4 model GPT4, + /// Optimized GPT-4 model for better performance GPT4o, + /// Default model - Mini version of optimized GPT-4 for faster processing #[default] GPT4oMini } impl Model { + /// Counts the number of tokens in the given text for the current model. + /// This is used to ensure we stay within the model's token limits. + /// + /// # Arguments + /// * `text` - The text to count tokens for + /// + /// # Returns + /// * `Result` - The number of tokens or an error pub fn count_tokens(&self, text: &str) -> Result { + profile!("Count tokens"); + let model_str: &str = self.into(); Ok( self .context_size() - .saturating_sub(get_completion_max_tokens(self.into(), text)?) + .saturating_sub(get_completion_max_tokens(model_str, text)?) ) } + /// Gets the maximum context size for the current model. + /// + /// # Returns + /// * `usize` - The maximum number of tokens the model can process pub fn context_size(&self) -> usize { - get_context_size(self.into()) + profile!("Get context size"); + let model_str: &str = self.into(); + get_context_size(model_str) } - pub(crate) fn truncate(&self, diff: &str, max_tokens: usize) -> Result { - self.walk_truncate(diff, max_tokens, usize::MAX) + /// Truncates the given text to fit within the specified token limit. + /// + /// # Arguments + /// * `text` - The text to truncate + /// * `max_tokens` - The maximum number of tokens allowed + /// + /// # Returns + /// * `Result` - The truncated text or an error + pub(crate) fn truncate(&self, text: &str, max_tokens: usize) -> Result { + profile!("Truncate text"); + self.walk_truncate(text, max_tokens, usize::MAX) } - pub(crate) fn walk_truncate(&self, diff: &str, max_tokens: usize, within: usize) -> Result { - log::debug!("max_tokens: {}", max_tokens); - log::debug!("diff: {}", diff); - log::debug!("within: {}", within); + /// Recursively truncates text to fit within token limits while maintaining coherence. + /// Uses a binary search-like approach to find the optimal truncation point. + /// + /// # Arguments + /// * `text` - The text to truncate + /// * `max_tokens` - The maximum number of tokens allowed + /// * `within` - The maximum allowed deviation from target token count + /// + /// # Returns + /// * `Result` - The truncated text or an error + pub(crate) fn walk_truncate(&self, text: &str, max_tokens: usize, within: usize) -> Result { + profile!("Walk truncate iteration"); + log::debug!("max_tokens: {}, within: {}", max_tokens, within); + + let truncated = { + profile!("Split and join text"); + text + .split_whitespace() + .take(max_tokens) + .collect::>() + .join(" ") + }; - let str = diff - .split_whitespace() - .take(max_tokens) - .collect::>() - .join(" "); - let offset = self.count_tokens(&str)?.saturating_sub(max_tokens); + let token_count = self.count_tokens(&truncated)?; + let offset = token_count.saturating_sub(max_tokens); if offset > within || offset == 0 { - Ok(str) // TODO: check if this is correct + Ok(truncated) } else { - self.walk_truncate(diff, max_tokens + offset, within) + // Recursively adjust token count to get closer to target + self.walk_truncate(text, max_tokens + offset, within) } } } @@ -59,9 +107,9 @@ impl Model { impl From<&Model> for &str { fn from(model: &Model) -> Self { match model { - Model::GPT4o => GPT4O, - Model::GPT4 => GPT4, - Model::GPT4oMini => GPT4OMINI + Model::GPT4o => MODEL_GPT4_OPTIMIZED, + Model::GPT4 => MODEL_GPT4, + Model::GPT4oMini => MODEL_GPT4_MINI } } } @@ -71,10 +119,10 @@ impl FromStr for Model { fn from_str(s: &str) -> Result { match s.trim().to_lowercase().as_str() { - GPT4O => Ok(Model::GPT4o), - GPT4 => Ok(Model::GPT4), - GPT4OMINI => Ok(Model::GPT4oMini), - model => bail!("Invalid model: {}", model) + MODEL_GPT4_OPTIMIZED => Ok(Model::GPT4o), + MODEL_GPT4 => Ok(Model::GPT4), + MODEL_GPT4_MINI => Ok(Model::GPT4oMini), + model => bail!("Invalid model name: {}", model) } } } @@ -85,6 +133,7 @@ impl Display for Model { } } +// Implement conversion from string types to Model with fallback to default impl From<&str> for Model { fn from(s: &str) -> Self { s.parse().unwrap_or_default() diff --git a/src/openai.rs b/src/openai.rs index db16c1e..2a7e008 100644 --- a/src/openai.rs +++ b/src/openai.rs @@ -1,11 +1,15 @@ use async_openai::types::{ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs, CreateChatCompletionRequestArgs}; use async_openai::config::OpenAIConfig; use async_openai::Client; -use anyhow::{Context, Result}; +use async_openai::error::OpenAIError; +use anyhow::{anyhow, Context, Result}; +use colored::*; -use crate::config; +use crate::{config, profile}; use crate::model::Model; +const MAX_ATTEMPTS: usize = 3; + #[derive(Debug, Clone, PartialEq)] pub struct Response { pub response: String @@ -19,42 +23,204 @@ pub struct Request { pub model: Model } +/// Generates an improved commit message using the provided prompt and diff +pub async fn generate_commit_message(diff: &str, prompt: &str, file_context: &str, author: &str, date: &str) -> Result { + profile!("Generate commit message"); + let system_prompt = format!( + "You are an expert at writing clear, concise git commit messages. \ + Your task is to generate a commit message for the following code changes.\n\n\ + {}\n\n\ + Consider:\n\ + - Author: {}\n\ + - Date: {}\n\ + - Files changed: {}\n", + prompt, author, date, file_context + ); + + let response = call(Request { + system: system_prompt, + prompt: format!("Generate a commit message for this diff:\n\n{}", diff), + max_tokens: 256, + model: Model::GPT4oMini + }) + .await?; + + Ok(response.response.trim().to_string()) +} + +/// Scores a commit message against the original using AI evaluation +pub async fn score_commit_message(message: &str, original: &str) -> Result { + profile!("Score commit message"); + let system_prompt = "You are an expert at evaluating git commit messages. Score the following commit message on these criteria: + - Accuracy (0-1): How well does it describe the actual changes? + - Clarity (0-1): How clear and understandable is the message? + - Brevity (0-1): Is it concise while being informative? + - Categorization (0-1): Does it properly categorize the type of change? + + Return ONLY a JSON object containing these scores and brief feedback."; + + let response = call(Request { + system: system_prompt.to_string(), + prompt: format!("Original commit message:\n{}\n\nGenerated commit message:\n{}", original, message), + max_tokens: 512, + model: Model::GPT4oMini + }) + .await?; + + // Parse the JSON response to get the overall score + let parsed: serde_json::Value = serde_json::from_str(&response.response).context("Failed to parse scoring response as JSON")?; + + let accuracy = parsed["accuracy"].as_f64().unwrap_or(0.0) as f32; + let clarity = parsed["clarity"].as_f64().unwrap_or(0.0) as f32; + let brevity = parsed["brevity"].as_f64().unwrap_or(0.0) as f32; + let categorization = parsed["categorization"].as_f64().unwrap_or(0.0) as f32; + + Ok((accuracy + clarity + brevity + categorization) / 4.0) +} + +/// Optimizes a prompt based on performance metrics +pub async fn optimize_prompt(current_prompt: &str, performance_metrics: &str) -> Result { + profile!("Optimize prompt"); + let system_prompt = "You are an expert at optimizing prompts for AI systems. \ + Your task is to improve a prompt used for generating git commit messages \ + based on performance metrics. Return ONLY the improved prompt text."; + + let response = call(Request { + system: system_prompt.to_string(), + prompt: format!( + "Current prompt:\n{}\n\nPerformance metrics:\n{}\n\n\ + Suggest an improved version of this prompt that addresses any weaknesses \ + shown in the metrics while maintaining its strengths.", + current_prompt, performance_metrics + ), + max_tokens: 1024, + model: Model::GPT4oMini + }) + .await?; + + Ok(response.response.trim().to_string()) +} + +fn truncate_to_fit(text: &str, max_tokens: usize, model: &Model) -> Result { + let token_count = model.count_tokens(text)?; + if token_count <= max_tokens { + return Ok(text.to_string()); + } + + let lines: Vec<&str> = text.lines().collect(); + + // Try increasingly aggressive truncation until we fit + for attempt in 0..MAX_ATTEMPTS { + let portion_size = match attempt { + 0 => lines.len() / 8, // First try: Keep 25% (12.5% each end) + 1 => lines.len() / 12, // Second try: Keep ~16% (8% each end) + _ => lines.len() / 20 // Final try: Keep 10% (5% each end) + }; + + let mut truncated = Vec::new(); + truncated.extend(lines.iter().take(portion_size)); + truncated.push("... (truncated for length) ..."); + truncated.extend(lines.iter().rev().take(portion_size).rev()); + + let result = truncated.join("\n"); + let new_token_count = model.count_tokens(&result)?; + + if new_token_count <= max_tokens { + return Ok(result); + } + } + + // If all attempts failed, return a minimal version + let mut minimal = Vec::new(); + minimal.extend(lines.iter().take(lines.len() / 50)); + minimal.push("... (severely truncated for length) ..."); + minimal.extend(lines.iter().rev().take(lines.len() / 50).rev()); + Ok(minimal.join("\n")) +} + pub async fn call(request: Request) -> Result { - let api_key = config::APP - .openai_api_key - .clone() - .context("Failed to get OpenAI API key, please run `git-ai config set openai-api")?; + profile!("OpenAI API call"); + let api_key = config::APP.openai_api_key.clone().context(format!( + "{} OpenAI API key not found.\n Run: {}", + "ERROR:".bold().bright_red(), + "git-ai config set openai-api-key ".yellow() + ))?; let config = OpenAIConfig::new().with_api_key(api_key); let client = Client::with_config(config); + // Calculate available tokens using model's context size + let system_tokens = request.model.count_tokens(&request.system)?; + let model_context_size = request.model.context_size(); + let available_tokens = model_context_size.saturating_sub(system_tokens + request.max_tokens as usize); + + // Truncate prompt if needed + let truncated_prompt = truncate_to_fit(&request.prompt, available_tokens, &request.model)?; + let request = CreateChatCompletionRequestArgs::default() - .model(request.model.to_string()) .max_tokens(request.max_tokens) + .model(request.model.to_string()) .messages([ ChatCompletionRequestSystemMessageArgs::default() .content(request.system) .build()? .into(), ChatCompletionRequestUserMessageArgs::default() - .content(request.prompt) + .content(truncated_prompt) .build()? .into() ]) .build()?; - let chat = client.chat().create(request).await?; - - let choise = chat - .choices - .first() - .context(format!("Failed to get response: {:?}", chat))?; + { + profile!("OpenAI request/response"); + let response = match client.chat().create(request).await { + Ok(response) => response, + Err(err) => { + let error_msg = match err { + OpenAIError::ApiError(e) => + format!( + "{} {}\n {}\n\nDetails:\n {}\n\nSuggested Actions:\n 1. {}\n 2. {}\n 3. {}", + "ERROR:".bold().bright_red(), + "OpenAI API error:".bright_white(), + e.message.dimmed(), + "Failed to create chat completion.".dimmed(), + "Ensure your OpenAI API key is valid".yellow(), + "Check your account credits".yellow(), + "Verify OpenAI service availability".yellow() + ), + OpenAIError::Reqwest(e) => + format!( + "{} {}\n {}\n\nDetails:\n {}\n\nSuggested Actions:\n 1. {}\n 2. {}", + "ERROR:".bold().bright_red(), + "Network error:".bright_white(), + e.to_string().dimmed(), + "Failed to connect to OpenAI service.".dimmed(), + "Check your internet connection".yellow(), + "Verify OpenAI service is not experiencing downtime".yellow() + ), + _ => + format!( + "{} {}\n {}\n\nDetails:\n {}", + "ERROR:".bold().bright_red(), + "Unexpected error:".bright_white(), + err.to_string().dimmed(), + "An unexpected error occurred while communicating with OpenAI.".dimmed() + ), + }; + return Err(anyhow!(error_msg)); + } + }; - let response = choise - .message - .content - .clone() - .context("Failed to get response text")?; + let content = response + .choices + .first() + .context("No choices returned")? + .message + .content + .clone() + .context("No content returned")?; - Ok(Response { response }) + Ok(Response { response: content }) + } }