From bea2fbff43939bd3c7d000a31a9197c7daa2bf72 Mon Sep 17 00:00:00 2001 From: Martin Treml Date: Wed, 31 Jul 2024 17:05:22 +0200 Subject: [PATCH 1/4] Increase evaluation volume size for kubernetes to avoid running out of disk space with unexpected bloated log files --- docs/kubernetes/volume.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kubernetes/volume.yml b/docs/kubernetes/volume.yml index 805f3761..92015301 100644 --- a/docs/kubernetes/volume.yml +++ b/docs/kubernetes/volume.yml @@ -9,4 +9,4 @@ spec: - ReadWriteMany # Ensure that the access mode is "ReadWriteMany". resources: requests: - storage: 5Gi + storage: 50Gi From b0227463a1d025278804025532eb21211ac21dd9 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Mon, 9 Sep 2024 11:17:26 +0200 Subject: [PATCH 2/4] Cleanup obsolete scripts because they are not used anymore --- scripts/docker-evaluation/docker.go | 108 ------------------------- scripts/evaluation/sequential-light.go | 91 --------------------- 2 files changed, 199 deletions(-) delete mode 100644 scripts/docker-evaluation/docker.go delete mode 100644 scripts/evaluation/sequential-light.go diff --git a/scripts/docker-evaluation/docker.go b/scripts/docker-evaluation/docker.go deleted file mode 100644 index 427ac60c..00000000 --- a/scripts/docker-evaluation/docker.go +++ /dev/null @@ -1,108 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/symflower/eval-dev-quality/provider" -) - -var models = []string{ - "openrouter/01-ai/yi-34b-chat", - "openrouter/01-ai/yi-6b", - "openrouter/allenai/olmo-7b-instruct", - "openrouter/alpindale/goliath-120b", - "openrouter/anthropic/claude-3-haiku", - "openrouter/anthropic/claude-3-haiku:beta", - "openrouter/anthropic/claude-3-opus", - "openrouter/anthropic/claude-3-opus:beta", - "openrouter/anthropic/claude-3-sonnet", - "openrouter/anthropic/claude-3-sonnet:beta", - "openrouter/anthropic/claude-instant-1.1", - "openrouter/austism/chronos-hermes-13b", - "openrouter/bigcode/starcoder2-15b-instruct", - "openrouter/cognitivecomputations/dolphin-mixtral-8x22b", - "openrouter/cognitivecomputations/dolphin-mixtral-8x7b", - "openrouter/cohere/command-r-plus", - "openrouter/databricks/dbrx-instruct", - "openrouter/deepseek/deepseek-chat", - "openrouter/deepseek/deepseek-coder", - "openrouter/fireworks/firellava-13b", - "openrouter/google/gemini-flash-1.5", - "openrouter/google/gemini-pro-1.5", - "openrouter/google/gemma-7b-it", - "openrouter/google/palm-2-chat-bison", - "openrouter/google/palm-2-codechat-bison", - "openrouter/huggingfaceh4/zephyr-7b-beta", - "openrouter/intel/neural-chat-7b", - "openrouter/jondurbin/airoboros-l2-70b", - "openrouter/liuhaotian/llava-13b", - "openrouter/liuhaotian/llava-yi-34b", - "openrouter/meta-llama/codellama-34b-instruct", - "openrouter/meta-llama/codellama-70b-instruct", - "openrouter/meta-llama/llama-3-70b-instruct", - "openrouter/meta-llama/llama-3-8b-instruct", - "openrouter/microsoft/phi-3-medium-128k-instruct", - "openrouter/microsoft/phi-3-medium-4k-instruct", - "openrouter/microsoft/phi-3-mini-128k-instruct", - "openrouter/microsoft/wizardlm-2-7b", - "openrouter/microsoft/wizardlm-2-8x22b", - "openrouter/mistralai/mistral-7b-instruct-v0.3", - "openrouter/mistralai/mistral-large", - "openrouter/mistralai/mistral-medium", - "openrouter/mistralai/mistral-small", - "openrouter/mistralai/mistral-tiny", - "openrouter/mistralai/mixtral-8x22b-instruct", - "openrouter/mistralai/mixtral-8x7b-instruct", - "openrouter/neversleep/llama-3-lumimaid-70b", - "openrouter/neversleep/llama-3-lumimaid-8b", - "openrouter/nousresearch/hermes-2-pro-llama-3-8b", - "openrouter/nousresearch/nous-capybara-34b", - "openrouter/nousresearch/nous-capybara-7b", - "openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo", - "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo", - "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft", - "openrouter/nousresearch/nous-hermes-yi-34b", - "openrouter/open-orca/mistral-7b-openorca", - "openrouter/openai/gpt-4", - "openrouter/openai/gpt-4-turbo", - "openrouter/openai/gpt-4o", - "openrouter/openchat/openchat-8b", - "openrouter/perplexity/llama-3-sonar-large-32k-chat", - "openrouter/perplexity/llama-3-sonar-small-32k-chat", - "openrouter/phind/phind-codellama-34b", - "openrouter/qwen/qwen-110b-chat", - "openrouter/qwen/qwen-14b-chat", - "openrouter/qwen/qwen-2-72b-instruct", - "openrouter/qwen/qwen-32b-chat", - "openrouter/qwen/qwen-4b-chat", - "openrouter/qwen/qwen-72b-chat", - "openrouter/qwen/qwen-7b-chat", - "openrouter/recursal/eagle-7b", - "openrouter/recursal/rwkv-5-3b-ai-town", - "openrouter/rwkv/rwkv-5-world-3b", - "openrouter/snowflake/snowflake-arctic-instruct", - "openrouter/teknium/openhermes-2-mistral-7b", - "openrouter/teknium/openhermes-2.5-mistral-7b", - "openrouter/togethercomputer/stripedhyena-hessian-7b", - "openrouter/togethercomputer/stripedhyena-nous-7b", - "openrouter/undi95/toppy-m-7b", - "openrouter/xwin-lm/xwin-lm-70b", -} - -func main() { - - for _, model := range models { - - modelIdentifier := strings.Split(model, provider.ProviderModelSeparator)[2] - repo := "light" - fmt.Printf("screen -dmS %s docker run -v ./:/home/ubuntu/evaluation -e PROVIDER_TOKEN ghcr.io/symflower/eval-dev-quality:docker-image eval-dev-quality evaluate --runs 5 --repository %s --repository %s --model %s --result-path /home/ubuntu/evaluation/%s\n", - modelIdentifier, - filepath.Join("golang", repo), - filepath.Join("java", repo), - model, modelIdentifier, - ) - } - -} diff --git a/scripts/evaluation/sequential-light.go b/scripts/evaluation/sequential-light.go deleted file mode 100644 index ab5f127e..00000000 --- a/scripts/evaluation/sequential-light.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "path/filepath" - "time" - - pkgerrors "github.com/pkg/errors" - "github.com/zimmski/osutil" - - "github.com/symflower/eval-dev-quality/log" - "github.com/symflower/eval-dev-quality/util" -) - -var models = []string{ - "openrouter/01-ai/yi-34b-chat", - "openrouter/anthropic/claude-2.1", - "openrouter/anthropic/claude-3-haiku", - "openrouter/anthropic/claude-3-sonnet", - "openrouter/cohere/command-r-plus", - "openrouter/cohere/command-r", - "openrouter/databricks/dbrx-instruct", - "openrouter/google/gemini-flash-1.5", - "openrouter/google/gemini-pro-1.5", - "openrouter/google/gemma-7b-it", - "openrouter/meta-llama/llama-2-70b-chat", - "openrouter/meta-llama/llama-3-70b-instruct", - "openrouter/meta-llama/llama-3-8b-instruct", - "openrouter/microsoft/wizardlm-2-7b", - "openrouter/microsoft/wizardlm-2-8x22b", - "openrouter/mistralai/mistral-7b-instruct", - "openrouter/mistralai/mistral-medium", - "openrouter/mistralai/mixtral-8x22b-instruct", - "openrouter/mistralai/mixtral-8x7b-instruct", - "openrouter/openai/gpt-3.5-turbo", - "openrouter/openai/gpt-4-turbo", - "openrouter/openai/gpt-4o", - "openrouter/qwen/qwen-2-72b-instruct", -} - -func main() { - logger := log.STDOUT() - - output, err := util.CommandWithResult(context.Background(), logger, &util.Command{ - Command: []string{ - "make", - "install-all", - }, - }) - if err != nil { - panic(pkgerrors.WithStack(pkgerrors.WithMessage(err, output))) - } - - progress := osutil.ProgressBar(os.Stdout, len(models), "Benchmarking") - defer progress.Exit() - - for _, model := range models { - progress.Describe("Benchmarking: " + model) - - err := benchmarkModel(logger, "evaluation-sequential-light-", "light", model, 150*time.Minute) - if err != nil { - fmt.Printf("Model %q misbehaved", model) - } - - progress.Add(1) - } -} - -func benchmarkModel(logger *log.Logger, resultPrefix string, repository string, id string, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - output, err := util.CommandWithResult(ctx, logger, &util.Command{ - Command: []string{ - "eval-dev-quality", - "evaluate", - "--runs", "5", - "--result-path", resultPrefix + log.CleanModelNameForFileSystem(id), - "--repository", filepath.Join("java", repository), - "--repository", filepath.Join("golang", repository), - "--model", id, - }, - }) - if err != nil { - return pkgerrors.WithStack(pkgerrors.WithMessage(err, output)) - } - - return nil -} From d3f61cc32e64a6c6dcb1d1ae0d56bfe7c073e4bc Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Mon, 9 Sep 2024 11:30:22 +0200 Subject: [PATCH 3/4] Scripts to find out max score and perform result aggregation --- scripts/evaluation/stats.sh | 146 ++++++++++++++++++++++++++++++++++++ scripts/max-score/main.go | 141 ++++++++++++++++++++++++++++++++++ scripts/max-score/run.sh | 27 +++++++ 3 files changed, 314 insertions(+) create mode 100755 scripts/evaluation/stats.sh create mode 100644 scripts/max-score/main.go create mode 100755 scripts/max-score/run.sh diff --git a/scripts/evaluation/stats.sh b/scripts/evaluation/stats.sh new file mode 100755 index 00000000..63feebc8 --- /dev/null +++ b/scripts/evaluation/stats.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Usage: script.sh + +# Check if csvkit is installed. +if ! command -v csvsql &> /dev/null +then + echo "Error: csvkit is not installed. Please install it using 'pip install csvkit'." + exit 1 +fi + +evaluation_file="${1%.*}" +evaluation_table=$(basename "$1" .csv) +meta_table=$(basename "$2" .csv) + +# SQL does not like hyphens in column names. +sed -i '1s/-/_/g' "$1" +sed -i '1s/-/_/g' "$2" + +csvsql --query "\ + SELECT \ + model_id, \ + language, \ + SUM(score) AS score, \ + SUM(coverage) AS coverage, \ + SUM(files_executed) AS files_executed, \ + SUM(files_executed_maximum_reachable) AS files_executed_maximum_reachable, \ + SUM(generate_tests_for_file_character_count) AS generate_tests_for_file_character_count, \ + SUM(processing_time) AS processing_time, \ + SUM(response_character_count) AS response_character_count, \ + SUM(response_no_error) AS response_no_error, \ + SUM(response_no_excess) AS response_no_excess, \ + SUM(response_with_code) AS response_with_code, \ + SUM(tests_passing) AS tests_passing \ + FROM $evaluation_table \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY model_id, language\ + " "$1" > "${evaluation_file}-by-language.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + SUM(score) AS score, \ + SUM(CASE WHEN language = 'golang' THEN score ELSE 0 END) AS golang_score, \ + SUM(CASE WHEN language = 'java' THEN score ELSE 0 END) AS java_score, \ + SUM(CASE WHEN language = 'ruby' THEN score ELSE 0 END) AS ruby_score \ + FROM $evaluation_table \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY model_id\ + " "$1" > "${evaluation_file}-by-language-score.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + SUM(score) AS score, \ + SUM(CASE WHEN task = 'write-tests' THEN score ELSE 0 END) AS write_tests_score, \ + SUM(CASE WHEN task = 'transpile' THEN score ELSE 0 END) AS transpile_score, \ + SUM(CASE WHEN task = 'code-repair' THEN score ELSE 0 END) AS code_repair_score \ + FROM $evaluation_table \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY model_id\ + " "$1" > "${evaluation_file}-by-task-score.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + task, \ + language, \ + SUM(files_executed_maximum_reachable) AS files \ + FROM $evaluation_table \ + GROUP BY model_id, task, language\ + " "$1" > "${evaluation_file}-cases.csv" + +csvsql --query "\ + SELECT \ + $evaluation_table.model_id, \ + model_name, \ + (completion + prompt + request) AS cost, \ + SUM(score) AS score, \ + SUM(coverage) AS coverage, \ + SUM(files_executed) AS files_executed, \ + SUM(files_executed_maximum_reachable) AS files_executed_maximum_reachable, \ + SUM(generate_tests_for_file_character_count) AS generate_tests_for_file_character_count, \ + SUM(processing_time) AS processing_time, \ + SUM(response_character_count) AS response_character_count, \ + SUM(response_no_error) AS response_no_error, \ + SUM(response_no_excess) AS response_no_excess, \ + SUM(response_with_code) AS response_with_code, \ + SUM(tests_passing) AS tests_passing \ + FROM $evaluation_table LEFT JOIN $meta_table ON $evaluation_table.model_id = $meta_table.model_id \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY $evaluation_table.model_id\ + " "$1" "$2" > "${evaluation_file}-by-model.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + task, \ + SUM(score) AS score, \ + SUM(coverage) AS coverage, \ + SUM(files_executed) AS files_executed, \ + SUM(files_executed_maximum_reachable) AS files_executed_maximum_reachable, \ + SUM(generate_tests_for_file_character_count) AS generate_tests_for_file_character_count, \ + SUM(processing_time) AS processing_time, \ + SUM(response_character_count) AS response_character_count, \ + SUM(response_no_error) AS response_no_error, \ + SUM(response_no_excess) AS response_no_excess, \ + SUM(response_with_code) AS response_with_code, \ + SUM(tests_passing) AS tests_passing \ + FROM $evaluation_table \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY model_id, task\ + " "$1" > "${evaluation_file}-by-task.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + task, \ + language, \ + SUM(score) AS score, \ + SUM(coverage) AS coverage, \ + SUM(files_executed) AS files_executed, \ + SUM(files_executed_maximum_reachable) AS files_executed_maximum_reachable, \ + SUM(generate_tests_for_file_character_count) AS generate_tests_for_file_character_count, \ + SUM(processing_time) AS processing_time, \ + SUM(response_character_count) AS response_character_count, \ + SUM(response_no_error) AS response_no_error, \ + SUM(response_no_excess) AS response_no_excess, \ + SUM(response_with_code) AS response_with_code, \ + SUM(tests_passing) AS tests_passing \ + FROM $evaluation_table \ + WHERE task NOT LIKE '%-symflower-fix' \ + GROUP BY model_id, task, language\ + " "$1" > "${evaluation_file}-by-task-by-language.csv" + +csvsql --query "\ + SELECT \ + model_id, \ + SUM(CASE WHEN task NOT LIKE '%-symflower-fix' THEN score ELSE 0 END) AS score, \ + SUM(CASE WHEN task LIKE '%-symflower-fix' THEN score ELSE 0 END) AS score_fix, \ + SUM(CASE WHEN task NOT LIKE '%-symflower-fix' THEN files_executed ELSE 0 END) AS files_executed, \ + SUM(CASE WHEN task LIKE '%-symflower-fix' THEN files_executed ELSE 0 END) AS files_executed_fix \ + FROM $evaluation_table \ + WHERE (task LIKE 'transpile%' OR task LIKE 'write-tests%') AND language = 'golang' \ + GROUP BY model_id\ + " "$1" > "${evaluation_file}-by-symflower-fix.csv" diff --git a/scripts/max-score/main.go b/scripts/max-score/main.go new file mode 100644 index 00000000..b3db23c5 --- /dev/null +++ b/scripts/max-score/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/exp/maps" +) + +var reTaskFileNameRepairAndWriteTest = regexp.MustCompile(`Given the following \w+ code file "(.*)" with`) +var reTaskFileNameTranspile = regexp.MustCompile(`The transpiled \w+ code file must have the package "(.*)"`) +var reTaskNameTranspileJava = regexp.MustCompile(`\s+class (\w+)\s*\{`) + +func parseTaskName(s string) (string, bool) { + if match := reTaskFileNameRepairAndWriteTest.FindStringSubmatch(s); match != nil { + return match[1], true + } else if match := reTaskFileNameTranspile.FindStringSubmatch(s); match != nil { + return match[1], true + } + + return "", false +} + +var reModelName = regexp.MustCompile(`Model "(.*)" responded`) + +func parseModelName(s string) (string, bool) { + if match := reModelName.FindStringSubmatch(s); match != nil { + return match[1], true + } + + return "", false +} + +var reCoverageObjects = regexp.MustCompile(`Executes tests with (\d+) coverage objects`) + +func parseCoverageObjects(s string) (int, bool) { + match := reCoverageObjects.FindStringSubmatch(s) + if match == nil { + return 0, false + } + coverageObjectsText := match[1] + coverageObjects, err := strconv.Atoi(coverageObjectsText) + if err != nil { + panic(fmt.Sprintf("cannot convert %q to integer (%q)", coverageObjectsText, s)) + } + + return coverageObjects, true +} + +var rePassingTests = regexp.MustCompile(`Executes tests with (\d+) tests passing`) + +func parsePassingTests(s string) (int, bool) { + match := rePassingTests.FindStringSubmatch(s) + if match == nil { + return 0, false + } + passingTestsText := match[1] + passingTests, err := strconv.Atoi(passingTestsText) + if err != nil { + panic(fmt.Sprintf("cannot convert %q to integer (%q)", passingTestsText, s)) + } + + return passingTests, true +} + +func main() { + logFileNames := os.Args[1:] + var logFileData []byte + + for _, logFileName := range logFileNames { + data, err := os.ReadFile(logFileName) + if err != nil { + panic(err) + } + logFileData = append(logFileData, data...) + } + + var currentTaskName string + var currentModelName string + bestCoverage := map[string]int{} + bestCoverageModel := map[string]string{} + bestTests := map[string]int{} + bestTestsModel := map[string]string{} + + logFileLines := strings.Split(string(logFileData), "\n") + fmt.Printf("Loaded %d log files (total of %d lines)\n", len(logFileNames), len(logFileLines)) + for _, line := range logFileLines { + if taskName, ok := parseTaskName(line); ok { + currentTaskName = taskName + } + // The package of Java is always "com.eval" so parse the task name from the class. + if match := reTaskNameTranspileJava.FindStringSubmatch(line); currentTaskName == "com.eval" && match != nil { + currentTaskName = match[1] + } + + if modelName, ok := parseModelName(line); ok { + currentModelName = modelName + } + + if currentCoverageObjects, ok := parseCoverageObjects(line); ok && currentCoverageObjects > bestCoverage[currentTaskName] { + bestCoverage[currentTaskName] = currentCoverageObjects + bestCoverageModel[currentTaskName] = currentModelName + } + + if currentPassingTests, ok := parsePassingTests(line); ok && currentPassingTests > bestTests[currentTaskName] { + bestTests[currentTaskName] = currentPassingTests + bestTestsModel[currentTaskName] = currentModelName + } + } + + if len(bestCoverage) > 0 { + taskNames := maps.Keys(bestCoverage) + sort.Strings(taskNames) + fmt.Println("Coverage") + sum := 0 + for _, taskName := range taskNames { + score := bestCoverage[taskName] + fmt.Printf(" - %s:%d (%q)\n", taskName, score, bestCoverageModel[taskName]) + sum += score + } + fmt.Printf("∑ = %d\n", sum) + } + + if len(bestTests) > 0 { + taskNames := maps.Keys(bestTests) + sort.Strings(taskNames) + fmt.Println("Tests Passing") + sum := 0 + for _, taskName := range taskNames { + score := bestTests[taskName] + fmt.Printf(" - %s:%d (%q)\n", taskName, score, bestTestsModel[taskName]) + sum += score + } + fmt.Printf("∑ = %d\n", sum) + + } +} diff --git a/scripts/max-score/run.sh b/scripts/max-score/run.sh new file mode 100755 index 00000000..5a79714d --- /dev/null +++ b/scripts/max-score/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Usage: script.sh + +echo "WRITE TESTS" +echo "java:" +find $1 -path "*/write-tests/*/java/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "ruby:" +find $1 -path "*/write-tests/*/ruby/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "golang:" +find $1 -path "*/write-tests/*/golang/*/*/evaluation.log" | xargs go run scripts/max-score/main.go + +echo "TRANSPILE" +echo "java:" +find $1 -path "*/transpile/*/java/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "ruby:" +find $1 -path "*/transpile/*/ruby/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "golang:" +find $1 -path "*/transpile/*/golang/*/*/evaluation.log" | xargs go run scripts/max-score/main.go + +echo "CODE REPAIR" +echo "java:" +find $1 -path "*/code-repair/*/java/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "ruby:" +find $1 -path "*/code-repair/*/ruby/*/*/evaluation.log" | xargs go run scripts/max-score/main.go +echo "golang:" +find $1 -path "*/code-repair/*/golang/*/*/evaluation.log" | xargs go run scripts/max-score/main.go From 3a71848c059bdb6fe9d4149f36dcee993a49a90e Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Mon, 9 Sep 2024 12:34:12 +0200 Subject: [PATCH 4/4] Results for v0.6 --- .gitignore | 2 +- docs/reports/v0.6/README.md | 3 + .../v0.6/evaluation-by-language-score.csv | 68 + docs/reports/v0.6/evaluation-by-language.csv | 202 + docs/reports/v0.6/evaluation-by-model.csv | 68 + .../v0.6/evaluation-by-symflower-fix.csv | 68 + .../v0.6/evaluation-by-task-by-language.csv | 525 ++ .../reports/v0.6/evaluation-by-task-score.csv | 68 + docs/reports/v0.6/evaluation-by-task.csv | 195 + docs/reports/v0.6/evaluation-cases.csv | 887 +++ docs/reports/v0.6/evaluation.csv | 5327 +++++++++++++++++ docs/reports/v0.6/max-score.txt | 141 + docs/reports/v0.6/meta.csv | 176 + scripts/evaluation/v0.6-openrouter.sh | 80 + 14 files changed, 7809 insertions(+), 1 deletion(-) create mode 100644 docs/reports/v0.6/README.md create mode 100644 docs/reports/v0.6/evaluation-by-language-score.csv create mode 100644 docs/reports/v0.6/evaluation-by-language.csv create mode 100644 docs/reports/v0.6/evaluation-by-model.csv create mode 100644 docs/reports/v0.6/evaluation-by-symflower-fix.csv create mode 100644 docs/reports/v0.6/evaluation-by-task-by-language.csv create mode 100644 docs/reports/v0.6/evaluation-by-task-score.csv create mode 100644 docs/reports/v0.6/evaluation-by-task.csv create mode 100644 docs/reports/v0.6/evaluation-cases.csv create mode 100644 docs/reports/v0.6/evaluation.csv create mode 100644 docs/reports/v0.6/max-score.txt create mode 100644 docs/reports/v0.6/meta.csv create mode 100755 scripts/evaluation/v0.6-openrouter.sh diff --git a/.gitignore b/.gitignore index 1ba0c87c..2b63cb78 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ go.work .envrc # Ignore auto-generated evaluation folders. -evaluation-* +evaluation-*/ diff --git a/docs/reports/v0.6/README.md b/docs/reports/v0.6/README.md new file mode 100644 index 00000000..cc558698 --- /dev/null +++ b/docs/reports/v0.6/README.md @@ -0,0 +1,3 @@ +This evaluation ran over several days and is actually assembled from several sub-evaluations. Hence, we only provide the `csv` results for now because the logs are distributed all over. + +The benchmark consisted of `5x` runs, **except** for the `code-repair` task, which contains only one usable run because of [a bug that is already fixed for later versions](https://github.com/symflower/eval-dev-quality/commit/19011da21c3196e346d363148bf83989dc1f2a88). diff --git a/docs/reports/v0.6/evaluation-by-language-score.csv b/docs/reports/v0.6/evaluation-by-language-score.csv new file mode 100644 index 00000000..4f634f60 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-language-score.csv @@ -0,0 +1,68 @@ +model_id,score,golang_score,java_score,ruby_score +openrouter/01-ai/yi-34b-chat,273.0,9.0,264.0,0.0 +openrouter/ai21/jamba-instruct,15606.0,15.0,7856.0,7735.0 +openrouter/alpindale/goliath-120b,2429.0,1265.0,13.0,1151.0 +openrouter/alpindale/magnum-72b,18583.0,780.0,9915.0,7888.0 +openrouter/anthropic/claude-3-haiku,34021.0,5137.0,16092.0,12792.0 +openrouter/anthropic/claude-3-opus,38485.0,7432.0,17939.0,13114.0 +openrouter/anthropic/claude-3-sonnet,29425.0,6419.0,12187.0,10819.0 +openrouter/anthropic/claude-3.5-sonnet,39540.0,7590.0,17948.0,14002.0 +openrouter/austism/chronos-hermes-13b,13583.0,3963.0,5421.0,4199.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,25969.0,4535.0,11786.0,9648.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,23843.0,4712.0,12113.0,7018.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,11602.0,1184.0,6698.0,3720.0 +openrouter/cohere/command-r-03-2024,14938.0,2139.0,8462.0,4337.0 +openrouter/cohere/command-r-08-2024,21549.0,6374.0,11442.0,3733.0 +openrouter/cohere/command-r-plus-04-2024,26797.0,3450.0,14993.0,8354.0 +openrouter/cohere/command-r-plus-08-2024,26312.0,3378.0,15137.0,7797.0 +openrouter/databricks/dbrx-instruct,14863.0,5819.0,9044.0,0.0 +openrouter/deepseek/deepseek-chat,38715.0,7302.0,17929.0,13484.0 +openrouter/deepseek/deepseek-coder,38263.0,7157.0,17166.0,13940.0 +openrouter/google/gemini-flash-1.5,25434.0,7245.0,4846.0,13343.0 +openrouter/google/gemini-pro-1.5,20955.0,7591.0,7161.0,6203.0 +openrouter/google/gemma-2-27b-it,24166.0,6514.0,13982.0,3670.0 +openrouter/google/gemma-2-9b-it,10019.0,1662.0,8346.0,11.0 +openrouter/google/palm-2-chat-bison,10545.0,4044.0,2364.0,4137.0 +openrouter/google/palm-2-codechat-bison,9463.0,4185.0,15.0,5263.0 +openrouter/meta-llama/llama-3-70b-instruct,32865.0,6641.0,14310.0,11914.0 +openrouter/meta-llama/llama-3-8b-instruct,9861.0,14.0,6225.0,3622.0 +openrouter/meta-llama/llama-3.1-405b-instruct,37535.0,7705.0,17080.0,12750.0 +openrouter/meta-llama/llama-3.1-70b-instruct,34328.0,6454.0,14852.0,13022.0 +openrouter/meta-llama/llama-3.1-8b-instruct,24111.0,4329.0,11618.0,8164.0 +openrouter/microsoft/phi-3-medium-128k-instruct,11145.0,1270.0,2009.0,7866.0 +openrouter/microsoft/phi-3-medium-4k-instruct,14576.0,4370.0,1527.0,8679.0 +openrouter/microsoft/phi-3-mini-128k-instruct,16317.0,1093.0,9579.0,5645.0 +openrouter/microsoft/wizardlm-2-7b,14293.0,929.0,9735.0,3629.0 +openrouter/microsoft/wizardlm-2-8x22b,27433.0,4909.0,16421.0,6103.0 +openrouter/mistralai/codestral-mamba,24507.0,6287.0,8979.0,9241.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,13879.0,2789.0,8137.0,2953.0 +openrouter/mistralai/mistral-large,36056.0,6899.0,17477.0,11680.0 +openrouter/mistralai/mistral-medium,30464.0,5287.0,15110.0,10067.0 +openrouter/mistralai/mistral-nemo,19162.0,1423.0,8615.0,9124.0 +openrouter/mistralai/mistral-small,20218.0,4505.0,15698.0,15.0 +openrouter/mistralai/mistral-tiny,8643.0,2407.0,6226.0,10.0 +openrouter/mistralai/mixtral-8x22b-instruct,29308.0,5941.0,15600.0,7767.0 +openrouter/mistralai/mixtral-8x7b-instruct,28497.0,4446.0,15072.0,8979.0 +openrouter/neversleep/llama-3-lumimaid-70b,18596.0,4050.0,8705.0,5841.0 +openrouter/neversleep/llama-3-lumimaid-8b,10618.0,13.0,6499.0,4106.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,16984.0,14.0,9064.0,7906.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,17210.0,11.0,10235.0,6964.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,19990.0,2183.0,11647.0,6160.0 +openrouter/nousresearch/nous-hermes-yi-34b,8206.0,3274.0,4919.0,13.0 +openrouter/openai/gpt-4-turbo,38422.0,7213.0,18170.0,13039.0 +openrouter/openai/gpt-4o,39005.0,6656.0,17900.0,14449.0 +openrouter/openai/gpt-4o-mini,39441.0,7182.0,17859.0,14400.0 +openrouter/openchat/openchat-8b,5376.0,3198.0,2167.0,11.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,35051.0,6853.0,15929.0,12269.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,7338.0,977.0,2167.0,4194.0 +openrouter/qwen/qwen-110b-chat,10980.0,4450.0,12.0,6518.0 +openrouter/qwen/qwen-2-72b-instruct,25290.0,15.0,14465.0,10810.0 +openrouter/qwen/qwen-2-7b-instruct,10750.0,15.0,6941.0,3794.0 +openrouter/qwen/qwen-72b-chat,27717.0,4033.0,13615.0,10069.0 +openrouter/recursal/eagle-7b,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,20.0,10.0,10.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,15881.0,15.0,10830.0,5036.0 +openrouter/togethercomputer/stripedhyena-nous-7b,12001.0,3487.0,5439.0,3075.0 +openrouter/undi95/toppy-m-7b,3026.0,2035.0,11.0,980.0 +openrouter/xwin-lm/xwin-lm-70b,5596.0,2589.0,2998.0,9.0 diff --git a/docs/reports/v0.6/evaluation-by-language.csv b/docs/reports/v0.6/evaluation-by-language.csv new file mode 100644 index 00000000..7cb41cd3 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-language.csv @@ -0,0 +1,202 @@ +model_id,language,score,coverage,files_executed,files_executed_maximum_reachable,generate_tests_for_file_character_count,processing_time,response_character_count,response_no_error,response_no_excess,response_with_code,tests_passing +openrouter/01-ai/yi-34b-chat,golang,9.0,0.0,0.0,5.0,5405.0,58053.0,5439.0,5.0,2.0,2.0,0.0 +openrouter/01-ai/yi-34b-chat,java,264.0,70.0,4.0,125.0,60672.0,1069765.0,104374.0,106.0,6.0,48.0,30.0 +openrouter/01-ai/yi-34b-chat,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/ai21/jamba-instruct,golang,15.0,0.0,0.0,5.0,1001.0,11333.0,1055.0,5.0,5.0,5.0,0.0 +openrouter/ai21/jamba-instruct,java,7856.0,5260.0,107.0,175.0,123112.0,755464.0,127096.0,174.0,159.0,166.0,1990.0 +openrouter/ai21/jamba-instruct,ruby,7735.0,5280.0,116.0,175.0,87707.0,661289.0,91758.0,175.0,163.0,171.0,1830.0 +openrouter/alpindale/goliath-120b,golang,1265.0,0.0,23.0,55.0,21346.0,597771.0,23416.0,55.0,46.0,51.0,1090.0 +openrouter/alpindale/goliath-120b,java,13.0,0.0,0.0,5.0,2501.0,58730.0,2557.0,5.0,4.0,4.0,0.0 +openrouter/alpindale/goliath-120b,ruby,1151.0,420.0,45.0,175.0,115374.0,2487163.0,123650.0,175.0,90.0,111.0,310.0 +openrouter/alpindale/magnum-72b,golang,780.0,210.0,12.0,125.0,94723.0,2716599.0,97941.0,125.0,120.0,123.0,190.0 +openrouter/alpindale/magnum-72b,java,9915.0,7030.0,122.0,175.0,142402.0,2878477.0,148779.0,175.0,165.0,173.0,2250.0 +openrouter/alpindale/magnum-72b,ruby,7888.0,6350.0,113.0,175.0,112651.0,1435301.0,117378.0,175.0,167.0,173.0,910.0 +openrouter/anthropic/claude-3-haiku,golang,5137.0,1990.0,107.0,175.0,120746.0,483176.0,128505.0,175.0,160.0,175.0,2530.0 +openrouter/anthropic/claude-3-haiku,java,16092.0,13120.0,162.0,175.0,140700.0,526473.0,161385.0,175.0,130.0,175.0,2330.0 +openrouter/anthropic/claude-3-haiku,ruby,12792.0,10230.0,167.0,175.0,103844.0,452693.0,124342.0,175.0,125.0,175.0,1920.0 +openrouter/anthropic/claude-3-opus,golang,7432.0,4020.0,164.0,175.0,108917.0,1691655.0,112944.0,175.0,148.0,175.0,2750.0 +openrouter/anthropic/claude-3-opus,java,17939.0,14950.0,175.0,175.0,148433.0,1833860.0,154944.0,175.0,124.0,175.0,2340.0 +openrouter/anthropic/claude-3-opus,ruby,13114.0,11010.0,173.0,175.0,106953.0,1630437.0,116662.0,175.0,131.0,175.0,1450.0 +openrouter/anthropic/claude-3-sonnet,golang,6419.0,3310.0,146.0,175.0,132633.0,989937.0,135342.0,175.0,173.0,175.0,2440.0 +openrouter/anthropic/claude-3-sonnet,java,12187.0,9200.0,123.0,175.0,160996.0,1003252.0,163844.0,175.0,174.0,175.0,2340.0 +openrouter/anthropic/claude-3-sonnet,ruby,10819.0,7970.0,152.0,175.0,107308.0,792804.0,109635.0,166.0,165.0,166.0,2200.0 +openrouter/anthropic/claude-3.5-sonnet,golang,7590.0,4100.0,167.0,175.0,112447.0,885125.0,114677.0,173.0,167.0,173.0,2810.0 +openrouter/anthropic/claude-3.5-sonnet,java,17948.0,15010.0,172.0,175.0,153755.0,963680.0,155819.0,172.0,172.0,172.0,2250.0 +openrouter/anthropic/claude-3.5-sonnet,ruby,14002.0,11440.0,175.0,175.0,119941.0,708009.0,122697.0,175.0,167.0,175.0,1870.0 +openrouter/austism/chronos-hermes-13b,golang,3963.0,1230.0,83.0,175.0,111119.0,1345385.0,132643.0,175.0,78.0,167.0,2230.0 +openrouter/austism/chronos-hermes-13b,java,5421.0,3210.0,77.0,175.0,134865.0,1334794.0,167227.0,175.0,68.0,171.0,1720.0 +openrouter/austism/chronos-hermes-13b,ruby,4199.0,2690.0,95.0,175.0,92171.0,1030120.0,117795.0,175.0,70.0,169.0,1000.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,4535.0,2800.0,106.0,175.0,112613.0,1079928.0,173739.0,175.0,39.0,175.0,1240.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,11786.0,9190.0,132.0,175.0,116357.0,831136.0,158445.0,175.0,65.0,174.0,2050.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,9648.0,8650.0,159.0,175.0,73802.0,794176.0,124872.0,175.0,50.0,174.0,440.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,4712.0,2040.0,96.0,175.0,104764.0,1445163.0,125734.0,175.0,116.0,175.0,2110.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,12113.0,9210.0,139.0,175.0,118906.0,1286699.0,138910.0,175.0,125.0,174.0,2290.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,7018.0,5510.0,108.0,175.0,82212.0,1035872.0,97091.0,164.0,132.0,164.0,940.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,1184.0,730.0,28.0,125.0,154319.0,928214.0,163537.0,125.0,66.0,85.0,150.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,6698.0,3990.0,86.0,175.0,212331.0,1077906.0,225705.0,175.0,134.0,153.0,2160.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,3720.0,2050.0,83.0,175.0,120304.0,1133288.0,133125.0,175.0,137.0,165.0,1110.0 +openrouter/cohere/command-r-03-2024,golang,2139.0,410.0,48.0,175.0,103163.0,404942.0,105356.0,175.0,153.0,153.0,1200.0 +openrouter/cohere/command-r-03-2024,java,8462.0,5820.0,107.0,175.0,118369.0,347982.0,120439.0,175.0,170.0,170.0,2020.0 +openrouter/cohere/command-r-03-2024,ruby,4337.0,3410.0,86.0,175.0,72030.0,269157.0,74058.0,175.0,163.0,163.0,340.0 +openrouter/cohere/command-r-08-2024,golang,6374.0,3320.0,149.0,175.0,138579.0,624227.0,140421.0,175.0,175.0,175.0,2380.0 +openrouter/cohere/command-r-08-2024,java,11442.0,8470.0,127.0,175.0,150810.0,545575.0,152912.0,175.0,175.0,175.0,2320.0 +openrouter/cohere/command-r-08-2024,ruby,3733.0,2220.0,78.0,175.0,104783.0,448164.0,106884.0,175.0,175.0,175.0,910.0 +openrouter/cohere/command-r-plus-04-2024,golang,3450.0,580.0,63.0,175.0,139950.0,1323021.0,142176.0,175.0,171.0,171.0,2290.0 +openrouter/cohere/command-r-plus-04-2024,java,14993.0,12080.0,160.0,175.0,182080.0,1175038.0,184171.0,175.0,174.0,174.0,2230.0 +openrouter/cohere/command-r-plus-04-2024,ruby,8354.0,6720.0,149.0,175.0,114816.0,842453.0,116917.0,175.0,175.0,175.0,960.0 +openrouter/cohere/command-r-plus-08-2024,golang,3378.0,190.0,63.0,175.0,133669.0,1236260.0,135841.0,175.0,175.0,175.0,2600.0 +openrouter/cohere/command-r-plus-08-2024,java,15137.0,12130.0,154.0,175.0,170811.0,1194004.0,173011.0,175.0,173.0,175.0,2330.0 +openrouter/cohere/command-r-plus-08-2024,ruby,7797.0,5680.0,117.0,175.0,116416.0,1004675.0,118608.0,175.0,171.0,174.0,1480.0 +openrouter/databricks/dbrx-instruct,golang,5819.0,2970.0,130.0,175.0,136877.0,822830.0,142444.0,175.0,162.0,172.0,2210.0 +openrouter/databricks/dbrx-instruct,java,9044.0,6170.0,110.0,175.0,136295.0,653927.0,150894.0,175.0,146.0,173.0,2270.0 +openrouter/databricks/dbrx-instruct,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/deepseek/deepseek-chat,golang,7302.0,4230.0,161.0,175.0,123061.0,2879287.0,125422.0,175.0,173.0,173.0,2390.0 +openrouter/deepseek/deepseek-chat,java,17929.0,14890.0,174.0,175.0,163823.0,2749171.0,165923.0,175.0,175.0,175.0,2340.0 +openrouter/deepseek/deepseek-chat,ruby,13484.0,10890.0,173.0,175.0,117719.0,2541389.0,119809.0,175.0,173.0,173.0,1900.0 +openrouter/deepseek/deepseek-coder,golang,7157.0,4090.0,158.0,175.0,134523.0,3325122.0,136840.0,175.0,172.0,172.0,2390.0 +openrouter/deepseek/deepseek-coder,java,17166.0,14140.0,168.0,175.0,201596.0,3649368.0,203666.0,174.0,172.0,172.0,2340.0 +openrouter/deepseek/deepseek-coder,ruby,13940.0,11140.0,175.0,175.0,121578.0,2415152.0,123678.0,175.0,175.0,175.0,2100.0 +openrouter/google/gemini-flash-1.5,golang,7245.0,3920.0,162.0,175.0,138397.0,548912.0,140556.0,175.0,174.0,174.0,2640.0 +openrouter/google/gemini-flash-1.5,java,4846.0,4300.0,31.0,125.0,131878.0,374654.0,133378.0,125.0,125.0,125.0,140.0 +openrouter/google/gemini-flash-1.5,ruby,13343.0,11240.0,172.0,175.0,104747.0,444353.0,106834.0,175.0,173.0,173.0,1410.0 +openrouter/google/gemini-pro-1.5,golang,7591.0,4010.0,166.0,175.0,106763.0,1288876.0,109034.0,175.0,175.0,175.0,2890.0 +openrouter/google/gemini-pro-1.5,java,7161.0,4210.0,86.0,175.0,125818.0,1253180.0,127922.0,175.0,175.0,175.0,2340.0 +openrouter/google/gemini-pro-1.5,ruby,6203.0,4320.0,90.0,175.0,118789.0,1512950.0,120904.0,175.0,174.0,174.0,1270.0 +openrouter/google/gemma-2-27b-it,golang,6514.0,3370.0,149.0,175.0,106425.0,831769.0,108931.0,175.0,175.0,175.0,2470.0 +openrouter/google/gemma-2-27b-it,java,13982.0,10970.0,147.0,175.0,134417.0,822344.0,136832.0,175.0,175.0,175.0,2340.0 +openrouter/google/gemma-2-27b-it,ruby,3670.0,1890.0,75.0,175.0,99516.0,746698.0,101916.0,175.0,175.0,175.0,1180.0 +openrouter/google/gemma-2-9b-it,golang,1662.0,1130.0,41.0,125.0,78677.0,945584.0,80512.0,125.0,107.0,109.0,150.0 +openrouter/google/gemma-2-9b-it,java,8346.0,6460.0,87.0,175.0,149059.0,1492104.0,151516.0,175.0,131.0,133.0,1360.0 +openrouter/google/gemma-2-9b-it,ruby,11.0,0.0,0.0,5.0,1602.0,36954.0,1679.0,5.0,3.0,3.0,0.0 +openrouter/google/palm-2-chat-bison,golang,4044.0,830.0,77.0,175.0,132049.0,517597.0,134043.0,175.0,166.0,166.0,2630.0 +openrouter/google/palm-2-chat-bison,java,2364.0,0.0,49.0,55.0,17389.0,92297.0,18104.0,55.0,55.0,55.0,2150.0 +openrouter/google/palm-2-chat-bison,ruby,4137.0,2620.0,72.0,175.0,110208.0,473692.0,112484.0,175.0,175.0,175.0,920.0 +openrouter/google/palm-2-codechat-bison,golang,4185.0,970.0,86.0,175.0,128370.0,492571.0,130346.0,175.0,172.0,172.0,2610.0 +openrouter/google/palm-2-codechat-bison,java,15.0,0.0,0.0,5.0,1007.0,7145.0,1072.0,5.0,5.0,5.0,0.0 +openrouter/google/palm-2-codechat-bison,ruby,5263.0,3650.0,80.0,175.0,112237.0,514985.0,114458.0,175.0,169.0,169.0,1020.0 +openrouter/meta-llama/llama-3-70b-instruct,golang,6641.0,3430.0,150.0,175.0,93934.0,1463927.0,95381.0,175.0,173.0,173.0,2540.0 +openrouter/meta-llama/llama-3-70b-instruct,java,14310.0,11340.0,155.0,175.0,140147.0,1519108.0,141571.0,175.0,175.0,175.0,2290.0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,11914.0,9890.0,159.0,175.0,98853.0,939230.0,100285.0,175.0,175.0,175.0,1340.0 +openrouter/meta-llama/llama-3-8b-instruct,golang,14.0,0.0,0.0,5.0,847.0,19782.0,1104.0,5.0,4.0,5.0,0.0 +openrouter/meta-llama/llama-3-8b-instruct,java,6225.0,3450.0,85.0,175.0,139374.0,736227.0,147957.0,175.0,151.0,164.0,2200.0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,3622.0,1690.0,72.0,175.0,106206.0,592496.0,109381.0,175.0,171.0,174.0,1340.0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,7705.0,4300.0,170.0,175.0,104171.0,1610982.0,105997.0,175.0,175.0,175.0,2710.0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,17080.0,14050.0,166.0,175.0,165058.0,1926263.0,169054.0,175.0,174.0,175.0,2340.0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,12750.0,10520.0,165.0,175.0,118180.0,1931153.0,120330.0,175.0,175.0,175.0,1540.0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,6454.0,3230.0,153.0,175.0,260833.0,2364478.0,263314.0,175.0,167.0,169.0,2560.0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,14852.0,12050.0,152.0,175.0,224096.0,1952909.0,234887.0,175.0,172.0,173.0,2130.0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,13022.0,10820.0,167.0,175.0,118587.0,1468729.0,120687.0,175.0,175.0,175.0,1510.0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,4329.0,1920.0,100.0,175.0,278721.0,919824.0,306491.0,174.0,156.0,169.0,1810.0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,11618.0,8850.0,128.0,175.0,364064.0,983263.0,401117.0,175.0,163.0,172.0,2130.0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,8164.0,5860.0,139.0,175.0,310418.0,971115.0,353513.0,175.0,162.0,168.0,1660.0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,1270.0,0.0,22.0,55.0,18638.0,274954.0,19843.0,55.0,35.0,38.0,1120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,2009.0,0.0,42.0,55.0,27009.0,301350.0,29600.0,55.0,40.0,42.0,1830.0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,7866.0,6750.0,145.0,175.0,129034.0,1471777.0,137611.0,175.0,130.0,146.0,520.0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,4370.0,1910.0,100.0,175.0,134135.0,885147.0,140848.0,175.0,161.0,174.0,1850.0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,1527.0,1010.0,15.0,125.0,150665.0,749973.0,155988.0,125.0,112.0,125.0,140.0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,8679.0,7500.0,160.0,174.0,110082.0,663278.0,117996.0,174.0,152.0,173.0,520.0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,1093.0,10.0,19.0,55.0,16851.0,109456.0,18266.0,55.0,49.0,50.0,910.0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,9579.0,6810.0,113.0,175.0,217703.0,1225251.0,273839.0,175.0,95.0,166.0,2220.0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,5645.0,4650.0,122.0,175.0,134455.0,891916.0,181910.0,175.0,89.0,159.0,450.0 +openrouter/microsoft/wizardlm-2-7b,golang,929.0,0.0,22.0,55.0,22471.0,228458.0,31562.0,55.0,47.0,55.0,750.0 +openrouter/microsoft/wizardlm-2-7b,java,9735.0,7110.0,103.0,175.0,346879.0,2310135.0,452960.0,175.0,53.0,174.0,2120.0 +openrouter/microsoft/wizardlm-2-7b,ruby,3629.0,2830.0,76.0,175.0,223892.0,1959331.0,322924.0,175.0,53.0,175.0,320.0 +openrouter/microsoft/wizardlm-2-8x22b,golang,4909.0,2300.0,121.0,175.0,167050.0,2926424.0,193571.0,175.0,109.0,174.0,2030.0 +openrouter/microsoft/wizardlm-2-8x22b,java,16421.0,13420.0,164.0,175.0,209892.0,3080161.0,224195.0,175.0,147.0,175.0,2340.0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,6103.0,4920.0,106.0,175.0,161609.0,2614728.0,200091.0,175.0,87.0,175.0,640.0 +openrouter/mistralai/codestral-mamba,golang,6287.0,3090.0,138.0,175.0,93189.0,515993.0,112137.0,175.0,129.0,175.0,2580.0 +openrouter/mistralai/codestral-mamba,java,8979.0,6080.0,108.0,175.0,120753.0,570553.0,148883.0,175.0,112.0,174.0,2330.0 +openrouter/mistralai/codestral-mamba,ruby,9241.0,7480.0,149.0,175.0,81170.0,441052.0,101907.0,175.0,132.0,175.0,1130.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,2789.0,1010.0,67.0,175.0,143239.0,1293449.0,198004.0,175.0,42.0,175.0,1320.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,8137.0,5550.0,110.0,175.0,181012.0,1144433.0,254280.0,175.0,28.0,174.0,2100.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,2953.0,2320.0,99.0,175.0,118195.0,796533.0,186304.0,175.0,18.0,171.0,170.0 +openrouter/mistralai/mistral-large,golang,6899.0,3390.0,145.0,175.0,98003.0,1510214.0,100194.0,175.0,174.0,175.0,2840.0 +openrouter/mistralai/mistral-large,java,17477.0,14480.0,172.0,175.0,147371.0,1553023.0,149471.0,175.0,175.0,175.0,2300.0 +openrouter/mistralai/mistral-large,ruby,11680.0,9370.0,156.0,175.0,110787.0,982503.0,112953.0,175.0,174.0,175.0,1630.0 +openrouter/mistralai/mistral-medium,golang,5287.0,2570.0,117.0,175.0,113370.0,1572750.0,150419.0,175.0,110.0,175.0,2140.0 +openrouter/mistralai/mistral-medium,java,15110.0,12160.0,159.0,175.0,164147.0,1741758.0,202761.0,175.0,101.0,175.0,2340.0 +openrouter/mistralai/mistral-medium,ruby,10067.0,8750.0,153.0,175.0,110585.0,1429671.0,163649.0,175.0,64.0,175.0,750.0 +openrouter/mistralai/mistral-nemo,golang,1423.0,910.0,36.0,125.0,96602.0,840261.0,100363.0,125.0,112.0,120.0,120.0 +openrouter/mistralai/mistral-nemo,java,8615.0,5990.0,98.0,175.0,134755.0,715359.0,139124.0,174.0,163.0,170.0,2020.0 +openrouter/mistralai/mistral-nemo,ruby,9124.0,7410.0,143.0,175.0,100304.0,695485.0,107864.0,175.0,154.0,172.0,1070.0 +openrouter/mistralai/mistral-small,golang,4505.0,1360.0,96.0,175.0,103233.0,891017.0,105966.0,175.0,169.0,175.0,2530.0 +openrouter/mistralai/mistral-small,java,15698.0,12680.0,161.0,175.0,141189.0,1044185.0,144553.0,175.0,167.0,175.0,2340.0 +openrouter/mistralai/mistral-small,ruby,15.0,0.0,0.0,5.0,781.0,6723.0,841.0,5.0,5.0,5.0,0.0 +openrouter/mistralai/mistral-tiny,golang,2407.0,540.0,44.0,175.0,117393.0,785353.0,150190.0,175.0,93.0,175.0,1380.0 +openrouter/mistralai/mistral-tiny,java,6226.0,3520.0,85.0,175.0,200665.0,815267.0,243708.0,175.0,71.0,175.0,2200.0 +openrouter/mistralai/mistral-tiny,ruby,10.0,0.0,0.0,5.0,1132.0,14285.0,4518.0,5.0,0.0,5.0,0.0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,5941.0,2910.0,133.0,175.0,113270.0,1218764.0,128940.0,175.0,138.0,175.0,2410.0 +openrouter/mistralai/mixtral-8x22b-instruct,java,15600.0,12700.0,158.0,175.0,140692.0,1096339.0,159166.0,175.0,132.0,175.0,2260.0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,7767.0,6680.0,124.0,175.0,100810.0,874097.0,128025.0,175.0,114.0,174.0,500.0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,4446.0,1850.0,101.0,175.0,133768.0,1100297.0,139797.0,175.0,143.0,157.0,2020.0 +openrouter/mistralai/mixtral-8x7b-instruct,java,15072.0,12310.0,152.0,175.0,181415.0,1203933.0,189848.0,175.0,125.0,170.0,2140.0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,8979.0,7730.0,125.0,175.0,112726.0,920493.0,118677.0,164.0,145.0,155.0,660.0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,4050.0,1810.0,89.0,175.0,93434.0,2139831.0,95688.0,175.0,107.0,109.0,1760.0 +openrouter/neversleep/llama-3-lumimaid-70b,java,8705.0,6230.0,105.0,175.0,130241.0,2051346.0,132211.0,175.0,137.0,138.0,1920.0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,5841.0,4490.0,103.0,175.0,86476.0,1710657.0,87849.0,175.0,106.0,107.0,860.0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,13.0,0.0,0.0,5.0,1285.0,13772.0,1326.0,5.0,4.0,4.0,0.0 +openrouter/neversleep/llama-3-lumimaid-8b,java,6499.0,3980.0,82.0,175.0,145299.0,1044696.0,149392.0,175.0,148.0,154.0,1960.0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,4106.0,2040.0,81.0,175.0,118642.0,816444.0,129412.0,175.0,166.0,174.0,1470.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,14.0,0.0,0.0,5.0,1902.0,8049.0,2105.0,5.0,4.0,5.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,9064.0,6490.0,109.0,175.0,126553.0,461587.0,148274.0,175.0,125.0,175.0,1990.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,7906.0,5770.0,147.0,175.0,87202.0,430468.0,111042.0,175.0,120.0,174.0,1520.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,11.0,0.0,0.0,5.0,1497.0,29170.0,2723.0,5.0,1.0,5.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,10235.0,7460.0,120.0,175.0,145221.0,1384779.0,158341.0,175.0,145.0,175.0,2160.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,6964.0,4600.0,129.0,175.0,103429.0,1209177.0,114868.0,175.0,146.0,174.0,1740.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,2183.0,0.0,38.0,55.0,17595.0,112771.0,28934.0,55.0,20.0,50.0,2020.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,11647.0,8800.0,135.0,175.0,220575.0,1549444.0,426354.0,175.0,67.0,170.0,2300.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,6160.0,4220.0,120.0,175.0,162317.0,850888.0,211479.0,175.0,61.0,174.0,1410.0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,3274.0,1220.0,71.0,175.0,159981.0,1634742.0,172864.0,175.0,129.0,149.0,1530.0 +openrouter/nousresearch/nous-hermes-yi-34b,java,4919.0,2520.0,71.0,175.0,201863.0,1517816.0,206729.0,175.0,135.0,148.0,1870.0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,13.0,0.0,0.0,5.0,740.0,9741.0,788.0,5.0,4.0,4.0,0.0 +openrouter/openai/gpt-4-turbo,golang,7213.0,3740.0,158.0,175.0,107248.0,1648942.0,109245.0,175.0,175.0,175.0,2790.0 +openrouter/openai/gpt-4-turbo,java,18170.0,15130.0,175.0,175.0,156677.0,1774860.0,158778.0,175.0,175.0,175.0,2340.0 +openrouter/openai/gpt-4-turbo,ruby,13039.0,11180.0,174.0,175.0,106271.0,1460994.0,108371.0,175.0,175.0,175.0,1160.0 +openrouter/openai/gpt-4o,golang,6656.0,3140.0,144.0,174.0,93032.0,747749.0,95315.0,174.0,174.0,174.0,2850.0 +openrouter/openai/gpt-4o,java,17900.0,14860.0,175.0,175.0,145425.0,862006.0,147525.0,175.0,175.0,175.0,2340.0 +openrouter/openai/gpt-4o,ruby,14449.0,11520.0,174.0,175.0,108779.0,620681.0,110882.0,175.0,175.0,175.0,2230.0 +openrouter/openai/gpt-4o-mini,golang,7182.0,3610.0,157.0,175.0,101802.0,721566.0,103576.0,175.0,175.0,175.0,2890.0 +openrouter/openai/gpt-4o-mini,java,17859.0,14820.0,174.0,175.0,172655.0,853504.0,174755.0,175.0,175.0,175.0,2340.0 +openrouter/openai/gpt-4o-mini,ruby,14400.0,11350.0,175.0,175.0,123933.0,638322.0,126033.0,175.0,175.0,175.0,2350.0 +openrouter/openchat/openchat-8b,golang,3198.0,1030.0,69.0,175.0,95117.0,639275.0,116099.0,173.0,87.0,169.0,1670.0 +openrouter/openchat/openchat-8b,java,2167.0,0.0,46.0,55.0,21833.0,129744.0,26831.0,54.0,14.0,53.0,2000.0 +openrouter/openchat/openchat-8b,ruby,11.0,0.0,0.0,5.0,1113.0,11076.0,2327.0,5.0,1.0,5.0,0.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,6853.0,3580.0,158.0,175.0,95269.0,896399.0,96711.0,175.0,175.0,175.0,2590.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,15929.0,13070.0,161.0,175.0,139917.0,966404.0,141525.0,173.0,172.0,173.0,2180.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,12269.0,10310.0,164.0,175.0,97354.0,1931765.0,98762.0,175.0,175.0,175.0,1270.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,977.0,480.0,21.0,125.0,218758.0,894488.0,237812.0,125.0,37.0,124.0,190.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,2167.0,0.0,46.0,55.0,22928.0,66908.0,25643.0,55.0,41.0,55.0,1970.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,4194.0,1750.0,79.0,175.0,114382.0,358102.0,127977.0,173.0,110.0,172.0,1910.0 +openrouter/qwen/qwen-110b-chat,golang,4450.0,1980.0,103.0,175.0,100890.0,951592.0,123603.0,175.0,127.0,175.0,1890.0 +openrouter/qwen/qwen-110b-chat,java,12.0,0.0,0.0,5.0,2012.0,19794.0,3559.0,5.0,2.0,5.0,0.0 +openrouter/qwen/qwen-110b-chat,ruby,6518.0,4000.0,135.0,175.0,119921.0,843719.0,126194.0,175.0,163.0,175.0,1870.0 +openrouter/qwen/qwen-2-72b-instruct,golang,15.0,0.0,0.0,5.0,891.0,9751.0,961.0,5.0,5.0,5.0,0.0 +openrouter/qwen/qwen-2-72b-instruct,java,14465.0,11620.0,153.0,175.0,125124.0,1235350.0,127621.0,175.0,173.0,174.0,2170.0 +openrouter/qwen/qwen-2-72b-instruct,ruby,10810.0,8870.0,144.0,175.0,100314.0,1176482.0,105991.0,175.0,166.0,175.0,1280.0 +openrouter/qwen/qwen-2-7b-instruct,golang,15.0,0.0,0.0,5.0,1441.0,9898.0,1511.0,5.0,5.0,5.0,0.0 +openrouter/qwen/qwen-2-7b-instruct,java,6941.0,4090.0,90.0,175.0,148818.0,663029.0,154427.0,175.0,171.0,175.0,2240.0 +openrouter/qwen/qwen-2-7b-instruct,ruby,3794.0,2100.0,72.0,175.0,131526.0,644070.0,133907.0,175.0,172.0,175.0,1100.0 +openrouter/qwen/qwen-72b-chat,golang,4033.0,2250.0,101.0,175.0,108654.0,1089763.0,122773.0,175.0,122.0,175.0,1210.0 +openrouter/qwen/qwen-72b-chat,java,13615.0,10720.0,146.0,175.0,145471.0,1023319.0,151294.0,175.0,159.0,175.0,2240.0 +openrouter/qwen/qwen-72b-chat,ruby,10069.0,8910.0,157.0,175.0,99786.0,743118.0,113861.0,168.0,116.0,168.0,550.0 +openrouter/recursal/eagle-7b,golang,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/eagle-7b,java,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/eagle-7b,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,golang,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,java,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,golang,10.0,0.0,0.0,5.0,3894.0,59802.0,4127.0,5.0,1.0,4.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,java,10.0,0.0,0.0,5.0,653.0,14125.0,775.0,5.0,2.0,3.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,15.0,0.0,0.0,5.0,1590.0,9612.0,1660.0,5.0,5.0,5.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,10830.0,8180.0,128.0,175.0,132730.0,536387.0,143038.0,175.0,136.0,171.0,2040.0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,5036.0,4040.0,80.0,175.0,91524.0,477963.0,99684.0,175.0,150.0,171.0,420.0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,3487.0,1020.0,73.0,175.0,106716.0,554960.0,127403.0,175.0,138.0,171.0,1910.0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,5439.0,4940.0,57.0,125.0,90719.0,386370.0,105971.0,125.0,88.0,119.0,110.0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,3075.0,2090.0,83.0,175.0,69725.0,384843.0,81519.0,173.0,130.0,159.0,440.0 +openrouter/undi95/toppy-m-7b,golang,2035.0,1540.0,49.0,125.0,99017.0,617190.0,171956.0,125.0,48.0,123.0,150.0 +openrouter/undi95/toppy-m-7b,java,11.0,0.0,0.0,5.0,3512.0,16855.0,5145.0,5.0,1.0,5.0,0.0 +openrouter/undi95/toppy-m-7b,ruby,980.0,420.0,56.0,175.0,93995.0,555801.0,187341.0,175.0,35.0,174.0,120.0 +openrouter/xwin-lm/xwin-lm-70b,golang,2589.0,420.0,52.0,175.0,95626.0,3531298.0,116960.0,175.0,41.0,111.0,1790.0 +openrouter/xwin-lm/xwin-lm-70b,java,2998.0,380.0,57.0,175.0,132606.0,3460467.0,158389.0,175.0,39.0,117.0,2230.0 +openrouter/xwin-lm/xwin-lm-70b,ruby,9.0,0.0,0.0,5.0,1044.0,71799.0,3346.0,5.0,0.0,4.0,0.0 diff --git a/docs/reports/v0.6/evaluation-by-model.csv b/docs/reports/v0.6/evaluation-by-model.csv new file mode 100644 index 00000000..a16cfe80 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-model.csv @@ -0,0 +1,68 @@ +model_id,model_name,cost,score,coverage,files_executed,files_executed_maximum_reachable,generate_tests_for_file_character_count,processing_time,response_character_count,response_no_error,response_no_excess,response_with_code,tests_passing +openrouter/01-ai/yi-34b-chat,Yi 34B Chat,1.44e-06,273.0,70.0,4.0,131.0,66077.0,1127818.0,109813.0,111.0,8.0,50.0,30.0 +openrouter/ai21/jamba-instruct,AI21: Jamba Instruct,1.2e-06,15606.0,10540.0,223.0,355.0,211820.0,1428086.0,219909.0,354.0,327.0,342.0,3820.0 +openrouter/alpindale/goliath-120b,Goliath 120B,1.875e-05,2429.0,420.0,68.0,235.0,139221.0,3143664.0,149623.0,235.0,140.0,166.0,1400.0 +openrouter/alpindale/magnum-72b,Magnum 72B,8.25e-06,18583.0,13590.0,247.0,475.0,349776.0,7030377.0,364098.0,475.0,452.0,469.0,3350.0 +openrouter/anthropic/claude-3-haiku,Anthropic: Claude 3 Haiku,1.5e-06,34021.0,25340.0,436.0,525.0,365290.0,1462342.0,414232.0,525.0,415.0,525.0,6780.0 +openrouter/anthropic/claude-3-opus,Anthropic: Claude 3 Opus,8.999999999999999e-05,38485.0,29980.0,512.0,525.0,364303.0,5155952.0,384550.0,525.0,403.0,525.0,6540.0 +openrouter/anthropic/claude-3-sonnet,Anthropic: Claude 3 Sonnet,1.8e-05,29425.0,20480.0,421.0,525.0,400937.0,2785993.0,408821.0,516.0,512.0,516.0,6980.0 +openrouter/anthropic/claude-3.5-sonnet,Anthropic: Claude 3.5 Sonnet,1.8e-05,39540.0,30550.0,514.0,525.0,386143.0,2556814.0,393193.0,520.0,506.0,520.0,6930.0 +openrouter/austism/chronos-hermes-13b,Chronos Hermes 13B v2,2.6e-07,13583.0,7130.0,255.0,525.0,338155.0,3710299.0,417665.0,525.0,216.0,507.0,4950.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,Dolphin Llama 3 70B 🐬,7.5e-07,25969.0,20640.0,397.0,525.0,302772.0,2705240.0,457056.0,525.0,154.0,523.0,3730.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,Dolphin 2.9.2 Mixtral 8x22B 🐬,1.8e-06,23843.0,16760.0,343.0,525.0,305882.0,3767734.0,361735.0,514.0,373.0,513.0,5340.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,Dolphin 2.6 Mixtral 8x7B 🐬,1e-06,11602.0,6770.0,197.0,475.0,486954.0,3139408.0,522367.0,475.0,337.0,403.0,3420.0 +openrouter/cohere/command-r-03-2024,Cohere: Command R (03-2024),2e-06,14938.0,9640.0,241.0,525.0,293562.0,1022081.0,299853.0,525.0,486.0,486.0,3560.0 +openrouter/cohere/command-r-08-2024,Cohere: Command R (08-2024),7.5e-07,21549.0,14010.0,354.0,525.0,394172.0,1617966.0,400217.0,525.0,525.0,525.0,5610.0 +openrouter/cohere/command-r-plus-04-2024,Cohere: Command R+ (04-2024),1.8e-05,26797.0,19380.0,372.0,525.0,436846.0,3340512.0,443264.0,525.0,520.0,520.0,5480.0 +openrouter/cohere/command-r-plus-08-2024,Cohere: Command R+ (08-2024),1.25e-05,26312.0,18000.0,334.0,525.0,420896.0,3434939.0,427460.0,525.0,519.0,524.0,6410.0 +openrouter/databricks/dbrx-instruct,Databricks: DBRX 132B Instruct,2.16e-06,14863.0,9140.0,240.0,351.0,273172.0,1476757.0,293338.0,350.0,308.0,345.0,4480.0 +openrouter/deepseek/deepseek-chat,DeepSeek-V2 Chat,4.2e-07,38715.0,30010.0,508.0,525.0,404603.0,8169847.0,411154.0,525.0,521.0,521.0,6630.0 +openrouter/deepseek/deepseek-coder,DeepSeek-Coder-V2,4.2e-07,38263.0,29370.0,501.0,525.0,457697.0,9389642.0,464184.0,524.0,519.0,519.0,6830.0 +openrouter/google/gemini-flash-1.5,Google: Gemini Flash 1.5,1.875e-07,25434.0,19460.0,365.0,475.0,375022.0,1367919.0,380768.0,475.0,472.0,472.0,4190.0 +openrouter/google/gemini-pro-1.5,Google: Gemini Pro 1.5,1e-05,20955.0,12540.0,342.0,525.0,351370.0,4055006.0,357860.0,525.0,524.0,524.0,6500.0 +openrouter/google/gemma-2-27b-it,Google: Gemma 2 27B,5.4e-07,24166.0,16230.0,371.0,525.0,340358.0,2400811.0,347679.0,525.0,525.0,525.0,5990.0 +openrouter/google/gemma-2-9b-it,Google: Gemma 2 9B,1.2e-07,10019.0,7590.0,128.0,305.0,229338.0,2474642.0,233707.0,305.0,241.0,245.0,1510.0 +openrouter/google/palm-2-chat-bison,Google: PaLM 2 Chat,7.5e-07,10545.0,3450.0,198.0,405.0,259646.0,1083586.0,264631.0,405.0,396.0,396.0,5700.0 +openrouter/google/palm-2-codechat-bison,Google: PaLM 2 Code Chat,7.5e-07,9463.0,4620.0,166.0,355.0,241614.0,1014701.0,245876.0,355.0,346.0,346.0,3630.0 +openrouter/meta-llama/llama-3-70b-instruct,Meta: Llama 3 70B Instruct,7.5e-07,32865.0,24660.0,464.0,525.0,332934.0,3922265.0,337237.0,525.0,523.0,523.0,6170.0 +openrouter/meta-llama/llama-3-8b-instruct,Meta: Llama 3 8B Instruct,1.1e-07,9861.0,5140.0,157.0,355.0,246427.0,1348505.0,258442.0,355.0,326.0,343.0,3540.0 +openrouter/meta-llama/llama-3.1-405b-instruct,Meta: Llama 3.1 405B Instruct,5e-06,37535.0,28870.0,501.0,525.0,387409.0,5468398.0,395381.0,525.0,524.0,525.0,6590.0 +openrouter/meta-llama/llama-3.1-70b-instruct,Meta: Llama 3.1 70B Instruct,7.5e-07,34328.0,26100.0,472.0,525.0,603516.0,5786116.0,618888.0,525.0,514.0,517.0,6200.0 +openrouter/meta-llama/llama-3.1-8b-instruct,Meta: Llama 3.1 8B Instruct,1.1e-07,24111.0,16630.0,367.0,525.0,953203.0,2874202.0,1061121.0,524.0,481.0,509.0,5600.0 +openrouter/microsoft/phi-3-medium-128k-instruct,Phi-3 Medium 128K Instruct,2e-06,11145.0,6750.0,209.0,285.0,174681.0,2048081.0,187054.0,285.0,205.0,226.0,3470.0 +openrouter/microsoft/phi-3-medium-4k-instruct,Phi-3 Medium 4K Instruct,2.8e-07,14576.0,10420.0,275.0,474.0,394882.0,2298398.0,414832.0,474.0,425.0,472.0,2510.0 +openrouter/microsoft/phi-3-mini-128k-instruct,Phi-3 Mini 128K Instruct,2e-07,16317.0,11470.0,254.0,405.0,369009.0,2226623.0,474015.0,405.0,233.0,375.0,3580.0 +openrouter/microsoft/wizardlm-2-7b,WizardLM-2 7B,1.1e-07,14293.0,9940.0,201.0,405.0,593242.0,4497924.0,807446.0,405.0,153.0,404.0,3190.0 +openrouter/microsoft/wizardlm-2-8x22b,WizardLM-2 8x22B,1e-06,27433.0,20640.0,391.0,525.0,538551.0,8621313.0,617857.0,525.0,343.0,524.0,5010.0 +openrouter/mistralai/codestral-mamba,Mistral: Codestral Mamba,5e-07,24507.0,16650.0,395.0,525.0,295112.0,1527598.0,362927.0,525.0,373.0,524.0,6040.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,Mistral: Mistral 7B Instruct v0.3,1.1e-07,13879.0,8880.0,276.0,525.0,442446.0,3234415.0,638588.0,525.0,88.0,520.0,3590.0 +openrouter/mistralai/mistral-large,Mistral Large,1.2e-05,36056.0,27240.0,473.0,525.0,356161.0,4045740.0,362618.0,525.0,523.0,525.0,6770.0 +openrouter/mistralai/mistral-medium,Mistral Medium,1.08e-05,30464.0,23480.0,429.0,525.0,388102.0,4744179.0,516829.0,525.0,275.0,525.0,5230.0 +openrouter/mistralai/mistral-nemo,Mistral: Mistral Nemo,3.4e-07,19162.0,14310.0,277.0,475.0,331661.0,2251105.0,347351.0,474.0,429.0,462.0,3210.0 +openrouter/mistralai/mistral-small,Mistral Small,8e-06,20218.0,14040.0,257.0,355.0,245203.0,1941925.0,251360.0,355.0,341.0,355.0,4870.0 +openrouter/mistralai/mistral-tiny,Mistral Tiny,5e-07,8643.0,4060.0,129.0,355.0,319190.0,1614905.0,398416.0,355.0,164.0,355.0,3580.0 +openrouter/mistralai/mixtral-8x22b-instruct,Mistral: Mixtral 8x22B Instruct,1.3e-06,29308.0,22290.0,415.0,525.0,354772.0,3189200.0,416131.0,525.0,384.0,524.0,5170.0 +openrouter/mistralai/mixtral-8x7b-instruct,Mixtral 8x7B Instruct,4.8e-07,28497.0,21890.0,378.0,525.0,427909.0,3224723.0,448322.0,514.0,413.0,482.0,4820.0 +openrouter/neversleep/llama-3-lumimaid-70b,Llama 3 Lumimaid 70B,7.875e-06,18596.0,12530.0,297.0,525.0,310151.0,5901834.0,315748.0,525.0,350.0,354.0,4540.0 +openrouter/neversleep/llama-3-lumimaid-8b,Llama 3 Lumimaid 8B,1.3125000000000001e-06,10618.0,6020.0,163.0,355.0,265226.0,1874912.0,280130.0,355.0,318.0,332.0,3430.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,NousResearch: Hermes 2 Pro - Llama-3 8B,2.8e-07,16984.0,12260.0,256.0,355.0,215657.0,900104.0,261421.0,355.0,249.0,354.0,3510.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,Nous: Hermes 2 Theta 8B,1.3125000000000001e-06,17210.0,12060.0,249.0,355.0,250147.0,2623126.0,275932.0,355.0,292.0,354.0,3900.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,Nous: Hermes 2 Mixtral 8x7B DPO,9e-07,19990.0,13020.0,293.0,405.0,400487.0,2513103.0,666767.0,405.0,148.0,394.0,5730.0 +openrouter/nousresearch/nous-hermes-yi-34b,Nous: Hermes 2 Yi 34B,1.44e-06,8206.0,3740.0,142.0,355.0,362584.0,3162299.0,380381.0,355.0,268.0,301.0,3400.0 +openrouter/openai/gpt-4-turbo,OpenAI: GPT-4 Turbo,4e-05,38422.0,30050.0,507.0,525.0,370196.0,4884796.0,376394.0,525.0,525.0,525.0,6290.0 +openrouter/openai/gpt-4o,OpenAI: GPT-4o,2e-05,39005.0,29520.0,493.0,524.0,347236.0,2230436.0,353722.0,524.0,524.0,524.0,7420.0 +openrouter/openai/gpt-4o-mini,OpenAI: GPT-4o-mini,7.5e-07,39441.0,29780.0,506.0,525.0,398390.0,2213392.0,404364.0,525.0,525.0,525.0,7580.0 +openrouter/openchat/openchat-8b,OpenChat 3.6 8B,1.1e-07,5376.0,1030.0,115.0,235.0,118063.0,780095.0,145257.0,232.0,102.0,227.0,3670.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,Perplexity: Llama3 Sonar 70B,2e-06,35051.0,26960.0,483.0,525.0,332540.0,3794568.0,336998.0,523.0,522.0,523.0,6040.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,Perplexity: Llama3 Sonar 8B,4e-07,7338.0,2230.0,146.0,355.0,356068.0,1319498.0,391432.0,353.0,188.0,351.0,4070.0 +openrouter/qwen/qwen-110b-chat,Qwen 1.5 110B Chat,3.24e-06,10980.0,5980.0,238.0,355.0,222823.0,1815105.0,253356.0,355.0,292.0,355.0,3760.0 +openrouter/qwen/qwen-2-72b-instruct,Qwen 2 72B Instruct,7.5e-07,25290.0,20490.0,297.0,355.0,226329.0,2421583.0,234573.0,355.0,344.0,354.0,3450.0 +openrouter/qwen/qwen-2-7b-instruct,Qwen 2 7B Instruct,1.1e-07,10750.0,6190.0,162.0,355.0,281785.0,1316997.0,289845.0,355.0,348.0,355.0,3340.0 +openrouter/qwen/qwen-72b-chat,Qwen 1.5 72B Chat,1.62e-06,27717.0,21880.0,404.0,525.0,353911.0,2856200.0,387928.0,518.0,397.0,518.0,4000.0 +openrouter/recursal/eagle-7b,RWKV v5: Eagle 7B,0.0,0.0,0.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,RWKV v5 3B AI Town,0.0,0.0,0.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,RWKV v5 World 3B,0.0,20.0,0.0,0.0,11.0,4547.0,73927.0,4902.0,10.0,3.0,7.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,OpenHermes 2.5 Mistral 7B,3.4e-07,15881.0,12220.0,208.0,355.0,225844.0,1023962.0,244382.0,355.0,291.0,347.0,2460.0 +openrouter/togethercomputer/stripedhyena-nous-7b,StripedHyena Nous 7B,3.6e-07,12001.0,8050.0,213.0,475.0,267160.0,1326173.0,314893.0,473.0,356.0,449.0,2460.0 +openrouter/undi95/toppy-m-7b,Toppy M 7B,1.4e-07,3026.0,1960.0,105.0,305.0,196524.0,1189846.0,364442.0,305.0,84.0,302.0,270.0 +openrouter/xwin-lm/xwin-lm-70b,Xwin 70B,7.5e-06,5596.0,800.0,109.0,355.0,229276.0,7063564.0,278695.0,355.0,80.0,232.0,4020.0 diff --git a/docs/reports/v0.6/evaluation-by-symflower-fix.csv b/docs/reports/v0.6/evaluation-by-symflower-fix.csv new file mode 100644 index 00000000..f4449b9d --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-symflower-fix.csv @@ -0,0 +1,68 @@ +model_id,score,score_fix,files_executed,files_executed_fix +openrouter/01-ai/yi-34b-chat,9.0,9.0,0.0,0.0 +openrouter/ai21/jamba-instruct,15.0,15.0,0.0,0.0 +openrouter/alpindale/goliath-120b,1265.0,1483.0,23.0,31.0 +openrouter/alpindale/magnum-72b,570.0,911.0,7.0,18.0 +openrouter/anthropic/claude-3-haiku,4928.0,5830.0,102.0,124.0 +openrouter/anthropic/claude-3-opus,7222.0,7477.0,159.0,164.0 +openrouter/anthropic/claude-3-sonnet,6209.0,7402.0,141.0,164.0 +openrouter/anthropic/claude-3.5-sonnet,7380.0,7584.0,162.0,166.0 +openrouter/austism/chronos-hermes-13b,3788.0,4560.0,79.0,101.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,4329.0,5329.0,101.0,121.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,4502.0,5485.0,91.0,114.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,1015.0,1439.0,24.0,38.0 +openrouter/cohere/command-r-03-2024,1959.0,2391.0,43.0,55.0 +openrouter/cohere/command-r-08-2024,6287.0,6646.0,147.0,156.0 +openrouter/cohere/command-r-plus-04-2024,3240.0,3454.0,58.0,62.0 +openrouter/cohere/command-r-plus-08-2024,3168.0,3629.0,58.0,69.0 +openrouter/databricks/dbrx-instruct,5609.0,6891.0,125.0,157.0 +openrouter/deepseek/deepseek-chat,7092.0,7684.0,156.0,168.0 +openrouter/deepseek/deepseek-coder,6947.0,7580.0,153.0,166.0 +openrouter/google/gemini-flash-1.5,7035.0,7392.0,157.0,164.0 +openrouter/google/gemini-pro-1.5,7381.0,7565.0,161.0,165.0 +openrouter/google/gemma-2-27b-it,6304.0,7569.0,144.0,169.0 +openrouter/google/gemma-2-9b-it,1495.0,1894.0,37.0,46.0 +openrouter/google/palm-2-chat-bison,3834.0,4667.0,72.0,95.0 +openrouter/google/palm-2-codechat-bison,3975.0,5165.0,81.0,111.0 +openrouter/meta-llama/llama-3-70b-instruct,6431.0,7064.0,145.0,158.0 +openrouter/meta-llama/llama-3-8b-instruct,14.0,14.0,0.0,0.0 +openrouter/meta-llama/llama-3.1-405b-instruct,7495.0,7536.0,165.0,166.0 +openrouter/meta-llama/llama-3.1-70b-instruct,6245.0,6459.0,148.0,152.0 +openrouter/meta-llama/llama-3.1-8b-instruct,4120.0,4683.0,95.0,108.0 +openrouter/microsoft/phi-3-medium-128k-instruct,1270.0,1565.0,22.0,27.0 +openrouter/microsoft/phi-3-medium-4k-instruct,4160.0,5345.0,95.0,120.0 +openrouter/microsoft/phi-3-mini-128k-instruct,1093.0,1175.0,19.0,21.0 +openrouter/microsoft/wizardlm-2-7b,929.0,1927.0,22.0,40.0 +openrouter/microsoft/wizardlm-2-8x22b,4740.0,6211.0,117.0,148.0 +openrouter/mistralai/codestral-mamba,6079.0,6499.0,133.0,143.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,2584.0,4686.0,62.0,114.0 +openrouter/mistralai/mistral-large,6689.0,7242.0,140.0,153.0 +openrouter/mistralai/mistral-medium,5078.0,6651.0,112.0,145.0 +openrouter/mistralai/mistral-nemo,1285.0,1747.0,33.0,45.0 +openrouter/mistralai/mistral-small,4295.0,5194.0,91.0,110.0 +openrouter/mistralai/mistral-tiny,2198.0,4990.0,39.0,111.0 +openrouter/mistralai/mixtral-8x22b-instruct,5764.0,6815.0,129.0,150.0 +openrouter/mistralai/mixtral-8x7b-instruct,4323.0,5298.0,98.0,123.0 +openrouter/neversleep/llama-3-lumimaid-70b,3875.0,4918.0,85.0,108.0 +openrouter/neversleep/llama-3-lumimaid-8b,13.0,13.0,0.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,14.0,14.0,0.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,11.0,11.0,0.0,0.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,2183.0,2570.0,38.0,45.0 +openrouter/nousresearch/nous-hermes-yi-34b,3064.0,3845.0,66.0,87.0 +openrouter/openai/gpt-4-turbo,7003.0,7790.0,153.0,170.0 +openrouter/openai/gpt-4o,6446.0,6895.0,139.0,148.0 +openrouter/openai/gpt-4o-mini,6972.0,7707.0,152.0,167.0 +openrouter/openchat/openchat-8b,3071.0,4574.0,66.0,99.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,6643.0,7234.0,153.0,164.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,768.0,1414.0,16.0,32.0 +openrouter/qwen/qwen-110b-chat,4240.0,5637.0,98.0,125.0 +openrouter/qwen/qwen-2-72b-instruct,15.0,27.0,0.0,2.0 +openrouter/qwen/qwen-2-7b-instruct,15.0,15.0,0.0,0.0 +openrouter/qwen/qwen-72b-chat,3823.0,6436.0,96.0,149.0 +openrouter/recursal/eagle-7b,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,10.0,10.0,0.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,15.0,15.0,0.0,0.0 +openrouter/togethercomputer/stripedhyena-nous-7b,3277.0,3894.0,68.0,85.0 +openrouter/undi95/toppy-m-7b,1869.0,2179.0,45.0,55.0 +openrouter/xwin-lm/xwin-lm-70b,2538.0,4112.0,51.0,95.0 diff --git a/docs/reports/v0.6/evaluation-by-task-by-language.csv b/docs/reports/v0.6/evaluation-by-task-by-language.csv new file mode 100644 index 00000000..2ee4bd04 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-task-by-language.csv @@ -0,0 +1,525 @@ +model_id,task,language,score,coverage,files_executed,files_executed_maximum_reachable,generate_tests_for_file_character_count,processing_time,response_character_count,response_no_error,response_no_excess,response_with_code,tests_passing +openrouter/01-ai/yi-34b-chat,code-repair,java,39.0,0.0,1.0,5.0,690.0,28124.0,2411.0,5.0,0.0,3.0,30.0 +openrouter/01-ai/yi-34b-chat,write-tests,golang,9.0,0.0,0.0,5.0,5405.0,58053.0,5439.0,5.0,2.0,2.0,0.0 +openrouter/01-ai/yi-34b-chat,write-tests,java,225.0,70.0,3.0,120.0,59982.0,1041641.0,101963.0,101.0,6.0,45.0,0.0 +openrouter/01-ai/yi-34b-chat,write-tests,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/ai21/jamba-instruct,code-repair,java,126.0,0.0,4.0,5.0,1397.0,11768.0,1527.0,5.0,3.0,4.0,110.0 +openrouter/ai21/jamba-instruct,code-repair,ruby,130.0,0.0,5.0,5.0,695.0,9137.0,760.0,5.0,5.0,5.0,110.0 +openrouter/ai21/jamba-instruct,transpile,java,2069.0,0.0,45.0,50.0,18180.0,147212.0,19212.0,50.0,46.0,48.0,1880.0 +openrouter/ai21/jamba-instruct,transpile,ruby,1909.0,0.0,46.0,50.0,15391.0,150191.0,17485.0,50.0,43.0,50.0,1720.0 +openrouter/ai21/jamba-instruct,write-tests,golang,15.0,0.0,0.0,5.0,1001.0,11333.0,1055.0,5.0,5.0,5.0,0.0 +openrouter/ai21/jamba-instruct,write-tests,java,5661.0,5260.0,58.0,120.0,103535.0,596484.0,106357.0,119.0,110.0,114.0,0.0 +openrouter/ai21/jamba-instruct,write-tests,ruby,5696.0,5280.0,65.0,120.0,71621.0,501961.0,73513.0,120.0,115.0,116.0,0.0 +openrouter/alpindale/goliath-120b,code-repair,ruby,136.0,0.0,4.0,5.0,609.0,27301.0,1543.0,5.0,2.0,5.0,120.0 +openrouter/alpindale/goliath-120b,transpile,golang,1251.0,0.0,23.0,50.0,20499.0,564043.0,22127.0,50.0,42.0,46.0,1090.0 +openrouter/alpindale/goliath-120b,transpile,ruby,353.0,0.0,23.0,50.0,20430.0,493632.0,22679.0,50.0,42.0,48.0,190.0 +openrouter/alpindale/goliath-120b,write-tests,golang,14.0,0.0,0.0,5.0,847.0,33728.0,1289.0,5.0,4.0,5.0,0.0 +openrouter/alpindale/goliath-120b,write-tests,java,13.0,0.0,0.0,5.0,2501.0,58730.0,2557.0,5.0,4.0,4.0,0.0 +openrouter/alpindale/goliath-120b,write-tests,ruby,662.0,420.0,18.0,120.0,94335.0,1966230.0,99428.0,120.0,46.0,58.0,0.0 +openrouter/alpindale/magnum-72b,code-repair,golang,210.0,0.0,5.0,5.0,657.0,22282.0,721.0,5.0,5.0,5.0,190.0 +openrouter/alpindale/magnum-72b,code-repair,java,160.0,0.0,5.0,5.0,1019.0,30874.0,1079.0,5.0,5.0,5.0,140.0 +openrouter/alpindale/magnum-72b,code-repair,ruby,160.0,0.0,5.0,5.0,628.0,11655.0,688.0,5.0,5.0,5.0,140.0 +openrouter/alpindale/magnum-72b,transpile,java,2307.0,0.0,48.0,50.0,17547.0,237647.0,18174.0,50.0,49.0,50.0,2110.0 +openrouter/alpindale/magnum-72b,transpile,ruby,949.0,0.0,33.0,50.0,17049.0,260798.0,19778.0,50.0,46.0,50.0,770.0 +openrouter/alpindale/magnum-72b,write-tests,golang,570.0,210.0,7.0,120.0,94066.0,2694317.0,97220.0,120.0,115.0,118.0,0.0 +openrouter/alpindale/magnum-72b,write-tests,java,7448.0,7030.0,69.0,120.0,123836.0,2609956.0,129526.0,120.0,111.0,118.0,0.0 +openrouter/alpindale/magnum-72b,write-tests,ruby,6779.0,6350.0,75.0,120.0,94974.0,1162848.0,96912.0,120.0,116.0,118.0,0.0 +openrouter/anthropic/claude-3-haiku,code-repair,golang,209.0,0.0,5.0,5.0,648.0,7332.0,897.0,5.0,4.0,5.0,190.0 +openrouter/anthropic/claude-3-haiku,code-repair,java,159.0,0.0,5.0,5.0,1050.0,11935.0,1295.0,5.0,4.0,5.0,140.0 +openrouter/anthropic/claude-3-haiku,code-repair,ruby,168.0,0.0,5.0,5.0,620.0,8828.0,803.0,5.0,3.0,5.0,150.0 +openrouter/anthropic/claude-3-haiku,transpile,golang,2533.0,0.0,44.0,50.0,14757.0,93775.0,15370.0,50.0,49.0,50.0,2340.0 +openrouter/anthropic/claude-3-haiku,transpile,java,2390.0,0.0,50.0,50.0,22119.0,95436.0,22719.0,50.0,50.0,50.0,2190.0 +openrouter/anthropic/claude-3-haiku,transpile,ruby,1969.0,0.0,49.0,50.0,15179.0,95373.0,15779.0,50.0,50.0,50.0,1770.0 +openrouter/anthropic/claude-3-haiku,write-tests,golang,2395.0,1990.0,58.0,120.0,105341.0,382069.0,112238.0,120.0,107.0,120.0,0.0 +openrouter/anthropic/claude-3-haiku,write-tests,java,13543.0,13120.0,107.0,120.0,117531.0,419102.0,137371.0,120.0,76.0,120.0,0.0 +openrouter/anthropic/claude-3-haiku,write-tests,ruby,10655.0,10230.0,113.0,120.0,88045.0,348492.0,107760.0,120.0,72.0,120.0,0.0 +openrouter/anthropic/claude-3-opus,code-repair,golang,210.0,0.0,5.0,5.0,638.0,17261.0,708.0,5.0,5.0,5.0,190.0 +openrouter/anthropic/claude-3-opus,code-repair,java,160.0,0.0,5.0,5.0,943.0,22385.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/anthropic/claude-3-opus,code-repair,ruby,160.0,0.0,5.0,5.0,623.0,16460.0,683.0,5.0,5.0,5.0,140.0 +openrouter/anthropic/claude-3-opus,transpile,golang,2743.0,0.0,48.0,50.0,14684.0,264611.0,15878.0,50.0,35.0,50.0,2560.0 +openrouter/anthropic/claude-3-opus,transpile,java,2380.0,0.0,50.0,50.0,18247.0,287990.0,19839.0,50.0,30.0,50.0,2200.0 +openrouter/anthropic/claude-3-opus,transpile,ruby,1492.0,0.0,50.0,50.0,16177.0,297445.0,17559.0,50.0,32.0,50.0,1310.0 +openrouter/anthropic/claude-3-opus,write-tests,golang,4479.0,4020.0,111.0,120.0,93595.0,1409783.0,96358.0,120.0,108.0,120.0,0.0 +openrouter/anthropic/claude-3-opus,write-tests,java,15399.0,14950.0,120.0,120.0,129243.0,1523485.0,134102.0,120.0,89.0,120.0,0.0 +openrouter/anthropic/claude-3-opus,write-tests,ruby,11462.0,11010.0,118.0,120.0,90153.0,1316532.0,98420.0,120.0,94.0,120.0,0.0 +openrouter/anthropic/claude-3-sonnet,code-repair,golang,210.0,0.0,5.0,5.0,648.0,10908.0,702.0,5.0,5.0,5.0,190.0 +openrouter/anthropic/claude-3-sonnet,code-repair,java,160.0,0.0,5.0,5.0,1104.0,14984.0,1164.0,5.0,5.0,5.0,140.0 +openrouter/anthropic/claude-3-sonnet,code-repair,ruby,170.0,0.0,5.0,5.0,716.0,10473.0,776.0,5.0,5.0,5.0,150.0 +openrouter/anthropic/claude-3-sonnet,transpile,golang,2441.0,0.0,41.0,50.0,14727.0,131789.0,15227.0,50.0,50.0,50.0,2250.0 +openrouter/anthropic/claude-3-sonnet,transpile,java,2400.0,0.0,50.0,50.0,18865.0,148089.0,19465.0,50.0,50.0,50.0,2200.0 +openrouter/anthropic/claude-3-sonnet,transpile,ruby,2249.0,0.0,49.0,50.0,14966.0,154623.0,15566.0,50.0,50.0,50.0,2050.0 +openrouter/anthropic/claude-3-sonnet,write-tests,golang,3768.0,3310.0,100.0,120.0,117258.0,847240.0,119413.0,120.0,118.0,120.0,0.0 +openrouter/anthropic/claude-3-sonnet,write-tests,java,9627.0,9200.0,68.0,120.0,141027.0,840179.0,143215.0,120.0,119.0,120.0,0.0 +openrouter/anthropic/claude-3-sonnet,write-tests,ruby,8400.0,7970.0,98.0,120.0,91626.0,627708.0,93293.0,111.0,110.0,111.0,0.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,golang,210.0,0.0,5.0,5.0,648.0,14999.0,718.0,5.0,5.0,5.0,190.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,java,160.0,0.0,5.0,5.0,943.0,18055.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,ruby,170.0,0.0,5.0,5.0,620.0,9919.0,680.0,5.0,5.0,5.0,150.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,golang,2816.0,0.0,49.0,50.0,12858.0,134822.0,13348.0,49.0,49.0,49.0,2620.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,java,2302.0,0.0,48.0,50.0,20418.0,150258.0,20994.0,48.0,48.0,48.0,2110.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,ruby,1920.0,0.0,50.0,50.0,15811.0,134184.0,16411.0,50.0,50.0,50.0,1720.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,golang,4564.0,4100.0,113.0,120.0,98941.0,735304.0,100611.0,119.0,113.0,119.0,0.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,java,15486.0,15010.0,119.0,120.0,132394.0,795367.0,133822.0,119.0,119.0,119.0,0.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,ruby,11912.0,11440.0,120.0,120.0,103510.0,563906.0,105606.0,120.0,112.0,120.0,0.0 +openrouter/austism/chronos-hermes-13b,code-repair,golang,175.0,0.0,4.0,5.0,792.0,15674.0,1647.0,5.0,1.0,5.0,160.0 +openrouter/austism/chronos-hermes-13b,code-repair,java,155.0,0.0,5.0,5.0,1101.0,26808.0,3169.0,5.0,0.0,5.0,140.0 +openrouter/austism/chronos-hermes-13b,code-repair,ruby,135.0,0.0,5.0,5.0,607.0,15423.0,1955.0,5.0,0.0,5.0,120.0 +openrouter/austism/chronos-hermes-13b,transpile,golang,2235.0,0.0,40.0,50.0,15822.0,166874.0,17883.0,50.0,27.0,48.0,2070.0 +openrouter/austism/chronos-hermes-13b,transpile,java,1732.0,0.0,37.0,50.0,20774.0,192945.0,24093.0,50.0,19.0,46.0,1580.0 +openrouter/austism/chronos-hermes-13b,transpile,ruby,1042.0,0.0,33.0,50.0,16286.0,189267.0,19334.0,50.0,30.0,49.0,880.0 +openrouter/austism/chronos-hermes-13b,write-tests,golang,1553.0,1230.0,39.0,120.0,94505.0,1162837.0,113113.0,120.0,50.0,114.0,0.0 +openrouter/austism/chronos-hermes-13b,write-tests,java,3534.0,3210.0,35.0,120.0,112990.0,1115041.0,139965.0,120.0,49.0,120.0,0.0 +openrouter/austism/chronos-hermes-13b,write-tests,ruby,3022.0,2690.0,57.0,120.0,75278.0,825430.0,96506.0,120.0,40.0,115.0,0.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,golang,206.0,0.0,5.0,5.0,646.0,10196.0,1532.0,5.0,1.0,5.0,190.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,java,135.0,0.0,4.0,5.0,968.0,13320.0,1708.0,5.0,1.0,5.0,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,ruby,135.0,0.0,4.0,5.0,615.0,7599.0,800.0,5.0,2.0,4.0,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,golang,1180.0,0.0,21.0,50.0,14963.0,203406.0,32475.0,50.0,9.0,50.0,1050.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,java,2101.0,0.0,44.0,50.0,18726.0,150995.0,24535.0,50.0,28.0,49.0,1930.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,ruby,500.0,0.0,47.0,50.0,14642.0,151490.0,20510.0,50.0,33.0,50.0,320.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,golang,3149.0,2800.0,80.0,120.0,97004.0,866326.0,139732.0,120.0,29.0,120.0,0.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,java,9550.0,9190.0,84.0,120.0,96663.0,666821.0,132202.0,120.0,36.0,120.0,0.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,ruby,9013.0,8650.0,108.0,120.0,58545.0,635087.0,103562.0,120.0,15.0,120.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,golang,210.0,0.0,5.0,5.0,643.0,11714.0,705.0,5.0,5.0,5.0,190.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,java,159.0,0.0,5.0,5.0,1045.0,17055.0,1210.0,5.0,4.0,5.0,140.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,ruby,169.0,0.0,5.0,5.0,600.0,11335.0,732.0,5.0,4.0,5.0,150.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,golang,2100.0,0.0,38.0,50.0,14723.0,210146.0,17642.0,50.0,42.0,50.0,1920.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,java,2346.0,0.0,49.0,50.0,16718.0,241285.0,18869.0,50.0,47.0,50.0,2150.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,ruby,958.0,0.0,35.0,50.0,14334.0,220562.0,18310.0,47.0,39.0,47.0,790.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,golang,2402.0,2040.0,53.0,120.0,89398.0,1223303.0,107387.0,120.0,69.0,120.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,java,9608.0,9210.0,85.0,120.0,101143.0,1028359.0,118831.0,120.0,74.0,119.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,ruby,5891.0,5510.0,68.0,120.0,67278.0,803975.0,78049.0,112.0,89.0,112.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,golang,169.0,0.0,4.0,5.0,735.0,9146.0,805.0,5.0,5.0,5.0,150.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,java,106.0,0.0,3.0,5.0,1700.0,9985.0,1750.0,5.0,4.0,4.0,90.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,ruby,140.0,0.0,5.0,5.0,629.0,6905.0,689.0,5.0,5.0,5.0,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile,java,2263.0,0.0,47.0,50.0,23548.0,224545.0,24355.0,50.0,47.0,49.0,2070.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile,ruby,1180.0,0.0,43.0,50.0,14712.0,162913.0,15995.0,50.0,47.0,50.0,990.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,golang,1015.0,730.0,24.0,120.0,153584.0,919068.0,162732.0,120.0,61.0,80.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,java,4329.0,3990.0,36.0,120.0,187083.0,843376.0,199600.0,120.0,83.0,100.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,ruby,2400.0,2050.0,35.0,120.0,104963.0,963470.0,116441.0,120.0,85.0,110.0,0.0 +openrouter/cohere/command-r-03-2024,code-repair,golang,180.0,0.0,5.0,5.0,712.0,4385.0,782.0,5.0,5.0,5.0,160.0 +openrouter/cohere/command-r-03-2024,code-repair,java,160.0,0.0,5.0,5.0,1160.0,4768.0,1220.0,5.0,5.0,5.0,140.0 +openrouter/cohere/command-r-03-2024,code-repair,ruby,109.0,0.0,4.0,5.0,728.0,5125.0,788.0,5.0,5.0,5.0,90.0 +openrouter/cohere/command-r-03-2024,transpile,golang,1168.0,0.0,22.0,50.0,15568.0,61464.0,15984.0,50.0,28.0,28.0,1040.0 +openrouter/cohere/command-r-03-2024,transpile,java,2065.0,0.0,45.0,50.0,19922.0,66481.0,20492.0,50.0,45.0,45.0,1880.0 +openrouter/cohere/command-r-03-2024,transpile,ruby,411.0,0.0,35.0,50.0,12753.0,54914.0,13281.0,50.0,38.0,38.0,250.0 +openrouter/cohere/command-r-03-2024,write-tests,golang,791.0,410.0,21.0,120.0,86883.0,339093.0,88590.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-03-2024,write-tests,java,6237.0,5820.0,57.0,120.0,97287.0,276733.0,98727.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-03-2024,write-tests,ruby,3817.0,3410.0,47.0,120.0,58549.0,209118.0,59989.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-08-2024,code-repair,golang,87.0,0.0,2.0,5.0,680.0,4823.0,750.0,5.0,5.0,5.0,70.0 +openrouter/cohere/command-r-08-2024,code-repair,java,160.0,0.0,5.0,5.0,1167.0,5878.0,1227.0,5.0,5.0,5.0,140.0 +openrouter/cohere/command-r-08-2024,code-repair,ruby,160.0,0.0,5.0,5.0,600.0,4337.0,660.0,5.0,5.0,5.0,140.0 +openrouter/cohere/command-r-08-2024,transpile,golang,2505.0,0.0,45.0,50.0,15216.0,76493.0,15716.0,50.0,50.0,50.0,2310.0 +openrouter/cohere/command-r-08-2024,transpile,java,2380.0,0.0,50.0,50.0,19832.0,98784.0,20432.0,50.0,50.0,50.0,2180.0 +openrouter/cohere/command-r-08-2024,transpile,ruby,964.0,0.0,44.0,50.0,14401.0,71965.0,15001.0,50.0,50.0,50.0,770.0 +openrouter/cohere/command-r-08-2024,write-tests,golang,3782.0,3320.0,102.0,120.0,122683.0,542911.0,123955.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-08-2024,write-tests,java,8902.0,8470.0,72.0,120.0,129811.0,440913.0,131253.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-08-2024,write-tests,ruby,2609.0,2220.0,29.0,120.0,89782.0,371862.0,91223.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,golang,210.0,0.0,5.0,5.0,645.0,9533.0,711.0,5.0,5.0,5.0,190.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,java,160.0,0.0,5.0,5.0,1050.0,8784.0,1110.0,5.0,5.0,5.0,140.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,ruby,170.0,0.0,5.0,5.0,612.0,7975.0,672.0,5.0,5.0,5.0,150.0 +openrouter/cohere/command-r-plus-04-2024,transpile,golang,2284.0,0.0,40.0,50.0,16549.0,163178.0,17037.0,50.0,47.0,47.0,2100.0 +openrouter/cohere/command-r-plus-04-2024,transpile,java,2290.0,0.0,50.0,50.0,18468.0,142372.0,19068.0,50.0,50.0,50.0,2090.0 +openrouter/cohere/command-r-plus-04-2024,transpile,ruby,1010.0,0.0,50.0,50.0,14995.0,136347.0,15595.0,50.0,50.0,50.0,810.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,golang,956.0,580.0,18.0,120.0,122756.0,1150310.0,124428.0,120.0,119.0,119.0,0.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,java,12543.0,12080.0,105.0,120.0,162562.0,1023882.0,163993.0,120.0,119.0,119.0,0.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,ruby,7174.0,6720.0,94.0,120.0,99209.0,698131.0,100650.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,golang,210.0,0.0,5.0,5.0,646.0,7484.0,696.0,5.0,5.0,5.0,190.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,java,160.0,0.0,5.0,5.0,1137.0,10420.0,1197.0,5.0,5.0,5.0,140.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,ruby,169.0,0.0,5.0,5.0,593.0,6426.0,681.0,5.0,4.0,5.0,150.0 +openrouter/cohere/command-r-plus-08-2024,transpile,golang,2607.0,0.0,47.0,50.0,14550.0,154729.0,15050.0,50.0,50.0,50.0,2410.0 +openrouter/cohere/command-r-plus-08-2024,transpile,java,2388.0,0.0,50.0,50.0,20667.0,157880.0,21367.0,50.0,48.0,50.0,2190.0 +openrouter/cohere/command-r-plus-08-2024,transpile,ruby,1522.0,0.0,46.0,50.0,13908.0,157446.0,14572.0,50.0,47.0,49.0,1330.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,golang,561.0,190.0,11.0,120.0,118473.0,1074047.0,120095.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,java,12589.0,12130.0,99.0,120.0,149007.0,1025704.0,150447.0,120.0,120.0,120.0,0.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,ruby,6106.0,5680.0,66.0,120.0,101915.0,840803.0,103355.0,120.0,120.0,120.0,0.0 +openrouter/databricks/dbrx-instruct,code-repair,golang,210.0,0.0,5.0,5.0,693.0,9162.0,780.0,5.0,5.0,5.0,190.0 +openrouter/databricks/dbrx-instruct,code-repair,java,160.0,0.0,5.0,5.0,1223.0,6726.0,1283.0,5.0,5.0,5.0,140.0 +openrouter/databricks/dbrx-instruct,transpile,golang,2203.0,0.0,39.0,50.0,15523.0,100806.0,16550.0,50.0,44.0,50.0,2020.0 +openrouter/databricks/dbrx-instruct,transpile,java,2316.0,0.0,49.0,50.0,20868.0,102756.0,23017.0,50.0,38.0,49.0,2130.0 +openrouter/databricks/dbrx-instruct,write-tests,golang,3406.0,2970.0,86.0,120.0,120661.0,712862.0,125114.0,120.0,113.0,117.0,0.0 +openrouter/databricks/dbrx-instruct,write-tests,java,6568.0,6170.0,56.0,120.0,114204.0,544445.0,126594.0,120.0,103.0,119.0,0.0 +openrouter/databricks/dbrx-instruct,write-tests,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/deepseek/deepseek-chat,code-repair,golang,210.0,0.0,5.0,5.0,638.0,16506.0,708.0,5.0,5.0,5.0,190.0 +openrouter/deepseek/deepseek-chat,code-repair,java,160.0,0.0,5.0,5.0,943.0,23376.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/deepseek/deepseek-chat,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,19269.0,679.0,5.0,5.0,5.0,150.0 +openrouter/deepseek/deepseek-chat,transpile,golang,2390.0,0.0,40.0,50.0,14869.0,387595.0,15501.0,50.0,50.0,50.0,2200.0 +openrouter/deepseek/deepseek-chat,transpile,java,2400.0,0.0,50.0,50.0,17235.0,443244.0,17835.0,50.0,50.0,50.0,2200.0 +openrouter/deepseek/deepseek-chat,transpile,ruby,1950.0,0.0,50.0,50.0,15562.0,381113.0,16162.0,50.0,50.0,50.0,1750.0 +openrouter/deepseek/deepseek-chat,write-tests,golang,4702.0,4230.0,116.0,120.0,107554.0,2475186.0,109213.0,120.0,118.0,118.0,0.0 +openrouter/deepseek/deepseek-chat,write-tests,java,15369.0,14890.0,119.0,120.0,145645.0,2282551.0,147085.0,120.0,120.0,120.0,0.0 +openrouter/deepseek/deepseek-chat,write-tests,ruby,11364.0,10890.0,118.0,120.0,101538.0,2141007.0,102968.0,120.0,118.0,118.0,0.0 +openrouter/deepseek/deepseek-coder,code-repair,golang,210.0,0.0,5.0,5.0,638.0,17580.0,708.0,5.0,5.0,5.0,190.0 +openrouter/deepseek/deepseek-coder,code-repair,java,160.0,0.0,5.0,5.0,943.0,26771.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/deepseek/deepseek-coder,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,17018.0,679.0,5.0,5.0,5.0,150.0 +openrouter/deepseek/deepseek-coder,transpile,golang,2390.0,0.0,40.0,50.0,15059.0,367719.0,15659.0,50.0,50.0,50.0,2200.0 +openrouter/deepseek/deepseek-coder,transpile,java,2400.0,0.0,50.0,50.0,17236.0,421890.0,17836.0,50.0,50.0,50.0,2200.0 +openrouter/deepseek/deepseek-coder,transpile,ruby,2150.0,0.0,50.0,50.0,15326.0,341024.0,15926.0,50.0,50.0,50.0,1950.0 +openrouter/deepseek/deepseek-coder,write-tests,golang,4557.0,4090.0,113.0,120.0,118826.0,2939823.0,120473.0,120.0,117.0,117.0,0.0 +openrouter/deepseek/deepseek-coder,write-tests,java,14606.0,14140.0,113.0,120.0,183417.0,3200707.0,184827.0,119.0,117.0,117.0,0.0 +openrouter/deepseek/deepseek-coder,write-tests,ruby,11620.0,11140.0,120.0,120.0,105633.0,2057110.0,107073.0,120.0,120.0,120.0,0.0 +openrouter/google/gemini-flash-1.5,code-repair,golang,210.0,0.0,5.0,5.0,638.0,6048.0,708.0,5.0,5.0,5.0,190.0 +openrouter/google/gemini-flash-1.5,code-repair,java,160.0,0.0,5.0,5.0,943.0,8134.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/google/gemini-flash-1.5,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,6737.0,679.0,5.0,5.0,5.0,150.0 +openrouter/google/gemini-flash-1.5,transpile,golang,2645.0,0.0,45.0,50.0,13153.0,80877.0,13654.0,50.0,50.0,50.0,2450.0 +openrouter/google/gemini-flash-1.5,transpile,ruby,1459.0,0.0,49.0,50.0,15792.0,94699.0,16392.0,50.0,50.0,50.0,1260.0 +openrouter/google/gemini-flash-1.5,write-tests,golang,4390.0,3920.0,112.0,120.0,124606.0,461987.0,126194.0,120.0,119.0,119.0,0.0 +openrouter/google/gemini-flash-1.5,write-tests,java,4686.0,4300.0,26.0,120.0,130935.0,366520.0,132375.0,120.0,120.0,120.0,0.0 +openrouter/google/gemini-flash-1.5,write-tests,ruby,11714.0,11240.0,118.0,120.0,88336.0,342917.0,89763.0,120.0,118.0,118.0,0.0 +openrouter/google/gemini-pro-1.5,code-repair,golang,210.0,0.0,5.0,5.0,638.0,14215.0,708.0,5.0,5.0,5.0,190.0 +openrouter/google/gemini-pro-1.5,code-repair,java,160.0,0.0,5.0,5.0,948.0,13827.0,1008.0,5.0,5.0,5.0,140.0 +openrouter/google/gemini-pro-1.5,code-repair,ruby,170.0,0.0,5.0,5.0,604.0,19835.0,664.0,5.0,5.0,5.0,150.0 +openrouter/google/gemini-pro-1.5,transpile,golang,2900.0,0.0,50.0,50.0,13214.0,344547.0,13715.0,50.0,50.0,50.0,2700.0 +openrouter/google/gemini-pro-1.5,transpile,java,2400.0,0.0,50.0,50.0,15964.0,406601.0,16564.0,50.0,50.0,50.0,2200.0 +openrouter/google/gemini-pro-1.5,transpile,ruby,1317.0,0.0,49.0,50.0,39713.0,430448.0,40304.0,50.0,49.0,49.0,1120.0 +openrouter/google/gemini-pro-1.5,write-tests,golang,4481.0,4010.0,111.0,120.0,92911.0,930114.0,94611.0,120.0,120.0,120.0,0.0 +openrouter/google/gemini-pro-1.5,write-tests,java,4601.0,4210.0,31.0,120.0,108906.0,832752.0,110350.0,120.0,120.0,120.0,0.0 +openrouter/google/gemini-pro-1.5,write-tests,ruby,4716.0,4320.0,36.0,120.0,78472.0,1062667.0,79936.0,120.0,120.0,120.0,0.0 +openrouter/google/gemma-2-27b-it,code-repair,golang,210.0,0.0,5.0,5.0,654.0,7728.0,732.0,5.0,5.0,5.0,190.0 +openrouter/google/gemma-2-27b-it,code-repair,java,160.0,0.0,5.0,5.0,942.0,9161.0,1008.0,5.0,5.0,5.0,140.0 +openrouter/google/gemma-2-27b-it,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,7618.0,688.0,5.0,5.0,5.0,150.0 +openrouter/google/gemma-2-27b-it,transpile,golang,2472.0,0.0,42.0,50.0,13815.0,123204.0,14418.0,50.0,50.0,50.0,2280.0 +openrouter/google/gemma-2-27b-it,transpile,java,2400.0,0.0,50.0,50.0,16172.0,138418.0,16857.0,50.0,50.0,50.0,2200.0 +openrouter/google/gemma-2-27b-it,transpile,ruby,1229.0,0.0,49.0,50.0,16603.0,138423.0,17296.0,50.0,50.0,50.0,1030.0 +openrouter/google/gemma-2-27b-it,write-tests,golang,3832.0,3370.0,102.0,120.0,91956.0,700837.0,93781.0,120.0,120.0,120.0,0.0 +openrouter/google/gemma-2-27b-it,write-tests,java,11422.0,10970.0,92.0,120.0,117303.0,674765.0,118967.0,120.0,120.0,120.0,0.0 +openrouter/google/gemma-2-27b-it,write-tests,ruby,2271.0,1890.0,21.0,120.0,82294.0,600657.0,83932.0,120.0,120.0,120.0,0.0 +openrouter/google/gemma-2-9b-it,code-repair,golang,167.0,0.0,4.0,5.0,641.0,13119.0,699.0,5.0,4.0,4.0,150.0 +openrouter/google/gemma-2-9b-it,code-repair,java,61.0,0.0,2.0,5.0,445.0,10545.0,488.0,5.0,2.0,2.0,50.0 +openrouter/google/gemma-2-9b-it,transpile,java,1459.0,0.0,30.0,50.0,15267.0,195055.0,16253.0,50.0,34.0,35.0,1310.0 +openrouter/google/gemma-2-9b-it,write-tests,golang,1495.0,1130.0,37.0,120.0,78036.0,932465.0,79813.0,120.0,103.0,105.0,0.0 +openrouter/google/gemma-2-9b-it,write-tests,java,6826.0,6460.0,55.0,120.0,133347.0,1286504.0,134775.0,120.0,95.0,96.0,0.0 +openrouter/google/gemma-2-9b-it,write-tests,ruby,11.0,0.0,0.0,5.0,1602.0,36954.0,1679.0,5.0,3.0,3.0,0.0 +openrouter/google/palm-2-chat-bison,code-repair,golang,210.0,0.0,5.0,5.0,641.0,7098.0,716.0,5.0,5.0,5.0,190.0 +openrouter/google/palm-2-chat-bison,code-repair,ruby,150.0,0.0,5.0,5.0,671.0,7513.0,736.0,5.0,5.0,5.0,130.0 +openrouter/google/palm-2-chat-bison,transpile,golang,2636.0,0.0,46.0,50.0,16561.0,90936.0,17111.0,50.0,50.0,50.0,2440.0 +openrouter/google/palm-2-chat-bison,transpile,java,2349.0,0.0,49.0,50.0,16249.0,84847.0,16899.0,50.0,50.0,50.0,2150.0 +openrouter/google/palm-2-chat-bison,transpile,ruby,958.0,0.0,18.0,50.0,18469.0,98485.0,19119.0,50.0,50.0,50.0,790.0 +openrouter/google/palm-2-chat-bison,write-tests,golang,1198.0,830.0,26.0,120.0,114847.0,419563.0,116216.0,120.0,111.0,111.0,0.0 +openrouter/google/palm-2-chat-bison,write-tests,java,15.0,0.0,0.0,5.0,1140.0,7450.0,1205.0,5.0,5.0,5.0,0.0 +openrouter/google/palm-2-chat-bison,write-tests,ruby,3029.0,2620.0,49.0,120.0,91068.0,367694.0,92629.0,120.0,120.0,120.0,0.0 +openrouter/google/palm-2-codechat-bison,code-repair,golang,210.0,0.0,5.0,5.0,641.0,5747.0,712.0,5.0,5.0,5.0,190.0 +openrouter/google/palm-2-codechat-bison,code-repair,ruby,170.0,0.0,5.0,5.0,650.0,5965.0,715.0,5.0,5.0,5.0,150.0 +openrouter/google/palm-2-codechat-bison,transpile,golang,2618.0,0.0,48.0,50.0,16110.0,81878.0,16660.0,50.0,50.0,50.0,2420.0 +openrouter/google/palm-2-codechat-bison,transpile,ruby,1048.0,0.0,30.0,50.0,17461.0,108222.0,18102.0,50.0,49.0,49.0,870.0 +openrouter/google/palm-2-codechat-bison,write-tests,golang,1357.0,970.0,33.0,120.0,111619.0,404946.0,112974.0,120.0,117.0,117.0,0.0 +openrouter/google/palm-2-codechat-bison,write-tests,java,15.0,0.0,0.0,5.0,1007.0,7145.0,1072.0,5.0,5.0,5.0,0.0 +openrouter/google/palm-2-codechat-bison,write-tests,ruby,4045.0,3650.0,45.0,120.0,94126.0,400798.0,95641.0,120.0,115.0,115.0,0.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,golang,210.0,0.0,5.0,5.0,648.0,18227.0,688.0,5.0,5.0,5.0,190.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,java,160.0,0.0,5.0,5.0,1051.0,13923.0,1091.0,5.0,5.0,5.0,140.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,12899.0,659.0,5.0,5.0,5.0,150.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,golang,2544.0,0.0,44.0,50.0,14442.0,222267.0,14854.0,50.0,50.0,50.0,2350.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,java,2350.0,0.0,50.0,50.0,19575.0,227967.0,19979.0,50.0,50.0,50.0,2150.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,ruby,1390.0,0.0,50.0,50.0,16480.0,175591.0,16896.0,50.0,50.0,50.0,1190.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,golang,3887.0,3430.0,101.0,120.0,78844.0,1223433.0,79839.0,120.0,118.0,118.0,0.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,java,11800.0,11340.0,100.0,120.0,119521.0,1277218.0,120501.0,120.0,120.0,120.0,0.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,ruby,10354.0,9890.0,104.0,120.0,81754.0,750740.0,82730.0,120.0,120.0,120.0,0.0 +openrouter/meta-llama/llama-3-8b-instruct,code-repair,java,160.0,0.0,5.0,5.0,1187.0,7168.0,1235.0,5.0,5.0,5.0,140.0 +openrouter/meta-llama/llama-3-8b-instruct,code-repair,ruby,109.0,0.0,4.0,5.0,617.0,8242.0,657.0,5.0,5.0,5.0,90.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile,java,2258.0,0.0,48.0,50.0,19316.0,167683.0,19784.0,50.0,50.0,50.0,2060.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile,ruby,1437.0,0.0,37.0,50.0,14374.0,112609.0,14919.0,50.0,50.0,50.0,1250.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,golang,14.0,0.0,0.0,5.0,847.0,19782.0,1104.0,5.0,4.0,5.0,0.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,java,3807.0,3450.0,32.0,120.0,118871.0,561376.0,126938.0,120.0,96.0,109.0,0.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,ruby,2076.0,1690.0,31.0,120.0,91215.0,471645.0,93805.0,120.0,116.0,119.0,0.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,golang,210.0,0.0,5.0,5.0,638.0,12359.0,696.0,5.0,5.0,5.0,190.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,java,160.0,0.0,5.0,5.0,965.0,15599.0,1027.0,5.0,5.0,5.0,140.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,ruby,170.0,0.0,5.0,5.0,668.0,14896.0,728.0,5.0,5.0,5.0,150.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,golang,2720.0,0.0,50.0,50.0,15006.0,243459.0,15524.0,50.0,50.0,50.0,2520.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,java,2400.0,0.0,50.0,50.0,18565.0,283061.0,19187.0,50.0,50.0,50.0,2200.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,ruby,1590.0,0.0,50.0,50.0,16439.0,320929.0,17057.0,50.0,50.0,50.0,1390.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,golang,4775.0,4300.0,115.0,120.0,88527.0,1355164.0,89777.0,120.0,120.0,120.0,0.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,java,14520.0,14050.0,111.0,120.0,145528.0,1627603.0,148840.0,120.0,119.0,120.0,0.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,ruby,10990.0,10520.0,110.0,120.0,101073.0,1595328.0,102545.0,120.0,120.0,120.0,0.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,golang,209.0,0.0,5.0,5.0,681.0,15260.0,925.0,5.0,4.0,5.0,190.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,java,160.0,0.0,5.0,5.0,965.0,10412.0,1025.0,5.0,5.0,5.0,140.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,ruby,170.0,0.0,5.0,5.0,621.0,11136.0,681.0,5.0,5.0,5.0,150.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,golang,2566.0,0.0,47.0,50.0,15370.0,209099.0,16293.0,50.0,49.0,50.0,2370.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,java,2186.0,0.0,46.0,50.0,19124.0,237667.0,19724.0,50.0,50.0,50.0,1990.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,ruby,1559.0,0.0,49.0,50.0,15820.0,270464.0,16424.0,50.0,50.0,50.0,1360.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,golang,3679.0,3230.0,101.0,120.0,244782.0,2140119.0,246096.0,120.0,114.0,114.0,0.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,java,12506.0,12050.0,101.0,120.0,204007.0,1704830.0,214138.0,120.0,117.0,118.0,0.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,ruby,11293.0,10820.0,113.0,120.0,102146.0,1187129.0,103582.0,120.0,120.0,120.0,0.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,golang,209.0,0.0,5.0,5.0,714.0,6307.0,1108.0,5.0,4.0,5.0,190.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,java,160.0,0.0,5.0,5.0,980.0,6182.0,1040.0,5.0,5.0,5.0,140.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,ruby,108.0,0.0,4.0,5.0,851.0,12319.0,1162.0,5.0,4.0,5.0,90.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,golang,1800.0,0.0,35.0,50.0,15757.0,90009.0,17263.0,50.0,46.0,49.0,1620.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,java,2183.0,0.0,46.0,50.0,19127.0,105181.0,31058.0,50.0,47.0,50.0,1990.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,ruby,1762.0,0.0,45.0,50.0,15308.0,97543.0,16539.0,50.0,47.0,50.0,1570.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,golang,2320.0,1920.0,60.0,120.0,262250.0,823508.0,288120.0,119.0,106.0,115.0,0.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,java,9275.0,8850.0,77.0,120.0,343957.0,871900.0,369019.0,120.0,111.0,117.0,0.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,ruby,6294.0,5860.0,90.0,120.0,294259.0,861253.0,335812.0,120.0,111.0,113.0,0.0 +openrouter/microsoft/phi-3-medium-128k-instruct,code-repair,ruby,138.0,0.0,5.0,5.0,625.0,7878.0,983.0,5.0,3.0,5.0,120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,golang,1258.0,0.0,22.0,50.0,17113.0,242250.0,17796.0,50.0,32.0,34.0,1120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,java,1996.0,0.0,42.0,50.0,23816.0,246515.0,24320.0,50.0,37.0,37.0,1830.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,ruby,576.0,0.0,42.0,50.0,14969.0,161323.0,17503.0,50.0,40.0,44.0,400.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,golang,12.0,0.0,0.0,5.0,1525.0,32704.0,2047.0,5.0,3.0,4.0,0.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,java,13.0,0.0,0.0,5.0,3193.0,54835.0,5280.0,5.0,3.0,5.0,0.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,ruby,7152.0,6750.0,98.0,120.0,113440.0,1302576.0,119125.0,120.0,87.0,97.0,0.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,golang,210.0,0.0,5.0,5.0,670.0,6677.0,745.0,5.0,5.0,5.0,190.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,java,159.0,0.0,5.0,5.0,1328.0,10047.0,1835.0,5.0,4.0,5.0,140.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,ruby,140.0,0.0,5.0,5.0,613.0,6106.0,678.0,5.0,5.0,5.0,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile,golang,1842.0,0.0,32.0,50.0,16506.0,110932.0,17064.0,50.0,50.0,50.0,1660.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile,ruby,588.0,0.0,45.0,50.0,14770.0,107235.0,16243.0,50.0,44.0,49.0,400.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,golang,2318.0,1910.0,63.0,120.0,116959.0,767538.0,123039.0,120.0,106.0,119.0,0.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,java,1368.0,1010.0,10.0,120.0,149337.0,739926.0,154153.0,120.0,108.0,120.0,0.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,ruby,7951.0,7500.0,110.0,119.0,94699.0,549937.0,101075.0,119.0,103.0,119.0,0.0 +openrouter/microsoft/phi-3-mini-128k-instruct,code-repair,java,159.0,0.0,5.0,5.0,1142.0,11745.0,1361.0,5.0,4.0,5.0,140.0 +openrouter/microsoft/phi-3-mini-128k-instruct,code-repair,ruby,137.0,0.0,5.0,5.0,756.0,9112.0,1477.0,5.0,2.0,5.0,120.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,golang,1068.0,0.0,18.0,50.0,15192.0,95385.0,15701.0,50.0,45.0,45.0,910.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,java,2275.0,0.0,47.0,50.0,19924.0,102574.0,20566.0,50.0,49.0,49.0,2080.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,ruby,483.0,0.0,25.0,50.0,15073.0,125036.0,18771.0,50.0,35.0,43.0,330.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,golang,25.0,10.0,1.0,5.0,1659.0,14071.0,2565.0,5.0,4.0,5.0,0.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,java,7145.0,6810.0,61.0,120.0,196637.0,1110932.0,251912.0,120.0,42.0,112.0,0.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,ruby,5025.0,4650.0,92.0,120.0,118626.0,757768.0,161662.0,120.0,52.0,111.0,0.0 +openrouter/microsoft/wizardlm-2-7b,code-repair,java,159.0,0.0,5.0,5.0,1375.0,13024.0,1644.0,5.0,4.0,5.0,140.0 +openrouter/microsoft/wizardlm-2-7b,code-repair,ruby,77.0,0.0,4.0,5.0,1005.0,13726.0,2285.0,5.0,3.0,5.0,60.0 +openrouter/microsoft/wizardlm-2-7b,transpile,golang,919.0,0.0,22.0,50.0,18505.0,170788.0,20615.0,50.0,47.0,50.0,750.0 +openrouter/microsoft/wizardlm-2-7b,transpile,java,2175.0,0.0,46.0,50.0,22664.0,164673.0,23363.0,50.0,49.0,50.0,1980.0 +openrouter/microsoft/wizardlm-2-7b,transpile,ruby,436.0,0.0,26.0,50.0,17687.0,137849.0,18287.0,50.0,50.0,50.0,260.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,golang,10.0,0.0,0.0,5.0,3966.0,57670.0,10947.0,5.0,0.0,5.0,0.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,java,7401.0,7110.0,52.0,120.0,322840.0,2132438.0,427953.0,120.0,0.0,119.0,0.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,ruby,3116.0,2830.0,46.0,120.0,205200.0,1807756.0,302352.0,120.0,0.0,120.0,0.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,golang,169.0,0.0,4.0,5.0,649.0,24824.0,724.0,5.0,5.0,5.0,150.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,java,160.0,0.0,5.0,5.0,1295.0,19954.0,1360.0,5.0,5.0,5.0,140.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,ruby,170.0,0.0,5.0,5.0,655.0,12597.0,720.0,5.0,5.0,5.0,150.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,golang,2067.0,0.0,37.0,50.0,14574.0,298942.0,15114.0,50.0,50.0,50.0,1880.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,java,2400.0,0.0,50.0,50.0,20487.0,280882.0,21131.0,50.0,50.0,50.0,2200.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,ruby,680.0,0.0,40.0,50.0,17857.0,325057.0,18500.0,50.0,50.0,50.0,490.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,golang,2673.0,2300.0,80.0,120.0,151827.0,2602658.0,177733.0,120.0,54.0,119.0,0.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,java,13861.0,13420.0,109.0,120.0,188110.0,2779325.0,201704.0,120.0,92.0,120.0,0.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,ruby,5253.0,4920.0,61.0,120.0,143097.0,2277074.0,180871.0,120.0,32.0,120.0,0.0 +openrouter/mistralai/codestral-mamba,code-repair,golang,208.0,0.0,5.0,5.0,703.0,5404.0,1154.0,5.0,3.0,5.0,190.0 +openrouter/mistralai/codestral-mamba,code-repair,java,157.0,0.0,5.0,5.0,1150.0,8932.0,2648.0,5.0,2.0,5.0,140.0 +openrouter/mistralai/codestral-mamba,code-repair,ruby,138.0,0.0,5.0,5.0,618.0,5973.0,1352.0,5.0,3.0,5.0,120.0 +openrouter/mistralai/codestral-mamba,transpile,golang,2578.0,0.0,44.0,50.0,14810.0,105416.0,17200.0,50.0,44.0,50.0,2390.0 +openrouter/mistralai/codestral-mamba,transpile,java,2377.0,0.0,50.0,50.0,17513.0,149997.0,21049.0,50.0,38.0,49.0,2190.0 +openrouter/mistralai/codestral-mamba,transpile,ruby,1197.0,0.0,44.0,50.0,15692.0,95234.0,19592.0,50.0,43.0,50.0,1010.0 +openrouter/mistralai/codestral-mamba,write-tests,golang,3501.0,3090.0,89.0,120.0,77676.0,405173.0,93783.0,120.0,82.0,120.0,0.0 +openrouter/mistralai/codestral-mamba,write-tests,java,6445.0,6080.0,53.0,120.0,102090.0,411624.0,125186.0,120.0,72.0,120.0,0.0 +openrouter/mistralai/codestral-mamba,write-tests,ruby,7906.0,7480.0,100.0,120.0,64860.0,339845.0,80963.0,120.0,86.0,120.0,0.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,golang,205.0,0.0,5.0,5.0,839.0,8513.0,2087.0,5.0,0.0,5.0,190.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,java,155.0,0.0,5.0,5.0,1136.0,12230.0,2543.0,5.0,0.0,5.0,140.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,ruby,134.0,0.0,4.0,5.0,689.0,11373.0,1839.0,5.0,0.0,5.0,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,golang,1278.0,0.0,25.0,50.0,17256.0,133063.0,29043.0,50.0,23.0,50.0,1130.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,java,2126.0,0.0,46.0,50.0,21827.0,145898.0,32713.0,50.0,20.0,50.0,1960.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,ruby,200.0,0.0,39.0,50.0,16364.0,143572.0,29620.0,50.0,11.0,50.0,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,golang,1306.0,1010.0,37.0,120.0,125144.0,1151873.0,166874.0,120.0,19.0,120.0,0.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,java,5856.0,5550.0,59.0,120.0,158049.0,986305.0,219024.0,120.0,8.0,119.0,0.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,ruby,2619.0,2320.0,56.0,120.0,101142.0,641588.0,154845.0,120.0,7.0,116.0,0.0 +openrouter/mistralai/mistral-large,code-repair,golang,210.0,0.0,5.0,5.0,672.0,15465.0,734.0,5.0,5.0,5.0,190.0 +openrouter/mistralai/mistral-large,code-repair,java,160.0,0.0,5.0,5.0,1090.0,15946.0,1150.0,5.0,5.0,5.0,140.0 +openrouter/mistralai/mistral-large,code-repair,ruby,170.0,0.0,5.0,5.0,620.0,7542.0,680.0,5.0,5.0,5.0,150.0 +openrouter/mistralai/mistral-large,transpile,golang,2849.0,0.0,49.0,50.0,14706.0,177546.0,15234.0,50.0,50.0,50.0,2650.0 +openrouter/mistralai/mistral-large,transpile,java,2359.0,0.0,49.0,50.0,18808.0,209061.0,19408.0,50.0,50.0,50.0,2160.0 +openrouter/mistralai/mistral-large,transpile,ruby,1679.0,0.0,49.0,50.0,15728.0,154756.0,16328.0,50.0,50.0,50.0,1480.0 +openrouter/mistralai/mistral-large,write-tests,golang,3840.0,3390.0,91.0,120.0,82625.0,1317203.0,84226.0,120.0,119.0,120.0,0.0 +openrouter/mistralai/mistral-large,write-tests,java,14958.0,14480.0,118.0,120.0,127473.0,1328016.0,128913.0,120.0,120.0,120.0,0.0 +openrouter/mistralai/mistral-large,write-tests,ruby,9831.0,9370.0,102.0,120.0,94439.0,820205.0,95945.0,120.0,119.0,120.0,0.0 +openrouter/mistralai/mistral-medium,code-repair,golang,209.0,0.0,5.0,5.0,656.0,10116.0,876.0,5.0,4.0,5.0,190.0 +openrouter/mistralai/mistral-medium,code-repair,java,156.0,0.0,5.0,5.0,1212.0,18736.0,1829.0,5.0,1.0,5.0,140.0 +openrouter/mistralai/mistral-medium,code-repair,ruby,149.0,0.0,5.0,5.0,614.0,11297.0,1127.0,5.0,4.0,5.0,130.0 +openrouter/mistralai/mistral-medium,transpile,golang,2122.0,0.0,35.0,50.0,14168.0,266718.0,25232.0,50.0,37.0,50.0,1950.0 +openrouter/mistralai/mistral-medium,transpile,java,2388.0,0.0,50.0,50.0,20580.0,250096.0,28023.0,50.0,38.0,50.0,2200.0 +openrouter/mistralai/mistral-medium,transpile,ruby,795.0,0.0,50.0,50.0,17294.0,246944.0,28762.0,50.0,25.0,50.0,620.0 +openrouter/mistralai/mistral-medium,write-tests,golang,2956.0,2570.0,77.0,120.0,98546.0,1295916.0,124311.0,120.0,69.0,120.0,0.0 +openrouter/mistralai/mistral-medium,write-tests,java,12566.0,12160.0,104.0,120.0,142355.0,1472926.0,172909.0,120.0,62.0,120.0,0.0 +openrouter/mistralai/mistral-medium,write-tests,ruby,9123.0,8750.0,98.0,120.0,92677.0,1171430.0,133760.0,120.0,35.0,120.0,0.0 +openrouter/mistralai/mistral-nemo,code-repair,golang,138.0,0.0,3.0,5.0,666.0,9594.0,716.0,5.0,5.0,5.0,120.0 +openrouter/mistralai/mistral-nemo,code-repair,java,139.0,0.0,4.0,5.0,996.0,7845.0,1056.0,5.0,5.0,5.0,120.0 +openrouter/mistralai/mistral-nemo,code-repair,ruby,170.0,0.0,5.0,5.0,539.0,7921.0,587.0,5.0,5.0,5.0,150.0 +openrouter/mistralai/mistral-nemo,transpile,java,2091.0,0.0,43.0,50.0,17807.0,127986.0,18342.0,50.0,49.0,49.0,1900.0 +openrouter/mistralai/mistral-nemo,transpile,ruby,1104.0,0.0,37.0,50.0,13759.0,130463.0,15892.0,50.0,47.0,50.0,920.0 +openrouter/mistralai/mistral-nemo,write-tests,golang,1285.0,910.0,33.0,120.0,95936.0,830667.0,99647.0,120.0,107.0,115.0,0.0 +openrouter/mistralai/mistral-nemo,write-tests,java,6385.0,5990.0,51.0,120.0,115952.0,579528.0,119726.0,119.0,109.0,116.0,0.0 +openrouter/mistralai/mistral-nemo,write-tests,ruby,7850.0,7410.0,101.0,120.0,86006.0,557101.0,91385.0,120.0,102.0,117.0,0.0 +openrouter/mistralai/mistral-small,code-repair,golang,210.0,0.0,5.0,5.0,652.0,6235.0,714.0,5.0,5.0,5.0,190.0 +openrouter/mistralai/mistral-small,code-repair,java,160.0,0.0,5.0,5.0,1245.0,9430.0,1305.0,5.0,5.0,5.0,140.0 +openrouter/mistralai/mistral-small,transpile,golang,2535.0,0.0,45.0,50.0,14937.0,170286.0,15467.0,50.0,50.0,50.0,2340.0 +openrouter/mistralai/mistral-small,transpile,java,2398.0,0.0,50.0,50.0,19805.0,167455.0,20499.0,50.0,48.0,50.0,2200.0 +openrouter/mistralai/mistral-small,write-tests,golang,1760.0,1360.0,46.0,120.0,87644.0,714496.0,89785.0,120.0,114.0,120.0,0.0 +openrouter/mistralai/mistral-small,write-tests,java,13140.0,12680.0,106.0,120.0,120139.0,867300.0,122749.0,120.0,114.0,120.0,0.0 +openrouter/mistralai/mistral-small,write-tests,ruby,15.0,0.0,0.0,5.0,781.0,6723.0,841.0,5.0,5.0,5.0,0.0 +openrouter/mistralai/mistral-tiny,code-repair,golang,209.0,0.0,5.0,5.0,961.0,5803.0,1299.0,5.0,4.0,5.0,190.0 +openrouter/mistralai/mistral-tiny,code-repair,java,156.0,0.0,5.0,5.0,1228.0,7502.0,2129.0,5.0,1.0,5.0,140.0 +openrouter/mistralai/mistral-tiny,transpile,golang,1345.0,0.0,24.0,50.0,16201.0,161663.0,23673.0,50.0,31.0,50.0,1190.0 +openrouter/mistralai/mistral-tiny,transpile,java,2251.0,0.0,48.0,50.0,21067.0,102573.0,22563.0,50.0,43.0,50.0,2060.0 +openrouter/mistralai/mistral-tiny,write-tests,golang,853.0,540.0,15.0,120.0,100231.0,617887.0,125218.0,120.0,58.0,120.0,0.0 +openrouter/mistralai/mistral-tiny,write-tests,java,3819.0,3520.0,32.0,120.0,178370.0,705192.0,219016.0,120.0,27.0,120.0,0.0 +openrouter/mistralai/mistral-tiny,write-tests,ruby,10.0,0.0,0.0,5.0,1132.0,14285.0,4518.0,5.0,0.0,5.0,0.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,golang,177.0,0.0,4.0,5.0,643.0,10295.0,1364.0,5.0,3.0,5.0,160.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,java,158.0,0.0,5.0,5.0,1149.0,11742.0,1538.0,5.0,3.0,5.0,140.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,ruby,168.0,0.0,5.0,5.0,582.0,7714.0,1085.0,5.0,3.0,5.0,150.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,golang,2439.0,0.0,43.0,50.0,14756.0,119059.0,15882.0,50.0,46.0,50.0,2250.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,java,2318.0,0.0,48.0,50.0,19281.0,139105.0,19929.0,50.0,50.0,50.0,2120.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,ruby,544.0,0.0,46.0,50.0,15982.0,145662.0,17156.0,50.0,48.0,50.0,350.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,golang,3325.0,2910.0,86.0,120.0,97871.0,1089410.0,111694.0,120.0,89.0,120.0,0.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,java,13124.0,12700.0,105.0,120.0,120262.0,945492.0,137699.0,120.0,79.0,120.0,0.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,ruby,7055.0,6680.0,73.0,120.0,84246.0,720721.0,109784.0,120.0,63.0,119.0,0.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,golang,123.0,0.0,3.0,5.0,618.0,10121.0,1477.0,5.0,1.0,4.0,110.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,java,156.0,0.0,5.0,5.0,1107.0,17114.0,2381.0,5.0,1.0,5.0,140.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,ruby,133.0,0.0,4.0,5.0,609.0,12832.0,1837.0,5.0,0.0,4.0,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,golang,2080.0,0.0,36.0,50.0,14553.0,121219.0,15157.0,50.0,41.0,43.0,1910.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,java,2161.0,0.0,45.0,50.0,20368.0,158629.0,23353.0,50.0,16.0,50.0,2000.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,ruby,702.0,0.0,34.0,50.0,14159.0,119066.0,14710.0,46.0,41.0,41.0,540.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,golang,2243.0,1850.0,62.0,120.0,118597.0,968957.0,123163.0,120.0,101.0,110.0,0.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,java,12755.0,12310.0,102.0,120.0,159940.0,1028190.0,164114.0,120.0,108.0,115.0,0.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,ruby,8144.0,7730.0,87.0,120.0,97958.0,788595.0,102130.0,113.0,104.0,110.0,0.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,golang,175.0,0.0,4.0,5.0,674.0,18903.0,716.0,5.0,3.0,3.0,160.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,java,137.0,0.0,4.0,5.0,2054.0,36699.0,2102.0,5.0,4.0,4.0,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,ruby,110.0,0.0,5.0,5.0,645.0,22443.0,705.0,5.0,5.0,5.0,90.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,golang,1701.0,0.0,31.0,50.0,16086.0,377227.0,16233.0,50.0,10.0,10.0,1600.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,java,1935.0,0.0,42.0,50.0,21647.0,380899.0,22208.0,50.0,21.0,22.0,1800.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,ruby,900.0,0.0,37.0,50.0,15408.0,354485.0,15719.0,50.0,21.0,22.0,770.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,golang,2174.0,1810.0,54.0,120.0,76674.0,1743701.0,78739.0,120.0,94.0,96.0,0.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,java,6633.0,6230.0,59.0,120.0,106540.0,1633748.0,107901.0,120.0,112.0,112.0,0.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,ruby,4831.0,4490.0,61.0,120.0,70423.0,1333729.0,71425.0,120.0,80.0,80.0,0.0 +openrouter/neversleep/llama-3-lumimaid-8b,code-repair,java,138.0,0.0,4.0,5.0,991.0,18847.0,1295.0,5.0,4.0,5.0,120.0 +openrouter/neversleep/llama-3-lumimaid-8b,code-repair,ruby,109.0,0.0,4.0,5.0,669.0,9557.0,729.0,5.0,5.0,5.0,90.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile,java,2019.0,0.0,43.0,50.0,19480.0,143386.0,20295.0,50.0,42.0,44.0,1840.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile,ruby,1570.0,0.0,40.0,50.0,13874.0,143164.0,14446.0,50.0,50.0,50.0,1380.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,golang,13.0,0.0,0.0,5.0,1285.0,13772.0,1326.0,5.0,4.0,4.0,0.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,java,4342.0,3980.0,35.0,120.0,124828.0,882463.0,127802.0,120.0,102.0,105.0,0.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,ruby,2427.0,2040.0,37.0,120.0,104099.0,663723.0,114237.0,120.0,111.0,119.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,code-repair,java,129.0,0.0,4.0,5.0,1145.0,5353.0,1205.0,5.0,5.0,5.0,110.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,code-repair,ruby,166.0,0.0,5.0,5.0,640.0,11060.0,1563.0,5.0,1.0,5.0,150.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile,java,2059.0,0.0,45.0,50.0,18677.0,99035.0,25580.0,50.0,34.0,50.0,1880.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile,ruby,1541.0,0.0,44.0,50.0,15944.0,100574.0,22327.0,50.0,28.0,49.0,1370.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,golang,14.0,0.0,0.0,5.0,1902.0,8049.0,2105.0,5.0,4.0,5.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,java,6876.0,6490.0,60.0,120.0,106731.0,357199.0,121489.0,120.0,86.0,120.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,ruby,6199.0,5770.0,98.0,120.0,70618.0,318834.0,87152.0,120.0,91.0,120.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,code-repair,java,158.0,0.0,5.0,5.0,1192.0,20367.0,1977.0,5.0,3.0,5.0,140.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,code-repair,ruby,166.0,0.0,5.0,5.0,587.0,19130.0,1657.0,5.0,1.0,5.0,150.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile,java,2210.0,0.0,47.0,50.0,18164.0,232433.0,19839.0,50.0,43.0,50.0,2020.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile,ruby,1774.0,0.0,47.0,50.0,14325.0,230792.0,17348.0,50.0,37.0,50.0,1590.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,golang,11.0,0.0,0.0,5.0,1497.0,29170.0,2723.0,5.0,1.0,5.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,java,7867.0,7460.0,68.0,120.0,125865.0,1131979.0,136525.0,120.0,99.0,120.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,ruby,5024.0,4600.0,77.0,120.0,88517.0,959255.0,95863.0,120.0,108.0,119.0,0.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,code-repair,java,155.0,0.0,5.0,5.0,1182.0,57191.0,12693.0,5.0,0.0,5.0,140.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,code-repair,ruby,116.0,0.0,5.0,5.0,621.0,7564.0,1909.0,5.0,1.0,5.0,100.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,golang,2170.0,0.0,38.0,50.0,15970.0,100374.0,26300.0,50.0,17.0,45.0,2020.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,java,2326.0,0.0,50.0,50.0,20312.0,401047.0,124421.0,50.0,16.0,50.0,2160.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,ruby,1457.0,0.0,40.0,50.0,14700.0,134411.0,31433.0,50.0,8.0,49.0,1310.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,golang,13.0,0.0,0.0,5.0,1625.0,12397.0,2634.0,5.0,3.0,5.0,0.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,java,9166.0,8800.0,80.0,120.0,199081.0,1091206.0,289240.0,120.0,51.0,115.0,0.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,ruby,4587.0,4220.0,75.0,120.0,146996.0,708913.0,178137.0,120.0,52.0,120.0,0.0 +openrouter/nousresearch/nous-hermes-yi-34b,code-repair,golang,210.0,0.0,5.0,5.0,648.0,6398.0,718.0,5.0,5.0,5.0,190.0 +openrouter/nousresearch/nous-hermes-yi-34b,code-repair,java,128.0,0.0,4.0,5.0,1075.0,10556.0,1254.0,5.0,4.0,5.0,110.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile,golang,1471.0,0.0,28.0,50.0,15568.0,188842.0,24335.0,50.0,22.0,31.0,1340.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile,java,1923.0,0.0,41.0,50.0,20723.0,144219.0,21404.0,50.0,33.0,39.0,1760.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,golang,1593.0,1220.0,38.0,120.0,143765.0,1439502.0,147811.0,120.0,102.0,113.0,0.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,java,2868.0,2520.0,26.0,120.0,180065.0,1363041.0,184071.0,120.0,98.0,104.0,0.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,ruby,13.0,0.0,0.0,5.0,740.0,9741.0,788.0,5.0,4.0,4.0,0.0 +openrouter/openai/gpt-4-turbo,code-repair,golang,210.0,0.0,5.0,5.0,638.0,14361.0,704.0,5.0,5.0,5.0,190.0 +openrouter/openai/gpt-4-turbo,code-repair,java,160.0,0.0,5.0,5.0,1051.0,21517.0,1111.0,5.0,5.0,5.0,140.0 +openrouter/openai/gpt-4-turbo,code-repair,ruby,170.0,0.0,5.0,5.0,620.0,12703.0,680.0,5.0,5.0,5.0,150.0 +openrouter/openai/gpt-4-turbo,transpile,golang,2800.0,0.0,50.0,50.0,13789.0,244996.0,14345.0,50.0,50.0,50.0,2600.0 +openrouter/openai/gpt-4-turbo,transpile,java,2400.0,0.0,50.0,50.0,20329.0,298560.0,20929.0,50.0,50.0,50.0,2200.0 +openrouter/openai/gpt-4-turbo,transpile,ruby,1210.0,0.0,50.0,50.0,17025.0,286077.0,17625.0,50.0,50.0,50.0,1010.0 +openrouter/openai/gpt-4-turbo,write-tests,golang,4203.0,3740.0,103.0,120.0,92821.0,1389585.0,94196.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4-turbo,write-tests,java,15610.0,15130.0,120.0,120.0,135297.0,1454783.0,136738.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4-turbo,write-tests,ruby,11659.0,11180.0,119.0,120.0,88626.0,1162214.0,90066.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4o,code-repair,golang,210.0,0.0,5.0,5.0,638.0,9700.0,708.0,5.0,5.0,5.0,190.0 +openrouter/openai/gpt-4o,code-repair,java,160.0,0.0,5.0,5.0,943.0,10034.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/openai/gpt-4o,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,7471.0,679.0,5.0,5.0,5.0,150.0 +openrouter/openai/gpt-4o,transpile,golang,2860.0,0.0,50.0,50.0,14628.0,105358.0,15184.0,50.0,50.0,50.0,2660.0 +openrouter/openai/gpt-4o,transpile,java,2400.0,0.0,50.0,50.0,18666.0,112115.0,19266.0,50.0,50.0,50.0,2200.0 +openrouter/openai/gpt-4o,transpile,ruby,2279.0,0.0,49.0,50.0,15367.0,126994.0,15967.0,50.0,50.0,50.0,2080.0 +openrouter/openai/gpt-4o,write-tests,golang,3586.0,3140.0,89.0,119.0,77766.0,632691.0,79423.0,119.0,119.0,119.0,0.0 +openrouter/openai/gpt-4o,write-tests,java,15340.0,14860.0,120.0,120.0,125816.0,739857.0,127256.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4o,write-tests,ruby,12000.0,11520.0,120.0,120.0,92793.0,486216.0,94236.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4o-mini,code-repair,golang,210.0,0.0,5.0,5.0,638.0,7751.0,708.0,5.0,5.0,5.0,190.0 +openrouter/openai/gpt-4o-mini,code-repair,java,160.0,0.0,5.0,5.0,943.0,9047.0,1003.0,5.0,5.0,5.0,140.0 +openrouter/openai/gpt-4o-mini,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,9109.0,679.0,5.0,5.0,5.0,150.0 +openrouter/openai/gpt-4o-mini,transpile,golang,2900.0,0.0,50.0,50.0,14088.0,109723.0,14592.0,50.0,50.0,50.0,2700.0 +openrouter/openai/gpt-4o-mini,transpile,java,2400.0,0.0,50.0,50.0,17196.0,128589.0,17796.0,50.0,50.0,50.0,2200.0 +openrouter/openai/gpt-4o-mini,transpile,ruby,2400.0,0.0,50.0,50.0,14099.0,128161.0,14699.0,50.0,50.0,50.0,2200.0 +openrouter/openai/gpt-4o-mini,write-tests,golang,4072.0,3610.0,102.0,120.0,87076.0,604092.0,88276.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4o-mini,write-tests,java,15299.0,14820.0,119.0,120.0,154516.0,715868.0,155956.0,120.0,120.0,120.0,0.0 +openrouter/openai/gpt-4o-mini,write-tests,ruby,11830.0,11350.0,120.0,120.0,109215.0,501052.0,110655.0,120.0,120.0,120.0,0.0 +openrouter/openchat/openchat-8b,code-repair,golang,127.0,0.0,3.0,5.0,555.0,8171.0,1469.0,5.0,4.0,5.0,110.0 +openrouter/openchat/openchat-8b,transpile,golang,1708.0,0.0,31.0,50.0,14404.0,100334.0,16674.0,49.0,22.0,46.0,1560.0 +openrouter/openchat/openchat-8b,transpile,java,2155.0,0.0,46.0,50.0,19325.0,115551.0,23075.0,49.0,12.0,48.0,2000.0 +openrouter/openchat/openchat-8b,write-tests,golang,1363.0,1030.0,35.0,120.0,80158.0,530770.0,97956.0,119.0,61.0,118.0,0.0 +openrouter/openchat/openchat-8b,write-tests,java,12.0,0.0,0.0,5.0,2508.0,14193.0,3756.0,5.0,2.0,5.0,0.0 +openrouter/openchat/openchat-8b,write-tests,ruby,11.0,0.0,0.0,5.0,1113.0,11076.0,2327.0,5.0,1.0,5.0,0.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,golang,210.0,0.0,5.0,5.0,648.0,6610.0,688.0,5.0,5.0,5.0,190.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,java,126.0,0.0,4.0,5.0,816.0,8059.0,848.0,4.0,4.0,4.0,110.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,12930.0,659.0,5.0,5.0,5.0,150.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,golang,2595.0,0.0,45.0,50.0,14721.0,146636.0,15123.0,50.0,50.0,50.0,2400.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,java,2265.0,0.0,48.0,50.0,18800.0,161010.0,19200.0,49.0,49.0,49.0,2070.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,ruby,1319.0,0.0,49.0,50.0,16691.0,366122.0,17095.0,50.0,50.0,50.0,1120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,golang,4048.0,3580.0,108.0,120.0,79900.0,743153.0,80900.0,120.0,120.0,120.0,0.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,java,13538.0,13070.0,109.0,120.0,120301.0,797335.0,121477.0,120.0,119.0,120.0,0.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,ruby,10780.0,10310.0,110.0,120.0,80044.0,1552713.0,81008.0,120.0,120.0,120.0,0.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,code-repair,golang,209.0,0.0,5.0,5.0,712.0,4512.0,908.0,5.0,4.0,5.0,190.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,code-repair,ruby,108.0,0.0,5.0,5.0,602.0,6359.0,898.0,5.0,3.0,5.0,90.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile,java,2156.0,0.0,46.0,50.0,19290.0,59018.0,21277.0,50.0,40.0,50.0,1970.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile,ruby,2009.0,0.0,47.0,50.0,13230.0,59274.0,15346.0,50.0,42.0,50.0,1820.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,golang,768.0,480.0,16.0,120.0,218046.0,889976.0,236904.0,120.0,33.0,119.0,0.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,java,11.0,0.0,0.0,5.0,3638.0,7890.0,4366.0,5.0,1.0,5.0,0.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,ruby,2077.0,1750.0,27.0,120.0,100550.0,292469.0,111733.0,118.0,65.0,117.0,0.0 +openrouter/qwen/qwen-110b-chat,code-repair,golang,210.0,0.0,5.0,5.0,648.0,8022.0,718.0,5.0,5.0,5.0,190.0 +openrouter/qwen/qwen-110b-chat,code-repair,ruby,170.0,0.0,5.0,5.0,631.0,6847.0,691.0,5.0,5.0,5.0,150.0 +openrouter/qwen/qwen-110b-chat,transpile,golang,1884.0,0.0,35.0,50.0,14191.0,133055.0,15017.0,50.0,49.0,50.0,1700.0 +openrouter/qwen/qwen-110b-chat,transpile,ruby,1919.0,0.0,50.0,50.0,12910.0,124917.0,13723.0,50.0,49.0,50.0,1720.0 +openrouter/qwen/qwen-110b-chat,write-tests,golang,2356.0,1980.0,63.0,120.0,86051.0,810515.0,107868.0,120.0,73.0,120.0,0.0 +openrouter/qwen/qwen-110b-chat,write-tests,java,12.0,0.0,0.0,5.0,2012.0,19794.0,3559.0,5.0,2.0,5.0,0.0 +openrouter/qwen/qwen-110b-chat,write-tests,ruby,4429.0,4000.0,80.0,120.0,106380.0,711955.0,111780.0,120.0,109.0,120.0,0.0 +openrouter/qwen/qwen-2-72b-instruct,code-repair,java,160.0,0.0,5.0,5.0,1068.0,11064.0,1128.0,5.0,5.0,5.0,140.0 +openrouter/qwen/qwen-2-72b-instruct,code-repair,ruby,170.0,0.0,5.0,5.0,619.0,8609.0,679.0,5.0,5.0,5.0,150.0 +openrouter/qwen/qwen-2-72b-instruct,transpile,java,2226.0,0.0,47.0,50.0,19290.0,232274.0,20296.0,50.0,49.0,50.0,2030.0 +openrouter/qwen/qwen-2-72b-instruct,transpile,ruby,1315.0,0.0,37.0,50.0,15041.0,202397.0,16548.0,50.0,48.0,50.0,1130.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,golang,15.0,0.0,0.0,5.0,891.0,9751.0,961.0,5.0,5.0,5.0,0.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,java,12079.0,11620.0,101.0,120.0,104766.0,992012.0,106197.0,120.0,119.0,119.0,0.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,ruby,9325.0,8870.0,102.0,120.0,84654.0,965476.0,88764.0,120.0,113.0,120.0,0.0 +openrouter/qwen/qwen-2-7b-instruct,code-repair,java,139.0,0.0,4.0,5.0,1216.0,7184.0,1276.0,5.0,5.0,5.0,120.0 +openrouter/qwen/qwen-2-7b-instruct,code-repair,ruby,139.0,0.0,4.0,5.0,626.0,6810.0,682.0,5.0,5.0,5.0,120.0 +openrouter/qwen/qwen-2-7b-instruct,transpile,java,2318.0,0.0,48.0,50.0,21240.0,109871.0,21848.0,50.0,50.0,50.0,2120.0 +openrouter/qwen/qwen-2-7b-instruct,transpile,ruby,1155.0,0.0,26.0,50.0,13124.0,96679.0,13723.0,50.0,49.0,50.0,980.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,golang,15.0,0.0,0.0,5.0,1441.0,9898.0,1511.0,5.0,5.0,5.0,0.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,java,4484.0,4090.0,38.0,120.0,126362.0,545974.0,131303.0,120.0,116.0,120.0,0.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,ruby,2500.0,2100.0,42.0,120.0,117776.0,540581.0,119502.0,120.0,118.0,120.0,0.0 +openrouter/qwen/qwen-72b-chat,code-repair,golang,210.0,0.0,5.0,5.0,667.0,9129.0,737.0,5.0,5.0,5.0,190.0 +openrouter/qwen/qwen-72b-chat,code-repair,java,158.0,0.0,5.0,5.0,1064.0,13650.0,1350.0,5.0,3.0,5.0,140.0 +openrouter/qwen/qwen-72b-chat,code-repair,ruby,169.0,0.0,5.0,5.0,624.0,7637.0,793.0,5.0,4.0,5.0,150.0 +openrouter/qwen/qwen-72b-chat,transpile,golang,1190.0,0.0,22.0,50.0,14642.0,135369.0,15707.0,50.0,48.0,50.0,1020.0 +openrouter/qwen/qwen-72b-chat,transpile,java,2298.0,0.0,48.0,50.0,20623.0,142160.0,21223.0,50.0,50.0,50.0,2100.0 +openrouter/qwen/qwen-72b-chat,transpile,ruby,583.0,0.0,42.0,50.0,13493.0,118732.0,14069.0,47.0,47.0,47.0,400.0 +openrouter/qwen/qwen-72b-chat,write-tests,golang,2633.0,2250.0,74.0,120.0,93345.0,945265.0,106329.0,120.0,69.0,120.0,0.0 +openrouter/qwen/qwen-72b-chat,write-tests,java,11159.0,10720.0,93.0,120.0,123784.0,867509.0,128721.0,120.0,106.0,120.0,0.0 +openrouter/qwen/qwen-72b-chat,write-tests,ruby,9317.0,8910.0,110.0,120.0,85669.0,616749.0,98999.0,116.0,65.0,116.0,0.0 +openrouter/recursal/eagle-7b,write-tests,golang,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/eagle-7b,write-tests,java,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/eagle-7b,write-tests,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,golang,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,java,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,golang,10.0,0.0,0.0,5.0,3894.0,59802.0,4127.0,5.0,1.0,4.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,java,10.0,0.0,0.0,5.0,653.0,14125.0,775.0,5.0,2.0,3.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,ruby,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,code-repair,java,96.0,0.0,3.0,5.0,828.0,7490.0,1792.0,5.0,3.0,5.0,80.0 +openrouter/teknium/openhermes-2.5-mistral-7b,code-repair,ruby,108.0,0.0,4.0,5.0,659.0,5724.0,794.0,5.0,4.0,5.0,90.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile,java,2130.0,0.0,46.0,50.0,19835.0,107623.0,23535.0,50.0,28.0,46.0,1960.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile,ruby,494.0,0.0,21.0,50.0,15208.0,100377.0,16639.0,50.0,45.0,48.0,330.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,golang,15.0,0.0,0.0,5.0,1590.0,9612.0,1660.0,5.0,5.0,5.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,java,8604.0,8180.0,79.0,120.0,112067.0,421274.0,117711.0,120.0,105.0,120.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,ruby,4434.0,4040.0,55.0,120.0,75657.0,371862.0,82251.0,120.0,101.0,118.0,0.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,golang,210.0,0.0,5.0,5.0,664.0,6031.0,735.0,5.0,5.0,5.0,190.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,java,127.0,0.0,4.0,5.0,956.0,7414.0,1362.0,5.0,3.0,5.0,110.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,ruby,109.0,0.0,4.0,5.0,681.0,5270.0,835.0,5.0,5.0,5.0,90.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile,golang,1896.0,0.0,34.0,50.0,14769.0,89385.0,17394.0,50.0,43.0,49.0,1720.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile,ruby,520.0,0.0,35.0,50.0,12349.0,82248.0,15816.0,50.0,39.0,46.0,350.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,golang,1381.0,1020.0,34.0,120.0,91283.0,459544.0,109274.0,120.0,90.0,117.0,0.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,java,5312.0,4940.0,53.0,120.0,89763.0,378956.0,104609.0,120.0,85.0,114.0,0.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,ruby,2446.0,2090.0,44.0,120.0,56695.0,297325.0,64868.0,118.0,86.0,108.0,0.0 +openrouter/undi95/toppy-m-7b,code-repair,golang,166.0,0.0,4.0,5.0,850.0,8209.0,1708.0,5.0,2.0,5.0,150.0 +openrouter/undi95/toppy-m-7b,code-repair,ruby,136.0,0.0,5.0,5.0,659.0,6597.0,1661.0,5.0,1.0,5.0,120.0 +openrouter/undi95/toppy-m-7b,transpile,ruby,156.0,0.0,45.0,50.0,15827.0,140873.0,51945.0,50.0,11.0,50.0,0.0 +openrouter/undi95/toppy-m-7b,write-tests,golang,1869.0,1540.0,45.0,120.0,98167.0,608981.0,170248.0,120.0,46.0,118.0,0.0 +openrouter/undi95/toppy-m-7b,write-tests,java,11.0,0.0,0.0,5.0,3512.0,16855.0,5145.0,5.0,1.0,5.0,0.0 +openrouter/undi95/toppy-m-7b,write-tests,ruby,688.0,420.0,6.0,120.0,77509.0,408331.0,133735.0,120.0,23.0,119.0,0.0 +openrouter/xwin-lm/xwin-lm-70b,code-repair,golang,51.0,0.0,1.0,5.0,844.0,51674.0,2428.0,5.0,0.0,5.0,40.0 +openrouter/xwin-lm/xwin-lm-70b,code-repair,java,155.0,0.0,5.0,5.0,1271.0,51966.0,2683.0,5.0,0.0,5.0,140.0 +openrouter/xwin-lm/xwin-lm-70b,transpile,golang,1916.0,0.0,34.0,50.0,18303.0,602825.0,21614.0,50.0,32.0,50.0,1750.0 +openrouter/xwin-lm/xwin-lm-70b,transpile,java,2276.0,0.0,49.0,50.0,20537.0,531451.0,23136.0,50.0,37.0,50.0,2090.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,golang,622.0,420.0,17.0,120.0,76479.0,2876799.0,92918.0,120.0,9.0,56.0,0.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,java,567.0,380.0,3.0,120.0,110798.0,2877050.0,132570.0,120.0,2.0,62.0,0.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,ruby,9.0,0.0,0.0,5.0,1044.0,71799.0,3346.0,5.0,0.0,4.0,0.0 diff --git a/docs/reports/v0.6/evaluation-by-task-score.csv b/docs/reports/v0.6/evaluation-by-task-score.csv new file mode 100644 index 00000000..3a07e260 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-task-score.csv @@ -0,0 +1,68 @@ +model_id,score,write_tests_score,transpile_score,code_repair_score +openrouter/01-ai/yi-34b-chat,273.0,234.0,0,39.0 +openrouter/ai21/jamba-instruct,15606.0,11372.0,3978.0,256.0 +openrouter/alpindale/goliath-120b,2429.0,689.0,1604.0,136.0 +openrouter/alpindale/magnum-72b,18583.0,14797.0,3256.0,530.0 +openrouter/anthropic/claude-3-haiku,34021.0,26593.0,6892.0,536.0 +openrouter/anthropic/claude-3-opus,38485.0,31340.0,6615.0,530.0 +openrouter/anthropic/claude-3-sonnet,29425.0,21795.0,7090.0,540.0 +openrouter/anthropic/claude-3.5-sonnet,39540.0,31962.0,7038.0,540.0 +openrouter/austism/chronos-hermes-13b,13583.0,8109.0,5009.0,465.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,25969.0,21712.0,3781.0,476.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,23843.0,17901.0,5404.0,538.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,11602.0,7744.0,3443.0,415.0 +openrouter/cohere/command-r-03-2024,14938.0,10845.0,3644.0,449.0 +openrouter/cohere/command-r-08-2024,21549.0,15293.0,5849.0,407.0 +openrouter/cohere/command-r-plus-04-2024,26797.0,20673.0,5584.0,540.0 +openrouter/cohere/command-r-plus-08-2024,26312.0,19256.0,6517.0,539.0 +openrouter/databricks/dbrx-instruct,14863.0,9974.0,4519.0,370.0 +openrouter/deepseek/deepseek-chat,38715.0,31435.0,6740.0,540.0 +openrouter/deepseek/deepseek-coder,38263.0,30783.0,6940.0,540.0 +openrouter/google/gemini-flash-1.5,25434.0,20790.0,4104.0,540.0 +openrouter/google/gemini-pro-1.5,20955.0,13798.0,6617.0,540.0 +openrouter/google/gemma-2-27b-it,24166.0,17525.0,6101.0,540.0 +openrouter/google/gemma-2-9b-it,10019.0,8332.0,1459.0,228.0 +openrouter/google/palm-2-chat-bison,10545.0,4242.0,5943.0,360.0 +openrouter/google/palm-2-codechat-bison,9463.0,5417.0,3666.0,380.0 +openrouter/meta-llama/llama-3-70b-instruct,32865.0,26041.0,6284.0,540.0 +openrouter/meta-llama/llama-3-8b-instruct,9861.0,5897.0,3695.0,269.0 +openrouter/meta-llama/llama-3.1-405b-instruct,37535.0,30285.0,6710.0,540.0 +openrouter/meta-llama/llama-3.1-70b-instruct,34328.0,27478.0,6311.0,539.0 +openrouter/meta-llama/llama-3.1-8b-instruct,24111.0,17889.0,5745.0,477.0 +openrouter/microsoft/phi-3-medium-128k-instruct,11145.0,7177.0,3830.0,138.0 +openrouter/microsoft/phi-3-medium-4k-instruct,14576.0,11637.0,2430.0,509.0 +openrouter/microsoft/phi-3-mini-128k-instruct,16317.0,12195.0,3826.0,296.0 +openrouter/microsoft/wizardlm-2-7b,14293.0,10527.0,3530.0,236.0 +openrouter/microsoft/wizardlm-2-8x22b,27433.0,21787.0,5147.0,499.0 +openrouter/mistralai/codestral-mamba,24507.0,17852.0,6152.0,503.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,13879.0,9781.0,3604.0,494.0 +openrouter/mistralai/mistral-large,36056.0,28629.0,6887.0,540.0 +openrouter/mistralai/mistral-medium,30464.0,24645.0,5305.0,514.0 +openrouter/mistralai/mistral-nemo,19162.0,15520.0,3195.0,447.0 +openrouter/mistralai/mistral-small,20218.0,14915.0,4933.0,370.0 +openrouter/mistralai/mistral-tiny,8643.0,4682.0,3596.0,365.0 +openrouter/mistralai/mixtral-8x22b-instruct,29308.0,23504.0,5301.0,503.0 +openrouter/mistralai/mixtral-8x7b-instruct,28497.0,23142.0,4943.0,412.0 +openrouter/neversleep/llama-3-lumimaid-70b,18596.0,13638.0,4536.0,422.0 +openrouter/neversleep/llama-3-lumimaid-8b,10618.0,6782.0,3589.0,247.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,16984.0,13089.0,3600.0,295.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,17210.0,12902.0,3984.0,324.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,19990.0,13766.0,5953.0,271.0 +openrouter/nousresearch/nous-hermes-yi-34b,8206.0,4474.0,3394.0,338.0 +openrouter/openai/gpt-4-turbo,38422.0,31472.0,6410.0,540.0 +openrouter/openai/gpt-4o,39005.0,30926.0,7539.0,540.0 +openrouter/openai/gpt-4o-mini,39441.0,31201.0,7700.0,540.0 +openrouter/openchat/openchat-8b,5376.0,1386.0,3863.0,127.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,35051.0,28366.0,6179.0,506.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,7338.0,2856.0,4165.0,317.0 +openrouter/qwen/qwen-110b-chat,10980.0,6797.0,3803.0,380.0 +openrouter/qwen/qwen-2-72b-instruct,25290.0,21419.0,3541.0,330.0 +openrouter/qwen/qwen-2-7b-instruct,10750.0,6999.0,3473.0,278.0 +openrouter/qwen/qwen-72b-chat,27717.0,23109.0,4071.0,537.0 +openrouter/recursal/eagle-7b,0.0,0.0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,0.0,0.0,0,0 +openrouter/rwkv/rwkv-5-world-3b,20.0,20.0,0,0 +openrouter/teknium/openhermes-2.5-mistral-7b,15881.0,13053.0,2624.0,204.0 +openrouter/togethercomputer/stripedhyena-nous-7b,12001.0,9139.0,2416.0,446.0 +openrouter/undi95/toppy-m-7b,3026.0,2568.0,156.0,302.0 +openrouter/xwin-lm/xwin-lm-70b,5596.0,1198.0,4192.0,206.0 diff --git a/docs/reports/v0.6/evaluation-by-task.csv b/docs/reports/v0.6/evaluation-by-task.csv new file mode 100644 index 00000000..f0efc1d8 --- /dev/null +++ b/docs/reports/v0.6/evaluation-by-task.csv @@ -0,0 +1,195 @@ +model_id,task,score,coverage,files_executed,files_executed_maximum_reachable,generate_tests_for_file_character_count,processing_time,response_character_count,response_no_error,response_no_excess,response_with_code,tests_passing +openrouter/01-ai/yi-34b-chat,code-repair,39.0,0.0,1.0,5.0,690.0,28124.0,2411.0,5.0,0.0,3.0,30.0 +openrouter/01-ai/yi-34b-chat,write-tests,234.0,70.0,3.0,126.0,65387.0,1099694.0,107402.0,106.0,8.0,47.0,0.0 +openrouter/ai21/jamba-instruct,code-repair,256.0,0.0,9.0,10.0,2092.0,20905.0,2287.0,10.0,8.0,9.0,220.0 +openrouter/ai21/jamba-instruct,transpile,3978.0,0.0,91.0,100.0,33571.0,297403.0,36697.0,100.0,89.0,98.0,3600.0 +openrouter/ai21/jamba-instruct,write-tests,11372.0,10540.0,123.0,245.0,176157.0,1109778.0,180925.0,244.0,230.0,235.0,0.0 +openrouter/alpindale/goliath-120b,code-repair,136.0,0.0,4.0,5.0,609.0,27301.0,1543.0,5.0,2.0,5.0,120.0 +openrouter/alpindale/goliath-120b,transpile,1604.0,0.0,46.0,100.0,40929.0,1057675.0,44806.0,100.0,84.0,94.0,1280.0 +openrouter/alpindale/goliath-120b,write-tests,689.0,420.0,18.0,130.0,97683.0,2058688.0,103274.0,130.0,54.0,67.0,0.0 +openrouter/alpindale/magnum-72b,code-repair,530.0,0.0,15.0,15.0,2304.0,64811.0,2488.0,15.0,15.0,15.0,470.0 +openrouter/alpindale/magnum-72b,transpile,3256.0,0.0,81.0,100.0,34596.0,498445.0,37952.0,100.0,95.0,100.0,2880.0 +openrouter/alpindale/magnum-72b,write-tests,14797.0,13590.0,151.0,360.0,312876.0,6467121.0,323658.0,360.0,342.0,354.0,0.0 +openrouter/anthropic/claude-3-haiku,code-repair,536.0,0.0,15.0,15.0,2318.0,28095.0,2995.0,15.0,11.0,15.0,480.0 +openrouter/anthropic/claude-3-haiku,transpile,6892.0,0.0,143.0,150.0,52055.0,284584.0,53868.0,150.0,149.0,150.0,6300.0 +openrouter/anthropic/claude-3-haiku,write-tests,26593.0,25340.0,278.0,360.0,310917.0,1149663.0,357369.0,360.0,255.0,360.0,0.0 +openrouter/anthropic/claude-3-opus,code-repair,530.0,0.0,15.0,15.0,2204.0,56106.0,2394.0,15.0,15.0,15.0,470.0 +openrouter/anthropic/claude-3-opus,transpile,6615.0,0.0,148.0,150.0,49108.0,850046.0,53276.0,150.0,97.0,150.0,6070.0 +openrouter/anthropic/claude-3-opus,write-tests,31340.0,29980.0,349.0,360.0,312991.0,4249800.0,328880.0,360.0,291.0,360.0,0.0 +openrouter/anthropic/claude-3-sonnet,code-repair,540.0,0.0,15.0,15.0,2468.0,36365.0,2642.0,15.0,15.0,15.0,480.0 +openrouter/anthropic/claude-3-sonnet,transpile,7090.0,0.0,140.0,150.0,48558.0,434501.0,50258.0,150.0,150.0,150.0,6500.0 +openrouter/anthropic/claude-3-sonnet,write-tests,21795.0,20480.0,266.0,360.0,349911.0,2315127.0,355921.0,351.0,347.0,351.0,0.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,540.0,0.0,15.0,15.0,2211.0,42973.0,2401.0,15.0,15.0,15.0,480.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,7038.0,0.0,147.0,150.0,49087.0,419264.0,50753.0,147.0,147.0,147.0,6450.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,31962.0,30550.0,352.0,360.0,334845.0,2094577.0,340039.0,358.0,344.0,358.0,0.0 +openrouter/austism/chronos-hermes-13b,code-repair,465.0,0.0,14.0,15.0,2500.0,57905.0,6771.0,15.0,1.0,15.0,420.0 +openrouter/austism/chronos-hermes-13b,transpile,5009.0,0.0,110.0,150.0,52882.0,549086.0,61310.0,150.0,76.0,143.0,4530.0 +openrouter/austism/chronos-hermes-13b,write-tests,8109.0,7130.0,131.0,360.0,282773.0,3103308.0,349584.0,360.0,139.0,349.0,0.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,476.0,0.0,13.0,15.0,2229.0,31115.0,4040.0,15.0,4.0,14.0,430.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,3781.0,0.0,112.0,150.0,48331.0,505891.0,77520.0,150.0,70.0,149.0,3300.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,21712.0,20640.0,272.0,360.0,252212.0,2168234.0,375496.0,360.0,80.0,360.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,538.0,0.0,15.0,15.0,2288.0,40104.0,2647.0,15.0,13.0,15.0,480.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,5404.0,0.0,122.0,150.0,45775.0,671993.0,54821.0,147.0,128.0,147.0,4860.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,17901.0,16760.0,206.0,360.0,257819.0,3055637.0,304267.0,352.0,232.0,351.0,0.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,415.0,0.0,12.0,15.0,3064.0,26036.0,3244.0,15.0,14.0,14.0,360.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile,3443.0,0.0,90.0,100.0,38260.0,387458.0,40350.0,100.0,94.0,99.0,3060.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,7744.0,6770.0,95.0,360.0,445630.0,2725914.0,478773.0,360.0,229.0,290.0,0.0 +openrouter/cohere/command-r-03-2024,code-repair,449.0,0.0,14.0,15.0,2600.0,14278.0,2790.0,15.0,15.0,15.0,390.0 +openrouter/cohere/command-r-03-2024,transpile,3644.0,0.0,102.0,150.0,48243.0,182859.0,49757.0,150.0,111.0,111.0,3170.0 +openrouter/cohere/command-r-03-2024,write-tests,10845.0,9640.0,125.0,360.0,242719.0,824944.0,247306.0,360.0,360.0,360.0,0.0 +openrouter/cohere/command-r-08-2024,code-repair,407.0,0.0,12.0,15.0,2447.0,15038.0,2637.0,15.0,15.0,15.0,350.0 +openrouter/cohere/command-r-08-2024,transpile,5849.0,0.0,139.0,150.0,49449.0,247242.0,51149.0,150.0,150.0,150.0,5260.0 +openrouter/cohere/command-r-08-2024,write-tests,15293.0,14010.0,203.0,360.0,342276.0,1355686.0,346431.0,360.0,360.0,360.0,0.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,540.0,0.0,15.0,15.0,2307.0,26292.0,2493.0,15.0,15.0,15.0,480.0 +openrouter/cohere/command-r-plus-04-2024,transpile,5584.0,0.0,140.0,150.0,50012.0,441897.0,51700.0,150.0,147.0,147.0,5000.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,20673.0,19380.0,217.0,360.0,384527.0,2872323.0,389071.0,360.0,358.0,358.0,0.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,539.0,0.0,15.0,15.0,2376.0,24330.0,2574.0,15.0,14.0,15.0,480.0 +openrouter/cohere/command-r-plus-08-2024,transpile,6517.0,0.0,143.0,150.0,49125.0,470055.0,50989.0,150.0,145.0,149.0,5930.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,19256.0,18000.0,176.0,360.0,369395.0,2940554.0,373897.0,360.0,360.0,360.0,0.0 +openrouter/databricks/dbrx-instruct,code-repair,370.0,0.0,10.0,10.0,1916.0,15888.0,2063.0,10.0,10.0,10.0,330.0 +openrouter/databricks/dbrx-instruct,transpile,4519.0,0.0,88.0,100.0,36391.0,203562.0,39567.0,100.0,82.0,99.0,4150.0 +openrouter/databricks/dbrx-instruct,write-tests,9974.0,9140.0,142.0,241.0,234865.0,1257307.0,251708.0,240.0,216.0,236.0,0.0 +openrouter/deepseek/deepseek-chat,code-repair,540.0,0.0,15.0,15.0,2200.0,59151.0,2390.0,15.0,15.0,15.0,480.0 +openrouter/deepseek/deepseek-chat,transpile,6740.0,0.0,140.0,150.0,47666.0,1211952.0,49498.0,150.0,150.0,150.0,6150.0 +openrouter/deepseek/deepseek-chat,write-tests,31435.0,30010.0,353.0,360.0,354737.0,6898744.0,359266.0,360.0,356.0,356.0,0.0 +openrouter/deepseek/deepseek-coder,code-repair,540.0,0.0,15.0,15.0,2200.0,61369.0,2390.0,15.0,15.0,15.0,480.0 +openrouter/deepseek/deepseek-coder,transpile,6940.0,0.0,140.0,150.0,47621.0,1130633.0,49421.0,150.0,150.0,150.0,6350.0 +openrouter/deepseek/deepseek-coder,write-tests,30783.0,29370.0,346.0,360.0,407876.0,8197640.0,412373.0,359.0,354.0,354.0,0.0 +openrouter/google/gemini-flash-1.5,code-repair,540.0,0.0,15.0,15.0,2200.0,20919.0,2390.0,15.0,15.0,15.0,480.0 +openrouter/google/gemini-flash-1.5,transpile,4104.0,0.0,94.0,100.0,28945.0,175576.0,30046.0,100.0,100.0,100.0,3710.0 +openrouter/google/gemini-flash-1.5,write-tests,20790.0,19460.0,256.0,360.0,343877.0,1171424.0,348332.0,360.0,357.0,357.0,0.0 +openrouter/google/gemini-pro-1.5,code-repair,540.0,0.0,15.0,15.0,2190.0,47877.0,2380.0,15.0,15.0,15.0,480.0 +openrouter/google/gemini-pro-1.5,transpile,6617.0,0.0,149.0,150.0,68891.0,1181596.0,70583.0,150.0,149.0,149.0,6020.0 +openrouter/google/gemini-pro-1.5,write-tests,13798.0,12540.0,178.0,360.0,280289.0,2825533.0,284897.0,360.0,360.0,360.0,0.0 +openrouter/google/gemma-2-27b-it,code-repair,540.0,0.0,15.0,15.0,2215.0,24507.0,2428.0,15.0,15.0,15.0,480.0 +openrouter/google/gemma-2-27b-it,transpile,6101.0,0.0,141.0,150.0,46590.0,400045.0,48571.0,150.0,150.0,150.0,5510.0 +openrouter/google/gemma-2-27b-it,write-tests,17525.0,16230.0,215.0,360.0,291553.0,1976259.0,296680.0,360.0,360.0,360.0,0.0 +openrouter/google/gemma-2-9b-it,code-repair,228.0,0.0,6.0,10.0,1086.0,23664.0,1187.0,10.0,6.0,6.0,200.0 +openrouter/google/gemma-2-9b-it,transpile,1459.0,0.0,30.0,50.0,15267.0,195055.0,16253.0,50.0,34.0,35.0,1310.0 +openrouter/google/gemma-2-9b-it,write-tests,8332.0,7590.0,92.0,245.0,212985.0,2255923.0,216267.0,245.0,201.0,204.0,0.0 +openrouter/google/palm-2-chat-bison,code-repair,360.0,0.0,10.0,10.0,1312.0,14611.0,1452.0,10.0,10.0,10.0,320.0 +openrouter/google/palm-2-chat-bison,transpile,5943.0,0.0,113.0,150.0,51279.0,274268.0,53129.0,150.0,150.0,150.0,5380.0 +openrouter/google/palm-2-chat-bison,write-tests,4242.0,3450.0,75.0,245.0,207055.0,794707.0,210050.0,245.0,236.0,236.0,0.0 +openrouter/google/palm-2-codechat-bison,code-repair,380.0,0.0,10.0,10.0,1291.0,11712.0,1427.0,10.0,10.0,10.0,340.0 +openrouter/google/palm-2-codechat-bison,transpile,3666.0,0.0,78.0,100.0,33571.0,190100.0,34762.0,100.0,99.0,99.0,3290.0 +openrouter/google/palm-2-codechat-bison,write-tests,5417.0,4620.0,78.0,245.0,206752.0,812889.0,209687.0,245.0,237.0,237.0,0.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,540.0,0.0,15.0,15.0,2318.0,45049.0,2438.0,15.0,15.0,15.0,480.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,6284.0,0.0,144.0,150.0,50497.0,625825.0,51729.0,150.0,150.0,150.0,5690.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,26041.0,24660.0,305.0,360.0,280119.0,3251391.0,283070.0,360.0,358.0,358.0,0.0 +openrouter/meta-llama/llama-3-8b-instruct,code-repair,269.0,0.0,9.0,10.0,1804.0,15410.0,1892.0,10.0,10.0,10.0,230.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile,3695.0,0.0,85.0,100.0,33690.0,280292.0,34703.0,100.0,100.0,100.0,3310.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,5897.0,5140.0,63.0,245.0,210933.0,1052803.0,221847.0,245.0,216.0,233.0,0.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,540.0,0.0,15.0,15.0,2271.0,42854.0,2451.0,15.0,15.0,15.0,480.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,6710.0,0.0,150.0,150.0,50010.0,847449.0,51768.0,150.0,150.0,150.0,6110.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,30285.0,28870.0,336.0,360.0,335128.0,4578095.0,341162.0,360.0,359.0,360.0,0.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,539.0,0.0,15.0,15.0,2267.0,36808.0,2631.0,15.0,14.0,15.0,480.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,6311.0,0.0,142.0,150.0,50314.0,717230.0,52441.0,150.0,149.0,150.0,5720.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,27478.0,26100.0,315.0,360.0,550935.0,5032078.0,563816.0,360.0,351.0,352.0,0.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,477.0,0.0,14.0,15.0,2545.0,24808.0,3310.0,15.0,13.0,15.0,420.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,5745.0,0.0,126.0,150.0,50192.0,292733.0,64860.0,150.0,140.0,149.0,5180.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,17889.0,16630.0,227.0,360.0,900466.0,2556661.0,992951.0,359.0,328.0,345.0,0.0 +openrouter/microsoft/phi-3-medium-128k-instruct,code-repair,138.0,0.0,5.0,5.0,625.0,7878.0,983.0,5.0,3.0,5.0,120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,3830.0,0.0,106.0,150.0,55898.0,650088.0,59619.0,150.0,109.0,115.0,3350.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,7177.0,6750.0,98.0,130.0,118158.0,1390115.0,126452.0,130.0,93.0,106.0,0.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,509.0,0.0,15.0,15.0,2611.0,22830.0,3258.0,15.0,14.0,15.0,450.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile,2430.0,0.0,77.0,100.0,31276.0,218167.0,33307.0,100.0,94.0,99.0,2060.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,11637.0,10420.0,183.0,359.0,360995.0,2057401.0,378267.0,359.0,317.0,358.0,0.0 +openrouter/microsoft/phi-3-mini-128k-instruct,code-repair,296.0,0.0,10.0,10.0,1898.0,20857.0,2838.0,10.0,6.0,10.0,260.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,3826.0,0.0,90.0,150.0,50189.0,322995.0,55038.0,150.0,129.0,137.0,3320.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,12195.0,11470.0,154.0,245.0,316922.0,1882771.0,416139.0,245.0,98.0,228.0,0.0 +openrouter/microsoft/wizardlm-2-7b,code-repair,236.0,0.0,9.0,10.0,2380.0,26750.0,3929.0,10.0,7.0,10.0,200.0 +openrouter/microsoft/wizardlm-2-7b,transpile,3530.0,0.0,94.0,150.0,58856.0,473310.0,62265.0,150.0,146.0,150.0,2990.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,10527.0,9940.0,98.0,245.0,532006.0,3997864.0,741252.0,245.0,0.0,244.0,0.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,499.0,0.0,14.0,15.0,2599.0,57375.0,2804.0,15.0,15.0,15.0,440.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,5147.0,0.0,127.0,150.0,52918.0,904881.0,54745.0,150.0,150.0,150.0,4570.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,21787.0,20640.0,250.0,360.0,483034.0,7659057.0,560308.0,360.0,178.0,359.0,0.0 +openrouter/mistralai/codestral-mamba,code-repair,503.0,0.0,15.0,15.0,2471.0,20309.0,5154.0,15.0,8.0,15.0,450.0 +openrouter/mistralai/codestral-mamba,transpile,6152.0,0.0,138.0,150.0,48015.0,350647.0,57841.0,150.0,125.0,149.0,5590.0 +openrouter/mistralai/codestral-mamba,write-tests,17852.0,16650.0,242.0,360.0,244626.0,1156642.0,299932.0,360.0,240.0,360.0,0.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,494.0,0.0,14.0,15.0,2664.0,32116.0,6469.0,15.0,0.0,15.0,450.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,3604.0,0.0,110.0,150.0,55447.0,422533.0,91376.0,150.0,54.0,150.0,3140.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,9781.0,8880.0,152.0,360.0,384335.0,2779766.0,540743.0,360.0,34.0,355.0,0.0 +openrouter/mistralai/mistral-large,code-repair,540.0,0.0,15.0,15.0,2382.0,38953.0,2564.0,15.0,15.0,15.0,480.0 +openrouter/mistralai/mistral-large,transpile,6887.0,0.0,147.0,150.0,49242.0,541363.0,50970.0,150.0,150.0,150.0,6290.0 +openrouter/mistralai/mistral-large,write-tests,28629.0,27240.0,311.0,360.0,304537.0,3465424.0,309084.0,360.0,358.0,360.0,0.0 +openrouter/mistralai/mistral-medium,code-repair,514.0,0.0,15.0,15.0,2482.0,40149.0,3832.0,15.0,9.0,15.0,460.0 +openrouter/mistralai/mistral-medium,transpile,5305.0,0.0,135.0,150.0,52042.0,763758.0,82017.0,150.0,100.0,150.0,4770.0 +openrouter/mistralai/mistral-medium,write-tests,24645.0,23480.0,279.0,360.0,333578.0,3940272.0,430980.0,360.0,166.0,360.0,0.0 +openrouter/mistralai/mistral-nemo,code-repair,447.0,0.0,12.0,15.0,2201.0,25360.0,2359.0,15.0,15.0,15.0,390.0 +openrouter/mistralai/mistral-nemo,transpile,3195.0,0.0,80.0,100.0,31566.0,258449.0,34234.0,100.0,96.0,99.0,2820.0 +openrouter/mistralai/mistral-nemo,write-tests,15520.0,14310.0,185.0,360.0,297894.0,1967296.0,310758.0,359.0,318.0,348.0,0.0 +openrouter/mistralai/mistral-small,code-repair,370.0,0.0,10.0,10.0,1897.0,15665.0,2019.0,10.0,10.0,10.0,330.0 +openrouter/mistralai/mistral-small,transpile,4933.0,0.0,95.0,100.0,34742.0,337741.0,35966.0,100.0,98.0,100.0,4540.0 +openrouter/mistralai/mistral-small,write-tests,14915.0,14040.0,152.0,245.0,208564.0,1588519.0,213375.0,245.0,233.0,245.0,0.0 +openrouter/mistralai/mistral-tiny,code-repair,365.0,0.0,10.0,10.0,2189.0,13305.0,3428.0,10.0,5.0,10.0,330.0 +openrouter/mistralai/mistral-tiny,transpile,3596.0,0.0,72.0,100.0,37268.0,264236.0,46236.0,100.0,74.0,100.0,3250.0 +openrouter/mistralai/mistral-tiny,write-tests,4682.0,4060.0,47.0,245.0,279733.0,1337364.0,348752.0,245.0,85.0,245.0,0.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,503.0,0.0,14.0,15.0,2374.0,29751.0,3987.0,15.0,9.0,15.0,450.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,5301.0,0.0,137.0,150.0,50019.0,403826.0,52967.0,150.0,144.0,150.0,4720.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,23504.0,22290.0,264.0,360.0,302379.0,2755623.0,359177.0,360.0,231.0,359.0,0.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,412.0,0.0,12.0,15.0,2334.0,40067.0,5695.0,15.0,2.0,13.0,370.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,4943.0,0.0,115.0,150.0,49080.0,398914.0,53220.0,146.0,98.0,134.0,4450.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,23142.0,21890.0,251.0,360.0,376495.0,2785742.0,389407.0,353.0,313.0,335.0,0.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,422.0,0.0,13.0,15.0,3373.0,78045.0,3523.0,15.0,12.0,12.0,370.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,4536.0,0.0,110.0,150.0,53141.0,1112611.0,54160.0,150.0,52.0,54.0,4170.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,13638.0,12530.0,174.0,360.0,253637.0,4711178.0,258065.0,360.0,286.0,288.0,0.0 +openrouter/neversleep/llama-3-lumimaid-8b,code-repair,247.0,0.0,8.0,10.0,1660.0,28404.0,2024.0,10.0,9.0,10.0,210.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile,3589.0,0.0,83.0,100.0,33354.0,286550.0,34741.0,100.0,92.0,94.0,3220.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,6782.0,6020.0,72.0,245.0,230212.0,1559958.0,243365.0,245.0,217.0,228.0,0.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,code-repair,295.0,0.0,9.0,10.0,1785.0,16413.0,2768.0,10.0,6.0,10.0,260.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile,3600.0,0.0,89.0,100.0,34621.0,199609.0,47907.0,100.0,62.0,99.0,3250.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,13089.0,12260.0,158.0,245.0,179251.0,684082.0,210746.0,245.0,181.0,245.0,0.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,code-repair,324.0,0.0,10.0,10.0,1779.0,39497.0,3634.0,10.0,4.0,10.0,290.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile,3984.0,0.0,94.0,100.0,32489.0,463225.0,37187.0,100.0,80.0,100.0,3610.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,12902.0,12060.0,145.0,245.0,215879.0,2120404.0,235111.0,245.0,208.0,244.0,0.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,code-repair,271.0,0.0,10.0,10.0,1803.0,64755.0,14602.0,10.0,1.0,10.0,240.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,5953.0,0.0,128.0,150.0,50982.0,635832.0,182154.0,150.0,41.0,144.0,5490.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,13766.0,13020.0,155.0,245.0,347702.0,1812516.0,470011.0,245.0,106.0,240.0,0.0 +openrouter/nousresearch/nous-hermes-yi-34b,code-repair,338.0,0.0,9.0,10.0,1723.0,16954.0,1972.0,10.0,9.0,10.0,300.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile,3394.0,0.0,69.0,100.0,36291.0,333061.0,45739.0,100.0,55.0,70.0,3100.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,4474.0,3740.0,64.0,245.0,324570.0,2812284.0,332670.0,245.0,204.0,221.0,0.0 +openrouter/openai/gpt-4-turbo,code-repair,540.0,0.0,15.0,15.0,2309.0,48581.0,2495.0,15.0,15.0,15.0,480.0 +openrouter/openai/gpt-4-turbo,transpile,6410.0,0.0,150.0,150.0,51143.0,829633.0,52899.0,150.0,150.0,150.0,5810.0 +openrouter/openai/gpt-4-turbo,write-tests,31472.0,30050.0,342.0,360.0,316744.0,4006582.0,321000.0,360.0,360.0,360.0,0.0 +openrouter/openai/gpt-4o,code-repair,540.0,0.0,15.0,15.0,2200.0,27205.0,2390.0,15.0,15.0,15.0,480.0 +openrouter/openai/gpt-4o,transpile,7539.0,0.0,149.0,150.0,48661.0,344467.0,50417.0,150.0,150.0,150.0,6940.0 +openrouter/openai/gpt-4o,write-tests,30926.0,29520.0,329.0,359.0,296375.0,1858764.0,300915.0,359.0,359.0,359.0,0.0 +openrouter/openai/gpt-4o-mini,code-repair,540.0,0.0,15.0,15.0,2200.0,25907.0,2390.0,15.0,15.0,15.0,480.0 +openrouter/openai/gpt-4o-mini,transpile,7700.0,0.0,150.0,150.0,45383.0,366473.0,47087.0,150.0,150.0,150.0,7100.0 +openrouter/openai/gpt-4o-mini,write-tests,31201.0,29780.0,341.0,360.0,350807.0,1821012.0,354887.0,360.0,360.0,360.0,0.0 +openrouter/openchat/openchat-8b,code-repair,127.0,0.0,3.0,5.0,555.0,8171.0,1469.0,5.0,4.0,5.0,110.0 +openrouter/openchat/openchat-8b,transpile,3863.0,0.0,77.0,100.0,33729.0,215885.0,39749.0,98.0,34.0,94.0,3560.0 +openrouter/openchat/openchat-8b,write-tests,1386.0,1030.0,35.0,130.0,83779.0,556039.0,104039.0,129.0,64.0,128.0,0.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,506.0,0.0,14.0,15.0,2083.0,27599.0,2195.0,14.0,14.0,14.0,450.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,6179.0,0.0,142.0,150.0,50212.0,673768.0,51418.0,149.0,149.0,149.0,5590.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,28366.0,26960.0,327.0,360.0,280245.0,3093201.0,283385.0,360.0,359.0,360.0,0.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,code-repair,317.0,0.0,10.0,10.0,1314.0,10871.0,1806.0,10.0,7.0,10.0,280.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile,4165.0,0.0,93.0,100.0,32520.0,118292.0,36623.0,100.0,82.0,100.0,3790.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,2856.0,2230.0,43.0,245.0,322234.0,1190335.0,353003.0,243.0,99.0,241.0,0.0 +openrouter/qwen/qwen-110b-chat,code-repair,380.0,0.0,10.0,10.0,1279.0,14869.0,1409.0,10.0,10.0,10.0,340.0 +openrouter/qwen/qwen-110b-chat,transpile,3803.0,0.0,85.0,100.0,27101.0,257972.0,28740.0,100.0,98.0,100.0,3420.0 +openrouter/qwen/qwen-110b-chat,write-tests,6797.0,5980.0,143.0,245.0,194443.0,1542264.0,223207.0,245.0,184.0,245.0,0.0 +openrouter/qwen/qwen-2-72b-instruct,code-repair,330.0,0.0,10.0,10.0,1687.0,19673.0,1807.0,10.0,10.0,10.0,290.0 +openrouter/qwen/qwen-2-72b-instruct,transpile,3541.0,0.0,84.0,100.0,34331.0,434671.0,36844.0,100.0,97.0,100.0,3160.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,21419.0,20490.0,203.0,245.0,190311.0,1967239.0,195922.0,245.0,237.0,244.0,0.0 +openrouter/qwen/qwen-2-7b-instruct,code-repair,278.0,0.0,8.0,10.0,1842.0,13994.0,1958.0,10.0,10.0,10.0,240.0 +openrouter/qwen/qwen-2-7b-instruct,transpile,3473.0,0.0,74.0,100.0,34364.0,206550.0,35571.0,100.0,99.0,100.0,3100.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,6999.0,6190.0,80.0,245.0,245579.0,1096453.0,252316.0,245.0,239.0,245.0,0.0 +openrouter/qwen/qwen-72b-chat,code-repair,537.0,0.0,15.0,15.0,2355.0,30416.0,2880.0,15.0,12.0,15.0,480.0 +openrouter/qwen/qwen-72b-chat,transpile,4071.0,0.0,112.0,150.0,48758.0,396261.0,50999.0,147.0,145.0,147.0,3520.0 +openrouter/qwen/qwen-72b-chat,write-tests,23109.0,21880.0,277.0,360.0,302798.0,2429523.0,334049.0,356.0,240.0,356.0,0.0 +openrouter/recursal/eagle-7b,write-tests,0.0,0.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,0.0,0.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,20.0,0.0,0.0,11.0,4547.0,73927.0,4902.0,10.0,3.0,7.0,0.0 +openrouter/teknium/openhermes-2.5-mistral-7b,code-repair,204.0,0.0,7.0,10.0,1487.0,13214.0,2586.0,10.0,7.0,10.0,170.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile,2624.0,0.0,67.0,100.0,35043.0,208000.0,40174.0,100.0,73.0,94.0,2290.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,13053.0,12220.0,134.0,245.0,189314.0,802748.0,201622.0,245.0,211.0,243.0,0.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,446.0,0.0,13.0,15.0,2301.0,18715.0,2932.0,15.0,13.0,15.0,390.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile,2416.0,0.0,69.0,100.0,27118.0,171633.0,33210.0,100.0,82.0,95.0,2070.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,9139.0,8050.0,131.0,360.0,237741.0,1135825.0,278751.0,358.0,261.0,339.0,0.0 +openrouter/undi95/toppy-m-7b,code-repair,302.0,0.0,9.0,10.0,1509.0,14806.0,3369.0,10.0,3.0,10.0,270.0 +openrouter/undi95/toppy-m-7b,transpile,156.0,0.0,45.0,50.0,15827.0,140873.0,51945.0,50.0,11.0,50.0,0.0 +openrouter/undi95/toppy-m-7b,write-tests,2568.0,1960.0,51.0,245.0,179188.0,1034167.0,309128.0,245.0,70.0,242.0,0.0 +openrouter/xwin-lm/xwin-lm-70b,code-repair,206.0,0.0,6.0,10.0,2115.0,103640.0,5111.0,10.0,0.0,10.0,180.0 +openrouter/xwin-lm/xwin-lm-70b,transpile,4192.0,0.0,83.0,100.0,38840.0,1134276.0,44750.0,100.0,69.0,100.0,3840.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,1198.0,800.0,20.0,245.0,188321.0,5825648.0,228834.0,245.0,11.0,122.0,0.0 diff --git a/docs/reports/v0.6/evaluation-cases.csv b/docs/reports/v0.6/evaluation-cases.csv new file mode 100644 index 00000000..bc8b6c5b --- /dev/null +++ b/docs/reports/v0.6/evaluation-cases.csv @@ -0,0 +1,887 @@ +model_id,task,language,files +openrouter/01-ai/yi-34b-chat,code-repair,java,5.0 +openrouter/01-ai/yi-34b-chat,write-tests,golang,5.0 +openrouter/01-ai/yi-34b-chat,write-tests,java,120.0 +openrouter/01-ai/yi-34b-chat,write-tests,ruby,1.0 +openrouter/01-ai/yi-34b-chat,write-tests-symflower-fix,golang,5.0 +openrouter/01-ai/yi-34b-chat,write-tests-symflower-fix,java,120.0 +openrouter/01-ai/yi-34b-chat,write-tests-symflower-fix,ruby,1.0 +openrouter/ai21/jamba-instruct,code-repair,java,5.0 +openrouter/ai21/jamba-instruct,code-repair,ruby,5.0 +openrouter/ai21/jamba-instruct,transpile,java,50.0 +openrouter/ai21/jamba-instruct,transpile,ruby,50.0 +openrouter/ai21/jamba-instruct,transpile-symflower-fix,java,50.0 +openrouter/ai21/jamba-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/ai21/jamba-instruct,write-tests,golang,5.0 +openrouter/ai21/jamba-instruct,write-tests,java,120.0 +openrouter/ai21/jamba-instruct,write-tests,ruby,120.0 +openrouter/ai21/jamba-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/ai21/jamba-instruct,write-tests-symflower-fix,java,120.0 +openrouter/ai21/jamba-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/alpindale/goliath-120b,code-repair,ruby,5.0 +openrouter/alpindale/goliath-120b,transpile,golang,50.0 +openrouter/alpindale/goliath-120b,transpile,ruby,50.0 +openrouter/alpindale/goliath-120b,transpile-symflower-fix,golang,50.0 +openrouter/alpindale/goliath-120b,transpile-symflower-fix,ruby,50.0 +openrouter/alpindale/goliath-120b,write-tests,golang,5.0 +openrouter/alpindale/goliath-120b,write-tests,java,5.0 +openrouter/alpindale/goliath-120b,write-tests,ruby,120.0 +openrouter/alpindale/goliath-120b,write-tests-symflower-fix,golang,5.0 +openrouter/alpindale/goliath-120b,write-tests-symflower-fix,java,5.0 +openrouter/alpindale/goliath-120b,write-tests-symflower-fix,ruby,120.0 +openrouter/alpindale/magnum-72b,code-repair,golang,5.0 +openrouter/alpindale/magnum-72b,code-repair,java,5.0 +openrouter/alpindale/magnum-72b,code-repair,ruby,5.0 +openrouter/alpindale/magnum-72b,transpile,java,50.0 +openrouter/alpindale/magnum-72b,transpile,ruby,50.0 +openrouter/alpindale/magnum-72b,transpile-symflower-fix,java,50.0 +openrouter/alpindale/magnum-72b,transpile-symflower-fix,ruby,50.0 +openrouter/alpindale/magnum-72b,write-tests,golang,120.0 +openrouter/alpindale/magnum-72b,write-tests,java,120.0 +openrouter/alpindale/magnum-72b,write-tests,ruby,120.0 +openrouter/alpindale/magnum-72b,write-tests-symflower-fix,golang,120.0 +openrouter/alpindale/magnum-72b,write-tests-symflower-fix,java,120.0 +openrouter/alpindale/magnum-72b,write-tests-symflower-fix,ruby,120.0 +openrouter/anthropic/claude-3-haiku,code-repair,golang,5.0 +openrouter/anthropic/claude-3-haiku,code-repair,java,5.0 +openrouter/anthropic/claude-3-haiku,code-repair,ruby,5.0 +openrouter/anthropic/claude-3-haiku,transpile,golang,50.0 +openrouter/anthropic/claude-3-haiku,transpile,java,50.0 +openrouter/anthropic/claude-3-haiku,transpile,ruby,50.0 +openrouter/anthropic/claude-3-haiku,transpile-symflower-fix,golang,50.0 +openrouter/anthropic/claude-3-haiku,transpile-symflower-fix,java,50.0 +openrouter/anthropic/claude-3-haiku,transpile-symflower-fix,ruby,50.0 +openrouter/anthropic/claude-3-haiku,write-tests,golang,120.0 +openrouter/anthropic/claude-3-haiku,write-tests,java,120.0 +openrouter/anthropic/claude-3-haiku,write-tests,ruby,120.0 +openrouter/anthropic/claude-3-haiku,write-tests-symflower-fix,golang,120.0 +openrouter/anthropic/claude-3-haiku,write-tests-symflower-fix,java,120.0 +openrouter/anthropic/claude-3-haiku,write-tests-symflower-fix,ruby,120.0 +openrouter/anthropic/claude-3-opus,code-repair,golang,5.0 +openrouter/anthropic/claude-3-opus,code-repair,java,5.0 +openrouter/anthropic/claude-3-opus,code-repair,ruby,5.0 +openrouter/anthropic/claude-3-opus,transpile,golang,50.0 +openrouter/anthropic/claude-3-opus,transpile,java,50.0 +openrouter/anthropic/claude-3-opus,transpile,ruby,50.0 +openrouter/anthropic/claude-3-opus,transpile-symflower-fix,golang,50.0 +openrouter/anthropic/claude-3-opus,transpile-symflower-fix,java,50.0 +openrouter/anthropic/claude-3-opus,transpile-symflower-fix,ruby,50.0 +openrouter/anthropic/claude-3-opus,write-tests,golang,120.0 +openrouter/anthropic/claude-3-opus,write-tests,java,120.0 +openrouter/anthropic/claude-3-opus,write-tests,ruby,120.0 +openrouter/anthropic/claude-3-opus,write-tests-symflower-fix,golang,120.0 +openrouter/anthropic/claude-3-opus,write-tests-symflower-fix,java,120.0 +openrouter/anthropic/claude-3-opus,write-tests-symflower-fix,ruby,120.0 +openrouter/anthropic/claude-3-sonnet,code-repair,golang,5.0 +openrouter/anthropic/claude-3-sonnet,code-repair,java,5.0 +openrouter/anthropic/claude-3-sonnet,code-repair,ruby,5.0 +openrouter/anthropic/claude-3-sonnet,transpile,golang,50.0 +openrouter/anthropic/claude-3-sonnet,transpile,java,50.0 +openrouter/anthropic/claude-3-sonnet,transpile,ruby,50.0 +openrouter/anthropic/claude-3-sonnet,transpile-symflower-fix,golang,50.0 +openrouter/anthropic/claude-3-sonnet,transpile-symflower-fix,java,50.0 +openrouter/anthropic/claude-3-sonnet,transpile-symflower-fix,ruby,50.0 +openrouter/anthropic/claude-3-sonnet,write-tests,golang,120.0 +openrouter/anthropic/claude-3-sonnet,write-tests,java,120.0 +openrouter/anthropic/claude-3-sonnet,write-tests,ruby,120.0 +openrouter/anthropic/claude-3-sonnet,write-tests-symflower-fix,golang,120.0 +openrouter/anthropic/claude-3-sonnet,write-tests-symflower-fix,java,120.0 +openrouter/anthropic/claude-3-sonnet,write-tests-symflower-fix,ruby,120.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,golang,5.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,java,5.0 +openrouter/anthropic/claude-3.5-sonnet,code-repair,ruby,5.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,golang,50.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,java,50.0 +openrouter/anthropic/claude-3.5-sonnet,transpile,ruby,50.0 +openrouter/anthropic/claude-3.5-sonnet,transpile-symflower-fix,golang,50.0 +openrouter/anthropic/claude-3.5-sonnet,transpile-symflower-fix,java,50.0 +openrouter/anthropic/claude-3.5-sonnet,transpile-symflower-fix,ruby,50.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,golang,120.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,java,120.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests,ruby,120.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests-symflower-fix,golang,120.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests-symflower-fix,java,120.0 +openrouter/anthropic/claude-3.5-sonnet,write-tests-symflower-fix,ruby,120.0 +openrouter/austism/chronos-hermes-13b,code-repair,golang,5.0 +openrouter/austism/chronos-hermes-13b,code-repair,java,5.0 +openrouter/austism/chronos-hermes-13b,code-repair,ruby,5.0 +openrouter/austism/chronos-hermes-13b,transpile,golang,50.0 +openrouter/austism/chronos-hermes-13b,transpile,java,50.0 +openrouter/austism/chronos-hermes-13b,transpile,ruby,50.0 +openrouter/austism/chronos-hermes-13b,transpile-symflower-fix,golang,50.0 +openrouter/austism/chronos-hermes-13b,transpile-symflower-fix,java,50.0 +openrouter/austism/chronos-hermes-13b,transpile-symflower-fix,ruby,50.0 +openrouter/austism/chronos-hermes-13b,write-tests,golang,120.0 +openrouter/austism/chronos-hermes-13b,write-tests,java,120.0 +openrouter/austism/chronos-hermes-13b,write-tests,ruby,120.0 +openrouter/austism/chronos-hermes-13b,write-tests-symflower-fix,golang,120.0 +openrouter/austism/chronos-hermes-13b,write-tests-symflower-fix,java,120.0 +openrouter/austism/chronos-hermes-13b,write-tests-symflower-fix,ruby,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,golang,5.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,java,5.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,code-repair,ruby,5.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,golang,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,java,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile,ruby,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile-symflower-fix,golang,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile-symflower-fix,java,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,transpile-symflower-fix,ruby,50.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,golang,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,java,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests,ruby,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests-symflower-fix,golang,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests-symflower-fix,java,120.0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,write-tests-symflower-fix,ruby,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,golang,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,java,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,code-repair,ruby,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,golang,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,java,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile,ruby,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile-symflower-fix,golang,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile-symflower-fix,java,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,transpile-symflower-fix,ruby,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,golang,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,java,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests,ruby,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests-symflower-fix,golang,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests-symflower-fix,java,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,write-tests-symflower-fix,ruby,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,golang,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,java,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,code-repair,ruby,5.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile,java,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile,ruby,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile-symflower-fix,java,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,transpile-symflower-fix,ruby,50.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,golang,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,java,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests,ruby,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests-symflower-fix,golang,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests-symflower-fix,java,120.0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,write-tests-symflower-fix,ruby,120.0 +openrouter/cohere/command-r-03-2024,code-repair,golang,5.0 +openrouter/cohere/command-r-03-2024,code-repair,java,5.0 +openrouter/cohere/command-r-03-2024,code-repair,ruby,5.0 +openrouter/cohere/command-r-03-2024,transpile,golang,50.0 +openrouter/cohere/command-r-03-2024,transpile,java,50.0 +openrouter/cohere/command-r-03-2024,transpile,ruby,50.0 +openrouter/cohere/command-r-03-2024,transpile-symflower-fix,golang,50.0 +openrouter/cohere/command-r-03-2024,transpile-symflower-fix,java,50.0 +openrouter/cohere/command-r-03-2024,transpile-symflower-fix,ruby,50.0 +openrouter/cohere/command-r-03-2024,write-tests,golang,120.0 +openrouter/cohere/command-r-03-2024,write-tests,java,120.0 +openrouter/cohere/command-r-03-2024,write-tests,ruby,120.0 +openrouter/cohere/command-r-03-2024,write-tests-symflower-fix,golang,120.0 +openrouter/cohere/command-r-03-2024,write-tests-symflower-fix,java,120.0 +openrouter/cohere/command-r-03-2024,write-tests-symflower-fix,ruby,120.0 +openrouter/cohere/command-r-08-2024,code-repair,golang,5.0 +openrouter/cohere/command-r-08-2024,code-repair,java,5.0 +openrouter/cohere/command-r-08-2024,code-repair,ruby,5.0 +openrouter/cohere/command-r-08-2024,transpile,golang,50.0 +openrouter/cohere/command-r-08-2024,transpile,java,50.0 +openrouter/cohere/command-r-08-2024,transpile,ruby,50.0 +openrouter/cohere/command-r-08-2024,transpile-symflower-fix,golang,50.0 +openrouter/cohere/command-r-08-2024,transpile-symflower-fix,java,50.0 +openrouter/cohere/command-r-08-2024,transpile-symflower-fix,ruby,50.0 +openrouter/cohere/command-r-08-2024,write-tests,golang,120.0 +openrouter/cohere/command-r-08-2024,write-tests,java,120.0 +openrouter/cohere/command-r-08-2024,write-tests,ruby,120.0 +openrouter/cohere/command-r-08-2024,write-tests-symflower-fix,golang,120.0 +openrouter/cohere/command-r-08-2024,write-tests-symflower-fix,java,120.0 +openrouter/cohere/command-r-08-2024,write-tests-symflower-fix,ruby,120.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,golang,5.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,java,5.0 +openrouter/cohere/command-r-plus-04-2024,code-repair,ruby,5.0 +openrouter/cohere/command-r-plus-04-2024,transpile,golang,50.0 +openrouter/cohere/command-r-plus-04-2024,transpile,java,50.0 +openrouter/cohere/command-r-plus-04-2024,transpile,ruby,50.0 +openrouter/cohere/command-r-plus-04-2024,transpile-symflower-fix,golang,50.0 +openrouter/cohere/command-r-plus-04-2024,transpile-symflower-fix,java,50.0 +openrouter/cohere/command-r-plus-04-2024,transpile-symflower-fix,ruby,50.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,golang,120.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,java,120.0 +openrouter/cohere/command-r-plus-04-2024,write-tests,ruby,120.0 +openrouter/cohere/command-r-plus-04-2024,write-tests-symflower-fix,golang,120.0 +openrouter/cohere/command-r-plus-04-2024,write-tests-symflower-fix,java,120.0 +openrouter/cohere/command-r-plus-04-2024,write-tests-symflower-fix,ruby,120.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,golang,5.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,java,5.0 +openrouter/cohere/command-r-plus-08-2024,code-repair,ruby,5.0 +openrouter/cohere/command-r-plus-08-2024,transpile,golang,50.0 +openrouter/cohere/command-r-plus-08-2024,transpile,java,50.0 +openrouter/cohere/command-r-plus-08-2024,transpile,ruby,50.0 +openrouter/cohere/command-r-plus-08-2024,transpile-symflower-fix,golang,50.0 +openrouter/cohere/command-r-plus-08-2024,transpile-symflower-fix,java,50.0 +openrouter/cohere/command-r-plus-08-2024,transpile-symflower-fix,ruby,50.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,golang,120.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,java,120.0 +openrouter/cohere/command-r-plus-08-2024,write-tests,ruby,120.0 +openrouter/cohere/command-r-plus-08-2024,write-tests-symflower-fix,golang,120.0 +openrouter/cohere/command-r-plus-08-2024,write-tests-symflower-fix,java,120.0 +openrouter/cohere/command-r-plus-08-2024,write-tests-symflower-fix,ruby,120.0 +openrouter/databricks/dbrx-instruct,code-repair,golang,5.0 +openrouter/databricks/dbrx-instruct,code-repair,java,5.0 +openrouter/databricks/dbrx-instruct,transpile,golang,50.0 +openrouter/databricks/dbrx-instruct,transpile,java,50.0 +openrouter/databricks/dbrx-instruct,transpile-symflower-fix,golang,50.0 +openrouter/databricks/dbrx-instruct,transpile-symflower-fix,java,50.0 +openrouter/databricks/dbrx-instruct,write-tests,golang,120.0 +openrouter/databricks/dbrx-instruct,write-tests,java,120.0 +openrouter/databricks/dbrx-instruct,write-tests,ruby,1.0 +openrouter/databricks/dbrx-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/databricks/dbrx-instruct,write-tests-symflower-fix,java,120.0 +openrouter/databricks/dbrx-instruct,write-tests-symflower-fix,ruby,1.0 +openrouter/deepseek/deepseek-chat,code-repair,golang,5.0 +openrouter/deepseek/deepseek-chat,code-repair,java,5.0 +openrouter/deepseek/deepseek-chat,code-repair,ruby,5.0 +openrouter/deepseek/deepseek-chat,transpile,golang,50.0 +openrouter/deepseek/deepseek-chat,transpile,java,50.0 +openrouter/deepseek/deepseek-chat,transpile,ruby,50.0 +openrouter/deepseek/deepseek-chat,transpile-symflower-fix,golang,50.0 +openrouter/deepseek/deepseek-chat,transpile-symflower-fix,java,50.0 +openrouter/deepseek/deepseek-chat,transpile-symflower-fix,ruby,50.0 +openrouter/deepseek/deepseek-chat,write-tests,golang,120.0 +openrouter/deepseek/deepseek-chat,write-tests,java,120.0 +openrouter/deepseek/deepseek-chat,write-tests,ruby,120.0 +openrouter/deepseek/deepseek-chat,write-tests-symflower-fix,golang,120.0 +openrouter/deepseek/deepseek-chat,write-tests-symflower-fix,java,120.0 +openrouter/deepseek/deepseek-chat,write-tests-symflower-fix,ruby,120.0 +openrouter/deepseek/deepseek-coder,code-repair,golang,5.0 +openrouter/deepseek/deepseek-coder,code-repair,java,5.0 +openrouter/deepseek/deepseek-coder,code-repair,ruby,5.0 +openrouter/deepseek/deepseek-coder,transpile,golang,50.0 +openrouter/deepseek/deepseek-coder,transpile,java,50.0 +openrouter/deepseek/deepseek-coder,transpile,ruby,50.0 +openrouter/deepseek/deepseek-coder,transpile-symflower-fix,golang,50.0 +openrouter/deepseek/deepseek-coder,transpile-symflower-fix,java,50.0 +openrouter/deepseek/deepseek-coder,transpile-symflower-fix,ruby,50.0 +openrouter/deepseek/deepseek-coder,write-tests,golang,120.0 +openrouter/deepseek/deepseek-coder,write-tests,java,120.0 +openrouter/deepseek/deepseek-coder,write-tests,ruby,120.0 +openrouter/deepseek/deepseek-coder,write-tests-symflower-fix,golang,120.0 +openrouter/deepseek/deepseek-coder,write-tests-symflower-fix,java,120.0 +openrouter/deepseek/deepseek-coder,write-tests-symflower-fix,ruby,120.0 +openrouter/google/gemini-flash-1.5,code-repair,golang,5.0 +openrouter/google/gemini-flash-1.5,code-repair,java,5.0 +openrouter/google/gemini-flash-1.5,code-repair,ruby,5.0 +openrouter/google/gemini-flash-1.5,transpile,golang,50.0 +openrouter/google/gemini-flash-1.5,transpile,ruby,50.0 +openrouter/google/gemini-flash-1.5,transpile-symflower-fix,golang,50.0 +openrouter/google/gemini-flash-1.5,transpile-symflower-fix,ruby,50.0 +openrouter/google/gemini-flash-1.5,write-tests,golang,120.0 +openrouter/google/gemini-flash-1.5,write-tests,java,120.0 +openrouter/google/gemini-flash-1.5,write-tests,ruby,120.0 +openrouter/google/gemini-flash-1.5,write-tests-symflower-fix,golang,120.0 +openrouter/google/gemini-flash-1.5,write-tests-symflower-fix,java,120.0 +openrouter/google/gemini-flash-1.5,write-tests-symflower-fix,ruby,120.0 +openrouter/google/gemini-pro-1.5,code-repair,golang,5.0 +openrouter/google/gemini-pro-1.5,code-repair,java,5.0 +openrouter/google/gemini-pro-1.5,code-repair,ruby,5.0 +openrouter/google/gemini-pro-1.5,transpile,golang,50.0 +openrouter/google/gemini-pro-1.5,transpile,java,50.0 +openrouter/google/gemini-pro-1.5,transpile,ruby,50.0 +openrouter/google/gemini-pro-1.5,transpile-symflower-fix,golang,50.0 +openrouter/google/gemini-pro-1.5,transpile-symflower-fix,java,50.0 +openrouter/google/gemini-pro-1.5,transpile-symflower-fix,ruby,50.0 +openrouter/google/gemini-pro-1.5,write-tests,golang,120.0 +openrouter/google/gemini-pro-1.5,write-tests,java,120.0 +openrouter/google/gemini-pro-1.5,write-tests,ruby,120.0 +openrouter/google/gemini-pro-1.5,write-tests-symflower-fix,golang,120.0 +openrouter/google/gemini-pro-1.5,write-tests-symflower-fix,java,120.0 +openrouter/google/gemini-pro-1.5,write-tests-symflower-fix,ruby,120.0 +openrouter/google/gemma-2-27b-it,code-repair,golang,5.0 +openrouter/google/gemma-2-27b-it,code-repair,java,5.0 +openrouter/google/gemma-2-27b-it,code-repair,ruby,5.0 +openrouter/google/gemma-2-27b-it,transpile,golang,50.0 +openrouter/google/gemma-2-27b-it,transpile,java,50.0 +openrouter/google/gemma-2-27b-it,transpile,ruby,50.0 +openrouter/google/gemma-2-27b-it,transpile-symflower-fix,golang,50.0 +openrouter/google/gemma-2-27b-it,transpile-symflower-fix,java,50.0 +openrouter/google/gemma-2-27b-it,transpile-symflower-fix,ruby,50.0 +openrouter/google/gemma-2-27b-it,write-tests,golang,120.0 +openrouter/google/gemma-2-27b-it,write-tests,java,120.0 +openrouter/google/gemma-2-27b-it,write-tests,ruby,120.0 +openrouter/google/gemma-2-27b-it,write-tests-symflower-fix,golang,120.0 +openrouter/google/gemma-2-27b-it,write-tests-symflower-fix,java,120.0 +openrouter/google/gemma-2-27b-it,write-tests-symflower-fix,ruby,120.0 +openrouter/google/gemma-2-9b-it,code-repair,golang,5.0 +openrouter/google/gemma-2-9b-it,code-repair,java,5.0 +openrouter/google/gemma-2-9b-it,transpile,java,50.0 +openrouter/google/gemma-2-9b-it,transpile-symflower-fix,java,50.0 +openrouter/google/gemma-2-9b-it,write-tests,golang,120.0 +openrouter/google/gemma-2-9b-it,write-tests,java,120.0 +openrouter/google/gemma-2-9b-it,write-tests,ruby,5.0 +openrouter/google/gemma-2-9b-it,write-tests-symflower-fix,golang,120.0 +openrouter/google/gemma-2-9b-it,write-tests-symflower-fix,java,120.0 +openrouter/google/gemma-2-9b-it,write-tests-symflower-fix,ruby,5.0 +openrouter/google/palm-2-chat-bison,code-repair,golang,5.0 +openrouter/google/palm-2-chat-bison,code-repair,ruby,5.0 +openrouter/google/palm-2-chat-bison,transpile,golang,50.0 +openrouter/google/palm-2-chat-bison,transpile,java,50.0 +openrouter/google/palm-2-chat-bison,transpile,ruby,50.0 +openrouter/google/palm-2-chat-bison,transpile-symflower-fix,golang,50.0 +openrouter/google/palm-2-chat-bison,transpile-symflower-fix,java,50.0 +openrouter/google/palm-2-chat-bison,transpile-symflower-fix,ruby,50.0 +openrouter/google/palm-2-chat-bison,write-tests,golang,120.0 +openrouter/google/palm-2-chat-bison,write-tests,java,5.0 +openrouter/google/palm-2-chat-bison,write-tests,ruby,120.0 +openrouter/google/palm-2-chat-bison,write-tests-symflower-fix,golang,120.0 +openrouter/google/palm-2-chat-bison,write-tests-symflower-fix,java,5.0 +openrouter/google/palm-2-chat-bison,write-tests-symflower-fix,ruby,120.0 +openrouter/google/palm-2-codechat-bison,code-repair,golang,5.0 +openrouter/google/palm-2-codechat-bison,code-repair,ruby,5.0 +openrouter/google/palm-2-codechat-bison,transpile,golang,50.0 +openrouter/google/palm-2-codechat-bison,transpile,ruby,50.0 +openrouter/google/palm-2-codechat-bison,transpile-symflower-fix,golang,50.0 +openrouter/google/palm-2-codechat-bison,transpile-symflower-fix,ruby,50.0 +openrouter/google/palm-2-codechat-bison,write-tests,golang,120.0 +openrouter/google/palm-2-codechat-bison,write-tests,java,5.0 +openrouter/google/palm-2-codechat-bison,write-tests,ruby,120.0 +openrouter/google/palm-2-codechat-bison,write-tests-symflower-fix,golang,120.0 +openrouter/google/palm-2-codechat-bison,write-tests-symflower-fix,java,5.0 +openrouter/google/palm-2-codechat-bison,write-tests-symflower-fix,ruby,120.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,golang,5.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,java,5.0 +openrouter/meta-llama/llama-3-70b-instruct,code-repair,ruby,5.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,golang,50.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,java,50.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile,ruby,50.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile-symflower-fix,java,50.0 +openrouter/meta-llama/llama-3-70b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,golang,120.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,java,120.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests,ruby,120.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/meta-llama/llama-3-70b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/meta-llama/llama-3-8b-instruct,code-repair,java,5.0 +openrouter/meta-llama/llama-3-8b-instruct,code-repair,ruby,5.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile,java,50.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile,ruby,50.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile-symflower-fix,java,50.0 +openrouter/meta-llama/llama-3-8b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,golang,5.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,java,120.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests,ruby,120.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/meta-llama/llama-3-8b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,golang,5.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,java,5.0 +openrouter/meta-llama/llama-3.1-405b-instruct,code-repair,ruby,5.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,golang,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,java,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile,ruby,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile-symflower-fix,java,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,golang,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,java,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests,ruby,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/meta-llama/llama-3.1-405b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,golang,5.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,java,5.0 +openrouter/meta-llama/llama-3.1-70b-instruct,code-repair,ruby,5.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,golang,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,java,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile,ruby,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile-symflower-fix,java,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,golang,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,java,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests,ruby,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/meta-llama/llama-3.1-70b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,golang,5.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,java,5.0 +openrouter/meta-llama/llama-3.1-8b-instruct,code-repair,ruby,5.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,golang,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,java,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile,ruby,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile-symflower-fix,java,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,golang,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,java,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests,ruby,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/meta-llama/llama-3.1-8b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,code-repair,ruby,5.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,golang,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,java,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile,ruby,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile-symflower-fix,golang,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile-symflower-fix,java,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,golang,5.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,java,5.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests,ruby,120.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests-symflower-fix,java,5.0 +openrouter/microsoft/phi-3-medium-128k-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,golang,5.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,java,5.0 +openrouter/microsoft/phi-3-medium-4k-instruct,code-repair,ruby,5.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile,golang,50.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile,ruby,50.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile-symflower-fix,golang,50.0 +openrouter/microsoft/phi-3-medium-4k-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,golang,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,java,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests,ruby,119.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests-symflower-fix,java,120.0 +openrouter/microsoft/phi-3-medium-4k-instruct,write-tests-symflower-fix,ruby,119.0 +openrouter/microsoft/phi-3-mini-128k-instruct,code-repair,java,5.0 +openrouter/microsoft/phi-3-mini-128k-instruct,code-repair,ruby,5.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,golang,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,java,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile,ruby,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile-symflower-fix,golang,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile-symflower-fix,java,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,golang,5.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,java,120.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests,ruby,120.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests-symflower-fix,java,120.0 +openrouter/microsoft/phi-3-mini-128k-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/microsoft/wizardlm-2-7b,code-repair,java,5.0 +openrouter/microsoft/wizardlm-2-7b,code-repair,ruby,5.0 +openrouter/microsoft/wizardlm-2-7b,transpile,golang,50.0 +openrouter/microsoft/wizardlm-2-7b,transpile,java,50.0 +openrouter/microsoft/wizardlm-2-7b,transpile,ruby,50.0 +openrouter/microsoft/wizardlm-2-7b,transpile-symflower-fix,golang,50.0 +openrouter/microsoft/wizardlm-2-7b,transpile-symflower-fix,java,50.0 +openrouter/microsoft/wizardlm-2-7b,transpile-symflower-fix,ruby,50.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,golang,5.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,java,120.0 +openrouter/microsoft/wizardlm-2-7b,write-tests,ruby,120.0 +openrouter/microsoft/wizardlm-2-7b,write-tests-symflower-fix,golang,5.0 +openrouter/microsoft/wizardlm-2-7b,write-tests-symflower-fix,java,120.0 +openrouter/microsoft/wizardlm-2-7b,write-tests-symflower-fix,ruby,120.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,golang,5.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,java,5.0 +openrouter/microsoft/wizardlm-2-8x22b,code-repair,ruby,5.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,golang,50.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,java,50.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile,ruby,50.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile-symflower-fix,golang,50.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile-symflower-fix,java,50.0 +openrouter/microsoft/wizardlm-2-8x22b,transpile-symflower-fix,ruby,50.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,golang,120.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,java,120.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests,ruby,120.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests-symflower-fix,golang,120.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests-symflower-fix,java,120.0 +openrouter/microsoft/wizardlm-2-8x22b,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/codestral-mamba,code-repair,golang,5.0 +openrouter/mistralai/codestral-mamba,code-repair,java,5.0 +openrouter/mistralai/codestral-mamba,code-repair,ruby,5.0 +openrouter/mistralai/codestral-mamba,transpile,golang,50.0 +openrouter/mistralai/codestral-mamba,transpile,java,50.0 +openrouter/mistralai/codestral-mamba,transpile,ruby,50.0 +openrouter/mistralai/codestral-mamba,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/codestral-mamba,transpile-symflower-fix,java,50.0 +openrouter/mistralai/codestral-mamba,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/codestral-mamba,write-tests,golang,120.0 +openrouter/mistralai/codestral-mamba,write-tests,java,120.0 +openrouter/mistralai/codestral-mamba,write-tests,ruby,120.0 +openrouter/mistralai/codestral-mamba,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/codestral-mamba,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/codestral-mamba,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,golang,5.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,java,5.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,code-repair,ruby,5.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,golang,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,java,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile,ruby,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,golang,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,java,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests,ruby,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-7b-instruct-v0.3,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mistral-large,code-repair,golang,5.0 +openrouter/mistralai/mistral-large,code-repair,java,5.0 +openrouter/mistralai/mistral-large,code-repair,ruby,5.0 +openrouter/mistralai/mistral-large,transpile,golang,50.0 +openrouter/mistralai/mistral-large,transpile,java,50.0 +openrouter/mistralai/mistral-large,transpile,ruby,50.0 +openrouter/mistralai/mistral-large,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mistral-large,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-large,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mistral-large,write-tests,golang,120.0 +openrouter/mistralai/mistral-large,write-tests,java,120.0 +openrouter/mistralai/mistral-large,write-tests,ruby,120.0 +openrouter/mistralai/mistral-large,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-large,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-large,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mistral-medium,code-repair,golang,5.0 +openrouter/mistralai/mistral-medium,code-repair,java,5.0 +openrouter/mistralai/mistral-medium,code-repair,ruby,5.0 +openrouter/mistralai/mistral-medium,transpile,golang,50.0 +openrouter/mistralai/mistral-medium,transpile,java,50.0 +openrouter/mistralai/mistral-medium,transpile,ruby,50.0 +openrouter/mistralai/mistral-medium,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mistral-medium,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-medium,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mistral-medium,write-tests,golang,120.0 +openrouter/mistralai/mistral-medium,write-tests,java,120.0 +openrouter/mistralai/mistral-medium,write-tests,ruby,120.0 +openrouter/mistralai/mistral-medium,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-medium,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-medium,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mistral-nemo,code-repair,golang,5.0 +openrouter/mistralai/mistral-nemo,code-repair,java,5.0 +openrouter/mistralai/mistral-nemo,code-repair,ruby,5.0 +openrouter/mistralai/mistral-nemo,transpile,java,50.0 +openrouter/mistralai/mistral-nemo,transpile,ruby,50.0 +openrouter/mistralai/mistral-nemo,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-nemo,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mistral-nemo,write-tests,golang,120.0 +openrouter/mistralai/mistral-nemo,write-tests,java,120.0 +openrouter/mistralai/mistral-nemo,write-tests,ruby,120.0 +openrouter/mistralai/mistral-nemo,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-nemo,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-nemo,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mistral-small,code-repair,golang,5.0 +openrouter/mistralai/mistral-small,code-repair,java,5.0 +openrouter/mistralai/mistral-small,transpile,golang,50.0 +openrouter/mistralai/mistral-small,transpile,java,50.0 +openrouter/mistralai/mistral-small,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mistral-small,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-small,write-tests,golang,120.0 +openrouter/mistralai/mistral-small,write-tests,java,120.0 +openrouter/mistralai/mistral-small,write-tests,ruby,5.0 +openrouter/mistralai/mistral-small,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-small,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-small,write-tests-symflower-fix,ruby,5.0 +openrouter/mistralai/mistral-tiny,code-repair,golang,5.0 +openrouter/mistralai/mistral-tiny,code-repair,java,5.0 +openrouter/mistralai/mistral-tiny,transpile,golang,50.0 +openrouter/mistralai/mistral-tiny,transpile,java,50.0 +openrouter/mistralai/mistral-tiny,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mistral-tiny,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mistral-tiny,write-tests,golang,120.0 +openrouter/mistralai/mistral-tiny,write-tests,java,120.0 +openrouter/mistralai/mistral-tiny,write-tests,ruby,5.0 +openrouter/mistralai/mistral-tiny,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mistral-tiny,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mistral-tiny,write-tests-symflower-fix,ruby,5.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,golang,5.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,java,5.0 +openrouter/mistralai/mixtral-8x22b-instruct,code-repair,ruby,5.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,golang,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,java,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile,ruby,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,golang,120.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,java,120.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests,ruby,120.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mixtral-8x22b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,golang,5.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,java,5.0 +openrouter/mistralai/mixtral-8x7b-instruct,code-repair,ruby,5.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,golang,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,java,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile,ruby,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile-symflower-fix,golang,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile-symflower-fix,java,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,golang,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,java,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests,ruby,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests-symflower-fix,golang,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/mistralai/mixtral-8x7b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,golang,5.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,java,5.0 +openrouter/neversleep/llama-3-lumimaid-70b,code-repair,ruby,5.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,golang,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,java,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile,ruby,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile-symflower-fix,golang,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile-symflower-fix,java,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,transpile-symflower-fix,ruby,50.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,golang,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,java,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests,ruby,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests-symflower-fix,golang,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests-symflower-fix,java,120.0 +openrouter/neversleep/llama-3-lumimaid-70b,write-tests-symflower-fix,ruby,120.0 +openrouter/neversleep/llama-3-lumimaid-8b,code-repair,java,5.0 +openrouter/neversleep/llama-3-lumimaid-8b,code-repair,ruby,5.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile,java,50.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile,ruby,50.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile-symflower-fix,java,50.0 +openrouter/neversleep/llama-3-lumimaid-8b,transpile-symflower-fix,ruby,50.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,golang,5.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,java,120.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests,ruby,120.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests-symflower-fix,golang,5.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests-symflower-fix,java,120.0 +openrouter/neversleep/llama-3-lumimaid-8b,write-tests-symflower-fix,ruby,120.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,code-repair,java,5.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,code-repair,ruby,5.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile,java,50.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile,ruby,50.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile-symflower-fix,java,50.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,transpile-symflower-fix,ruby,50.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,golang,5.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,java,120.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests,ruby,120.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests-symflower-fix,golang,5.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests-symflower-fix,java,120.0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,write-tests-symflower-fix,ruby,120.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,code-repair,java,5.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,code-repair,ruby,5.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile,java,50.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile,ruby,50.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile-symflower-fix,java,50.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,transpile-symflower-fix,ruby,50.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,golang,5.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,java,120.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests,ruby,120.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests-symflower-fix,golang,5.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests-symflower-fix,java,120.0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,write-tests-symflower-fix,ruby,120.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,code-repair,java,5.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,code-repair,ruby,5.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,golang,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,java,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile,ruby,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile-symflower-fix,golang,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile-symflower-fix,java,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,transpile-symflower-fix,ruby,50.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,golang,5.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,java,120.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests,ruby,120.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests-symflower-fix,golang,5.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests-symflower-fix,java,120.0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,write-tests-symflower-fix,ruby,120.0 +openrouter/nousresearch/nous-hermes-yi-34b,code-repair,golang,5.0 +openrouter/nousresearch/nous-hermes-yi-34b,code-repair,java,5.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile,golang,50.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile,java,50.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile-symflower-fix,golang,50.0 +openrouter/nousresearch/nous-hermes-yi-34b,transpile-symflower-fix,java,50.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,golang,120.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,java,120.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests,ruby,5.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests-symflower-fix,golang,120.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests-symflower-fix,java,120.0 +openrouter/nousresearch/nous-hermes-yi-34b,write-tests-symflower-fix,ruby,5.0 +openrouter/openai/gpt-4-turbo,code-repair,golang,5.0 +openrouter/openai/gpt-4-turbo,code-repair,java,5.0 +openrouter/openai/gpt-4-turbo,code-repair,ruby,5.0 +openrouter/openai/gpt-4-turbo,transpile,golang,50.0 +openrouter/openai/gpt-4-turbo,transpile,java,50.0 +openrouter/openai/gpt-4-turbo,transpile,ruby,50.0 +openrouter/openai/gpt-4-turbo,transpile-symflower-fix,golang,50.0 +openrouter/openai/gpt-4-turbo,transpile-symflower-fix,java,50.0 +openrouter/openai/gpt-4-turbo,transpile-symflower-fix,ruby,50.0 +openrouter/openai/gpt-4-turbo,write-tests,golang,120.0 +openrouter/openai/gpt-4-turbo,write-tests,java,120.0 +openrouter/openai/gpt-4-turbo,write-tests,ruby,120.0 +openrouter/openai/gpt-4-turbo,write-tests-symflower-fix,golang,120.0 +openrouter/openai/gpt-4-turbo,write-tests-symflower-fix,java,120.0 +openrouter/openai/gpt-4-turbo,write-tests-symflower-fix,ruby,120.0 +openrouter/openai/gpt-4o,code-repair,golang,5.0 +openrouter/openai/gpt-4o,code-repair,java,5.0 +openrouter/openai/gpt-4o,code-repair,ruby,5.0 +openrouter/openai/gpt-4o,transpile,golang,50.0 +openrouter/openai/gpt-4o,transpile,java,50.0 +openrouter/openai/gpt-4o,transpile,ruby,50.0 +openrouter/openai/gpt-4o,transpile-symflower-fix,golang,50.0 +openrouter/openai/gpt-4o,transpile-symflower-fix,java,50.0 +openrouter/openai/gpt-4o,transpile-symflower-fix,ruby,50.0 +openrouter/openai/gpt-4o,write-tests,golang,119.0 +openrouter/openai/gpt-4o,write-tests,java,120.0 +openrouter/openai/gpt-4o,write-tests,ruby,120.0 +openrouter/openai/gpt-4o,write-tests-symflower-fix,golang,119.0 +openrouter/openai/gpt-4o,write-tests-symflower-fix,java,120.0 +openrouter/openai/gpt-4o,write-tests-symflower-fix,ruby,120.0 +openrouter/openai/gpt-4o-mini,code-repair,golang,5.0 +openrouter/openai/gpt-4o-mini,code-repair,java,5.0 +openrouter/openai/gpt-4o-mini,code-repair,ruby,5.0 +openrouter/openai/gpt-4o-mini,transpile,golang,50.0 +openrouter/openai/gpt-4o-mini,transpile,java,50.0 +openrouter/openai/gpt-4o-mini,transpile,ruby,50.0 +openrouter/openai/gpt-4o-mini,transpile-symflower-fix,golang,50.0 +openrouter/openai/gpt-4o-mini,transpile-symflower-fix,java,50.0 +openrouter/openai/gpt-4o-mini,transpile-symflower-fix,ruby,50.0 +openrouter/openai/gpt-4o-mini,write-tests,golang,120.0 +openrouter/openai/gpt-4o-mini,write-tests,java,120.0 +openrouter/openai/gpt-4o-mini,write-tests,ruby,120.0 +openrouter/openai/gpt-4o-mini,write-tests-symflower-fix,golang,120.0 +openrouter/openai/gpt-4o-mini,write-tests-symflower-fix,java,120.0 +openrouter/openai/gpt-4o-mini,write-tests-symflower-fix,ruby,120.0 +openrouter/openchat/openchat-8b,code-repair,golang,5.0 +openrouter/openchat/openchat-8b,transpile,golang,50.0 +openrouter/openchat/openchat-8b,transpile,java,50.0 +openrouter/openchat/openchat-8b,transpile-symflower-fix,golang,50.0 +openrouter/openchat/openchat-8b,transpile-symflower-fix,java,50.0 +openrouter/openchat/openchat-8b,write-tests,golang,120.0 +openrouter/openchat/openchat-8b,write-tests,java,5.0 +openrouter/openchat/openchat-8b,write-tests,ruby,5.0 +openrouter/openchat/openchat-8b,write-tests-symflower-fix,golang,120.0 +openrouter/openchat/openchat-8b,write-tests-symflower-fix,java,5.0 +openrouter/openchat/openchat-8b,write-tests-symflower-fix,ruby,5.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,golang,5.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,java,5.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,code-repair,ruby,5.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,golang,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,java,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile,ruby,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile-symflower-fix,golang,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile-symflower-fix,java,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,transpile-symflower-fix,ruby,50.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,golang,120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,java,120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests,ruby,120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests-symflower-fix,golang,120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests-symflower-fix,java,120.0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,write-tests-symflower-fix,ruby,120.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,code-repair,golang,5.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,code-repair,ruby,5.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile,java,50.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile,ruby,50.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile-symflower-fix,java,50.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,transpile-symflower-fix,ruby,50.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,golang,120.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,java,5.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests,ruby,120.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests-symflower-fix,golang,120.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests-symflower-fix,java,5.0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,write-tests-symflower-fix,ruby,120.0 +openrouter/qwen/qwen-110b-chat,code-repair,golang,5.0 +openrouter/qwen/qwen-110b-chat,code-repair,ruby,5.0 +openrouter/qwen/qwen-110b-chat,transpile,golang,50.0 +openrouter/qwen/qwen-110b-chat,transpile,ruby,50.0 +openrouter/qwen/qwen-110b-chat,transpile-symflower-fix,golang,50.0 +openrouter/qwen/qwen-110b-chat,transpile-symflower-fix,ruby,50.0 +openrouter/qwen/qwen-110b-chat,write-tests,golang,120.0 +openrouter/qwen/qwen-110b-chat,write-tests,java,5.0 +openrouter/qwen/qwen-110b-chat,write-tests,ruby,120.0 +openrouter/qwen/qwen-110b-chat,write-tests-symflower-fix,golang,120.0 +openrouter/qwen/qwen-110b-chat,write-tests-symflower-fix,java,5.0 +openrouter/qwen/qwen-110b-chat,write-tests-symflower-fix,ruby,120.0 +openrouter/qwen/qwen-2-72b-instruct,code-repair,java,5.0 +openrouter/qwen/qwen-2-72b-instruct,code-repair,ruby,5.0 +openrouter/qwen/qwen-2-72b-instruct,transpile,java,50.0 +openrouter/qwen/qwen-2-72b-instruct,transpile,ruby,50.0 +openrouter/qwen/qwen-2-72b-instruct,transpile-symflower-fix,java,50.0 +openrouter/qwen/qwen-2-72b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,golang,5.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,java,120.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests,ruby,120.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/qwen/qwen-2-72b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/qwen/qwen-2-7b-instruct,code-repair,java,5.0 +openrouter/qwen/qwen-2-7b-instruct,code-repair,ruby,5.0 +openrouter/qwen/qwen-2-7b-instruct,transpile,java,50.0 +openrouter/qwen/qwen-2-7b-instruct,transpile,ruby,50.0 +openrouter/qwen/qwen-2-7b-instruct,transpile-symflower-fix,java,50.0 +openrouter/qwen/qwen-2-7b-instruct,transpile-symflower-fix,ruby,50.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,golang,5.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,java,120.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests,ruby,120.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests-symflower-fix,golang,5.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests-symflower-fix,java,120.0 +openrouter/qwen/qwen-2-7b-instruct,write-tests-symflower-fix,ruby,120.0 +openrouter/qwen/qwen-72b-chat,code-repair,golang,5.0 +openrouter/qwen/qwen-72b-chat,code-repair,java,5.0 +openrouter/qwen/qwen-72b-chat,code-repair,ruby,5.0 +openrouter/qwen/qwen-72b-chat,transpile,golang,50.0 +openrouter/qwen/qwen-72b-chat,transpile,java,50.0 +openrouter/qwen/qwen-72b-chat,transpile,ruby,50.0 +openrouter/qwen/qwen-72b-chat,transpile-symflower-fix,golang,50.0 +openrouter/qwen/qwen-72b-chat,transpile-symflower-fix,java,50.0 +openrouter/qwen/qwen-72b-chat,transpile-symflower-fix,ruby,50.0 +openrouter/qwen/qwen-72b-chat,write-tests,golang,120.0 +openrouter/qwen/qwen-72b-chat,write-tests,java,120.0 +openrouter/qwen/qwen-72b-chat,write-tests,ruby,120.0 +openrouter/qwen/qwen-72b-chat,write-tests-symflower-fix,golang,120.0 +openrouter/qwen/qwen-72b-chat,write-tests-symflower-fix,java,120.0 +openrouter/qwen/qwen-72b-chat,write-tests-symflower-fix,ruby,120.0 +openrouter/recursal/eagle-7b,write-tests,golang,1.0 +openrouter/recursal/eagle-7b,write-tests,java,1.0 +openrouter/recursal/eagle-7b,write-tests,ruby,1.0 +openrouter/recursal/eagle-7b,write-tests-symflower-fix,golang,1.0 +openrouter/recursal/eagle-7b,write-tests-symflower-fix,java,1.0 +openrouter/recursal/eagle-7b,write-tests-symflower-fix,ruby,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,golang,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,java,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests,ruby,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests-symflower-fix,golang,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests-symflower-fix,java,1.0 +openrouter/recursal/rwkv-5-3b-ai-town,write-tests-symflower-fix,ruby,1.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,golang,5.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,java,5.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests,ruby,1.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests-symflower-fix,golang,5.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests-symflower-fix,java,5.0 +openrouter/rwkv/rwkv-5-world-3b,write-tests-symflower-fix,ruby,1.0 +openrouter/teknium/openhermes-2.5-mistral-7b,code-repair,java,5.0 +openrouter/teknium/openhermes-2.5-mistral-7b,code-repair,ruby,5.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile,java,50.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile,ruby,50.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile-symflower-fix,java,50.0 +openrouter/teknium/openhermes-2.5-mistral-7b,transpile-symflower-fix,ruby,50.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,golang,5.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,java,120.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests,ruby,120.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests-symflower-fix,golang,5.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests-symflower-fix,java,120.0 +openrouter/teknium/openhermes-2.5-mistral-7b,write-tests-symflower-fix,ruby,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,golang,5.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,java,5.0 +openrouter/togethercomputer/stripedhyena-nous-7b,code-repair,ruby,5.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile,golang,50.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile,ruby,50.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile-symflower-fix,golang,50.0 +openrouter/togethercomputer/stripedhyena-nous-7b,transpile-symflower-fix,ruby,50.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,golang,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,java,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests,ruby,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests-symflower-fix,golang,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests-symflower-fix,java,120.0 +openrouter/togethercomputer/stripedhyena-nous-7b,write-tests-symflower-fix,ruby,120.0 +openrouter/undi95/toppy-m-7b,code-repair,golang,5.0 +openrouter/undi95/toppy-m-7b,code-repair,ruby,5.0 +openrouter/undi95/toppy-m-7b,transpile,ruby,50.0 +openrouter/undi95/toppy-m-7b,transpile-symflower-fix,ruby,50.0 +openrouter/undi95/toppy-m-7b,write-tests,golang,120.0 +openrouter/undi95/toppy-m-7b,write-tests,java,5.0 +openrouter/undi95/toppy-m-7b,write-tests,ruby,120.0 +openrouter/undi95/toppy-m-7b,write-tests-symflower-fix,golang,120.0 +openrouter/undi95/toppy-m-7b,write-tests-symflower-fix,java,5.0 +openrouter/undi95/toppy-m-7b,write-tests-symflower-fix,ruby,120.0 +openrouter/xwin-lm/xwin-lm-70b,code-repair,golang,5.0 +openrouter/xwin-lm/xwin-lm-70b,code-repair,java,5.0 +openrouter/xwin-lm/xwin-lm-70b,transpile,golang,50.0 +openrouter/xwin-lm/xwin-lm-70b,transpile,java,50.0 +openrouter/xwin-lm/xwin-lm-70b,transpile-symflower-fix,golang,50.0 +openrouter/xwin-lm/xwin-lm-70b,transpile-symflower-fix,java,50.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,golang,120.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,java,120.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests,ruby,5.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests-symflower-fix,golang,120.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests-symflower-fix,java,120.0 +openrouter/xwin-lm/xwin-lm-70b,write-tests-symflower-fix,ruby,5.0 diff --git a/docs/reports/v0.6/evaluation.csv b/docs/reports/v0.6/evaluation.csv new file mode 100644 index 00000000..1321a885 --- /dev/null +++ b/docs/reports/v0.6/evaluation.csv @@ -0,0 +1,5327 @@ +model_id,language,repository,task,score,coverage,files_executed,files_executed_maximum_reachable,generate_tests_for_file_character_count,processing_time,response_character_count,response_no_error,response_no_excess,response_with_code,tests_passing +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,1,0,0,1,306,2386,307,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,1,0,0,1,3904,34071,3905,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,1,0,0,1,817,14286,818,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,3,0,0,1,139,4389,154,1,1,1,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,3,0,0,1,239,2921,255,1,1,1,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,306,2386,307,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,3904,34071,3905,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,817,14286,818,1,0,0,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,139,4389,154,1,1,1,0 +openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,239,2921,255,1,1,1,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,26,0,0,23,6565,80275,7270,20,0,6,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,28,0,0,23,20370,320136,33654,19,1,8,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,29,0,0,23,7053,109712,10842,19,1,9,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,32,0,0,23,15243,278594,24556,19,1,12,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,99,70,2,23,10330,149108,12103,19,2,6,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests-symflower-fix,26,0,0,23,6565,80275,7270,20,0,6,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests-symflower-fix,28,0,0,23,20370,320136,33654,19,1,8,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests-symflower-fix,29,0,0,23,7053,109712,10842,19,1,9,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests-symflower-fix,32,0,0,23,15243,278594,24556,19,1,12,0 +openrouter/01-ai/yi-34b-chat,java,java/light,write-tests-symflower-fix,99,70,2,23,10330,149108,12103,19,2,6,0 +openrouter/01-ai/yi-34b-chat,java,java/mistakes,code-repair,39,0,1,5,690,28124,2411,5,0,3,30 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,1,0,0,1,24,945,28,1,0,0,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,2,0,0,1,137,85247,11651,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,2,0,0,1,23,11193,1268,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,2,0,0,1,96,4054,437,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,4,0,1,1,141,2377,154,1,1,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests-symflower-fix,1,0,0,1,24,945,28,1,0,0,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,137,85247,11651,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,23,11193,1268,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,96,4054,437,1,0,1,0 +openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests-symflower-fix,4,0,1,1,141,2377,154,1,1,1,0 +openrouter/01-ai/yi-34b-chat,ruby,ruby/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/01-ai/yi-34b-chat,ruby,ruby/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests,3,0,0,1,127,1654,138,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests,3,0,0,1,190,1774,200,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests,3,0,0,1,210,3541,221,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests,3,0,0,1,401,3018,412,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests,3,0,0,1,73,1346,84,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,127,1654,138,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,190,1774,200,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,210,3541,221,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,401,3018,412,1,1,1,0 +openrouter/ai21/jamba-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,73,1346,84,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests,1056,980,10,23,24708,139686,25164,23,21,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests,1336,1260,12,23,19304,108816,19578,22,21,21,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests,1448,1370,13,23,18625,111061,19468,23,20,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests,835,760,8,23,16965,100775,17254,23,22,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests,927,850,11,23,21339,118785,22234,23,21,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests-symflower-fix,1056,980,10,23,24708,139686,25164,23,21,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests-symflower-fix,1336,1260,12,23,19304,108816,19578,22,21,21,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests-symflower-fix,1448,1370,13,23,18625,111061,19468,23,20,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests-symflower-fix,835,760,8,23,16965,100775,17254,23,22,22,0 +openrouter/ai21/jamba-instruct,java,java/light,write-tests-symflower-fix,927,850,11,23,21339,118785,22234,23,21,22,0 +openrouter/ai21/jamba-instruct,java,java/mistakes,code-repair,126,0,4,5,1397,11768,1527,5,3,4,110 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests,14,10,1,1,1451,6446,1464,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests,14,10,1,1,237,2158,250,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests,14,10,1,1,239,2050,252,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests,14,10,1,1,319,2373,332,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests,3,0,0,1,348,4334,361,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,1451,6446,1464,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,237,2158,250,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,239,2050,252,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,319,2373,332,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,348,4334,361,1,1,1,0 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile,356,0,8,10,4057,29894,4175,10,9,9,320 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile,367,0,8,10,3530,29146,3695,10,9,10,330 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile,417,0,9,10,3625,28039,3742,10,9,9,380 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile,460,0,10,10,3626,31040,3756,10,10,10,420 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile,469,0,10,10,3342,29093,3844,10,9,10,430 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile-symflower-fix,356,0,8,10,4057,29894,4175,10,9,9,320 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile-symflower-fix,367,0,8,10,3530,29146,3695,10,9,10,330 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile-symflower-fix,417,0,9,10,3625,28039,3742,10,9,9,380 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile-symflower-fix,460,0,10,10,3626,31040,3756,10,10,10,420 +openrouter/ai21/jamba-instruct,java,java/transpile,transpile-symflower-fix,469,0,10,10,3342,29093,3844,10,9,10,430 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests,1086,1010,13,23,12392,85476,12649,23,20,20,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests,1120,1040,11,23,15770,107459,16061,23,23,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests,1575,1490,16,23,15589,111382,15886,23,23,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests,909,830,11,23,12801,88307,13500,23,22,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests,969,890,12,23,14244,99132,14529,23,22,22,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests-symflower-fix,1086,1010,13,23,12392,85476,12649,23,20,20,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests-symflower-fix,1120,1040,11,23,15770,107459,16061,23,23,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests-symflower-fix,1575,1490,16,23,15589,111382,15886,23,23,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests-symflower-fix,909,830,11,23,12801,88307,13500,23,22,23,0 +openrouter/ai21/jamba-instruct,ruby,ruby/light,write-tests-symflower-fix,969,890,12,23,14244,99132,14529,23,22,22,0 +openrouter/ai21/jamba-instruct,ruby,ruby/mistakes,code-repair,130,0,5,5,695,9137,760,5,5,5,110 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests,14,10,1,1,184,1997,197,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests,14,10,1,1,214,2556,225,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests,3,0,0,1,126,2020,139,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests,3,0,0,1,145,1775,158,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests,3,0,0,1,156,1857,169,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,184,1997,197,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,214,2556,225,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,126,2020,139,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,145,1775,158,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,156,1857,169,1,1,1,0 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile,358,0,9,10,3185,33943,3372,10,9,10,320 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile,369,0,9,10,2837,26397,2967,10,10,10,330 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile,385,0,9,10,2792,31301,4212,10,6,10,350 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile,388,0,9,10,2924,26703,3111,10,9,10,350 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile,409,0,10,10,3653,31847,3823,10,9,10,370 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile-symflower-fix,358,0,9,10,3185,33943,3372,10,9,10,320 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile-symflower-fix,369,0,9,10,2837,26397,2967,10,10,10,330 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile-symflower-fix,385,0,9,10,2792,31301,4212,10,6,10,350 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile-symflower-fix,388,0,9,10,2924,26703,3111,10,9,10,350 +openrouter/ai21/jamba-instruct,ruby,ruby/transpile,transpile-symflower-fix,409,0,10,10,3653,31847,3823,10,9,10,370 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,2,0,0,1,127,11170,515,1,0,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,3,0,0,1,149,5018,161,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,3,0,0,1,217,5053,229,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,3,0,0,1,256,9331,271,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,3,0,0,1,98,3156,113,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,149,5387,161,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,98,3523,113,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,217,5053,229,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,256,9331,271,1,1,1,0 +openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests-symflower-fix,3,0,1,1,127,11841,515,1,0,1,0 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile,164,0,4,10,3386,87335,3501,10,10,10,130 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile,243,0,5,10,3583,106926,4274,10,8,10,210 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile,250,0,4,10,4787,136580,5403,10,7,9,220 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile,291,0,5,10,4512,120742,4614,10,8,8,260 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile,303,0,5,10,4231,112460,4335,10,9,9,270 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile-symflower-fix,225,0,5,10,3386,88132,3501,10,10,10,190 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile-symflower-fix,282,0,6,10,4787,137695,5403,10,7,9,250 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile-symflower-fix,291,0,5,10,4512,120742,4614,10,8,8,260 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile-symflower-fix,294,0,6,10,3583,107724,4274,10,8,10,260 +openrouter/alpindale/goliath-120b,golang,golang/transpile,transpile-symflower-fix,354,0,6,10,4231,113080,4335,10,9,9,320 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests,1,0,0,1,1089,23420,1093,1,0,0,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests,3,0,0,1,186,4602,199,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests,3,0,0,1,339,8422,352,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests,3,0,0,1,360,7504,373,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests,3,0,0,1,527,14782,540,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests-symflower-fix,1,0,0,1,1089,23420,1093,1,0,0,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests-symflower-fix,3,0,0,1,186,4602,199,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests-symflower-fix,3,0,0,1,339,8422,352,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests-symflower-fix,3,0,0,1,360,7504,373,1,1,1,0 +openrouter/alpindale/goliath-120b,java,java/plain,write-tests-symflower-fix,3,0,0,1,527,14782,540,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests,134,90,3,23,19490,385043,19674,23,9,9,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests,193,150,4,23,18284,373669,19550,23,7,9,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests,195,150,3,23,18589,415482,19717,23,8,11,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests,45,0,2,23,18795,401026,20208,23,8,12,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests,47,0,2,23,18381,368813,19118,23,10,12,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests-symflower-fix,134,90,3,23,19490,385043,19674,23,9,9,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests-symflower-fix,193,150,4,23,18284,373669,19550,23,7,9,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests-symflower-fix,195,150,3,23,18589,415482,19717,23,8,11,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests-symflower-fix,45,0,2,23,18795,401026,20208,23,8,12,0 +openrouter/alpindale/goliath-120b,ruby,ruby/light,write-tests-symflower-fix,47,0,2,23,18381,368813,19118,23,10,12,0 +openrouter/alpindale/goliath-120b,ruby,ruby/mistakes,code-repair,136,0,4,5,609,27301,1543,5,2,5,120 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests,13,10,1,1,114,6261,419,1,0,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests,14,10,1,1,140,3532,157,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests,14,10,1,1,224,5166,241,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests,3,0,0,1,173,3698,186,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests,4,0,1,1,145,3540,158,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,114,6261,419,1,0,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3532,157,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,224,5166,241,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,173,3698,186,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,145,3540,158,1,1,1,0 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile,103,0,6,10,4577,109047,4997,10,8,9,70 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile,106,0,8,10,3488,107230,4340,10,8,10,70 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile,30,0,1,10,4197,92629,4485,10,9,10,0 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile,33,0,4,10,3816,81894,3984,10,9,10,0 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile,81,0,4,10,4352,102832,4873,10,8,9,50 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile-symflower-fix,103,0,6,10,4577,109047,4997,10,8,9,70 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile-symflower-fix,106,0,8,10,3488,107230,4340,10,8,10,70 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile-symflower-fix,30,0,1,10,4197,92629,4485,10,9,10,0 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile-symflower-fix,33,0,4,10,3816,81894,3984,10,9,10,0 +openrouter/alpindale/goliath-120b,ruby,ruby/transpile,transpile-symflower-fix,81,0,4,10,4352,102832,4873,10,8,9,50 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests,116,50,1,23,22080,597328,22375,23,21,21,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests,131,60,2,23,17700,469790,18024,23,23,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests,139,70,2,23,17996,451667,19198,23,21,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests,68,0,0,23,18929,547367,19870,23,22,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests,90,20,1,23,16544,604565,16866,23,23,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests-symflower-fix,137,70,2,23,22080,597615,22375,23,21,21,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests-symflower-fix,152,80,3,23,17700,470098,18024,23,23,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests-symflower-fix,171,100,3,23,18929,548589,19870,23,22,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests-symflower-fix,313,240,6,23,17996,453058,19198,23,21,23,0 +openrouter/alpindale/magnum-72b,golang,golang/light,write-tests-symflower-fix,90,20,1,23,16544,604565,16866,23,23,23,0 +openrouter/alpindale/magnum-72b,golang,golang/mistakes,code-repair,210,0,5,5,657,22282,721,5,5,5,190 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests,14,10,1,1,74,2034,88,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests,3,0,0,1,179,6124,193,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests,3,0,0,1,182,6988,196,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests,3,0,0,1,283,4498,297,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests,3,0,0,1,99,3956,113,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,283,4860,297,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2034,88,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,99,4306,113,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,179,6124,193,1,1,1,0 +openrouter/alpindale/magnum-72b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,182,6988,196,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests,1267,1190,10,23,22244,416549,23586,23,21,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests,1308,1230,12,23,24732,516139,25899,23,21,22,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests,1470,1390,13,23,25763,522333,26436,23,21,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests,1604,1520,16,23,24620,571039,25862,23,22,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests,1740,1660,14,23,25420,555030,26626,23,21,22,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests-symflower-fix,1267,1190,10,23,22244,416549,23586,23,21,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests-symflower-fix,1308,1230,12,23,24732,516139,25899,23,21,22,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests-symflower-fix,1470,1390,13,23,25763,522333,26436,23,21,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests-symflower-fix,1604,1520,16,23,24620,571039,25862,23,22,23,0 +openrouter/alpindale/magnum-72b,java,java/light,write-tests-symflower-fix,1740,1660,14,23,25420,555030,26626,23,21,22,0 +openrouter/alpindale/magnum-72b,java,java/mistakes,code-repair,160,0,5,5,1019,30874,1079,5,5,5,140 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests,14,10,1,1,192,6328,204,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests,14,10,1,1,213,10012,225,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests,14,10,1,1,230,3100,242,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests,14,10,1,1,230,5157,242,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests,3,0,0,1,192,4269,204,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests-symflower-fix,14,10,1,1,192,6328,204,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests-symflower-fix,14,10,1,1,213,10012,225,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests-symflower-fix,14,10,1,1,230,3100,242,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests-symflower-fix,14,10,1,1,230,5157,242,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/plain,write-tests-symflower-fix,3,0,0,1,192,4269,204,1,1,1,0 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile,388,0,8,10,3303,46037,3420,10,10,10,350 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile,479,0,10,10,3614,48527,3764,10,9,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile,480,0,10,10,3374,45363,3494,10,10,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile,480,0,10,10,3525,47442,3645,10,10,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile,480,0,10,10,3731,50278,3851,10,10,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile-symflower-fix,388,0,8,10,3303,46037,3420,10,10,10,350 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile-symflower-fix,479,0,10,10,3614,48527,3764,10,9,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile-symflower-fix,480,0,10,10,3374,45363,3494,10,10,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile-symflower-fix,480,0,10,10,3525,47442,3645,10,10,10,440 +openrouter/alpindale/magnum-72b,java,java/transpile,transpile-symflower-fix,480,0,10,10,3731,50278,3851,10,10,10,440 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests,1033,950,14,23,18000,225293,18272,23,23,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests,1120,1040,12,23,16665,201295,17265,23,22,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests,1252,1170,13,23,17558,217212,17830,23,23,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests,1570,1490,15,23,23805,289823,24064,23,21,21,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests,1756,1670,18,23,18154,217060,18629,23,22,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests-symflower-fix,1033,950,14,23,18000,225293,18272,23,23,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests-symflower-fix,1120,1040,12,23,16665,201295,17265,23,22,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests-symflower-fix,1252,1170,13,23,17558,217212,17830,23,23,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests-symflower-fix,1570,1490,15,23,23805,289823,24064,23,21,21,0 +openrouter/alpindale/magnum-72b,ruby,ruby/light,write-tests-symflower-fix,1756,1670,18,23,18154,217060,18629,23,22,23,0 +openrouter/alpindale/magnum-72b,ruby,ruby/mistakes,code-repair,160,0,5,5,628,11655,688,5,5,5,140 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests,14,10,1,1,123,1988,135,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests,14,10,1,1,149,2587,161,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests,14,10,1,1,225,3181,237,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests,3,0,0,1,116,2046,128,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests,3,0,0,1,179,2363,191,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,123,1988,135,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,149,2587,161,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,225,3181,237,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,116,2046,128,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,179,2363,191,1,1,1,0 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile,127,0,8,10,4664,67667,5368,10,9,10,90 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile,134,0,5,10,3221,47064,3475,10,9,10,100 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile,166,0,7,10,3047,50047,3619,10,9,10,130 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile,256,0,6,10,3076,44360,3197,10,10,10,220 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile,266,0,7,10,3041,51660,4119,10,9,10,230 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile-symflower-fix,127,0,8,10,4664,67667,5368,10,9,10,90 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile-symflower-fix,134,0,5,10,3221,47064,3475,10,9,10,100 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile-symflower-fix,166,0,7,10,3047,50047,3619,10,9,10,130 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile-symflower-fix,256,0,6,10,3076,44360,3197,10,10,10,220 +openrouter/alpindale/magnum-72b,ruby,ruby/transpile,transpile-symflower-fix,266,0,7,10,3041,51660,4119,10,9,10,230 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,846,760,21,23,19117,161231,19434,22,21,22,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,859,770,21,23,19074,127232,19393,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,878,790,21,23,21771,148215,22095,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,940,850,22,23,19620,157493,19947,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,972,880,23,23,18701,126066,18931,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests-symflower-fix,897,810,22,23,19117,161821,19434,22,21,22,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests-symflower-fix,910,820,22,23,19074,127608,19393,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests-symflower-fix,929,840,22,23,21771,148698,22095,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests-symflower-fix,972,880,23,23,18701,126066,18931,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests-symflower-fix,991,900,23,23,19620,157777,19947,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/mistakes,code-repair,210,0,5,5,648,14999,718,5,5,5,190 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,13,10,1,1,74,3212,183,1,0,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,14,10,1,1,116,2088,126,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,14,10,1,1,116,2358,126,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,14,10,1,1,116,3560,130,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,14,10,1,1,236,3849,246,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,3212,183,1,0,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,116,2088,126,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,116,2358,126,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,116,3560,130,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,236,3849,246,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile,516,0,9,10,2300,24424,2390,9,9,9,480 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile,560,0,10,10,2638,30309,2738,10,10,10,520 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile,580,0,10,10,2639,25942,2739,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile,580,0,10,10,2639,28806,2739,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile,580,0,10,10,2642,25341,2742,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile-symflower-fix,516,0,9,10,2300,24424,2390,9,9,9,480 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2638,30309,2738,10,10,10,520 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2639,25942,2739,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2639,28806,2739,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2642,25341,2742,10,10,10,540 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,2938,2850,22,23,25003,176580,25267,22,22,22,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,3102,3010,23,23,26791,167284,27067,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,3122,3030,23,23,25998,140475,26274,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,3122,3030,23,23,27543,147835,27819,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,3132,3040,23,23,26379,145867,26655,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests-symflower-fix,2938,2850,22,23,25003,176580,25267,22,22,22,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests-symflower-fix,3102,3010,23,23,26791,167284,27067,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests-symflower-fix,3122,3030,23,23,25998,140475,26274,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests-symflower-fix,3122,3030,23,23,27543,147835,27819,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests-symflower-fix,3132,3040,23,23,26379,145867,26655,23,23,23,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/mistakes,code-repair,160,0,5,5,943,18055,1003,5,5,5,140 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,14,10,1,1,136,2013,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,14,10,1,1,136,2027,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,14,10,1,1,136,2566,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,14,10,1,1,136,5170,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,14,10,1,1,136,5550,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,2013,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,2027,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,2566,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,5170,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,5550,148,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile,382,0,8,10,3315,26799,3411,8,8,8,350 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile,480,0,10,10,4117,25586,4237,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile,480,0,10,10,4117,28590,4237,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile,480,0,10,10,4381,31610,4501,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile,480,0,10,10,4488,37673,4608,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile-symflower-fix,382,0,8,10,3315,26799,3411,8,8,8,350 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,4117,25586,4237,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,4117,28590,4237,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,4381,31610,4501,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,4488,37673,4608,10,10,10,440 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests,2361,2270,23,23,20425,105501,20783,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests,2370,2280,23,23,20009,111983,20449,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests,2370,2280,23,23,20381,113365,20821,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests,2370,2280,23,23,21535,115631,21975,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests,2371,2280,23,23,20220,106986,20578,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests-symflower-fix,2361,2270,23,23,20425,105501,20783,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests-symflower-fix,2370,2280,23,23,20009,111983,20449,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests-symflower-fix,2370,2280,23,23,20381,113365,20821,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests-symflower-fix,2370,2280,23,23,21535,115631,21975,23,21,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/light,write-tests-symflower-fix,2371,2280,23,23,20220,106986,20578,23,22,23,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/mistakes,code-repair,170,0,5,5,620,9919,680,5,5,5,150 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,1804,152,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,1904,152,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,220,1962,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,220,2299,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,220,2471,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1804,152,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1904,152,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,220,1962,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,220,2299,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,220,2471,232,1,1,1,0 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile,330,0,10,10,3289,28958,3409,10,10,10,290 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile,350,0,10,10,3181,26285,3301,10,10,10,310 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile,380,0,10,10,3148,24901,3268,10,10,10,340 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile,430,0,10,10,3085,27504,3205,10,10,10,390 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile,430,0,10,10,3108,26536,3228,10,10,10,390 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile-symflower-fix,330,0,10,10,3289,28958,3409,10,10,10,290 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,3181,26285,3301,10,10,10,310 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile-symflower-fix,380,0,10,10,3148,24901,3268,10,10,10,340 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3085,27504,3205,10,10,10,390 +openrouter/anthropic/claude-3.5-sonnet,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3108,26536,3228,10,10,10,390 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,312,240,8,23,23019,88006,25889,23,18,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,314,240,7,23,20058,64817,20836,23,21,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,480,400,12,23,20148,68123,20873,23,22,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,580,500,13,23,21707,74900,22909,23,21,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,650,570,14,23,19943,77405,21195,23,20,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests-symflower-fix,517,440,13,23,23019,89693,25889,23,18,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests-symflower-fix,519,440,12,23,20058,66087,20836,23,21,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests-symflower-fix,582,500,14,23,20148,68700,20873,23,22,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests-symflower-fix,662,580,15,23,21707,75705,22909,23,21,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests-symflower-fix,712,630,16,23,19943,78252,21195,23,20,23,0 +openrouter/anthropic/claude-3-haiku,golang,golang/mistakes,code-repair,209,0,5,5,648,7332,897,5,4,5,190 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,14,10,1,1,79,1241,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,14,10,1,1,79,1339,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,14,10,1,1,79,1670,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,14,10,1,1,79,1806,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,3,0,0,1,150,2762,164,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1241,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1339,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1670,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1806,93,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,150,2762,164,1,1,1,0 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile,477,0,8,10,2925,15465,3082,10,9,10,440 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile,478,0,8,10,2877,23850,2985,10,10,10,440 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile,509,0,9,10,2946,15561,3062,10,10,10,470 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile,509,0,9,10,2988,17251,3096,10,10,10,470 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile,560,0,10,10,3021,21648,3145,10,10,10,520 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2946,16185,3062,10,10,10,500 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile-symflower-fix,559,0,10,10,2925,16766,3082,10,9,10,520 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2877,25240,2985,10,10,10,520 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2988,17919,3096,10,10,10,520 +openrouter/anthropic/claude-3-haiku,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,3021,21648,3145,10,10,10,520 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,2570,2490,19,23,22642,76200,26585,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,2598,2520,20,23,22366,77678,27016,23,12,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,2714,2630,21,23,24353,89517,27655,23,17,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,2791,2710,20,23,23887,88546,27315,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,2803,2720,22,23,23162,76596,27196,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests-symflower-fix,2570,2490,19,23,22642,76200,26585,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests-symflower-fix,2598,2520,20,23,22366,77678,27016,23,12,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests-symflower-fix,2714,2630,21,23,24353,89517,27655,23,17,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests-symflower-fix,2791,2710,20,23,23887,88546,27315,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/light,write-tests-symflower-fix,2803,2720,22,23,23162,76596,27196,23,15,23,0 +openrouter/anthropic/claude-3-haiku,java,java/mistakes,code-repair,159,0,5,5,1050,11935,1295,5,4,5,140 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,13,10,1,1,220,2544,441,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,13,10,1,1,223,1650,391,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,13,10,1,1,223,1924,293,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,14,10,1,1,223,2915,235,1,1,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,14,10,1,1,232,1532,244,1,1,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests-symflower-fix,13,10,1,1,220,2544,441,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests-symflower-fix,13,10,1,1,223,1650,391,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests-symflower-fix,13,10,1,1,223,1924,293,1,0,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests-symflower-fix,14,10,1,1,223,2915,235,1,1,1,0 +openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests-symflower-fix,14,10,1,1,232,1532,244,1,1,1,0 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile,470,0,10,10,4420,18158,4540,10,10,10,430 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile,480,0,10,10,4347,16773,4467,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile,480,0,10,10,4353,20050,4473,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile,480,0,10,10,4486,17370,4606,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile,480,0,10,10,4513,23085,4633,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile-symflower-fix,470,0,10,10,4420,18158,4540,10,10,10,430 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile-symflower-fix,480,0,10,10,4347,16773,4467,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile-symflower-fix,480,0,10,10,4353,20050,4473,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile-symflower-fix,480,0,10,10,4486,17370,4606,10,10,10,440 +openrouter/anthropic/claude-3-haiku,java,java/transpile,transpile-symflower-fix,480,0,10,10,4513,23085,4633,10,10,10,440 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests,1992,1910,20,23,15736,63005,18630,23,16,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests,2041,1960,21,23,18226,70636,21805,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests,2082,2000,22,23,17786,65681,21585,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests,2212,2130,22,23,18530,74078,23261,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests,2263,2180,23,23,17009,66550,20853,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests-symflower-fix,1992,1910,20,23,15736,63005,18630,23,16,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests-symflower-fix,2041,1960,21,23,18226,70636,21805,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests-symflower-fix,2082,2000,22,23,17786,65681,21585,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests-symflower-fix,2212,2130,22,23,18530,74078,23261,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/light,write-tests-symflower-fix,2263,2180,23,23,17009,66550,20853,23,14,23,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/mistakes,code-repair,168,0,5,5,620,8828,803,5,3,5,150 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests,13,10,1,1,140,1969,364,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests,13,10,1,1,147,1336,219,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests,13,10,1,1,157,1334,229,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests,13,10,1,1,157,1782,388,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests,13,10,1,1,157,2121,426,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,140,1969,364,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,147,1336,219,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,157,1334,229,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,157,1782,388,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,157,2121,426,1,0,1,0 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile,289,0,9,10,3125,19459,3245,10,10,10,250 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile,410,0,10,10,3021,20154,3141,10,10,10,370 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile,410,0,10,10,3036,20630,3156,10,10,10,370 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile,420,0,10,10,2950,16849,3070,10,10,10,380 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile,440,0,10,10,3047,18281,3167,10,10,10,400 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile-symflower-fix,289,0,9,10,3125,19459,3245,10,10,10,250 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile-symflower-fix,410,0,10,10,3021,20154,3141,10,10,10,370 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile-symflower-fix,410,0,10,10,3036,20630,3156,10,10,10,370 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile-symflower-fix,420,0,10,10,2950,16849,3070,10,10,10,380 +openrouter/anthropic/claude-3-haiku,ruby,ruby/transpile,transpile-symflower-fix,440,0,10,10,3047,18281,3167,10,10,10,400 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,806,720,20,23,17860,271052,18758,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,858,770,21,23,19010,279619,19368,23,21,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,887,800,21,23,17820,272233,18491,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,908,820,22,23,18865,279253,19378,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,950,860,22,23,19665,290198,19938,23,22,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests-symflower-fix,858,770,21,23,19010,279619,19368,23,21,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests-symflower-fix,908,820,22,23,18865,279253,19378,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests-symflower-fix,928,840,22,23,17820,272532,18491,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests-symflower-fix,938,850,22,23,17860,271873,18758,23,20,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests-symflower-fix,950,860,22,23,19665,290198,19938,23,22,23,0 +openrouter/anthropic/claude-3-opus,golang,golang/mistakes,code-repair,210,0,5,5,638,17261,708,5,5,5,190 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,14,10,1,1,75,3254,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,14,10,1,1,75,3265,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,14,10,1,1,75,3362,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,14,10,1,1,75,3486,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,14,10,1,1,75,4061,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,3254,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,3265,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,3362,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,3486,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,4061,85,1,1,1,0 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile,475,0,8,10,2994,50147,3275,10,7,10,440 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile,557,0,10,10,2844,57790,3075,10,7,10,520 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile,557,0,10,10,2959,50628,3180,10,7,10,520 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile,576,0,10,10,2927,56082,3214,10,6,10,540 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile,578,0,10,10,2960,49964,3134,10,8,10,540 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile-symflower-fix,557,0,10,10,2844,57790,3075,10,7,10,520 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile-symflower-fix,557,0,10,10,2959,50628,3180,10,7,10,520 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile-symflower-fix,557,0,10,10,2994,50860,3275,10,7,10,520 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile-symflower-fix,576,0,10,10,2927,56082,3214,10,6,10,540 +openrouter/anthropic/claude-3-opus,golang,golang/transpile,transpile-symflower-fix,578,0,10,10,2960,49964,3134,10,8,10,540 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests,3028,2940,23,23,25440,291759,26064,23,19,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests,3044,2960,23,23,26622,305771,27611,23,15,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests,3046,2960,23,23,25752,300819,26925,23,17,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests,3105,3020,23,23,24965,299033,25896,23,16,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests,3106,3020,23,23,25705,307909,26787,23,17,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests-symflower-fix,3028,2940,23,23,25440,291759,26064,23,19,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests-symflower-fix,3044,2960,23,23,26622,305771,27611,23,15,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests-symflower-fix,3046,2960,23,23,25752,300819,26925,23,17,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests-symflower-fix,3105,3020,23,23,24965,299033,25896,23,16,23,0 +openrouter/anthropic/claude-3-opus,java,java/light,write-tests-symflower-fix,3106,3020,23,23,25705,307909,26787,23,17,23,0 +openrouter/anthropic/claude-3-opus,java,java/mistakes,code-repair,160,0,5,5,943,22385,1003,5,5,5,140 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,14,10,1,1,136,2922,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,14,10,1,1,136,3093,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,14,10,1,1,136,4967,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,14,10,1,1,137,2867,149,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,14,10,1,1,214,4345,226,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,2922,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,3093,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,4967,148,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,2867,149,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/plain,write-tests-symflower-fix,14,10,1,1,214,4345,226,1,1,1,0 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile,473,0,10,10,3868,61139,4357,10,3,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile,476,0,10,10,3613,58082,3880,10,6,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile,476,0,10,10,3766,54230,4034,10,6,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile,477,0,10,10,3566,55606,3846,10,7,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile,478,0,10,10,3434,58933,3722,10,8,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile-symflower-fix,473,0,10,10,3868,61139,4357,10,3,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile-symflower-fix,476,0,10,10,3613,58082,3880,10,6,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile-symflower-fix,476,0,10,10,3766,54230,4034,10,6,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile-symflower-fix,477,0,10,10,3566,55606,3846,10,7,10,440 +openrouter/anthropic/claude-3-opus,java,java/transpile,transpile-symflower-fix,478,0,10,10,3434,58933,3722,10,8,10,440 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests,2167,2080,22,23,17767,253325,19491,23,19,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests,2236,2150,22,23,18144,263842,19429,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests,2307,2220,23,23,18714,268064,20409,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests,2336,2250,23,23,17297,259907,19205,23,17,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests,2347,2260,23,23,17529,251932,19040,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests-symflower-fix,2167,2080,22,23,17767,253325,19491,23,19,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests-symflower-fix,2236,2150,22,23,18144,263842,19429,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests-symflower-fix,2307,2220,23,23,18714,268064,20409,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests-symflower-fix,2336,2250,23,23,17297,259907,19205,23,17,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/light,write-tests-symflower-fix,2347,2260,23,23,17529,251932,19040,23,18,23,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/mistakes,code-repair,160,0,5,5,623,16460,683,5,5,5,140 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests,13,10,1,1,141,3967,237,1,0,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests,14,10,1,1,140,3732,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests,14,10,1,1,140,3850,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests,14,10,1,1,140,3931,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests,14,10,1,1,141,3982,153,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,141,3967,237,1,0,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3732,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3850,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3931,152,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,141,3982,153,1,1,1,0 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile,216,0,10,10,3300,58867,3567,10,6,10,180 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile,255,0,10,10,3301,66003,3619,10,5,10,220 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile,307,0,10,10,3226,59136,3521,10,7,10,270 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile,336,0,10,10,3161,56724,3475,10,6,10,300 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile,378,0,10,10,3189,56715,3377,10,8,10,340 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile-symflower-fix,216,0,10,10,3300,58867,3567,10,6,10,180 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile-symflower-fix,255,0,10,10,3301,66003,3619,10,5,10,220 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile-symflower-fix,307,0,10,10,3226,59136,3521,10,7,10,270 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile-symflower-fix,336,0,10,10,3161,56724,3475,10,6,10,300 +openrouter/anthropic/claude-3-opus,ruby,ruby/transpile,transpile-symflower-fix,378,0,10,10,3189,56715,3377,10,8,10,340 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,657,570,18,23,21811,158183,22049,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,706,620,18,23,23349,170701,23790,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,757,670,19,23,21006,149587,21964,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,789,700,20,23,21627,159951,21861,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,789,700,20,23,28785,194397,29019,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests-symflower-fix,808,720,20,23,23349,171337,23790,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests-symflower-fix,881,790,22,23,21811,159964,22049,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests-symflower-fix,891,800,22,23,21627,160550,21861,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests-symflower-fix,900,810,22,23,21006,150326,21964,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests-symflower-fix,952,860,23,23,28785,195590,29019,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/mistakes,code-repair,210,0,5,5,648,10908,702,5,5,5,190 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,14,10,1,1,224,4223,234,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,14,10,1,1,234,3405,244,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,14,10,1,1,74,1684,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,14,10,1,1,74,2204,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,14,10,1,1,74,2905,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,224,4223,234,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,234,3405,244,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1684,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2204,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2905,84,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile,478,0,8,10,2890,25286,2990,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile,478,0,8,10,2941,28440,3041,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile,478,0,8,10,2955,26448,3055,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile,478,0,8,10,2971,25923,3071,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile,529,0,9,10,2970,25692,3070,10,10,10,490 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2890,26784,2990,10,10,10,540 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2941,29295,3041,10,10,10,540 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2955,27724,3055,10,10,10,540 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2970,26226,3070,10,10,10,540 +openrouter/anthropic/claude-3-sonnet,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2971,26907,3071,10,10,10,540 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,1590,1510,11,23,28239,166165,28515,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,1861,1780,12,23,30091,175275,30367,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,1913,1830,14,23,24956,155399,25232,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,2051,1970,12,23,29182,167340,29458,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,2142,2060,14,23,27781,163709,28805,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests-symflower-fix,1590,1510,11,23,28239,166165,28515,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests-symflower-fix,1861,1780,12,23,30091,175275,30367,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests-symflower-fix,1913,1830,14,23,24956,155399,25232,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests-symflower-fix,2051,1970,12,23,29182,167340,29458,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests-symflower-fix,2142,2060,14,23,27781,163709,28805,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,java,java/mistakes,code-repair,160,0,5,5,1104,14984,1164,5,5,5,140 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,14,10,1,1,136,1799,148,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,14,10,1,1,137,1807,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,14,10,1,1,137,2171,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,14,10,1,1,137,2179,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,14,10,1,1,231,4335,243,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,1799,148,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1807,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,2171,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,2179,149,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests-symflower-fix,14,10,1,1,231,4335,243,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile,480,0,10,10,3486,29854,3606,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile,480,0,10,10,3577,28692,3697,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile,480,0,10,10,3577,28800,3697,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile,480,0,10,10,3850,30519,3970,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile,480,0,10,10,4375,30224,4495,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,3486,29854,3606,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,3577,28692,3697,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,3577,28800,3697,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,3850,30519,3970,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,java,java/transpile,transpile-symflower-fix,480,0,10,10,4375,30224,4495,10,10,10,440 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests,1638,1550,19,23,20367,136240,20643,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests,1772,1680,23,23,20047,133869,20323,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests,1920,1830,22,23,20392,142205,21003,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests,2149,2060,20,23,19386,128437,19662,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests,862,810,10,23,10743,76520,10911,14,14,14,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests-symflower-fix,1638,1550,19,23,20367,136240,20643,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests-symflower-fix,1772,1680,23,23,20047,133869,20323,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests-symflower-fix,1920,1830,22,23,20392,142205,21003,23,22,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests-symflower-fix,2149,2060,20,23,19386,128437,19662,23,23,23,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/light,write-tests-symflower-fix,862,810,10,23,10743,76520,10911,14,14,14,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/mistakes,code-repair,170,0,5,5,716,10473,776,5,5,5,150 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,1988,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,2032,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,2143,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests,14,10,1,1,140,2325,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests,3,0,0,1,131,1949,143,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1988,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,2032,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,2143,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,2325,152,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,131,1949,143,1,1,1,0 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile,419,0,9,10,2937,30648,3057,10,10,10,380 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile,420,0,10,10,3035,30922,3155,10,10,10,380 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile,470,0,10,10,2966,30125,3086,10,10,10,430 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile,470,0,10,10,2969,31103,3089,10,10,10,430 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile,470,0,10,10,3059,31825,3179,10,10,10,430 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile-symflower-fix,419,0,9,10,2937,30648,3057,10,10,10,380 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile-symflower-fix,420,0,10,10,3035,30922,3155,10,10,10,380 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,2966,30125,3086,10,10,10,430 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,2969,31103,3089,10,10,10,430 +openrouter/anthropic/claude-3-sonnet,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,3059,31825,3179,10,10,10,430 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,239,180,5,23,18876,225703,23527,23,9,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,258,200,6,23,19620,244887,24399,23,7,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,292,230,7,23,16535,202410,19254,23,10,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,354,290,8,23,19208,226032,21649,23,11,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,364,300,10,23,19392,249547,23220,23,10,21,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests-symflower-fix,299,240,7,23,19620,245297,24399,23,7,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests-symflower-fix,405,340,10,23,16535,204136,19254,23,10,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests-symflower-fix,416,350,10,23,19208,226868,21649,23,11,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests-symflower-fix,457,390,13,23,19392,250533,23220,23,10,21,0 +openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests-symflower-fix,475,410,11,23,18876,228754,23527,23,9,22,0 +openrouter/austism/chronos-hermes-13b,golang,golang/mistakes,code-repair,175,0,4,5,792,15674,1647,5,1,5,160 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,14,10,1,1,160,3960,175,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,14,10,1,1,247,2779,256,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,14,10,1,1,91,1687,106,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,2,0,0,1,126,2478,180,1,0,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,2,0,0,1,250,3354,347,1,0,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,126,2831,180,1,0,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,160,3960,175,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,247,2779,256,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,91,1687,106,1,1,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,250,3354,347,1,0,1,0 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile,381,0,7,10,2974,33226,3497,10,4,10,350 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile,413,0,7,10,3132,31654,3486,10,6,10,380 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile,470,0,8,10,3732,37913,4308,10,3,9,440 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile,484,0,9,10,2829,28865,3053,10,6,9,450 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile,487,0,9,10,3155,35216,3539,10,8,10,450 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile-symflower-fix,464,0,8,10,3132,33247,3486,10,6,10,430 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile-symflower-fix,484,0,10,10,2974,34725,3497,10,4,10,450 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile-symflower-fix,484,0,9,10,2829,28865,3053,10,6,9,450 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile-symflower-fix,498,0,10,10,3155,36148,3539,10,8,10,460 +openrouter/austism/chronos-hermes-13b,golang,golang/transpile,transpile-symflower-fix,521,0,9,10,3732,38279,4308,10,3,9,490 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests,553,490,6,23,22178,221587,28025,23,11,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests,561,500,4,23,24512,226774,28341,23,11,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests,711,650,8,23,23481,228272,29368,23,7,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests,821,760,7,23,20070,204363,25196,23,8,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests,866,800,8,23,20787,205986,25041,23,12,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests-symflower-fix,553,490,6,23,22178,221587,28025,23,11,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests-symflower-fix,561,500,4,23,24512,226774,28341,23,11,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests-symflower-fix,711,650,8,23,23481,228272,29368,23,7,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests-symflower-fix,821,760,7,23,20070,204363,25196,23,8,23,0 +openrouter/austism/chronos-hermes-13b,java,java/light,write-tests-symflower-fix,866,800,8,23,20787,205986,25041,23,12,23,0 +openrouter/austism/chronos-hermes-13b,java,java/mistakes,code-repair,155,0,5,5,1101,26808,3169,5,0,5,140 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,13,10,1,1,154,3799,477,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,2,0,0,1,366,5263,702,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,2,0,0,1,561,5839,847,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,2,0,0,1,669,8093,1221,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,3,0,1,1,212,5065,747,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests-symflower-fix,13,10,1,1,154,3799,477,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests-symflower-fix,2,0,0,1,366,5263,702,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests-symflower-fix,2,0,0,1,561,5839,847,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests-symflower-fix,2,0,0,1,669,8093,1221,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests-symflower-fix,3,0,1,1,212,5065,747,1,0,1,0 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile,209,0,4,10,3873,38195,4511,10,5,10,180 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile,340,0,8,10,4343,37959,4846,10,4,8,310 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile,368,0,8,10,4127,36409,4629,10,2,8,340 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile,393,0,8,10,4217,40423,4974,10,5,10,360 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile,422,0,9,10,4214,39959,5133,10,3,10,390 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile-symflower-fix,209,0,4,10,3873,38195,4511,10,5,10,180 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile-symflower-fix,340,0,8,10,4343,37959,4846,10,4,8,310 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile-symflower-fix,368,0,8,10,4127,36409,4629,10,2,8,340 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile-symflower-fix,393,0,8,10,4217,40423,4974,10,5,10,360 +openrouter/austism/chronos-hermes-13b,java,java/transpile,transpile-symflower-fix,422,0,9,10,4214,39959,5133,10,3,10,390 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests,419,360,10,23,17136,178396,21736,23,5,21,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests,486,420,10,23,12713,140018,15754,23,10,23,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests,646,580,11,23,16324,172699,19620,23,10,22,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests,696,630,12,23,15394,158897,18328,23,8,23,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests,743,680,12,23,12785,155184,18358,23,7,21,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests-symflower-fix,419,360,10,23,17136,178396,21736,23,5,21,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests-symflower-fix,486,420,10,23,12713,140018,15754,23,10,23,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests-symflower-fix,646,580,11,23,16324,172699,19620,23,10,22,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests-symflower-fix,696,630,12,23,15394,158897,18328,23,8,23,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/light,write-tests-symflower-fix,743,680,12,23,12785,155184,18358,23,7,21,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/mistakes,code-repair,135,0,5,5,607,15423,1955,5,0,5,120 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests,13,10,1,1,167,3966,565,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests,13,10,1,1,173,3948,542,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests,2,0,0,1,154,3594,403,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests,2,0,0,1,190,4414,602,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests,2,0,0,1,242,4314,598,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,167,3966,565,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,173,3948,542,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,154,3594,403,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,190,4414,602,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,242,4314,598,1,0,1,0 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile,136,0,6,10,3095,33110,3230,10,10,10,100 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile,189,0,7,10,3641,44639,5116,10,2,10,160 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile,303,0,7,10,3353,39025,3948,10,6,10,270 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile,345,0,9,10,2945,33827,3411,10,6,10,310 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile,69,0,4,10,3252,38666,3629,10,6,9,40 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile-symflower-fix,136,0,6,10,3095,33110,3230,10,10,10,100 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile-symflower-fix,189,0,7,10,3641,44639,5116,10,2,10,160 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile-symflower-fix,303,0,7,10,3353,39025,3948,10,6,10,270 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile-symflower-fix,345,0,9,10,2945,33827,3411,10,6,10,310 +openrouter/austism/chronos-hermes-13b,ruby,ruby/transpile,transpile-symflower-fix,69,0,4,10,3252,38666,3629,10,6,9,40 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests,505,440,13,23,18813,170449,26116,23,6,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests,578,510,15,23,20250,166417,26973,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests,613,550,15,23,19444,180692,29734,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests,628,560,15,23,19114,167656,27638,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests,760,690,17,23,19013,167322,27358,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests-symflower-fix,556,490,14,23,18813,170951,26116,23,6,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests-symflower-fix,782,710,19,23,19114,168769,27638,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests-symflower-fix,806,740,18,23,19444,181400,29734,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests-symflower-fix,853,780,20,23,19013,168271,27358,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/light,write-tests-symflower-fix,904,830,21,23,20250,168353,26973,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/mistakes,code-repair,206,0,5,5,646,10196,1532,5,1,5,190 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests,13,10,1,1,74,1863,245,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests,13,10,1,1,74,2005,367,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests,13,10,1,1,74,2233,364,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests,13,10,1,1,74,3130,571,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests,13,10,1,1,74,4559,366,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,1863,245,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,2005,367,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,2233,364,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,3130,571,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,4559,366,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile,174,0,3,10,3649,41987,7132,10,1,10,150 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile,175,0,3,10,3053,44607,7530,10,2,10,150 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile,258,0,5,10,3322,44667,6984,10,3,10,230 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile,286,0,5,10,2500,35321,5251,10,1,10,260 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile,287,0,5,10,2439,36824,5578,10,2,10,260 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile-symflower-fix,175,0,3,10,3053,44607,7530,10,2,10,150 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile-symflower-fix,258,0,5,10,3322,44667,6984,10,3,10,230 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile-symflower-fix,286,0,5,10,2500,35321,5251,10,1,10,260 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile-symflower-fix,296,0,5,10,3649,43738,7132,10,1,10,270 +openrouter/cognitivecomputations/dolphin-llama-3-70b,golang,golang/transpile,transpile-symflower-fix,348,0,6,10,2439,37548,5578,10,2,10,320 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests,1688,1620,13,23,20928,134401,28047,23,9,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests,1727,1660,16,23,19561,134980,27199,23,5,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests,1849,1780,16,23,17639,123298,24691,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests,2010,1940,16,23,20174,139696,26531,23,8,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests,2221,2150,19,23,17055,121381,23346,23,6,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests-symflower-fix,1688,1620,13,23,20928,134401,28047,23,9,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests-symflower-fix,1727,1660,16,23,19561,134980,27199,23,5,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests-symflower-fix,1849,1780,16,23,17639,123298,24691,23,7,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests-symflower-fix,2010,1940,16,23,20174,139696,26531,23,8,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/light,write-tests-symflower-fix,2221,2150,19,23,17055,121381,23346,23,6,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/mistakes,code-repair,135,0,4,5,968,13320,1708,5,1,5,120 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests,13,10,1,1,186,2321,465,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests,13,10,1,1,191,2367,407,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests,13,10,1,1,242,1854,322,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests,14,10,1,1,212,1439,224,1,1,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests,2,0,0,1,475,5084,970,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests-symflower-fix,13,10,1,1,186,2321,465,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests-symflower-fix,13,10,1,1,191,2367,407,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests-symflower-fix,13,10,1,1,242,1854,322,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests-symflower-fix,14,10,1,1,212,1439,224,1,1,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/plain,write-tests-symflower-fix,2,0,0,1,475,5084,970,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile,382,0,8,10,3631,29232,4574,10,4,10,350 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile,382,0,8,10,3894,33402,5674,10,4,10,350 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile,434,0,9,10,3390,31674,5051,10,6,9,400 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile,436,0,9,10,3723,28212,4797,10,7,10,400 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile,467,0,10,10,4088,28475,4439,10,7,10,430 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile-symflower-fix,382,0,8,10,3631,29232,4574,10,4,10,350 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile-symflower-fix,382,0,8,10,3894,33402,5674,10,4,10,350 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile-symflower-fix,434,0,9,10,3390,31674,5051,10,6,9,400 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile-symflower-fix,436,0,9,10,3723,28212,4797,10,7,10,400 +openrouter/cognitivecomputations/dolphin-llama-3-70b,java,java/transpile,transpile-symflower-fix,467,0,10,10,4088,28475,4439,10,7,10,430 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests,1678,1610,18,23,10914,109145,19109,23,4,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests,1732,1660,22,23,10611,117687,20918,23,4,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests,1778,1710,20,23,12235,116360,20746,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests,1799,1730,21,23,12286,163932,19818,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests,1960,1890,22,23,11718,113020,20530,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests-symflower-fix,1678,1610,18,23,10914,109145,19109,23,4,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests-symflower-fix,1732,1660,22,23,10611,117687,20918,23,4,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests-symflower-fix,1778,1710,20,23,12235,116360,20746,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests-symflower-fix,1799,1730,21,23,12286,163932,19818,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/light,write-tests-symflower-fix,1960,1890,22,23,11718,113020,20530,23,2,23,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/mistakes,code-repair,135,0,4,5,615,7599,800,5,2,4,120 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests,13,10,1,1,146,2986,526,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests,13,10,1,1,148,2851,468,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests,13,10,1,1,161,4459,842,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests,13,10,1,1,167,2965,433,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests,14,10,1,1,159,1682,172,1,1,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,146,2986,526,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,148,2851,468,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,161,4459,842,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,167,2965,433,1,0,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,159,1682,172,1,1,1,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile,125,0,9,10,2629,28381,3797,10,6,10,90 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile,137,0,10,10,3017,29296,4010,10,7,10,100 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile,68,0,9,10,3211,26841,3657,10,9,10,30 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile,84,0,10,10,3117,34900,5251,10,4,10,50 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile,86,0,9,10,2668,32072,3795,10,7,10,50 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile-symflower-fix,125,0,9,10,2629,28381,3797,10,6,10,90 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile-symflower-fix,137,0,10,10,3017,29296,4010,10,7,10,100 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile-symflower-fix,68,0,9,10,3211,26841,3657,10,9,10,30 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile-symflower-fix,84,0,10,10,3117,34900,5251,10,4,10,50 +openrouter/cognitivecomputations/dolphin-llama-3-70b,ruby,ruby/transpile,transpile-symflower-fix,86,0,9,10,2668,32072,3795,10,7,10,50 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,307,240,7,23,17691,242061,21182,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,458,390,10,23,17565,246939,21484,23,12,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,500,430,11,23,17794,237858,21698,23,13,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,502,430,12,23,17176,246187,20110,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,610,540,12,23,17995,233692,21572,23,12,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests-symflower-fix,583,510,13,23,17691,244074,21182,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests-symflower-fix,614,540,14,23,17176,246697,20110,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests-symflower-fix,644,570,15,23,17794,239253,21698,23,13,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests-symflower-fix,704,630,16,23,17995,234653,21572,23,12,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests-symflower-fix,713,640,15,23,17565,248796,21484,23,12,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/mistakes,code-repair,210,0,5,5,643,11714,705,5,5,5,190 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,13,10,1,1,106,2372,214,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,3,0,0,1,135,2328,149,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,3,0,0,1,197,3510,211,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,3,0,0,1,337,4000,351,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,3,0,0,1,402,4356,416,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,106,2372,214,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,135,2328,149,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,197,3510,211,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,337,4000,351,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,402,4356,416,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile,354,0,6,10,2688,42473,3501,10,8,10,320 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile,376,0,7,10,3220,42446,3367,10,9,10,340 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile,395,0,7,10,2757,44571,3670,10,8,10,360 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile,467,0,9,10,3180,42928,3979,10,8,10,430 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile,508,0,9,10,2878,37728,3125,10,9,10,470 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile-symflower-fix,376,0,7,10,3220,42446,3367,10,9,10,340 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile-symflower-fix,395,0,7,10,2757,44571,3670,10,8,10,360 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile-symflower-fix,456,0,8,10,2688,45040,3501,10,8,10,420 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile-symflower-fix,467,0,9,10,3180,42928,3979,10,8,10,430 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/transpile,transpile-symflower-fix,508,0,9,10,2878,37728,3125,10,9,10,470 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,1411,1340,14,23,23577,232299,27258,23,11,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,1950,1870,17,23,21489,215140,24227,23,17,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,1955,1880,16,23,14812,158298,17299,23,14,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,2047,1970,17,23,18971,187434,22359,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,2198,2120,18,23,21062,216737,25753,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests-symflower-fix,1411,1340,14,23,23577,232299,27258,23,11,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests-symflower-fix,1950,1870,17,23,21489,215140,24227,23,17,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests-symflower-fix,1955,1880,16,23,14812,158298,17299,23,14,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests-symflower-fix,2047,1970,17,23,18971,187434,22359,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests-symflower-fix,2198,2120,18,23,21062,216737,25753,23,14,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/mistakes,code-repair,159,0,5,5,1045,17055,1210,5,4,5,140 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,14,10,1,1,186,2796,198,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,14,10,1,1,236,2967,248,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,14,10,1,1,251,2580,263,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,2,0,0,1,329,7584,984,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,3,0,0,1,230,2524,242,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,186,2796,198,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,236,2967,248,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,251,2580,263,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests-symflower-fix,2,0,0,1,329,7584,984,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests-symflower-fix,3,0,0,1,230,2524,242,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile,439,0,9,10,3335,44953,3455,10,10,10,400 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile,470,0,10,10,3386,44153,3506,10,10,10,430 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile,478,0,10,10,3479,55394,4776,10,8,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile,479,0,10,10,3410,49201,3904,10,9,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile,480,0,10,10,3108,47584,3228,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile-symflower-fix,439,0,9,10,3335,44953,3455,10,10,10,400 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile-symflower-fix,470,0,10,10,3386,44153,3506,10,10,10,430 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile-symflower-fix,478,0,10,10,3479,55394,4776,10,8,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile-symflower-fix,479,0,10,10,3410,49201,3904,10,9,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,3108,47584,3228,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests,1027,950,12,23,12846,160310,15080,23,19,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests,1107,1030,12,23,15538,182220,17352,23,19,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests,1178,1100,14,23,14724,179386,17372,23,18,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests,1255,1200,12,23,9697,104282,10682,15,13,15,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests,1256,1180,13,23,13644,159830,16024,23,17,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests-symflower-fix,1027,950,12,23,12846,160310,15080,23,19,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests-symflower-fix,1107,1030,12,23,15538,182220,17352,23,19,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests-symflower-fix,1178,1100,14,23,14724,179386,17372,23,18,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests-symflower-fix,1255,1200,12,23,9697,104282,10682,15,13,15,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/light,write-tests-symflower-fix,1256,1180,13,23,13644,159830,16024,23,17,23,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/mistakes,code-repair,169,0,5,5,600,11335,732,5,4,5,150 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests,13,10,1,1,147,5105,566,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests,13,10,1,1,151,3395,406,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests,14,10,1,1,147,2417,159,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests,14,10,1,1,186,4127,198,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests,14,10,1,1,198,2903,210,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,147,5105,566,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,151,3395,406,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,2417,159,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,186,4127,198,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,198,2903,210,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile,135,0,7,10,3603,63759,5259,10,8,10,100 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile,164,0,7,10,3033,45142,3930,10,7,10,130 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile,175,0,5,10,1999,29019,2496,7,6,7,150 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile,215,0,7,10,2839,44631,3645,10,8,10,180 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile,269,0,9,10,2860,38011,2980,10,10,10,230 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile-symflower-fix,135,0,7,10,3603,63759,5259,10,8,10,100 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile-symflower-fix,164,0,7,10,3033,45142,3930,10,7,10,130 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile-symflower-fix,175,0,5,10,1999,29019,2496,7,6,7,150 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile-symflower-fix,215,0,7,10,2839,44631,3645,10,8,10,180 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,ruby,ruby/transpile,transpile-symflower-fix,269,0,9,10,2860,38011,2980,10,10,10,230 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests,130,80,4,23,33547,179289,34590,23,10,13,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests,196,140,4,23,25949,133067,29125,23,11,18,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests,197,140,5,23,31970,186528,32840,23,14,15,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests,210,160,5,23,33871,253388,35100,23,10,12,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests,258,200,5,23,26547,153677,28946,23,13,17,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests-symflower-fix,192,140,6,23,33547,179678,34590,23,10,13,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests-symflower-fix,241,190,6,23,33871,253677,35100,23,10,12,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests-symflower-fix,248,190,6,23,31970,186795,32840,23,14,15,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests-symflower-fix,340,280,7,23,26547,154684,28946,23,13,17,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/light,write-tests-symflower-fix,383,320,11,23,25949,134840,29125,23,11,18,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/mistakes,code-repair,169,0,4,5,735,9146,805,5,5,5,150 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,13,10,1,1,110,2463,379,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,2,0,0,1,950,5183,1070,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,3,0,0,1,254,2196,268,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,3,0,0,1,330,2586,344,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,3,0,0,1,56,691,70,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,110,2463,379,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,56,1183,70,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,950,5183,1070,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,254,2196,268,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,330,2586,344,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,1057,990,10,23,34971,173586,36068,23,16,18,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,619,560,5,23,38048,143625,39460,23,14,17,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,736,670,6,23,37010,148979,38251,23,17,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,927,860,6,23,37523,148079,38326,23,18,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,967,900,8,23,35728,203203,38482,23,16,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests-symflower-fix,1057,990,10,23,34971,173586,36068,23,16,18,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests-symflower-fix,619,560,5,23,38048,143625,39460,23,14,17,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests-symflower-fix,736,670,6,23,37010,148979,38251,23,17,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests-symflower-fix,927,860,6,23,37523,148079,38326,23,18,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests-symflower-fix,967,900,8,23,35728,203203,38482,23,16,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/mistakes,code-repair,106,0,3,5,1700,9985,1750,5,4,4,90 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,14,10,1,1,466,1603,478,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,2,0,0,1,288,1875,504,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,2,0,0,1,606,12966,4925,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,2,0,0,1,797,4887,1448,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,3,0,0,1,1646,4573,1658,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests-symflower-fix,14,10,1,1,466,1603,478,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,288,1875,504,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,606,12966,4925,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,797,4887,1448,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests-symflower-fix,3,0,0,1,1646,4573,1658,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile,386,0,8,10,7075,66455,7196,10,9,9,350 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile,437,0,9,10,4038,45457,4364,10,8,10,400 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile,480,0,10,10,4073,36051,4193,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile,480,0,10,10,4139,33848,4259,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile,480,0,10,10,4223,42734,4343,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile-symflower-fix,386,0,8,10,7075,66455,7196,10,9,9,350 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile-symflower-fix,437,0,9,10,4038,45457,4364,10,8,10,400 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4073,36051,4193,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4139,33848,4259,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4223,42734,4343,10,10,10,440 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests,245,180,4,23,17997,155533,20624,23,16,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests,343,280,6,23,21129,226325,24209,23,13,21,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests,346,280,5,23,19610,177466,22113,23,16,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests,650,580,10,23,23685,187689,26135,23,17,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests,780,710,8,23,21653,200616,22141,23,19,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests-symflower-fix,245,180,4,23,17997,155533,20624,23,16,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests-symflower-fix,343,280,6,23,21129,226325,24209,23,13,21,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests-symflower-fix,346,280,5,23,19610,177466,22113,23,16,22,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests-symflower-fix,650,580,10,23,23685,187689,26135,23,17,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/light,write-tests-symflower-fix,780,710,8,23,21653,200616,22141,23,19,20,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/mistakes,code-repair,140,0,5,5,629,6905,689,5,5,5,120 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests,13,10,1,1,147,4680,429,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests,14,10,1,1,147,3009,159,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests,3,0,0,1,153,2220,165,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests,3,0,0,1,200,3072,212,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests,3,0,0,1,242,2860,254,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,147,4680,429,1,0,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,3009,159,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,153,2220,165,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,200,3072,212,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,242,2860,254,1,1,1,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile,168,0,8,10,3183,36616,3304,10,10,10,130 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile,227,0,8,10,2910,29536,3245,10,9,10,190 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile,240,0,10,10,2710,28954,2830,10,10,10,200 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile,267,0,7,10,2863,31692,2983,10,10,10,230 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile,278,0,10,10,3046,36115,3633,10,8,10,240 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile-symflower-fix,168,0,8,10,3183,36616,3304,10,10,10,130 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile-symflower-fix,227,0,8,10,2910,29536,3245,10,9,10,190 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile-symflower-fix,240,0,10,10,2710,28954,2830,10,10,10,200 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile-symflower-fix,267,0,7,10,2863,31692,2983,10,10,10,230 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,ruby,ruby/transpile,transpile-symflower-fix,278,0,10,10,3046,36115,3633,10,8,10,240 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests,111,40,2,23,15533,60063,15856,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests,121,50,2,23,18646,69454,19001,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests,142,70,3,23,14476,66804,14798,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests,165,90,6,23,17862,68091,18181,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests,204,130,5,23,19895,71266,20213,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests-symflower-fix,173,100,4,23,18646,69970,19001,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests-symflower-fix,193,120,4,23,15533,60577,15856,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests-symflower-fix,206,130,7,23,17862,68460,18181,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests-symflower-fix,224,150,5,23,14476,67259,14798,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/light,write-tests-symflower-fix,286,210,7,23,19895,72251,20213,23,23,23,0 +openrouter/cohere/command-r-03-2024,golang,golang/mistakes,code-repair,180,0,5,5,712,4385,782,5,5,5,160 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests,14,10,1,1,129,765,143,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests,14,10,1,1,78,627,92,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests,14,10,1,1,80,621,94,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests,3,0,0,1,100,744,114,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests,3,0,0,1,84,658,98,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,129,765,143,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,78,627,92,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,80,621,94,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,100,744,114,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,84,658,98,1,1,1,0 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile,141,0,3,10,3207,11986,3283,10,4,4,120 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile,161,0,3,10,3208,12123,3284,10,4,4,140 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile,226,0,4,10,3057,11549,3141,10,6,6,200 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile,287,0,5,10,3078,13761,3162,10,6,6,260 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile,353,0,7,10,3018,12045,3114,10,8,8,320 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile-symflower-fix,172,0,4,10,3207,12188,3283,10,4,4,150 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile-symflower-fix,192,0,4,10,3208,12581,3284,10,4,4,170 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile-symflower-fix,226,0,4,10,3057,11549,3141,10,6,6,200 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile-symflower-fix,318,0,6,10,3078,14013,3162,10,6,6,290 +openrouter/cohere/command-r-03-2024,golang,golang/transpile,transpile-symflower-fix,353,0,7,10,3018,12045,3114,10,8,8,320 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests,1179,1100,10,23,20369,57274,20645,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests,1218,1140,9,23,19361,53895,19637,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests,1320,1240,11,23,20228,55443,20504,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests,1462,1380,13,23,18845,54602,19121,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests,999,920,10,23,17787,51734,18063,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests-symflower-fix,1179,1100,10,23,20369,57274,20645,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests-symflower-fix,1218,1140,9,23,19361,53895,19637,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests-symflower-fix,1320,1240,11,23,20228,55443,20504,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests-symflower-fix,1462,1380,13,23,18845,54602,19121,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/light,write-tests-symflower-fix,999,920,10,23,17787,51734,18063,23,23,23,0 +openrouter/cohere/command-r-03-2024,java,java/mistakes,code-repair,160,0,5,5,1160,4768,1220,5,5,5,140 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests,14,10,1,1,136,707,148,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests,14,10,1,1,136,871,148,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests,14,10,1,1,137,716,149,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests,14,10,1,1,137,719,149,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests,3,0,0,1,151,772,163,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,707,148,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,871,148,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,716,149,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,719,149,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/plain,write-tests-symflower-fix,3,0,0,1,151,772,163,1,1,1,0 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile,364,0,8,10,3964,12385,4072,10,8,8,330 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile,384,0,8,10,4035,13205,4143,10,8,8,350 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile,417,0,9,10,3942,14576,4056,10,9,9,380 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile,440,0,10,10,3963,13897,4083,10,10,10,400 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile,460,0,10,10,4018,12418,4138,10,10,10,420 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile-symflower-fix,364,0,8,10,3964,12385,4072,10,8,8,330 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile-symflower-fix,384,0,8,10,4035,13205,4143,10,8,8,350 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile-symflower-fix,417,0,9,10,3942,14576,4056,10,9,9,380 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile-symflower-fix,440,0,10,10,3963,13897,4083,10,10,10,400 +openrouter/cohere/command-r-03-2024,java,java/transpile,transpile-symflower-fix,460,0,10,10,4018,12418,4138,10,10,10,420 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests,1001,920,12,23,11224,37640,11500,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests,586,510,7,23,11494,39372,11770,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests,647,570,8,23,12093,44397,12369,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests,678,600,9,23,10734,36760,11010,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests,857,780,8,23,12272,47011,12548,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests-symflower-fix,1001,920,12,23,11224,37640,11500,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests-symflower-fix,586,510,7,23,11494,39372,11770,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests-symflower-fix,647,570,8,23,12093,44397,12369,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests-symflower-fix,678,600,9,23,10734,36760,11010,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/light,write-tests-symflower-fix,857,780,8,23,12272,47011,12548,23,23,23,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/mistakes,code-repair,109,0,4,5,728,5125,788,5,5,5,90 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests,14,10,1,1,145,732,157,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests,14,10,1,1,155,743,167,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests,14,10,1,1,155,756,167,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests,3,0,0,1,131,688,143,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests,3,0,0,1,146,1019,158,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,145,732,157,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,743,167,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,756,167,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,131,688,143,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,146,1019,158,1,1,1,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile,141,0,7,10,2682,12802,2784,10,7,7,110 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile,144,0,8,10,2350,10717,2458,10,8,8,110 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile,31,0,7,10,2543,10810,2645,10,7,7,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile,32,0,6,10,2701,10271,2809,10,8,8,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile,63,0,7,10,2477,10314,2585,10,8,8,30 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile-symflower-fix,141,0,7,10,2682,12802,2784,10,7,7,110 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile-symflower-fix,144,0,8,10,2350,10717,2458,10,8,8,110 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile-symflower-fix,31,0,7,10,2543,10810,2645,10,7,7,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile-symflower-fix,32,0,6,10,2701,10271,2809,10,8,8,0 +openrouter/cohere/command-r-03-2024,ruby,ruby/transpile,transpile-symflower-fix,63,0,7,10,2477,10314,2585,10,8,8,30 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests,555,470,16,23,25169,110834,25411,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests,667,580,18,23,23289,102859,23531,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests,769,680,20,23,26779,127502,27033,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests,800,710,21,23,24537,106810,24783,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests,921,830,22,23,22436,90892,22666,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests-symflower-fix,709,620,20,23,25169,112185,25411,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests-symflower-fix,769,680,20,23,23289,103467,23531,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests-symflower-fix,800,710,21,23,24537,106810,24783,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests-symflower-fix,831,740,22,23,26779,128315,27033,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/light,write-tests-symflower-fix,962,870,23,23,22436,91286,22666,23,23,23,0 +openrouter/cohere/command-r-08-2024,golang,golang/mistakes,code-repair,87,0,2,5,680,4823,750,5,5,5,70 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests,14,10,1,1,112,856,126,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests,14,10,1,1,136,1088,146,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests,14,10,1,1,74,665,84,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests,14,10,1,1,74,693,84,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests,14,10,1,1,77,712,91,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,112,856,126,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,136,1088,146,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,665,84,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,693,84,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,77,712,91,1,1,1,0 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile,448,0,8,10,2870,14840,2970,10,10,10,410 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile,499,0,9,10,3237,15709,3337,10,10,10,460 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile,509,0,9,10,2914,14537,3014,10,10,10,470 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile,509,0,9,10,3192,16794,3292,10,10,10,470 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile,540,0,10,10,3003,14613,3103,10,10,10,500 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile-symflower-fix,448,0,8,10,2870,14840,2970,10,10,10,410 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile-symflower-fix,499,0,9,10,3237,15709,3337,10,10,10,460 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile-symflower-fix,509,0,9,10,2914,14537,3014,10,10,10,470 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile-symflower-fix,509,0,9,10,3192,16794,3292,10,10,10,470 +openrouter/cohere/command-r-08-2024,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3003,14613,3103,10,10,10,500 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests,1339,1260,10,23,27721,95333,27997,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests,1492,1410,13,23,24940,83064,25217,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests,1833,1750,14,23,25550,86062,25827,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests,1892,1810,13,23,24238,83431,24514,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests,2276,2190,17,23,26137,87355,26413,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests-symflower-fix,1339,1260,10,23,27721,95333,27997,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests-symflower-fix,1492,1410,13,23,24940,83064,25217,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests-symflower-fix,1833,1750,14,23,25550,86062,25827,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests-symflower-fix,1892,1810,13,23,24238,83431,24514,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/light,write-tests-symflower-fix,2276,2190,17,23,26137,87355,26413,23,23,23,0 +openrouter/cohere/command-r-08-2024,java,java/mistakes,code-repair,160,0,5,5,1167,5878,1227,5,5,5,140 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests,14,10,1,1,224,1232,236,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests,14,10,1,1,238,1073,250,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests,14,10,1,1,238,1102,250,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests,14,10,1,1,255,1132,267,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests,14,10,1,1,270,1129,282,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1232,236,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,238,1073,250,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,238,1102,250,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,255,1132,267,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,270,1129,282,1,1,1,0 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile,470,0,10,10,3765,17062,3885,10,10,10,430 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile,470,0,10,10,3872,16421,3992,10,10,10,430 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile,480,0,10,10,3991,28078,4111,10,10,10,440 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile,480,0,10,10,3996,18546,4116,10,10,10,440 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile,480,0,10,10,4208,18677,4328,10,10,10,440 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile-symflower-fix,470,0,10,10,3765,17062,3885,10,10,10,430 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile-symflower-fix,470,0,10,10,3872,16421,3992,10,10,10,430 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile-symflower-fix,480,0,10,10,3991,28078,4111,10,10,10,440 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile-symflower-fix,480,0,10,10,3996,18546,4116,10,10,10,440 +openrouter/cohere/command-r-08-2024,java,java/transpile,transpile-symflower-fix,480,0,10,10,4208,18677,4328,10,10,10,440 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests,478,400,9,23,17771,71819,18047,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests,533,460,4,23,16162,67836,16438,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests,675,600,6,23,18202,74256,18478,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests,796,720,7,23,17152,74552,17428,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests,90,20,1,23,19600,77820,19877,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests-symflower-fix,478,400,9,23,17771,71819,18047,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests-symflower-fix,533,460,4,23,16162,67836,16438,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests-symflower-fix,675,600,6,23,18202,74256,18478,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests-symflower-fix,796,720,7,23,17152,74552,17428,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/light,write-tests-symflower-fix,90,20,1,23,19600,77820,19877,23,23,23,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/mistakes,code-repair,160,0,5,5,600,4337,660,5,5,5,140 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests,14,10,1,1,113,799,125,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests,14,10,1,1,142,895,154,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests,3,0,0,1,135,1141,147,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests,3,0,0,1,229,1390,241,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests,3,0,0,1,276,1354,288,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,113,799,125,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,142,895,154,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,135,1141,147,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,229,1390,241,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,276,1354,288,1,1,1,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile,159,0,9,10,2885,13957,3005,10,10,10,120 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile,228,0,8,10,2696,14212,2816,10,10,10,190 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile,259,0,9,10,2749,14255,2869,10,10,10,220 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile,280,0,10,10,3035,14686,3155,10,10,10,240 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile,38,0,8,10,3036,14855,3156,10,10,10,0 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile-symflower-fix,159,0,9,10,2885,13957,3005,10,10,10,120 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile-symflower-fix,228,0,8,10,2696,14212,2816,10,10,10,190 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile-symflower-fix,259,0,9,10,2749,14255,2869,10,10,10,220 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile-symflower-fix,280,0,10,10,3035,14686,3155,10,10,10,240 +openrouter/cohere/command-r-08-2024,ruby,ruby/transpile,transpile-symflower-fix,38,0,8,10,3036,14855,3156,10,10,10,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests,101,30,2,23,22553,215138,22875,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests,131,60,2,23,25067,251256,25389,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests,162,90,3,23,24536,241188,24858,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests,264,190,5,23,22651,204183,22973,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests,272,200,5,23,27336,231566,27650,23,22,22,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests-symflower-fix,101,30,2,23,22553,215138,22875,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests-symflower-fix,131,60,2,23,25067,251256,25389,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests-symflower-fix,162,90,3,23,24536,241188,24858,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests-symflower-fix,264,190,5,23,22651,204183,22973,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/light,write-tests-symflower-fix,272,200,5,23,27336,231566,27650,23,22,22,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/mistakes,code-repair,210,0,5,5,645,9533,711,5,5,5,190 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests,14,10,1,1,124,1100,138,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests,3,0,0,1,121,1706,135,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests,3,0,0,1,127,1552,141,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests,3,0,0,1,145,1482,159,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests,3,0,0,1,96,1139,110,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,124,1100,138,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,121,1706,135,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,127,1552,141,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,145,1482,159,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,96,1139,110,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile,387,0,7,10,3556,37346,3656,10,10,10,350 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile,425,0,7,10,3427,28853,3523,10,9,9,390 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile,464,0,8,10,2823,29086,2915,10,8,8,430 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile,499,0,9,10,4047,30968,4147,10,10,10,460 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile,509,0,9,10,2696,36925,2796,10,10,10,470 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile-symflower-fix,448,0,8,10,3556,37559,3656,10,10,10,410 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile-symflower-fix,464,0,8,10,2823,29086,2915,10,8,8,430 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile-symflower-fix,486,0,8,10,3427,29164,3523,10,9,9,450 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2696,37114,2796,10,10,10,500 +openrouter/cohere/command-r-plus-04-2024,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,4047,31308,4147,10,10,10,520 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests,2298,2210,19,23,31678,190716,31954,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests,2309,2220,20,23,28068,149905,28344,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests,2577,2490,20,23,39229,311044,39496,23,22,22,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests,2619,2530,20,23,30994,175425,31270,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests,2670,2580,21,23,31512,189015,31788,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests-symflower-fix,2298,2210,19,23,31678,190716,31954,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests-symflower-fix,2309,2220,20,23,28068,149905,28344,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests-symflower-fix,2577,2490,20,23,39229,311044,39496,23,22,22,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests-symflower-fix,2619,2530,20,23,30994,175425,31270,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/light,write-tests-symflower-fix,2670,2580,21,23,31512,189015,31788,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,java,java/mistakes,code-repair,160,0,5,5,1050,8784,1110,5,5,5,140 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests,14,10,1,1,185,1369,197,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests,14,10,1,1,224,1538,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests,14,10,1,1,224,1570,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests,14,10,1,1,224,1637,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests,14,10,1,1,224,1663,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,185,1369,197,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1538,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1570,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1637,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1663,236,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile,450,0,10,10,3639,32534,3759,10,10,10,410 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile,460,0,10,10,3552,30520,3672,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile,460,0,10,10,3693,28919,3813,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile,460,0,10,10,3705,24838,3825,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile,460,0,10,10,3879,25561,3999,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile-symflower-fix,450,0,10,10,3639,32534,3759,10,10,10,410 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile-symflower-fix,460,0,10,10,3552,30520,3672,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile-symflower-fix,460,0,10,10,3693,28919,3813,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile-symflower-fix,460,0,10,10,3705,24838,3825,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,java,java/transpile,transpile-symflower-fix,460,0,10,10,3879,25561,3999,10,10,10,420 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests,1177,1090,18,23,21618,157236,21894,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests,1255,1170,16,23,17484,118621,17760,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests,1518,1430,19,23,18262,136645,18538,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests,1527,1440,18,23,19556,147867,19833,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests,1627,1540,18,23,21505,131436,21781,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests-symflower-fix,1177,1090,18,23,21618,157236,21894,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests-symflower-fix,1255,1170,16,23,17484,118621,17760,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests-symflower-fix,1518,1430,19,23,18262,136645,18538,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests-symflower-fix,1527,1440,18,23,19556,147867,19833,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/light,write-tests-symflower-fix,1627,1540,18,23,21505,131436,21781,23,23,23,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/mistakes,code-repair,170,0,5,5,612,7975,672,5,5,5,150 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests,14,10,1,1,108,1063,120,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests,14,10,1,1,140,1202,152,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests,14,10,1,1,155,1232,167,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests,14,10,1,1,155,1386,167,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests,14,10,1,1,226,1443,238,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,108,1063,120,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1202,152,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,1232,167,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,1386,167,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,226,1443,238,1,1,1,0 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile,120,0,10,10,3182,25973,3302,10,10,10,80 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile,200,0,10,10,3174,24280,3294,10,10,10,160 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile,220,0,10,10,2848,22116,2968,10,10,10,180 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile,220,0,10,10,2874,38534,2994,10,10,10,180 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile,250,0,10,10,2917,25444,3037,10,10,10,210 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile-symflower-fix,120,0,10,10,3182,25973,3302,10,10,10,80 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile-symflower-fix,200,0,10,10,3174,24280,3294,10,10,10,160 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile-symflower-fix,220,0,10,10,2848,22116,2968,10,10,10,180 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile-symflower-fix,220,0,10,10,2874,38534,2994,10,10,10,180 +openrouter/cohere/command-r-plus-04-2024,ruby,ruby/transpile,transpile-symflower-fix,250,0,10,10,2917,25444,3037,10,10,10,210 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests,112,40,3,23,21673,219376,21987,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests,120,50,1,23,22792,214781,23114,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests,142,70,3,23,25697,204176,26004,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests,69,0,0,23,23318,182644,23629,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests,92,20,3,23,23948,240585,24258,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests-symflower-fix,161,90,2,23,23318,183116,23629,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests-symflower-fix,162,90,3,23,22792,215607,23114,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests-symflower-fix,184,110,5,23,25697,205029,26004,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests-symflower-fix,204,130,5,23,23948,241248,24258,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/light,write-tests-symflower-fix,285,210,6,23,21673,220316,21987,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/mistakes,code-repair,210,0,5,5,646,7484,696,5,5,5,190 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests,14,10,1,1,145,3315,155,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests,3,0,0,1,195,2190,209,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests,3,0,0,1,204,2625,214,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests,3,0,0,1,230,2018,244,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests,3,0,0,1,271,2337,281,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,145,3315,155,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,195,2190,209,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,204,2625,214,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,230,2018,244,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,271,2337,281,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile,478,0,8,10,2809,31310,2909,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile,509,0,9,10,2861,42321,2961,10,10,10,470 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile,540,0,10,10,2887,30544,2987,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile,540,0,10,10,2982,27516,3082,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile,540,0,10,10,3011,23038,3111,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile-symflower-fix,478,0,8,10,2809,31310,2909,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile-symflower-fix,509,0,9,10,2861,42321,2961,10,10,10,470 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2887,30544,2987,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2982,27516,3082,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3011,23038,3111,10,10,10,500 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests,2066,1980,17,23,30643,230883,30919,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests,2166,2080,17,23,27720,165339,27996,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests,2648,2560,19,23,27410,192228,27686,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests,2739,2650,20,23,31672,236796,31948,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests,2910,2820,21,23,30257,189175,30533,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests-symflower-fix,2066,1980,17,23,30643,230883,30919,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests-symflower-fix,2166,2080,17,23,27720,165339,27996,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests-symflower-fix,2648,2560,19,23,27410,192228,27686,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests-symflower-fix,2739,2650,20,23,31672,236796,31948,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/light,write-tests-symflower-fix,2910,2820,21,23,30257,189175,30533,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,java,java/mistakes,code-repair,160,0,5,5,1137,10420,1197,5,5,5,140 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests,14,10,1,1,236,2016,248,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests,14,10,1,1,237,2450,249,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests,14,10,1,1,244,1912,256,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests,14,10,1,1,330,2864,342,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests,4,0,1,1,258,2041,270,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,236,2016,248,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,237,2450,249,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,244,1912,256,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests-symflower-fix,14,10,1,1,330,2864,342,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/plain,write-tests-symflower-fix,4,0,1,1,258,2041,270,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile,470,0,10,10,4199,35068,4319,10,10,10,430 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile,479,0,10,10,4055,27207,4240,10,9,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile,479,0,10,10,4161,31516,4316,10,9,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile,480,0,10,10,4125,34938,4245,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile,480,0,10,10,4127,29151,4247,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile-symflower-fix,470,0,10,10,4199,35068,4319,10,10,10,430 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile-symflower-fix,479,0,10,10,4055,27207,4240,10,9,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile-symflower-fix,479,0,10,10,4161,31516,4316,10,9,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile-symflower-fix,480,0,10,10,4125,34938,4245,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,java,java/transpile,transpile-symflower-fix,480,0,10,10,4127,29151,4247,10,10,10,440 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests,1202,1120,13,23,20528,140695,20804,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests,1202,1120,13,23,20861,212293,21137,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests,1282,1200,13,23,18608,163189,18884,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests,1594,1510,15,23,20313,165945,20589,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests,778,700,9,23,20797,150937,21073,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests-symflower-fix,1202,1120,13,23,20528,140695,20804,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests-symflower-fix,1202,1120,13,23,20861,212293,21137,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests-symflower-fix,1282,1200,13,23,18608,163189,18884,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests-symflower-fix,1594,1510,15,23,20313,165945,20589,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/light,write-tests-symflower-fix,778,700,9,23,20797,150937,21073,23,23,23,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/mistakes,code-repair,169,0,5,5,593,6426,681,5,4,5,150 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests,14,10,1,1,112,1207,124,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests,14,10,1,1,203,1528,215,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests,14,10,1,1,216,1749,228,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests,3,0,0,1,127,1697,139,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests,3,0,0,1,150,1563,162,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,112,1207,124,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,203,1528,215,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,216,1749,228,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,127,1697,139,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,150,1563,162,1,1,1,0 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile,228,0,9,10,2895,59625,3049,10,9,10,190 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile,288,0,8,10,2762,23250,2882,10,10,10,250 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile,327,0,9,10,3135,29714,3243,10,9,9,290 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile,329,0,10,10,2637,24616,2799,10,9,10,290 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile,350,0,10,10,2479,20241,2599,10,10,10,310 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile-symflower-fix,228,0,9,10,2895,59625,3049,10,9,10,190 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile-symflower-fix,288,0,8,10,2762,23250,2882,10,10,10,250 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile-symflower-fix,327,0,9,10,3135,29714,3243,10,9,9,290 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile-symflower-fix,329,0,10,10,2637,24616,2799,10,9,10,290 +openrouter/cohere/command-r-plus-08-2024,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,2479,20241,2599,10,10,10,310 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,571,490,15,23,20359,137691,21197,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,602,520,16,23,20530,126272,22137,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,661,580,15,23,26381,116725,27345,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,694,610,16,23,23298,117347,23965,23,22,23,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,808,720,19,23,29284,205703,29594,23,23,23,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests-symflower-fix,766,680,20,23,20530,127999,22137,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests-symflower-fix,846,760,20,23,26381,119077,27345,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests-symflower-fix,847,760,21,23,20359,140545,21197,23,21,22,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests-symflower-fix,880,790,22,23,23298,119273,23965,23,22,23,0 +openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests-symflower-fix,992,900,23,23,29284,207206,29594,23,23,23,0 +openrouter/databricks/dbrx-instruct,golang,golang/mistakes,code-repair,210,0,5,5,693,9162,780,5,5,5,190 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,14,10,1,1,125,3084,135,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,14,10,1,1,141,2236,156,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,14,10,1,1,385,1570,399,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,14,10,1,1,79,1272,93,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,14,10,1,1,79,962,93,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,125,3084,135,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,141,2236,156,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,385,1570,399,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1272,93,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,962,93,1,1,1,0 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile,394,0,7,10,3040,15399,3366,10,7,10,360 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile,415,0,7,10,3021,25044,3260,10,8,10,380 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile,428,0,8,10,3222,21287,3354,10,10,10,390 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile,477,0,8,10,3142,19466,3336,10,9,10,440 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile,489,0,9,10,3098,19610,3234,10,10,10,450 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile-symflower-fix,428,0,8,10,3222,21287,3354,10,10,10,390 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile-symflower-fix,445,0,8,10,3040,16211,3366,10,7,10,410 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile-symflower-fix,538,0,10,10,3021,26648,3260,10,8,10,500 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile-symflower-fix,539,0,10,10,3142,20401,3336,10,9,10,500 +openrouter/databricks/dbrx-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3098,20141,3234,10,10,10,500 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests,1021,950,8,23,19270,110513,22718,23,18,22,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests,1044,970,8,23,26144,129974,27808,23,20,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests,1324,1250,10,23,23861,92044,27521,23,18,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests,1449,1370,13,23,24838,130886,27417,23,20,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests,1671,1590,13,23,19162,73814,20141,23,22,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests-symflower-fix,1021,950,8,23,19270,110513,22718,23,18,22,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests-symflower-fix,1044,970,8,23,26144,129974,27808,23,20,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests-symflower-fix,1324,1250,10,23,23861,92044,27521,23,18,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests-symflower-fix,1449,1370,13,23,24838,130886,27417,23,20,23,0 +openrouter/databricks/dbrx-instruct,java,java/light,write-tests-symflower-fix,1671,1590,13,23,19162,73814,20141,23,22,23,0 +openrouter/databricks/dbrx-instruct,java,java/mistakes,code-repair,160,0,5,5,1223,6726,1283,5,5,5,140 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,14,10,1,1,186,1175,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,14,10,1,1,186,1176,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,14,10,1,1,186,1339,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,14,10,1,1,186,901,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,3,0,0,1,185,2623,197,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,186,1175,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,186,1176,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,186,1339,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,186,901,198,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,185,2623,197,1,1,1,0 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile,426,0,9,10,4185,17823,4686,10,8,9,390 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile,467,0,10,10,4159,20968,4453,10,7,10,430 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile,469,0,10,10,4158,22945,4340,10,9,10,430 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile,476,0,10,10,4183,20030,4545,10,6,10,440 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile,478,0,10,10,4183,20990,4993,10,8,10,440 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile-symflower-fix,426,0,9,10,4185,17823,4686,10,8,9,390 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile-symflower-fix,467,0,10,10,4159,20968,4453,10,7,10,430 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile-symflower-fix,469,0,10,10,4158,22945,4340,10,9,10,430 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile-symflower-fix,476,0,10,10,4183,20030,4545,10,6,10,440 +openrouter/databricks/dbrx-instruct,java,java/transpile,transpile-symflower-fix,478,0,10,10,4183,20990,4993,10,8,10,440 +openrouter/databricks/dbrx-instruct,ruby,ruby/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/databricks/dbrx-instruct,ruby,ruby/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,868,780,21,23,24135,660672,24447,23,22,22,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,868,780,21,23,24707,584364,25018,23,22,22,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,962,870,23,23,19071,406016,19393,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,962,870,23,23,20383,434492,20705,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,972,880,23,23,18888,376731,19210,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests-symflower-fix,909,820,22,23,24135,660921,24447,23,22,22,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests-symflower-fix,909,820,22,23,24707,585163,25018,23,22,22,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests-symflower-fix,962,870,23,23,19071,406016,19393,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests-symflower-fix,962,870,23,23,20383,434492,20705,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests-symflower-fix,972,880,23,23,18888,376731,19210,23,23,23,0 +openrouter/deepseek/deepseek-chat,golang,golang/mistakes,code-repair,210,0,5,5,638,16506,708,5,5,5,190 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,14,10,1,1,74,2265,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,14,10,1,1,74,2572,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,14,10,1,1,74,2592,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,14,10,1,1,74,2626,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,14,10,1,1,74,2856,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2265,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2572,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2592,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2626,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2856,88,1,1,1,0 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile,478,0,8,10,2973,71584,3109,10,10,10,440 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile,478,0,8,10,2974,75150,3098,10,10,10,440 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile,478,0,8,10,2974,78289,3094,10,10,10,440 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile,478,0,8,10,2974,81151,3102,10,10,10,440 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile,478,0,8,10,2974,81421,3098,10,10,10,440 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2973,72071,3109,10,10,10,540 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2974,76589,3098,10,10,10,540 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2974,79223,3094,10,10,10,540 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2974,82626,3098,10,10,10,540 +openrouter/deepseek/deepseek-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2974,82887,3102,10,10,10,540 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests,3012,2920,23,23,29996,476552,30272,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests,3042,2950,23,23,28918,454798,29194,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests,3071,2980,22,23,28466,445693,28742,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests,3072,2980,23,23,30018,475540,30294,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests,3102,3010,23,23,27562,412002,27838,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests-symflower-fix,3012,2920,23,23,29996,476552,30272,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests-symflower-fix,3042,2950,23,23,28918,454798,29194,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests-symflower-fix,3071,2980,22,23,28466,445693,28742,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests-symflower-fix,3072,2980,23,23,30018,475540,30294,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/light,write-tests-symflower-fix,3102,3010,23,23,27562,412002,27838,23,23,23,0 +openrouter/deepseek/deepseek-chat,java,java/mistakes,code-repair,160,0,5,5,943,23376,1003,5,5,5,140 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,14,10,1,1,137,3359,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,14,10,1,1,137,3412,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,14,10,1,1,137,3610,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,14,10,1,1,137,3662,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,14,10,1,1,137,3923,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3359,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3412,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3610,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3662,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3923,149,1,1,1,0 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile,480,0,10,10,3385,88449,3505,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile,480,0,10,10,3385,95169,3505,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile,480,0,10,10,3417,84251,3537,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile,480,0,10,10,3508,85241,3628,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile,480,0,10,10,3540,90134,3660,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,3385,88449,3505,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,3385,95169,3505,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,3417,84251,3537,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,3508,85241,3628,10,10,10,440 +openrouter/deepseek/deepseek-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,3540,90134,3660,10,10,10,440 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests,1986,1900,21,23,19974,415098,20240,23,21,21,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests,2322,2230,23,23,19100,393763,19376,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests,2322,2230,23,23,19974,422065,20250,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests,2322,2230,23,23,20251,447107,20527,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests,2342,2250,23,23,21130,437201,21406,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests-symflower-fix,1986,1900,21,23,19974,415098,20240,23,21,21,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests-symflower-fix,2322,2230,23,23,19100,393763,19376,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests-symflower-fix,2322,2230,23,23,19974,422065,20250,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests-symflower-fix,2322,2230,23,23,20251,447107,20527,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,21130,437201,21406,23,23,23,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/mistakes,code-repair,170,0,5,5,619,19269,679,5,5,5,150 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests,14,10,1,1,169,4262,181,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests,14,10,1,1,235,5181,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests,14,10,1,1,235,5256,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests,14,10,1,1,235,5298,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests,14,10,1,1,235,5776,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,169,4262,181,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5181,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5256,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5298,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5776,247,1,1,1,0 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile,390,0,10,10,3093,75897,3213,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile,390,0,10,10,3098,76569,3218,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile,390,0,10,10,3118,77059,3238,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile,390,0,10,10,3123,74801,3243,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile,390,0,10,10,3130,76787,3250,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile-symflower-fix,390,0,10,10,3093,75897,3213,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile-symflower-fix,390,0,10,10,3098,76569,3218,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile-symflower-fix,390,0,10,10,3118,77059,3238,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile-symflower-fix,390,0,10,10,3123,74801,3243,10,10,10,350 +openrouter/deepseek/deepseek-chat,ruby,ruby/transpile,transpile-symflower-fix,390,0,10,10,3130,76787,3250,10,10,10,350 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,868,780,21,23,26070,776891,26381,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,868,780,21,23,26536,674474,26847,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,868,780,21,23,28689,620002,29000,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,911,820,22,23,18857,420739,19179,23,23,23,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,972,880,23,23,18304,430681,18626,23,23,23,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests-symflower-fix,909,820,22,23,26070,777133,26381,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests-symflower-fix,909,820,22,23,26536,674695,26847,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests-symflower-fix,909,820,22,23,28689,620409,29000,23,22,22,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests-symflower-fix,911,820,22,23,18857,420739,19179,23,23,23,0 +openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests-symflower-fix,972,880,23,23,18304,430681,18626,23,23,23,0 +openrouter/deepseek/deepseek-coder,golang,golang/mistakes,code-repair,210,0,5,5,638,17580,708,5,5,5,190 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,14,10,1,1,74,2550,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,14,10,1,1,74,2839,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,14,10,1,1,74,2951,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,14,10,1,1,74,3672,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,14,10,1,1,74,5024,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2550,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2839,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2951,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,3672,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,5024,88,1,1,1,0 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile,478,0,8,10,3010,70558,3130,10,10,10,440 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile,478,0,8,10,3010,74128,3126,10,10,10,440 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile,478,0,8,10,3013,72430,3129,10,10,10,440 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile,478,0,8,10,3013,74091,3133,10,10,10,440 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile,478,0,8,10,3013,76512,3141,10,10,10,440 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3010,71000,3130,10,10,10,540 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3010,74965,3126,10,10,10,540 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3013,74704,3129,10,10,10,540 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3013,75414,3133,10,10,10,540 +openrouter/deepseek/deepseek-coder,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3013,77357,3141,10,10,10,540 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests,2605,2520,20,23,45421,884381,45679,23,21,21,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests,2686,2600,20,23,31039,514769,31303,22,22,22,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests,3041,2950,22,23,35775,608981,36051,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests,3102,3010,23,23,34517,562429,34793,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests,3102,3010,23,23,35980,608629,36256,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests-symflower-fix,2605,2520,20,23,45421,884381,45679,23,21,21,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests-symflower-fix,2686,2600,20,23,31039,514769,31303,22,22,22,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests-symflower-fix,3041,2950,22,23,35775,608981,36051,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests-symflower-fix,3102,3010,23,23,34517,562429,34793,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/light,write-tests-symflower-fix,3102,3010,23,23,35980,608629,36256,23,23,23,0 +openrouter/deepseek/deepseek-coder,java,java/mistakes,code-repair,160,0,5,5,943,26771,1003,5,5,5,140 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,14,10,1,1,137,3594,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,14,10,1,1,137,3654,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,14,10,1,1,137,3840,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,14,10,1,1,137,4019,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,14,10,1,1,137,6411,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3594,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3654,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3840,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,4019,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,6411,149,1,1,1,0 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile,480,0,10,10,3421,85179,3541,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile,480,0,10,10,3424,81246,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile,480,0,10,10,3424,85367,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile,480,0,10,10,3424,87479,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile,480,0,10,10,3543,82619,3663,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile-symflower-fix,480,0,10,10,3421,85179,3541,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile-symflower-fix,480,0,10,10,3424,81246,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile-symflower-fix,480,0,10,10,3424,85367,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile-symflower-fix,480,0,10,10,3424,87479,3544,10,10,10,440 +openrouter/deepseek/deepseek-coder,java,java/transpile,transpile-symflower-fix,480,0,10,10,3543,82619,3663,10,10,10,440 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests,2252,2160,23,23,21362,411492,21638,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests,2272,2180,23,23,20806,404568,21082,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests,2342,2250,23,23,20762,411141,21038,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests,2342,2250,23,23,20794,394126,21070,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests,2342,2250,23,23,20939,414183,21215,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests-symflower-fix,2252,2160,23,23,21362,411492,21638,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests-symflower-fix,2272,2180,23,23,20806,404568,21082,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,20762,411141,21038,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,20794,394126,21070,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,20939,414183,21215,23,23,23,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/mistakes,code-repair,170,0,5,5,619,17018,679,5,5,5,150 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests,14,10,1,1,162,3768,174,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests,14,10,1,1,169,3712,181,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests,14,10,1,1,169,3798,181,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests,14,10,1,1,235,5101,247,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests,14,10,1,1,235,5221,247,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,162,3768,174,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,169,3712,181,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,169,3798,181,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5101,247,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,235,5221,247,1,1,1,0 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile,430,0,10,10,3029,68128,3149,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile,430,0,10,10,3066,67274,3186,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile,430,0,10,10,3066,68021,3186,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile,430,0,10,10,3081,67740,3201,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile,430,0,10,10,3084,69861,3204,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3029,68128,3149,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3066,67274,3186,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3066,68021,3186,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3081,67740,3201,10,10,10,390 +openrouter/deepseek/deepseek-coder,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,3084,69861,3204,10,10,10,390 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,800,710,21,23,22428,82538,22734,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,860,770,21,23,22185,77999,22500,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,868,780,21,23,30247,125006,30546,23,22,22,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,891,800,22,23,23348,82218,23655,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,901,810,22,23,26028,88357,26319,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests-symflower-fix,861,770,22,23,22428,82802,22734,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests-symflower-fix,868,780,21,23,30247,125006,30546,23,22,22,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests-symflower-fix,891,800,22,23,23348,82218,23655,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests-symflower-fix,901,810,22,23,26028,88357,26319,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests-symflower-fix,921,830,22,23,22185,78759,22500,23,23,23,0 +openrouter/google/gemini-flash-1.5,golang,golang/mistakes,code-repair,210,0,5,5,638,6048,708,5,5,5,190 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1106,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1149,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1159,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1199,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1256,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1106,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1149,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1159,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1199,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1256,88,1,1,1,0 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile,529,0,9,10,2623,15833,2723,10,10,10,490 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile,529,0,9,10,2628,15258,2729,10,10,10,490 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile,529,0,9,10,2632,18758,2732,10,10,10,490 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile,529,0,9,10,2633,15917,2733,10,10,10,490 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile,529,0,9,10,2637,15111,2737,10,10,10,490 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2637,15586,2737,10,10,10,520 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2623,16305,2723,10,10,10,540 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2628,15850,2729,10,10,10,540 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2632,19301,2732,10,10,10,540 +openrouter/google/gemini-flash-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2633,16127,2733,10,10,10,540 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests,1175,1100,6,23,28142,71295,28418,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests,833,760,4,23,26336,73263,26612,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests,863,790,4,23,25476,70616,25752,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests,874,800,5,23,23856,65347,24132,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests,904,830,5,23,26364,79806,26640,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests-symflower-fix,1175,1100,6,23,28142,71295,28418,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests-symflower-fix,833,760,4,23,26336,73263,26612,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests-symflower-fix,863,790,4,23,25476,70616,25752,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests-symflower-fix,874,800,5,23,23856,65347,24132,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/light,write-tests-symflower-fix,904,830,5,23,26364,79806,26640,23,23,23,0 +openrouter/google/gemini-flash-1.5,java,java/mistakes,code-repair,160,0,5,5,943,8134,1003,5,5,5,140 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,14,10,1,1,133,1279,145,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,14,10,1,1,184,1308,196,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,3,0,0,1,114,1197,126,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,3,0,0,1,165,1038,177,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,3,0,0,1,165,1371,177,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests-symflower-fix,14,10,1,1,133,1279,145,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests-symflower-fix,14,10,1,1,184,1308,196,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests-symflower-fix,3,0,0,1,114,1197,126,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests-symflower-fix,3,0,0,1,165,1038,177,1,1,1,0 +openrouter/google/gemini-flash-1.5,java,java/plain,write-tests-symflower-fix,3,0,0,1,165,1371,177,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests,2266,2180,21,23,17264,67495,17522,23,21,21,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests,2332,2240,23,23,17446,67920,17724,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests,2342,2250,23,23,17575,64441,17851,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests,2342,2250,23,23,17661,68653,17938,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests,2362,2270,23,23,17690,67810,17968,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests-symflower-fix,2266,2180,21,23,17264,67495,17522,23,21,21,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests-symflower-fix,2332,2240,23,23,17446,67920,17724,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,17575,64441,17851,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,17661,68653,17938,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/light,write-tests-symflower-fix,2362,2270,23,23,17690,67810,17968,23,23,23,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/mistakes,code-repair,170,0,5,5,619,6737,679,5,5,5,150 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests,14,10,1,1,140,1272,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests,14,10,1,1,140,1302,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests,14,10,1,1,140,1303,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests,14,10,1,1,140,1321,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests,14,10,1,1,140,1400,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1272,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1302,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1303,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1321,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1400,152,1,1,1,0 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile,259,0,9,10,3224,18835,3344,10,10,10,220 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile,260,0,10,10,3242,17975,3362,10,10,10,220 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile,310,0,10,10,3117,19796,3237,10,10,10,270 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile,310,0,10,10,3130,19156,3250,10,10,10,270 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile,320,0,10,10,3079,18937,3199,10,10,10,280 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile-symflower-fix,259,0,9,10,3224,18835,3344,10,10,10,220 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile-symflower-fix,260,0,10,10,3242,17975,3362,10,10,10,220 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile-symflower-fix,310,0,10,10,3117,19796,3237,10,10,10,270 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile-symflower-fix,310,0,10,10,3130,19156,3250,10,10,10,270 +openrouter/google/gemini-flash-1.5,ruby,ruby/transpile,transpile-symflower-fix,320,0,10,10,3079,18937,3199,10,10,10,280 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,819,730,20,23,17934,182392,18258,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,860,770,21,23,18191,177737,18518,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,860,770,21,23,19467,196079,19798,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,880,790,21,23,18218,178719,18541,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,992,900,23,23,18731,185035,19054,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests-symflower-fix,861,770,22,23,19467,196357,19798,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests-symflower-fix,880,790,21,23,17934,182704,18258,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests-symflower-fix,921,830,22,23,18191,177984,18518,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests-symflower-fix,941,850,22,23,18218,178955,18541,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests-symflower-fix,992,900,23,23,18731,185035,19054,23,23,23,0 +openrouter/google/gemini-pro-1.5,golang,golang/mistakes,code-repair,210,0,5,5,638,14215,708,5,5,5,190 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,14,10,1,1,74,1855,89,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,14,10,1,1,74,2002,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,14,10,1,1,74,2037,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,14,10,1,1,74,2055,89,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,14,10,1,1,74,2203,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1855,89,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2002,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2037,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2055,89,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2203,88,1,1,1,0 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile,580,0,10,10,2640,64365,2740,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile,580,0,10,10,2641,57970,2742,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile,580,0,10,10,2644,69843,2744,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile,580,0,10,10,2644,90727,2744,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile,580,0,10,10,2645,61642,2745,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2640,64365,2740,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2641,57970,2742,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2644,69843,2744,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2644,90727,2744,10,10,10,540 +openrouter/google/gemini-pro-1.5,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2645,61642,2745,10,10,10,540 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests,1115,1040,6,23,21477,163236,21754,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests,642,570,3,23,21477,159063,21754,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests,915,840,6,23,21049,164118,21325,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests,916,840,7,23,23043,176526,23321,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests,965,890,6,23,21022,156985,21298,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests-symflower-fix,1115,1040,6,23,21477,163236,21754,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests-symflower-fix,642,570,3,23,21477,159063,21754,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests-symflower-fix,915,840,6,23,21049,164118,21325,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests-symflower-fix,916,840,7,23,23043,176526,23321,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/light,write-tests-symflower-fix,965,890,6,23,21022,156985,21298,23,23,23,0 +openrouter/google/gemini-pro-1.5,java,java/mistakes,code-repair,160,0,5,5,948,13827,1008,5,5,5,140 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,14,10,1,1,133,2326,145,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,14,10,1,1,184,2402,196,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,14,10,1,1,184,3487,196,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,3,0,0,1,164,2341,176,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,3,0,0,1,173,2268,185,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests-symflower-fix,14,10,1,1,133,2326,145,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests-symflower-fix,14,10,1,1,184,2402,196,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests-symflower-fix,14,10,1,1,184,3487,196,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests-symflower-fix,3,0,0,1,164,2341,176,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/plain,write-tests-symflower-fix,3,0,0,1,173,2268,185,1,1,1,0 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile,480,0,10,10,3166,66162,3286,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile,480,0,10,10,3166,69992,3286,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile,480,0,10,10,3172,92457,3292,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile,480,0,10,10,3179,88686,3299,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile,480,0,10,10,3281,89304,3401,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile-symflower-fix,480,0,10,10,3166,66162,3286,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile-symflower-fix,480,0,10,10,3166,69992,3286,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile-symflower-fix,480,0,10,10,3172,92457,3292,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile-symflower-fix,480,0,10,10,3179,88686,3299,10,10,10,440 +openrouter/google/gemini-pro-1.5,java,java/transpile,transpile-symflower-fix,480,0,10,10,3281,89304,3401,10,10,10,440 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests,1016,940,7,23,14621,209661,14900,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests,1148,1070,9,23,16975,207981,17256,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests,723,650,4,23,14911,216462,15190,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests,875,800,6,23,15724,198783,16006,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests,928,850,9,23,15621,205218,15904,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests-symflower-fix,1016,940,7,23,14621,209661,14900,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests-symflower-fix,1148,1070,9,23,16975,207981,17256,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests-symflower-fix,723,650,4,23,14911,216462,15190,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests-symflower-fix,875,800,6,23,15724,198783,16006,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/light,write-tests-symflower-fix,928,850,9,23,15621,205218,15904,23,23,23,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/mistakes,code-repair,170,0,5,5,604,19835,664,5,5,5,150 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests,14,10,1,1,114,2701,126,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests,3,0,0,1,122,10090,134,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests,3,0,0,1,122,4407,134,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests,3,0,0,1,128,3323,140,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests,3,0,0,1,134,4041,146,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,114,2701,126,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,122,10090,134,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,122,4407,134,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,128,3323,140,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,134,4041,146,1,1,1,0 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile,197,0,9,10,26369,199595,26480,10,9,9,160 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile,260,0,10,10,3354,57371,3474,10,10,10,220 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile,260,0,10,10,3463,63984,3583,10,10,10,220 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile,290,0,10,10,3270,48642,3390,10,10,10,250 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile,310,0,10,10,3257,60856,3377,10,10,10,270 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile-symflower-fix,197,0,9,10,26369,199595,26480,10,9,9,160 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile-symflower-fix,260,0,10,10,3354,57371,3474,10,10,10,220 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile-symflower-fix,260,0,10,10,3463,63984,3583,10,10,10,220 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile-symflower-fix,290,0,10,10,3270,48642,3390,10,10,10,250 +openrouter/google/gemini-pro-1.5,ruby,ruby/transpile,transpile-symflower-fix,310,0,10,10,3257,60856,3377,10,10,10,270 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests,677,590,18,23,17935,141002,18273,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests,718,630,19,23,18038,135264,18389,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests,738,650,19,23,17713,137824,18058,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests,799,710,20,23,18375,135939,18737,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests,830,740,21,23,19525,145578,19874,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests-symflower-fix,891,800,22,23,18038,136386,18389,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests-symflower-fix,932,840,23,23,17935,143562,18273,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests-symflower-fix,942,850,23,23,19525,146523,19874,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests-symflower-fix,962,870,23,23,18375,136888,18737,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/light,write-tests-symflower-fix,972,880,23,23,17713,138999,18058,23,23,23,0 +openrouter/google/gemma-2-27b-it,golang,golang/mistakes,code-repair,210,0,5,5,654,7728,732,5,5,5,190 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests,14,10,1,1,74,1044,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests,14,10,1,1,74,1260,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests,14,10,1,1,74,945,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests,14,10,1,1,74,986,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests,14,10,1,1,74,995,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1044,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1260,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,945,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,986,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,995,90,1,1,1,0 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile,478,0,8,10,2668,22750,2791,10,10,10,440 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile,478,0,8,10,2710,22128,2828,10,10,10,440 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile,478,0,8,10,2718,21631,2838,10,10,10,440 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile,478,0,8,10,2773,31038,2897,10,10,10,440 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile,560,0,10,10,2946,25657,3064,10,10,10,520 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2668,24384,2791,10,10,10,520 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2710,23094,2828,10,10,10,520 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2718,22765,2838,10,10,10,520 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2773,32318,2897,10,10,10,520 +openrouter/google/gemma-2-27b-it,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2946,25657,3064,10,10,10,520 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests,1995,1910,16,23,23167,130173,23482,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests,2176,2090,17,23,23074,130267,23393,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests,2205,2120,16,23,23806,136343,24119,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests,2367,2280,18,23,23286,130953,23605,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests,2609,2520,20,23,22889,138383,23217,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests-symflower-fix,1995,1910,16,23,23167,130173,23482,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests-symflower-fix,2176,2090,17,23,23074,130267,23393,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests-symflower-fix,2205,2120,16,23,23806,136343,24119,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests-symflower-fix,2367,2280,18,23,23286,130953,23605,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/light,write-tests-symflower-fix,2609,2520,20,23,22889,138383,23217,23,23,23,0 +openrouter/google/gemma-2-27b-it,java,java/mistakes,code-repair,160,0,5,5,942,9161,1008,5,5,5,140 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests,14,10,1,1,187,1416,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests,14,10,1,1,187,1528,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests,14,10,1,1,187,1741,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests,14,10,1,1,247,1987,261,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests,14,10,1,1,273,1974,287,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1416,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1528,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1741,201,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,247,1987,261,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,273,1974,287,1,1,1,0 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile,480,0,10,10,3105,27492,3243,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile,480,0,10,10,3172,27128,3308,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile,480,0,10,10,3226,30316,3363,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile,480,0,10,10,3271,26846,3408,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile,480,0,10,10,3398,26636,3535,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile-symflower-fix,480,0,10,10,3105,27492,3243,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile-symflower-fix,480,0,10,10,3172,27128,3308,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile-symflower-fix,480,0,10,10,3226,30316,3363,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile-symflower-fix,480,0,10,10,3271,26846,3408,10,10,10,440 +openrouter/google/gemma-2-27b-it,java,java/transpile,transpile-symflower-fix,480,0,10,10,3398,26636,3535,10,10,10,440 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests,402,330,3,23,15877,111488,16187,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests,433,360,4,23,16099,116477,16413,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests,614,540,5,23,16163,122053,16481,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests,684,610,5,23,17643,130092,17956,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests,90,20,1,23,15790,110600,16105,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests-symflower-fix,402,330,3,23,15877,111488,16187,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests-symflower-fix,433,360,4,23,16099,116477,16413,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests-symflower-fix,614,540,5,23,16163,122053,16481,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests-symflower-fix,684,610,5,23,17643,130092,17956,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/light,write-tests-symflower-fix,90,20,1,23,15790,110600,16105,23,23,23,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/mistakes,code-repair,170,0,5,5,619,7618,688,5,5,5,150 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests,14,10,1,1,140,1288,154,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests,14,10,1,1,140,3873,154,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests,14,10,1,1,147,1948,161,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests,3,0,0,1,131,1477,143,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests,3,0,0,1,164,1361,178,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1288,154,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3873,154,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,1948,161,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,131,1477,143,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,164,1361,178,1,1,1,0 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile,170,0,10,10,3487,29478,3628,10,10,10,130 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile,219,0,9,10,3362,27961,3498,10,10,10,180 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile,220,0,10,10,3363,25867,3499,10,10,10,180 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile,270,0,10,10,3243,27978,3383,10,10,10,230 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile,350,0,10,10,3148,27139,3288,10,10,10,310 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile-symflower-fix,170,0,10,10,3487,29478,3628,10,10,10,130 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile-symflower-fix,219,0,9,10,3362,27961,3498,10,10,10,180 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile-symflower-fix,220,0,10,10,3363,25867,3499,10,10,10,180 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile-symflower-fix,270,0,10,10,3243,27978,3383,10,10,10,230 +openrouter/google/gemma-2-27b-it,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,3148,27139,3288,10,10,10,310 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests,207,140,4,23,15302,186920,15649,23,20,20,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests,263,190,8,23,14373,155299,14737,23,21,21,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests,291,220,6,23,16996,218906,17323,23,21,21,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests,329,260,8,23,14945,179402,15286,23,18,20,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests,379,310,10,23,15776,182328,16101,23,18,18,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests-symflower-fix,325,250,10,23,14373,155849,14737,23,21,21,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests-symflower-fix,329,260,8,23,14945,179402,15286,23,18,20,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests-symflower-fix,379,310,10,23,15776,182328,16101,23,18,18,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests-symflower-fix,380,310,7,23,15302,188432,15649,23,20,20,0 +openrouter/google/gemma-2-9b-it,golang,golang/light,write-tests-symflower-fix,455,380,10,23,16996,220561,17323,23,21,21,0 +openrouter/google/gemma-2-9b-it,golang,golang/mistakes,code-repair,167,0,4,5,641,13119,699,5,4,4,150 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests,14,10,1,1,74,1824,90,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests,3,0,0,1,146,2122,160,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests,3,0,0,1,147,1151,161,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests,3,0,0,1,193,2826,208,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests,3,0,0,1,84,1687,98,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1824,90,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,146,2122,160,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,147,1151,161,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,193,2826,208,1,1,1,0 +openrouter/google/gemma-2-9b-it,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,84,1687,98,1,1,1,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests,1264,1190,11,23,23840,224565,24117,23,20,20,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests,1557,1480,12,23,39513,407758,39805,23,21,21,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests,1580,1510,11,23,19341,171395,19617,23,18,18,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests,1696,1630,12,23,33623,298234,33873,23,15,16,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests,698,630,7,23,16192,168189,16480,23,19,19,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests-symflower-fix,1264,1190,11,23,23840,224565,24117,23,20,20,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests-symflower-fix,1557,1480,12,23,39513,407758,39805,23,21,21,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests-symflower-fix,1580,1510,11,23,19341,171395,19617,23,18,18,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests-symflower-fix,1696,1630,12,23,33623,298234,33873,23,15,16,0 +openrouter/google/gemma-2-9b-it,java,java/light,write-tests-symflower-fix,698,630,7,23,16192,168189,16480,23,19,19,0 +openrouter/google/gemma-2-9b-it,java,java/mistakes,code-repair,61,0,2,5,445,10545,488,5,2,2,50 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests,1,0,0,1,16,3794,22,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests,1,0,0,1,16,947,22,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests,1,0,0,1,436,9534,445,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests,14,10,1,1,183,949,195,1,1,1,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests,14,10,1,1,187,1139,199,1,1,1,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests-symflower-fix,1,0,0,1,16,3794,22,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests-symflower-fix,1,0,0,1,16,947,22,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests-symflower-fix,1,0,0,1,436,9534,445,1,0,0,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,183,949,195,1,1,1,0 +openrouter/google/gemma-2-9b-it,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1139,199,1,1,1,0 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile,245,0,5,10,3253,42963,3357,10,5,5,220 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile,279,0,6,10,2609,49740,3147,10,6,7,250 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile,292,0,6,10,3592,36702,3706,10,8,8,260 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile,310,0,6,10,2464,31666,2575,10,7,7,280 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile,333,0,7,10,3349,33984,3468,10,8,8,300 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile-symflower-fix,245,0,5,10,3253,42963,3357,10,5,5,220 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile-symflower-fix,279,0,6,10,2609,49740,3147,10,6,7,250 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile-symflower-fix,292,0,6,10,3592,36702,3706,10,8,8,260 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile-symflower-fix,310,0,6,10,2464,31666,2575,10,7,7,280 +openrouter/google/gemma-2-9b-it,java,java/transpile,transpile-symflower-fix,333,0,7,10,3349,33984,3468,10,8,8,300 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests,1,0,0,1,218,2886,221,1,0,0,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests,1,0,0,1,263,16208,269,1,0,0,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests,3,0,0,1,255,3212,272,1,1,1,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests,3,0,0,1,432,3628,469,1,1,1,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests,3,0,0,1,434,11020,448,1,1,1,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests-symflower-fix,1,0,0,1,218,2886,221,1,0,0,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests-symflower-fix,1,0,0,1,263,16208,269,1,0,0,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,255,3212,272,1,1,1,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,432,3628,469,1,1,1,0 +openrouter/google/gemma-2-9b-it,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,434,11020,448,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,159,90,4,23,23058,81248,23314,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,199,130,4,23,22984,82582,23249,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,228,160,5,23,23595,86554,23836,23,20,20,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,253,180,4,23,20825,76742,21098,23,23,23,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,311,240,6,23,23700,84527,23959,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests-symflower-fix,293,220,8,23,23058,83010,23314,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests-symflower-fix,311,240,8,23,23595,87244,23836,23,20,20,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests-symflower-fix,326,250,7,23,20825,77469,21098,23,23,23,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests-symflower-fix,352,280,7,23,22984,83592,23249,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests-symflower-fix,477,400,12,23,23700,86466,23959,23,21,21,0 +openrouter/google/palm-2-chat-bison,golang,golang/mistakes,code-repair,210,0,5,5,641,7098,716,5,5,5,190 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,14,10,1,1,93,1472,108,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,14,10,1,1,97,1455,112,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,14,10,1,1,97,1572,112,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,3,0,0,1,158,1774,173,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,3,0,0,1,240,1637,255,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,93,1472,108,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,97,1455,112,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,97,1572,112,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,158,1774,173,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,240,1637,255,1,1,1,0 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile,438,0,8,10,3389,16653,3499,10,10,10,400 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile,519,0,9,10,3071,20187,3181,10,10,10,480 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile,519,0,9,10,3288,18228,3398,10,10,10,480 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile,580,0,10,10,3211,16398,3321,10,10,10,540 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile,580,0,10,10,3602,19470,3712,10,10,10,540 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3389,18701,3499,10,10,10,500 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3071,20599,3181,10,10,10,540 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3211,16398,3321,10,10,10,540 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3288,18706,3398,10,10,10,540 +openrouter/google/palm-2-chat-bison,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3602,19470,3712,10,10,10,540 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,3,0,0,1,228,1339,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,3,0,0,1,228,1441,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,3,0,0,1,228,1535,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,3,0,0,1,228,1538,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,3,0,0,1,228,1597,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,228,1339,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,228,1441,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,228,1535,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,228,1538,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,228,1597,241,1,1,1,0 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile,429,0,9,10,3342,16352,3472,10,10,10,390 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile,480,0,10,10,3130,20035,3260,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile,480,0,10,10,3213,16266,3343,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile,480,0,10,10,3282,16096,3412,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile,480,0,10,10,3282,16098,3412,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile-symflower-fix,429,0,9,10,3342,16352,3472,10,10,10,390 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile-symflower-fix,480,0,10,10,3130,20035,3260,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile-symflower-fix,480,0,10,10,3213,16266,3343,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile-symflower-fix,480,0,10,10,3282,16096,3412,10,10,10,440 +openrouter/google/palm-2-chat-bison,java,java/transpile,transpile-symflower-fix,480,0,10,10,3282,16098,3412,10,10,10,440 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests,506,430,7,23,16707,79626,17006,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests,506,430,7,23,18885,66665,19184,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests,587,510,8,23,20868,83512,21167,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests,620,540,11,23,17087,64256,17387,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests,740,660,11,23,16642,66386,16941,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests-symflower-fix,506,430,7,23,16707,79626,17006,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests-symflower-fix,506,430,7,23,18885,66665,19184,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests-symflower-fix,587,510,8,23,20868,83512,21167,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests-symflower-fix,620,540,11,23,17087,64256,17387,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/light,write-tests-symflower-fix,740,660,11,23,16642,66386,16941,23,23,23,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/mistakes,code-repair,150,0,5,5,671,7513,736,5,5,5,130 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests,14,10,1,1,131,1402,144,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests,14,10,1,1,165,1391,178,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests,14,10,1,1,165,1546,178,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests,14,10,1,1,200,1499,213,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests,14,10,1,1,218,1411,231,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,131,1402,144,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,165,1391,178,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,165,1546,178,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,200,1499,213,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,218,1411,231,1,1,1,0 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile,132,0,2,10,3599,20040,3729,10,10,10,100 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile,183,0,3,10,3510,18445,3640,10,10,10,150 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile,185,0,5,10,4274,23597,4404,10,10,10,150 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile,224,0,4,10,3893,19484,4023,10,10,10,190 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile,234,0,4,10,3193,16919,3323,10,10,10,200 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile-symflower-fix,132,0,2,10,3599,20040,3729,10,10,10,100 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile-symflower-fix,183,0,3,10,3510,18445,3640,10,10,10,150 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile-symflower-fix,185,0,5,10,4274,23597,4404,10,10,10,150 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile-symflower-fix,224,0,4,10,3893,19484,4023,10,10,10,190 +openrouter/google/palm-2-chat-bison,ruby,ruby/transpile,transpile-symflower-fix,234,0,4,10,3193,16919,3323,10,10,10,200 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,199,130,4,23,22347,81895,22598,23,21,21,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,214,140,5,23,22916,81230,23183,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,285,210,6,23,19629,73367,19895,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,294,220,7,23,24005,82394,24259,23,22,22,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,295,220,6,23,22269,78540,22531,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests-symflower-fix,374,300,9,23,22347,83582,22598,23,21,21,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests-symflower-fix,377,300,8,23,19629,73940,19895,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests-symflower-fix,500,420,11,23,22269,79882,22531,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests-symflower-fix,533,450,14,23,22916,84109,23183,23,23,23,0 +openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests-symflower-fix,571,490,14,23,24005,86136,24259,23,22,22,0 +openrouter/google/palm-2-codechat-bison,golang,golang/mistakes,code-repair,210,0,5,5,641,5747,712,5,5,5,190 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,14,10,1,1,140,1532,151,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,14,10,1,1,74,1161,85,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,14,10,1,1,79,1629,90,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,14,10,1,1,79,1857,90,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,14,10,1,1,81,1341,92,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,140,1532,151,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1161,85,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1629,90,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1857,90,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,81,1341,92,1,1,1,0 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile,479,0,9,10,3188,21887,3298,10,10,10,440 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile,499,0,9,10,3834,16181,3944,10,10,10,460 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile,540,0,10,10,3059,14874,3169,10,10,10,500 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile,540,0,10,10,3090,14468,3200,10,10,10,500 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile,560,0,10,10,2939,14468,3049,10,10,10,520 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3059,14874,3169,10,10,10,500 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3090,14468,3200,10,10,10,500 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3188,22445,3298,10,10,10,500 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2939,14468,3049,10,10,10,520 +openrouter/google/palm-2-codechat-bison,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,3834,16601,3944,10,10,10,520 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,3,0,0,1,141,1250,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,3,0,0,1,141,1348,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,3,0,0,1,141,1355,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,3,0,0,1,141,1456,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,3,0,0,1,443,1736,456,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,141,1250,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,141,1348,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,141,1355,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,141,1456,154,1,1,1,0 +openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests-symflower-fix,3,0,0,1,443,1736,456,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests,673,600,8,23,19382,77583,19663,23,21,21,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests,794,720,7,23,18648,79367,18938,23,22,22,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests,817,740,8,23,16285,66271,16584,23,23,23,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests,848,770,9,23,17868,66576,18167,23,23,23,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests,854,780,9,23,20977,103858,21258,23,21,21,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests-symflower-fix,673,600,8,23,19382,77583,19663,23,21,21,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests-symflower-fix,794,720,7,23,18648,79367,18938,23,22,22,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests-symflower-fix,817,740,8,23,16285,66271,16584,23,23,23,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests-symflower-fix,848,770,9,23,17868,66576,18167,23,23,23,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/light,write-tests-symflower-fix,854,780,9,23,20977,103858,21258,23,21,21,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/mistakes,code-repair,170,0,5,5,650,5965,715,5,5,5,150 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests,14,10,1,1,136,1404,149,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests,14,10,1,1,136,1548,149,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests,14,10,1,1,254,1669,267,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests,14,10,1,1,290,1709,303,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests,3,0,0,1,150,813,163,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,136,1404,149,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,136,1548,149,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,254,1669,267,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,290,1709,303,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,150,813,163,1,1,1,0 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile,196,0,6,10,3497,23328,3627,10,10,10,160 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile,197,0,7,10,3471,24457,3601,10,10,10,160 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile,202,0,4,10,3495,19590,3616,10,9,9,170 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile,226,0,6,10,3369,21065,3499,10,10,10,190 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile,227,0,7,10,3629,19782,3759,10,10,10,190 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile-symflower-fix,196,0,6,10,3497,23328,3627,10,10,10,160 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile-symflower-fix,197,0,7,10,3471,24457,3601,10,10,10,160 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile-symflower-fix,202,0,4,10,3495,19590,3616,10,9,9,170 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile-symflower-fix,226,0,6,10,3369,21065,3499,10,10,10,190 +openrouter/google/palm-2-codechat-bison,ruby,ruby/transpile,transpile-symflower-fix,227,0,7,10,3629,19782,3759,10,10,10,190 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests,890,800,21,23,18810,253762,19056,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests,911,820,22,23,17580,280945,17818,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests,920,830,21,23,16883,271870,17121,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests,992,900,23,23,17043,267600,17279,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests,992,900,23,23,17819,270141,18055,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests-symflower-fix,890,800,21,23,18810,253762,19056,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests-symflower-fix,911,820,22,23,17580,280945,17818,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests-symflower-fix,961,870,22,23,16883,272069,17121,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests-symflower-fix,992,900,23,23,17043,267600,17279,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/light,write-tests-symflower-fix,992,900,23,23,17819,270141,18055,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/mistakes,code-repair,210,0,5,5,638,12359,696,5,5,5,190 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests,14,10,1,1,75,1637,85,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,1452,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,2112,91,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,2870,91,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests,14,10,1,1,80,2775,92,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,75,1637,85,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1452,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,2112,91,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,2870,91,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,80,2775,92,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile,540,0,10,10,2904,49338,3006,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile,540,0,10,10,2918,65838,3020,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile,540,0,10,10,3036,42217,3138,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile,540,0,10,10,3182,43656,3288,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile,560,0,10,10,2966,42410,3072,10,10,10,520 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2904,49338,3006,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2918,65838,3020,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3036,42217,3138,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3182,43656,3288,10,10,10,500 +openrouter/meta-llama/llama-3.1-405b-instruct,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2966,42410,3072,10,10,10,520 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests,2588,2500,19,23,27230,370973,27518,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests,2748,2660,20,23,27853,262703,29963,23,22,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests,2970,2880,21,23,30672,343630,30958,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests,3062,2970,23,23,28197,238766,28485,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests,3082,2990,23,23,30642,398792,30922,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests-symflower-fix,2588,2500,19,23,27230,370973,27518,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests-symflower-fix,2748,2660,20,23,27853,262703,29963,23,22,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests-symflower-fix,2970,2880,21,23,30672,343630,30958,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests-symflower-fix,3062,2970,23,23,28197,238766,28485,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/light,write-tests-symflower-fix,3082,2990,23,23,30642,398792,30922,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/mistakes,code-repair,160,0,5,5,965,15599,1027,5,5,5,140 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests,14,10,1,1,137,1779,149,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests,14,10,1,1,151,2463,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests,14,10,1,1,151,2751,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests,14,10,1,1,151,2770,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests,14,10,1,1,344,2976,356,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1779,149,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,151,2463,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,151,2751,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,151,2770,163,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,344,2976,356,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile,480,0,10,10,3521,49699,3647,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile,480,0,10,10,3696,57620,3818,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile,480,0,10,10,3705,51705,3827,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile,480,0,10,10,3770,85139,3894,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile,480,0,10,10,3873,38898,4001,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3521,49699,3647,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3696,57620,3818,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3705,51705,3827,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3770,85139,3894,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3873,38898,4001,10,10,10,440 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests,1897,1810,18,23,19379,304709,19661,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests,2190,2100,21,23,20778,390557,21056,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests,2200,2110,21,23,19616,308535,19894,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests,2271,2180,22,23,19630,286266,19914,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests,2362,2270,23,23,20666,289716,20954,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests-symflower-fix,1897,1810,18,23,19379,304709,19661,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests-symflower-fix,2190,2100,21,23,20778,390557,21056,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests-symflower-fix,2200,2110,21,23,19616,308535,19894,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests-symflower-fix,2271,2180,22,23,19630,286266,19914,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/light,write-tests-symflower-fix,2362,2270,23,23,20666,289716,20954,23,23,23,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/mistakes,code-repair,170,0,5,5,668,14896,728,5,5,5,150 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,140,1456,154,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,140,3049,152,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,140,3781,152,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,292,3144,304,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,292,4115,304,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1456,154,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3049,152,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3781,152,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,292,3144,304,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,292,4115,304,1,1,1,0 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile,280,0,10,10,3363,66665,3489,10,10,10,240 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile,290,0,10,10,3412,62370,3536,10,10,10,250 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile,300,0,10,10,3162,50343,3290,10,10,10,260 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile,340,0,10,10,3382,63592,3502,10,10,10,300 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile,380,0,10,10,3120,77959,3240,10,10,10,340 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile-symflower-fix,280,0,10,10,3363,66665,3489,10,10,10,240 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile-symflower-fix,290,0,10,10,3412,62370,3536,10,10,10,250 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile-symflower-fix,300,0,10,10,3162,50343,3290,10,10,10,260 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile-symflower-fix,340,0,10,10,3382,63592,3502,10,10,10,300 +openrouter/meta-llama/llama-3.1-405b-instruct,ruby,ruby/transpile,transpile-symflower-fix,380,0,10,10,3120,77959,3240,10,10,10,340 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests,581,500,16,23,75060,634025,75303,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests,714,630,19,23,75387,573165,75633,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests,726,640,19,23,23761,270888,24021,23,22,22,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests,737,650,20,23,49570,414670,49827,23,22,22,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests,862,770,23,23,19703,235532,19961,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests-symflower-fix,673,590,18,23,75060,635099,75303,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests-symflower-fix,726,640,19,23,23761,270888,24021,23,22,22,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests-symflower-fix,737,650,20,23,49570,414670,49827,23,22,22,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests-symflower-fix,775,690,20,23,75387,573430,75633,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/light,write-tests-symflower-fix,862,770,23,23,19703,235532,19961,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/mistakes,code-repair,209,0,5,5,681,15260,925,5,4,5,190 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,229,2255,239,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,74,1089,84,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,746,4963,756,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,1467,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests,3,0,0,1,173,2065,183,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,229,2255,239,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1089,84,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,746,4963,756,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1467,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,173,2065,183,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile,459,0,9,10,3205,38651,3305,10,10,10,420 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile,479,0,9,10,3115,38631,3223,10,10,10,440 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile,508,0,9,10,3023,39068,3526,10,9,10,470 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile,560,0,10,10,2850,42115,2958,10,10,10,520 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile,560,0,10,10,3177,50634,3281,10,10,10,520 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile-symflower-fix,459,0,9,10,3205,38651,3305,10,10,10,420 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile-symflower-fix,508,0,9,10,3023,39068,3526,10,9,10,470 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3115,38889,3223,10,10,10,500 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2850,42115,2958,10,10,10,520 +openrouter/meta-llama/llama-3.1-70b-instruct,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,3177,50634,3281,10,10,10,520 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests,2132,2050,17,23,79308,532001,79560,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests,2258,2170,19,23,29317,250558,29593,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests,2488,2400,19,23,30474,385307,30750,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests,2679,2590,20,23,31033,246875,31309,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests,2879,2790,21,23,32214,273904,41205,23,22,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests-symflower-fix,2132,2050,17,23,79308,532001,79560,23,21,21,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests-symflower-fix,2258,2170,19,23,29317,250558,29593,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests-symflower-fix,2488,2400,19,23,30474,385307,30750,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests-symflower-fix,2679,2590,20,23,31033,246875,31309,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/light,write-tests-symflower-fix,2879,2790,21,23,32214,273904,41205,23,22,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/mistakes,code-repair,160,0,5,5,965,10412,1025,5,5,5,140 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests,14,10,1,1,229,2690,241,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests,14,10,1,1,231,1593,243,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests,14,10,1,1,261,2048,273,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests,14,10,1,1,266,4740,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests,14,10,1,1,674,5114,686,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,229,2690,241,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,231,1593,243,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,261,2048,273,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,266,4740,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,674,5114,686,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile,388,0,8,10,3820,35285,3940,10,10,10,350 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile,419,0,9,10,3911,50290,4031,10,10,10,380 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile,429,0,9,10,3713,49831,3833,10,10,10,390 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile,470,0,10,10,3699,42161,3819,10,10,10,430 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile,480,0,10,10,3981,60100,4101,10,10,10,440 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile-symflower-fix,388,0,8,10,3820,35285,3940,10,10,10,350 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile-symflower-fix,419,0,9,10,3911,50290,4031,10,10,10,380 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile-symflower-fix,429,0,9,10,3713,49831,3833,10,10,10,390 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3699,42161,3819,10,10,10,430 +openrouter/meta-llama/llama-3.1-70b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3981,60100,4101,10,10,10,440 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests,2179,2090,20,23,18528,199926,18804,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests,2220,2130,21,23,20315,258297,20587,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests,2231,2140,22,23,20087,236211,20363,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests,2282,2190,23,23,21751,227777,22027,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests,2311,2220,22,23,20426,248417,20702,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2179,2090,20,23,18528,199926,18804,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2220,2130,21,23,20315,258297,20587,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2231,2140,22,23,20087,236211,20363,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2282,2190,23,23,21751,227777,22027,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2311,2220,22,23,20426,248417,20702,23,23,23,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/mistakes,code-repair,170,0,5,5,621,11136,681,5,5,5,150 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,163,2561,175,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,171,2061,183,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,208,3109,220,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,210,4724,222,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,287,4046,299,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,163,2561,175,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,171,2061,183,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,208,3109,220,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,210,4724,222,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,287,4046,299,1,1,1,0 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile,240,0,10,10,3561,43323,3681,10,10,10,200 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile,269,0,9,10,3241,48588,3361,10,10,10,230 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile,330,0,10,10,2887,56774,3007,10,10,10,290 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile,350,0,10,10,3151,49314,3271,10,10,10,310 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile,370,0,10,10,2980,72465,3104,10,10,10,330 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,240,0,10,10,3561,43323,3681,10,10,10,200 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,269,0,9,10,3241,48588,3361,10,10,10,230 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,330,0,10,10,2887,56774,3007,10,10,10,290 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,3151,49314,3271,10,10,10,310 +openrouter/meta-llama/llama-3.1-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,370,0,10,10,2980,72465,3104,10,10,10,330 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,438,360,11,23,25548,117825,44711,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,440,360,13,23,26679,125625,28801,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,466,390,12,23,21647,84938,21857,22,21,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,468,390,12,23,54760,142712,58149,23,20,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,485,410,11,23,131967,343184,132648,23,20,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,467,390,13,23,21647,85362,21857,22,21,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,499,420,13,23,54760,143145,58149,23,20,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,521,440,14,23,26679,125872,28801,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,526,450,12,23,131967,343401,132648,23,20,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,531,450,14,23,25548,118654,44711,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/mistakes,code-repair,209,0,5,5,714,6307,1108,5,4,5,190 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,1,0,0,1,30,779,33,1,0,0,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,714,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,2,0,0,1,437,2622,709,1,0,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,396,2092,406,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,707,3017,717,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,30,779,33,1,0,0,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,714,89,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,437,2622,709,1,0,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,396,2092,406,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,707,3017,717,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,275,0,6,10,3059,17168,3649,10,9,10,240 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,357,0,7,10,3053,16768,3155,10,10,10,320 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,375,0,7,10,2839,13692,2929,10,9,9,340 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,376,0,7,10,3323,22171,3694,10,9,10,340 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,417,0,8,10,3483,20210,3836,10,9,10,380 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,375,0,7,10,2839,13692,2929,10,9,9,340 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,418,0,8,10,3053,17258,3155,10,10,10,380 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,437,0,8,10,3323,22943,3694,10,9,10,400 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,438,0,9,10,3059,17839,3649,10,9,10,400 +openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,448,0,9,10,3483,20380,3836,10,9,10,410 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,1449,1370,12,23,111000,198664,117377,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,1570,1490,14,23,32430,150993,33325,23,21,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,1871,1790,14,23,40377,104164,40644,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,1941,1860,16,23,122129,302380,123864,23,20,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,2396,2310,18,23,36338,108508,52066,23,22,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,1449,1370,12,23,111000,198664,117377,23,21,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,1570,1490,14,23,32430,150993,33325,23,21,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,1871,1790,14,23,40377,104164,40644,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,1941,1860,16,23,122129,302380,123864,23,20,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,2396,2310,18,23,36338,108508,52066,23,22,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/mistakes,code-repair,160,0,5,5,980,6182,1040,5,5,5,140 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,14,10,1,1,266,1014,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,14,10,1,1,266,980,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,14,10,1,1,352,1000,364,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,3,0,0,1,355,1619,367,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,3,0,0,1,444,2578,456,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,266,1014,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,266,980,278,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,352,1000,364,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,355,1619,367,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,444,2578,456,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,376,0,8,10,4428,33806,15471,10,8,10,340 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,429,0,9,10,3857,18114,3977,10,10,10,390 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,438,0,9,10,4062,19610,4590,10,9,10,400 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,460,0,10,10,3294,16765,3414,10,10,10,420 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,480,0,10,10,3486,16886,3606,10,10,10,440 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,376,0,8,10,4428,33806,15471,10,8,10,340 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,429,0,9,10,3857,18114,3977,10,10,10,390 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,438,0,9,10,4062,19610,4590,10,9,10,400 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,460,0,10,10,3294,16765,3414,10,10,10,420 +openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3486,16886,3606,10,10,10,440 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests,1004,920,17,23,35340,95276,35605,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests,1068,990,14,23,67126,255127,106842,23,20,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests,1220,1140,18,23,144130,324676,145098,23,19,20,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests,1464,1380,17,23,21454,76125,21722,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests,1479,1390,20,23,24249,93658,24525,23,23,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,1004,920,17,23,35340,95276,35605,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,1068,990,14,23,67126,255127,106842,23,20,21,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,1220,1140,18,23,144130,324676,145098,23,19,20,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,1464,1380,17,23,21454,76125,21722,23,22,22,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,1479,1390,20,23,24249,93658,24525,23,23,23,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/mistakes,code-repair,108,0,4,5,851,12319,1162,5,4,5,90 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,304,1148,316,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,313,1367,325,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,395,1119,407,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,632,9197,644,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,316,3560,328,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,304,1148,316,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,313,1367,325,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,395,1119,407,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,632,9197,644,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,316,3560,328,1,1,1,0 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile,296,0,7,10,3034,19961,3493,10,9,10,260 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile,318,0,9,10,2968,17828,3349,10,9,10,280 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile,350,0,10,10,3378,19313,3494,10,10,10,310 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile,378,0,9,10,3044,18341,3199,10,9,10,340 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile,420,0,10,10,2884,22100,3004,10,10,10,380 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,296,0,7,10,3034,19961,3493,10,9,10,260 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,318,0,9,10,2968,17828,3349,10,9,10,280 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,3378,19313,3494,10,10,10,310 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,378,0,9,10,3044,18341,3199,10,9,10,340 +openrouter/meta-llama/llama-3.1-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,420,0,10,10,2884,22100,3004,10,10,10,380 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,696,610,17,23,15095,301874,15289,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,706,620,19,23,14968,198286,15153,23,22,22,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,775,690,18,23,16316,219787,16504,23,22,22,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,790,700,21,23,16612,268524,16804,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,850,760,21,23,15478,228089,15666,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests-symflower-fix,779,690,20,23,15095,302635,15289,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests-symflower-fix,787,700,20,23,14968,198557,15153,23,22,22,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests-symflower-fix,816,730,19,23,16316,219986,16504,23,22,22,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests-symflower-fix,841,750,22,23,16612,268770,16804,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests-symflower-fix,891,800,22,23,15478,228424,15666,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/mistakes,code-repair,210,0,5,5,648,18227,688,5,5,5,190 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,74,1160,82,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,74,2385,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,74,952,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,74,953,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,14,10,1,1,79,1423,89,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1160,82,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2385,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,952,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,953,84,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1423,89,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile,458,0,8,10,2943,50584,3023,10,10,10,420 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile,519,0,9,10,2641,47746,2725,10,10,10,480 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile,519,0,9,10,2978,47465,3060,10,10,10,480 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile,519,0,9,10,2994,36487,3076,10,10,10,480 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile,529,0,9,10,2886,39985,2970,10,10,10,490 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2886,40660,2970,10,10,10,520 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2641,48253,2725,10,10,10,540 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2943,51233,3023,10,10,10,540 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2978,48166,3060,10,10,10,540 +openrouter/meta-llama/llama-3-70b-instruct,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2994,37773,3076,10,10,10,540 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,2087,2000,18,23,22138,283956,22322,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,2117,2030,18,23,25284,270205,25472,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,2429,2340,20,23,23935,245349,24123,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,2529,2440,20,23,23725,245296,23913,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,2579,2490,20,23,23323,215431,23511,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests-symflower-fix,2087,2000,18,23,22138,283956,22322,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests-symflower-fix,2117,2030,18,23,25284,270205,25472,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests-symflower-fix,2429,2340,20,23,23935,245349,24123,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests-symflower-fix,2529,2440,20,23,23725,245296,23913,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests-symflower-fix,2579,2490,20,23,23323,215431,23511,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/mistakes,code-repair,160,0,5,5,1051,13923,1091,5,5,5,140 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,14,10,1,1,137,1993,145,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,14,10,1,1,137,4583,145,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,14,10,1,1,266,6019,278,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,14,10,1,1,289,2028,297,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,3,0,0,1,287,2358,295,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1993,145,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,4583,145,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,266,6019,278,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,289,2028,297,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,287,2358,295,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile,470,0,10,10,3834,36451,3914,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile,470,0,10,10,3872,41934,3952,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile,470,0,10,10,3894,57596,3974,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile,470,0,10,10,3925,42728,4009,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile,470,0,10,10,4050,49258,4130,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3834,36451,3914,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3872,41934,3952,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3894,57596,3974,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3925,42728,4009,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,4050,49258,4130,10,10,10,430 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests,1868,1780,19,23,15666,159427,15850,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests,1928,1840,19,23,16339,154713,16523,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests,2119,2030,20,23,15710,125808,15902,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests,2140,2050,21,23,16004,146154,16196,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests,2251,2160,22,23,17199,152323,17383,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,1868,1780,19,23,15666,159427,15850,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,1928,1840,19,23,16339,154713,16523,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2119,2030,20,23,15710,125808,15902,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2140,2050,21,23,16004,146154,16196,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/light,write-tests-symflower-fix,2251,2160,22,23,17199,152323,17383,23,23,23,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/mistakes,code-repair,170,0,5,5,619,12899,659,5,5,5,150 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,140,2676,148,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,152,1446,160,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,243,3373,251,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,145,1693,153,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,156,3127,164,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,2676,148,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,152,1446,160,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,243,3373,251,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,145,1693,153,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,156,3127,164,1,1,1,0 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile,120,0,10,10,3475,37538,3563,10,10,10,80 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile,230,0,10,10,3324,44823,3404,10,10,10,190 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile,320,0,10,10,3186,27042,3270,10,10,10,280 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile,340,0,10,10,3251,36219,3335,10,10,10,300 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile,380,0,10,10,3244,29969,3324,10,10,10,340 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,120,0,10,10,3475,37538,3563,10,10,10,80 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,230,0,10,10,3324,44823,3404,10,10,10,190 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,320,0,10,10,3186,27042,3270,10,10,10,280 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,340,0,10,10,3251,36219,3335,10,10,10,300 +openrouter/meta-llama/llama-3-70b-instruct,ruby,ruby/transpile,transpile-symflower-fix,380,0,10,10,3244,29969,3324,10,10,10,340 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,2,0,0,1,5,1516,201,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,130,1473,144,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,135,1793,149,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,254,2173,279,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,323,12827,331,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,5,1516,201,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,130,1473,144,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,135,1793,149,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,254,2173,279,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,323,12827,331,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,1061,990,8,23,21558,99978,23240,23,19,21,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,1304,1230,9,23,23318,82109,24420,23,20,22,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,391,330,4,23,19928,111054,23405,23,15,19,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,450,380,5,23,29483,168112,30144,23,20,22,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,577,510,5,23,23717,89219,24039,23,19,20,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests-symflower-fix,1061,990,8,23,21558,99978,23240,23,19,21,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests-symflower-fix,1304,1230,9,23,23318,82109,24420,23,20,22,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests-symflower-fix,391,330,4,23,19928,111054,23405,23,15,19,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests-symflower-fix,450,380,5,23,29483,168112,30144,23,20,22,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests-symflower-fix,577,510,5,23,23717,89219,24039,23,19,20,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/mistakes,code-repair,160,0,5,5,1187,7168,1235,5,5,5,140 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,14,10,1,1,178,1594,190,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,2,0,0,1,7,2735,547,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,2,0,0,1,7,4225,258,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,3,0,0,1,336,1341,348,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,3,0,0,1,339,1009,347,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,178,1594,190,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,7,2735,547,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,7,4225,258,1,0,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,336,1341,348,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,339,1009,347,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile,419,0,9,10,3716,44444,3824,10,10,10,380 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile,429,0,9,10,3819,16864,3915,10,10,10,390 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile,470,0,10,10,3864,40216,3960,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile,470,0,10,10,3908,20757,3988,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile,470,0,10,10,4009,45402,4097,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile-symflower-fix,419,0,9,10,3716,44444,3824,10,10,10,380 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile-symflower-fix,429,0,9,10,3819,16864,3915,10,10,10,390 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3864,40216,3960,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3908,20757,3988,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,4009,45402,4097,10,10,10,430 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests,223,150,5,23,17134,111750,17971,23,22,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests,263,190,6,23,16752,87496,17697,23,21,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests,412,340,5,23,15908,74644,16172,23,22,22,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests,455,380,6,23,22957,107915,23201,23,23,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests,697,620,8,23,17532,67754,17776,23,23,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,223,150,5,23,17134,111750,17971,23,22,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,263,190,6,23,16752,87496,17697,23,21,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,412,340,5,23,15908,74644,16172,23,22,22,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,455,380,6,23,22957,107915,23201,23,23,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/light,write-tests-symflower-fix,697,620,8,23,17532,67754,17776,23,23,23,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/mistakes,code-repair,109,0,4,5,617,8242,657,5,5,5,90 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,240,906,252,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,131,16665,143,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,137,1651,145,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,151,1237,163,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,273,1627,285,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,240,906,252,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,131,16665,143,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,137,1651,145,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,151,1237,163,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,273,1627,285,1,1,1,0 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile,256,0,6,10,2882,14591,3002,10,10,10,220 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile,257,0,7,10,2746,28695,2846,10,10,10,220 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile,298,0,8,10,3193,23078,3306,10,10,10,260 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile,307,0,7,10,2792,21337,2896,10,10,10,270 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile,319,0,9,10,2761,24908,2869,10,10,10,280 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,256,0,6,10,2882,14591,3002,10,10,10,220 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,257,0,7,10,2746,28695,2846,10,10,10,220 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,298,0,8,10,3193,23078,3306,10,10,10,260 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,307,0,7,10,2792,21337,2896,10,10,10,270 +openrouter/meta-llama/llama-3-8b-instruct,ruby,ruby/transpile,transpile-symflower-fix,319,0,9,10,2761,24908,2869,10,10,10,280 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,1,0,0,1,957,16629,958,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,2,0,0,1,78,4098,562,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,194,3864,205,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,198,5158,209,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,98,2955,113,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,957,16629,958,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,78,4098,562,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,194,3864,205,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,198,5158,209,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,98,2955,113,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile,184,0,3,10,4291,72620,4624,10,5,6,160 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile,226,0,4,10,3227,46505,3310,10,6,6,200 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile,270,0,5,10,3165,42164,3261,10,7,8,240 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile,279,0,5,10,3206,37687,3294,10,7,7,250 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile,299,0,5,10,3224,43274,3307,10,7,7,270 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile-symflower-fix,245,0,4,10,4291,73071,4624,10,5,6,220 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile-symflower-fix,279,0,5,10,3206,37687,3294,10,7,7,250 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile-symflower-fix,331,0,6,10,3165,42451,3261,10,7,8,300 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile-symflower-fix,338,0,6,10,3227,47179,3310,10,6,6,310 +openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/transpile,transpile-symflower-fix,360,0,6,10,3224,43670,3307,10,7,7,330 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,2,0,0,1,598,16223,2315,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,2,0,0,1,849,6993,1178,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,3,0,0,1,1081,19096,1095,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,3,0,0,1,205,4373,218,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,3,0,0,1,460,8150,474,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,598,16223,2315,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,849,6993,1178,1,0,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,1081,19096,1095,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,205,4373,218,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,460,8150,474,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile,300,0,6,10,5526,65695,5617,10,7,7,270 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile,345,0,7,10,5164,49075,5283,10,9,9,310 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile,419,0,9,10,4219,52904,4298,10,5,5,390 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile,466,0,10,10,4141,36197,4249,10,8,8,430 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile,466,0,10,10,4766,42644,4873,10,8,8,430 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile-symflower-fix,300,0,6,10,5526,65695,5617,10,7,7,270 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile-symflower-fix,345,0,7,10,5164,49075,5283,10,9,9,310 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile-symflower-fix,419,0,9,10,4219,52904,4298,10,5,5,390 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile-symflower-fix,466,0,10,10,4141,36197,4249,10,8,8,430 +openrouter/microsoft/phi-3-medium-128k-instruct,java,java/transpile,transpile-symflower-fix,466,0,10,10,4766,42644,4873,10,8,8,430 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests,1164,1080,20,23,21046,170887,22517,23,20,21,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests,1286,1210,15,23,21912,199416,22174,23,19,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests,1293,1220,18,23,30500,531157,31141,23,15,17,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests,1655,1580,20,23,17672,213860,20533,23,13,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests,1701,1620,21,23,21312,177560,21731,23,18,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1164,1080,20,23,21046,170887,22517,23,20,21,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1286,1210,15,23,21912,199416,22174,23,19,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1293,1220,18,23,30500,531157,31141,23,15,17,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1655,1580,20,23,17672,213860,20533,23,13,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1701,1620,21,23,21312,177560,21731,23,18,19,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/mistakes,code-repair,138,0,5,5,625,7878,983,5,3,5,120 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests,12,10,1,1,147,1608,149,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests,12,10,1,1,160,1579,162,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests,12,10,1,1,177,1652,178,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,242,2339,255,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests,3,0,0,1,272,2518,285,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,12,10,1,1,147,1608,149,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,12,10,1,1,160,1579,162,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,12,10,1,1,177,1652,178,1,0,0,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,242,2339,255,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,272,2518,285,1,1,1,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile,104,0,9,10,3133,36129,4725,10,7,8,70 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile,128,0,10,10,3073,26591,3196,10,9,9,90 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile,146,0,10,10,2866,29478,2974,10,8,8,110 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile,163,0,7,10,2819,29399,3391,10,7,9,130 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile,35,0,6,10,3078,39726,3217,10,9,10,0 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,104,0,9,10,3133,36129,4725,10,7,8,70 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,128,0,10,10,3073,26591,3196,10,9,9,90 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,146,0,10,10,2866,29478,2974,10,8,8,110 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,163,0,7,10,2819,29399,3391,10,7,9,130 +openrouter/microsoft/phi-3-medium-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,35,0,6,10,3078,39726,3217,10,9,10,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests,396,320,11,23,21954,150911,23693,23,19,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests,442,360,13,23,22287,139903,22640,23,23,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests,450,370,13,23,21053,144793,22013,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests,454,380,12,23,25984,174063,28010,23,17,22,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests,560,480,13,23,23505,146308,24430,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests-symflower-fix,585,500,16,23,22287,141295,22640,23,23,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests-symflower-fix,600,520,15,23,21954,152495,23693,23,19,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests-symflower-fix,603,520,16,23,21053,145577,22013,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests-symflower-fix,700,620,18,23,25984,176457,28010,23,17,22,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/light,write-tests-symflower-fix,775,690,18,23,23505,148786,24430,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/mistakes,code-repair,210,0,5,5,670,6677,745,5,5,5,190 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,3,0,0,1,298,1534,313,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,3,0,0,1,300,1746,315,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,3,0,0,1,437,2929,456,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,3,0,0,1,703,3007,714,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,4,0,1,1,438,2344,455,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,298,1534,313,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,300,1746,315,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,437,2929,456,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,703,3007,714,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests-symflower-fix,4,0,1,1,438,2344,455,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile,234,0,4,10,3289,21900,3403,10,10,10,200 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile,356,0,6,10,3351,21204,3461,10,10,10,320 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile,407,0,7,10,3271,23685,3381,10,10,10,370 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile,407,0,7,10,3304,22918,3418,10,10,10,370 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile,438,0,8,10,3291,21225,3401,10,10,10,400 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile-symflower-fix,295,0,5,10,3289,22177,3403,10,10,10,260 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile-symflower-fix,407,0,7,10,3271,23685,3381,10,10,10,370 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile-symflower-fix,438,0,8,10,3291,21225,3401,10,10,10,400 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile-symflower-fix,458,0,8,10,3351,21892,3461,10,10,10,420 +openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/transpile,transpile-symflower-fix,468,0,8,10,3304,23562,3418,10,10,10,430 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,116,50,1,23,26487,133455,28211,23,19,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,120,50,1,23,25749,123802,26048,23,23,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,269,200,2,23,31241,158724,32000,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,388,320,2,23,33816,172596,34636,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,450,380,3,23,28683,134555,29532,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests-symflower-fix,116,50,1,23,26487,133455,28211,23,19,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests-symflower-fix,120,50,1,23,25749,123802,26048,23,23,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests-symflower-fix,269,200,2,23,31241,158724,32000,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests-symflower-fix,388,320,2,23,33816,172596,34636,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests-symflower-fix,450,380,3,23,28683,134555,29532,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/mistakes,code-repair,159,0,5,5,1328,10047,1835,5,4,5,140 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,14,10,1,1,230,1720,243,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,2,0,0,1,293,2649,606,1,0,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,3,0,0,1,1528,6350,1541,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,3,0,0,1,578,2528,591,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,3,0,0,1,732,3547,745,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,230,1720,243,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,293,2649,606,1,0,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,1528,6350,1541,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,578,2528,591,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,732,3547,745,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests,1438,1350,21,23,20358,111293,21257,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests,1497,1410,21,23,19813,113488,20922,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests,1527,1440,21,23,16644,100232,18423,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests,1575,1490,21,23,19975,115633,21444,23,18,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests,1858,1770,22,23,17186,101389,18254,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests-symflower-fix,1438,1350,21,23,20358,111293,21257,23,21,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests-symflower-fix,1497,1410,21,23,19813,113488,20922,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests-symflower-fix,1527,1440,21,23,16644,100232,18423,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests-symflower-fix,1575,1490,21,23,19975,115633,21444,23,18,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/light,write-tests-symflower-fix,1858,1770,22,23,17186,101389,18254,23,20,23,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/mistakes,code-repair,140,0,5,5,613,6106,678,5,5,5,120 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,155,1299,168,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,155,1310,168,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,158,3595,171,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,255,1698,268,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,1299,168,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,1310,168,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,158,3595,171,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,255,1698,268,1,1,1,0 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile,137,0,9,10,2925,21363,3043,10,9,9,100 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile,240,0,10,10,2701,19644,2831,10,10,10,200 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile,58,0,10,10,2889,22054,3546,10,8,10,20 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile,66,0,8,10,2937,22747,3303,10,8,10,30 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile,87,0,8,10,3318,21427,3520,10,9,10,50 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile-symflower-fix,137,0,9,10,2925,21363,3043,10,9,9,100 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile-symflower-fix,240,0,10,10,2701,19644,2831,10,10,10,200 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile-symflower-fix,58,0,10,10,2889,22054,3546,10,8,10,20 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile-symflower-fix,66,0,8,10,2937,22747,3303,10,8,10,30 +openrouter/microsoft/phi-3-medium-4k-instruct,ruby,ruby/transpile,transpile-symflower-fix,87,0,8,10,3318,21427,3520,10,9,10,50 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,14,10,1,1,228,1495,239,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,2,0,0,1,668,7751,1530,1,0,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,169,1073,180,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,243,1578,254,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,3,0,0,1,351,2174,362,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,228,1495,239,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,668,7751,1530,1,0,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,169,1073,180,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,243,1578,254,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,351,2174,362,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile,110,0,2,10,3020,19351,3125,10,9,9,80 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile,161,0,3,10,3181,19579,3282,10,9,9,130 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile,189,0,3,10,2932,17858,3024,10,8,8,160 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile,234,0,4,10,3071,19617,3181,10,10,10,200 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile,374,0,6,10,2988,18980,3089,10,9,9,340 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile-symflower-fix,110,0,2,10,3020,19351,3125,10,9,9,80 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile-symflower-fix,202,0,4,10,3181,19799,3282,10,9,9,170 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile-symflower-fix,230,0,4,10,2932,18326,3024,10,8,8,200 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile-symflower-fix,234,0,4,10,3071,19617,3181,10,10,10,200 +openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/transpile,transpile-symflower-fix,374,0,6,10,2988,18980,3089,10,9,9,340 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests,1034,970,10,23,40416,223392,49983,23,9,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests,1100,1040,9,23,38279,230531,52822,23,6,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests,1450,1390,10,23,42373,235042,51802,23,6,21,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests,1615,1550,13,23,36037,174858,44778,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests,1888,1820,15,23,37161,235085,49620,23,8,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests-symflower-fix,1034,970,10,23,40416,223392,49983,23,9,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests-symflower-fix,1100,1040,9,23,38279,230531,52822,23,6,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests-symflower-fix,1450,1390,10,23,42373,235042,51802,23,6,21,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests-symflower-fix,1615,1550,13,23,36037,174858,44778,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/light,write-tests-symflower-fix,1888,1820,15,23,37161,235085,49620,23,8,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/mistakes,code-repair,159,0,5,5,1142,11745,1361,5,4,5,140 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,13,10,1,1,731,3896,1215,1,0,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,14,10,1,1,244,1577,257,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,14,10,1,1,459,2228,472,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,14,10,1,1,684,2801,697,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,3,0,0,1,253,1522,266,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests-symflower-fix,13,10,1,1,731,3896,1215,1,0,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,244,1577,257,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,459,2228,472,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,684,2801,697,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,253,1522,266,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile,427,0,9,10,3914,21631,4036,10,9,9,390 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile,439,0,9,10,4326,20693,4456,10,10,10,400 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile,449,0,9,10,3880,18908,4010,10,10,10,410 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile,480,0,10,10,3837,20867,3967,10,10,10,440 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile,480,0,10,10,3967,20475,4097,10,10,10,440 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile-symflower-fix,427,0,9,10,3914,21631,4036,10,9,9,390 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile-symflower-fix,439,0,9,10,4326,20693,4456,10,10,10,400 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile-symflower-fix,449,0,9,10,3880,18908,4010,10,10,10,410 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3837,20867,3967,10,10,10,440 +openrouter/microsoft/phi-3-mini-128k-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3967,20475,4097,10,10,10,440 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests,1044,970,20,23,27814,170636,35099,23,10,21,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests,1153,1080,17,23,21956,122210,29302,23,10,23,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests,1159,1090,17,23,23120,164462,32311,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests,716,650,14,23,21022,156175,28938,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests,883,810,19,23,23341,135771,34574,23,9,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1044,970,20,23,27814,170636,35099,23,10,21,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1153,1080,17,23,21956,122210,29302,23,10,23,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,1159,1090,17,23,23120,164462,32311,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,716,650,14,23,21022,156175,28938,23,9,20,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/light,write-tests-symflower-fix,883,810,19,23,23341,135771,34574,23,9,22,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/mistakes,code-repair,137,0,5,5,756,9112,1477,5,2,5,120 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,201,1445,214,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,213,1255,226,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,233,1544,246,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,298,2036,311,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests,14,10,1,1,428,2234,441,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,201,1445,214,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,213,1255,226,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,233,1544,246,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,298,2036,311,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,428,2234,441,1,1,1,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile,112,0,6,10,3139,20507,3247,10,8,8,80 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile,114,0,6,10,3171,30553,3290,10,9,9,80 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile,140,0,5,10,2789,20766,4068,10,6,9,110 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile,28,0,3,10,2959,30360,3454,10,7,8,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile,89,0,5,10,3015,22850,4712,10,5,9,60 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,112,0,6,10,3139,20507,3247,10,8,8,80 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,114,0,6,10,3171,30553,3290,10,9,9,80 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,140,0,5,10,2789,20766,4068,10,6,9,110 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,28,0,3,10,2959,30360,3454,10,7,8,0 +openrouter/microsoft/phi-3-mini-128k-instruct,ruby,ruby/transpile,transpile-symflower-fix,89,0,5,10,3015,22850,4712,10,5,9,60 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,2,0,0,1,1505,12423,2397,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,2,0,0,1,359,12297,2030,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,2,0,0,1,431,6221,2551,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,2,0,0,1,818,15199,2219,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,2,0,0,1,853,11530,1750,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,1505,12423,2397,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,359,12297,2030,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,431,6221,2551,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,818,15199,2219,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,853,11530,1750,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile,164,0,4,10,3589,29758,3689,10,10,10,130 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile,183,0,4,10,3974,32061,4694,10,9,10,150 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile,184,0,4,10,3488,32137,3588,10,10,10,150 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile,193,0,5,10,3638,40256,4728,10,8,10,160 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile,195,0,5,10,3816,36576,3916,10,10,10,160 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile-symflower-fix,347,0,7,10,3488,33190,3588,10,10,10,310 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile-symflower-fix,376,0,8,10,3638,41593,4728,10,8,10,340 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile-symflower-fix,378,0,8,10,3816,37413,3916,10,10,10,340 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile-symflower-fix,387,0,8,10,3974,33624,4694,10,9,10,350 +openrouter/microsoft/wizardlm-2-7b,golang,golang/transpile,transpile-symflower-fix,429,0,9,10,3589,31816,3689,10,10,10,390 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,1376,1320,10,23,61630,430867,81421,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,1545,1490,9,23,55872,411912,77356,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,1646,1590,10,23,62406,400728,80540,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,1848,1790,12,23,62176,384862,80737,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,954,900,9,23,74905,442039,94594,23,0,22,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests-symflower-fix,1376,1320,10,23,61630,430867,81421,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests-symflower-fix,1545,1490,9,23,55872,411912,77356,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests-symflower-fix,1646,1590,10,23,62406,400728,80540,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests-symflower-fix,1848,1790,12,23,62176,384862,80737,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests-symflower-fix,954,900,9,23,74905,442039,94594,23,0,22,0 +openrouter/microsoft/wizardlm-2-7b,java,java/mistakes,code-repair,159,0,5,5,1375,13024,1644,5,4,5,140 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,13,10,1,1,236,11715,1987,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,13,10,1,1,616,5274,2362,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,2,0,0,1,1610,19321,2684,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,2,0,0,1,1659,7861,3162,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,2,0,0,1,1730,17859,3110,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests-symflower-fix,13,10,1,1,236,11715,1987,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests-symflower-fix,13,10,1,1,616,5274,2362,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,1610,19321,2684,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,1659,7861,3162,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,1730,17859,3110,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile,378,0,8,10,4600,33211,4720,10,10,10,340 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile,419,0,9,10,4549,32490,4669,10,10,10,380 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile,419,0,9,10,4702,29132,4822,10,10,10,380 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile,479,0,10,10,4515,38048,4734,10,9,10,440 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile,480,0,10,10,4298,31792,4418,10,10,10,440 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile-symflower-fix,378,0,8,10,4600,33211,4720,10,10,10,340 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile-symflower-fix,419,0,9,10,4549,32490,4669,10,10,10,380 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile-symflower-fix,419,0,9,10,4702,29132,4822,10,10,10,380 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile-symflower-fix,479,0,10,10,4515,38048,4734,10,9,10,440 +openrouter/microsoft/wizardlm-2-7b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4298,31792,4418,10,10,10,440 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests,536,480,10,23,42227,393550,59235,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests,555,500,9,23,40928,333298,58306,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests,562,510,6,23,40517,357367,60093,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests,636,580,10,23,38942,326336,57573,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests,795,740,9,23,37827,346983,55946,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests-symflower-fix,536,480,10,23,42227,393550,59235,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests-symflower-fix,555,500,9,23,40928,333298,58306,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests-symflower-fix,562,510,6,23,40517,357367,60093,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests-symflower-fix,636,580,10,23,38942,326336,57573,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/light,write-tests-symflower-fix,795,740,9,23,37827,346983,55946,23,0,23,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/mistakes,code-repair,77,0,4,5,1005,13726,2285,5,3,5,60 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests,13,10,1,1,1112,10587,2287,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests,13,10,1,1,199,4025,1334,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests,2,0,0,1,1324,10144,2635,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests,2,0,0,1,1346,12686,2821,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests,2,0,0,1,778,12780,2122,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,1112,10587,2287,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,199,4025,1334,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,1324,10144,2635,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,1346,12686,2821,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,778,12780,2122,1,0,1,0 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile,115,0,5,10,3421,26334,3541,10,10,10,80 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile,126,0,6,10,3641,21753,3761,10,10,10,90 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile,65,0,5,10,3325,28199,3445,10,10,10,30 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile,65,0,5,10,3612,31156,3732,10,10,10,30 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile,65,0,5,10,3688,30407,3808,10,10,10,30 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile-symflower-fix,115,0,5,10,3421,26334,3541,10,10,10,80 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile-symflower-fix,126,0,6,10,3641,21753,3761,10,10,10,90 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile-symflower-fix,65,0,5,10,3325,28199,3445,10,10,10,30 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile-symflower-fix,65,0,5,10,3612,31156,3732,10,10,10,30 +openrouter/microsoft/wizardlm-2-7b,ruby,ruby/transpile,transpile-symflower-fix,65,0,5,10,3688,30407,3808,10,10,10,30 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,439,370,14,23,58605,759019,63588,23,10,22,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,461,390,14,23,22817,489357,27193,23,11,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,500,430,14,23,21666,408149,27265,23,10,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,511,440,15,23,22489,431292,27949,23,10,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,693,620,18,23,24860,491429,30056,23,9,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests-symflower-fix,635,560,18,23,22817,491200,27193,23,11,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests-symflower-fix,725,650,20,23,58605,761878,63588,23,10,22,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests-symflower-fix,734,660,19,23,24860,491742,30056,23,9,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests-symflower-fix,745,670,19,23,21666,410620,27265,23,10,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests-symflower-fix,797,720,21,23,22489,433981,27949,23,10,23,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/mistakes,code-repair,169,0,4,5,649,24824,724,5,5,5,150 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,13,10,1,1,296,13107,530,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,14,10,1,1,135,1904,150,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,14,10,1,1,293,1968,307,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,14,10,1,1,308,1394,322,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,14,10,1,1,358,5039,373,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,296,13107,530,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,135,1904,150,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,293,1968,307,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,308,1394,322,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,358,5039,373,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile,315,0,5,10,2896,40053,3003,10,10,10,280 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile,377,0,7,10,2934,44079,3043,10,10,10,340 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile,448,0,8,10,2858,44864,2966,10,10,10,410 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile,448,0,8,10,2972,71963,3079,10,10,10,410 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile,479,0,9,10,2914,97983,3023,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile-symflower-fix,448,0,8,10,2896,40938,3003,10,10,10,410 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile-symflower-fix,479,0,9,10,2914,97983,3023,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile-symflower-fix,499,0,9,10,2934,44844,3043,10,10,10,460 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2858,45937,2966,10,10,10,500 +openrouter/microsoft/wizardlm-2-8x22b,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2972,72706,3079,10,10,10,500 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,2535,2450,19,23,36433,657784,37569,23,20,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,2673,2590,20,23,39898,474236,42243,23,17,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,2766,2680,22,23,35992,473883,38642,23,18,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,2793,2710,21,23,38165,667437,41548,23,16,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,3025,2940,22,23,35882,475161,39511,23,17,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests-symflower-fix,2535,2450,19,23,36433,657784,37569,23,20,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests-symflower-fix,2673,2590,20,23,39898,474236,42243,23,17,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests-symflower-fix,2766,2680,22,23,35992,473883,38642,23,18,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests-symflower-fix,2793,2710,21,23,38165,667437,41548,23,16,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests-symflower-fix,3025,2940,22,23,35882,475161,39511,23,17,23,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/mistakes,code-repair,160,0,5,5,1295,19954,1360,5,5,5,140 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,13,10,1,1,317,12148,717,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,14,10,1,1,246,5875,259,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,14,10,1,1,287,4266,300,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,14,10,1,1,413,4943,426,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,14,10,1,1,477,3592,489,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests-symflower-fix,13,10,1,1,317,12148,717,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,246,5875,259,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,287,4266,300,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,413,4943,426,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests-symflower-fix,14,10,1,1,477,3592,489,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile,480,0,10,10,4059,53454,4188,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile,480,0,10,10,4063,49648,4193,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile,480,0,10,10,4074,53971,4202,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile,480,0,10,10,4096,44429,4225,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile,480,0,10,10,4195,79380,4323,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4059,53454,4188,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4063,49648,4193,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4074,53971,4202,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4096,44429,4225,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,java,java/transpile,transpile-symflower-fix,480,0,10,10,4195,79380,4323,10,10,10,440 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests,1011,950,12,23,27841,449275,36097,23,3,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests,1022,960,10,23,27977,406449,34627,23,6,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests,1104,1040,12,23,28957,480080,35763,23,6,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests,1105,1040,12,23,28301,437569,35585,23,7,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests,988,920,14,23,27652,446406,33376,23,8,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests-symflower-fix,1011,950,12,23,27841,449275,36097,23,3,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests-symflower-fix,1022,960,10,23,27977,406449,34627,23,6,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests-symflower-fix,1104,1040,12,23,28957,480080,35763,23,6,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests-symflower-fix,1105,1040,12,23,28301,437569,35585,23,7,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/light,write-tests-symflower-fix,988,920,14,23,27652,446406,33376,23,8,23,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/mistakes,code-repair,170,0,5,5,655,12597,720,5,5,5,150 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests,14,10,1,1,308,7699,321,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests,2,0,0,1,365,7701,1560,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests,2,0,0,1,540,18420,1470,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests,2,0,0,1,709,16793,1613,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests,3,0,0,1,447,6682,459,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,308,7699,321,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,365,7701,1560,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,540,18420,1470,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,709,16793,1613,1,0,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,447,6682,459,1,1,1,0 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile,128,0,8,10,3671,59019,3802,10,10,10,90 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile,137,0,7,10,3446,83222,3576,10,10,10,100 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile,178,0,8,10,3326,48056,3452,10,10,10,140 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile,189,0,9,10,3736,82519,3865,10,10,10,150 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile,48,0,8,10,3678,52241,3805,10,10,10,10 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile-symflower-fix,128,0,8,10,3671,59019,3802,10,10,10,90 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile-symflower-fix,137,0,7,10,3446,83222,3576,10,10,10,100 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile-symflower-fix,178,0,8,10,3326,48056,3452,10,10,10,140 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile-symflower-fix,189,0,9,10,3736,82519,3865,10,10,10,150 +openrouter/microsoft/wizardlm-2-8x22b,ruby,ruby/transpile,transpile-symflower-fix,48,0,8,10,3678,52241,3805,10,10,10,10 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests,603,530,15,23,16413,86225,22219,23,12,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests,610,530,16,23,16303,80294,18061,23,18,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests,721,640,17,23,14925,79892,17350,23,18,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests,749,670,18,23,14290,75120,17019,23,15,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests,751,670,18,23,15230,78466,17715,23,17,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests-symflower-fix,675,600,17,23,16413,86938,22219,23,12,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests-symflower-fix,721,640,17,23,14925,79892,17350,23,18,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests-symflower-fix,743,660,19,23,16303,81164,18061,23,18,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests-symflower-fix,782,700,19,23,15230,78964,17715,23,17,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/light,write-tests-symflower-fix,831,750,20,23,14290,75550,17019,23,15,23,0 +openrouter/mistralai/codestral-mamba,golang,golang/mistakes,code-repair,208,0,5,5,703,5404,1154,5,3,5,190 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests,13,10,1,1,74,1338,446,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests,13,10,1,1,77,1156,322,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests,13,10,1,1,79,1184,338,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests,14,10,1,1,194,856,208,1,1,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests,14,10,1,1,91,642,105,1,1,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,1338,446,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,77,1156,322,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,79,1184,338,1,0,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,194,856,208,1,1,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,91,642,105,1,1,1,0 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile,447,0,8,10,2743,18332,2888,10,9,10,410 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile,486,0,8,10,3057,21764,4164,10,8,10,450 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile,527,0,9,10,3176,25370,4044,10,8,10,490 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile,538,0,9,10,2924,17610,3078,10,9,10,500 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile,580,0,10,10,2910,22340,3026,10,10,10,540 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile-symflower-fix,508,0,9,10,2743,18784,2888,10,9,10,470 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile-symflower-fix,527,0,9,10,3057,22084,4164,10,8,10,490 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile-symflower-fix,527,0,9,10,3176,25370,4044,10,8,10,490 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile-symflower-fix,538,0,9,10,2924,17610,3078,10,9,10,500 +openrouter/mistralai/codestral-mamba,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2910,22340,3026,10,10,10,540 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests,1138,1070,10,23,23701,89782,28339,23,12,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests,1239,1170,9,23,19624,80589,24266,23,14,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests,1570,1500,12,23,19854,82161,24946,23,12,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests,1625,1550,13,23,19329,75495,23185,23,16,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests,837,770,7,23,18093,77260,22631,23,14,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests-symflower-fix,1138,1070,10,23,23701,89782,28339,23,12,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests-symflower-fix,1239,1170,9,23,19624,80589,24266,23,14,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests-symflower-fix,1570,1500,12,23,19854,82161,24946,23,12,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests-symflower-fix,1625,1550,13,23,19329,75495,23185,23,16,23,0 +openrouter/mistralai/codestral-mamba,java,java/light,write-tests-symflower-fix,837,770,7,23,18093,77260,22631,23,14,23,0 +openrouter/mistralai/codestral-mamba,java,java/mistakes,code-repair,157,0,5,5,1150,8932,2648,5,2,5,140 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests,13,10,1,1,418,1937,700,1,0,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests,14,10,1,1,232,993,244,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests,3,0,0,1,216,971,228,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests,3,0,0,1,306,1156,318,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests,3,0,0,1,317,1280,329,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests-symflower-fix,13,10,1,1,418,1937,700,1,0,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests-symflower-fix,14,10,1,1,232,993,244,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests-symflower-fix,3,0,0,1,216,971,228,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests-symflower-fix,3,0,0,1,306,1156,318,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/plain,write-tests-symflower-fix,3,0,0,1,317,1280,329,1,1,1,0 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile,467,0,10,10,3610,23648,4136,10,7,10,430 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile,476,0,10,10,3553,21204,4153,10,7,9,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile,477,0,10,10,3312,52962,4677,10,7,10,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile,478,0,10,10,3566,21634,4002,10,8,10,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile,479,0,10,10,3472,30549,4081,10,9,10,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile-symflower-fix,467,0,10,10,3610,23648,4136,10,7,10,430 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile-symflower-fix,476,0,10,10,3553,21204,4153,10,7,9,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile-symflower-fix,477,0,10,10,3312,52962,4677,10,7,10,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile-symflower-fix,478,0,10,10,3566,21634,4002,10,8,10,440 +openrouter/mistralai/codestral-mamba,java,java/transpile,transpile-symflower-fix,479,0,10,10,3472,30549,4081,10,9,10,440 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests,1199,1120,15,23,11555,57609,13449,23,18,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests,1364,1280,20,23,11921,64763,14748,23,18,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests,1663,1580,21,23,13154,69446,16070,23,16,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests,1689,1610,18,23,14030,70761,17876,23,15,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests,1931,1850,21,23,13331,70796,17891,23,14,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests-symflower-fix,1199,1120,15,23,11555,57609,13449,23,18,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests-symflower-fix,1364,1280,20,23,11921,64763,14748,23,18,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests-symflower-fix,1663,1580,21,23,13154,69446,16070,23,16,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests-symflower-fix,1689,1610,18,23,14030,70761,17876,23,15,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/light,write-tests-symflower-fix,1931,1850,21,23,13331,70796,17891,23,14,23,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/mistakes,code-repair,138,0,5,5,618,5973,1352,5,3,5,120 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests,14,10,1,1,158,949,170,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests,14,10,1,1,171,870,183,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests,14,10,1,1,176,980,188,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests,14,10,1,1,208,1981,220,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests,4,0,1,1,156,1690,168,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,158,949,170,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,171,870,183,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,176,980,188,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,208,1981,220,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,156,1690,168,1,1,1,0 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile,116,0,8,10,2843,18458,4276,10,8,10,80 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile,198,0,10,10,3054,17567,3247,10,8,10,160 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile,227,0,8,10,3444,19235,3994,10,9,10,190 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile,318,0,9,10,3169,20150,3731,10,9,10,280 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile,338,0,9,10,3182,19824,4344,10,9,10,300 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile-symflower-fix,116,0,8,10,2843,18458,4276,10,8,10,80 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile-symflower-fix,198,0,10,10,3054,17567,3247,10,8,10,160 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile-symflower-fix,227,0,8,10,3444,19235,3994,10,9,10,190 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile-symflower-fix,318,0,9,10,3169,20150,3731,10,9,10,280 +openrouter/mistralai/codestral-mamba,ruby,ruby/transpile,transpile-symflower-fix,338,0,9,10,3182,19824,4344,10,9,10,300 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,188,130,8,23,24492,183322,32235,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,207,150,7,23,21793,181508,29815,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,245,190,5,23,26860,209995,34457,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,265,210,7,23,23666,340685,31612,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,367,310,8,23,26338,217643,35254,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests-symflower-fix,321,260,11,23,24492,185183,32235,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests-symflower-fix,413,350,13,23,21793,183817,29815,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests-symflower-fix,462,400,13,23,26338,219234,35254,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests-symflower-fix,472,410,12,23,26860,212363,34457,23,4,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests-symflower-fix,603,540,15,23,23666,343431,31612,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/mistakes,code-repair,205,0,5,5,839,8513,2087,5,0,5,190 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,13,10,1,1,156,2849,516,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,13,10,1,1,79,1788,626,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,2,0,0,1,128,4583,706,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,3,0,0,1,691,5392,702,1,1,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,3,0,0,1,941,4108,951,1,1,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,156,2849,516,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,79,1788,626,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,128,4583,706,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,691,5392,702,1,1,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,941,4108,951,1,1,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile,178,0,3,10,3544,27181,6335,10,5,10,150 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile,180,0,4,10,3018,25768,4302,10,6,10,150 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile,210,0,4,10,3567,25951,6278,10,6,10,180 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile,339,0,7,10,3592,24986,6117,10,2,10,310 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile,371,0,7,10,3535,29177,6011,10,4,10,340 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile-symflower-fix,421,0,9,10,3592,25865,6117,10,2,10,390 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile-symflower-fix,425,0,9,10,3567,28220,6278,10,6,10,390 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile-symflower-fix,476,0,10,10,3018,27768,4302,10,6,10,440 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile-symflower-fix,524,0,10,10,3535,30332,6011,10,4,10,490 +openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/transpile,transpile-symflower-fix,535,0,10,10,3544,29799,6335,10,5,10,500 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,1079,1020,11,23,31306,192860,44551,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,1117,1060,11,23,35670,230056,46825,23,1,22,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,1158,1100,11,23,29784,175169,43478,23,1,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,1200,1140,11,23,28936,175269,39102,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,1248,1190,11,23,31009,199111,41199,23,1,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests-symflower-fix,1079,1020,11,23,31306,192860,44551,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests-symflower-fix,1117,1060,11,23,35670,230056,46825,23,1,22,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests-symflower-fix,1158,1100,11,23,29784,175169,43478,23,1,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests-symflower-fix,1200,1140,11,23,28936,175269,39102,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests-symflower-fix,1248,1190,11,23,31009,199111,41199,23,1,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/mistakes,code-repair,155,0,5,5,1136,12230,2543,5,0,5,140 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,13,10,1,1,237,1585,726,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,13,10,1,1,249,3488,1058,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,13,10,1,1,291,2031,453,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,13,10,1,1,339,2847,1030,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,2,0,0,1,228,3889,602,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests-symflower-fix,13,10,1,1,237,1585,726,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests-symflower-fix,13,10,1,1,249,3488,1058,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests-symflower-fix,13,10,1,1,291,2031,453,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests-symflower-fix,13,10,1,1,339,2847,1030,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests-symflower-fix,2,0,0,1,228,3889,602,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile,352,0,8,10,4418,28492,6505,10,4,10,320 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile,422,0,9,10,4387,27693,7453,10,3,10,390 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile,432,0,9,10,4253,27348,6634,10,3,10,400 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile,457,0,10,10,3978,24144,5475,10,7,10,420 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile,463,0,10,10,4791,38221,6646,10,3,10,430 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile-symflower-fix,352,0,8,10,4418,28492,6505,10,4,10,320 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile-symflower-fix,422,0,9,10,4387,27693,7453,10,3,10,390 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile-symflower-fix,432,0,9,10,4253,27348,6634,10,3,10,400 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile-symflower-fix,457,0,10,10,3978,24144,5475,10,7,10,420 +openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/transpile,transpile-symflower-fix,463,0,10,10,4791,38221,6646,10,3,10,430 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests,294,240,9,23,19062,114687,29124,23,2,20,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests,369,310,11,23,23285,140032,31946,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests,608,550,12,23,20198,115998,30818,23,0,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests,648,590,13,23,19605,133304,31439,23,0,22,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests,679,620,10,23,17483,116892,27592,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests-symflower-fix,294,240,9,23,19062,114687,29124,23,2,20,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests-symflower-fix,369,310,11,23,23285,140032,31946,23,2,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests-symflower-fix,608,550,12,23,20198,115998,30818,23,0,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests-symflower-fix,648,590,13,23,19605,133304,31439,23,0,22,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/light,write-tests-symflower-fix,679,620,10,23,17483,116892,27592,23,3,23,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/mistakes,code-repair,134,0,4,5,689,11373,1839,5,0,5,120 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests,13,10,1,1,278,3326,716,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests,2,0,0,1,131,2138,445,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests,2,0,0,1,138,4114,1100,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests,2,0,0,1,430,5385,763,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests,2,0,0,1,532,5712,902,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,278,3326,716,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,131,2138,445,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,138,4114,1100,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,430,5385,763,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,532,5712,902,1,0,1,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile,29,0,8,10,3135,27340,6475,10,1,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile,30,0,7,10,3142,24154,4799,10,3,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile,30,0,9,10,4042,31338,7233,10,1,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile,31,0,8,10,3055,26313,5074,10,3,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile,80,0,7,10,2990,34427,6039,10,3,10,50 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile-symflower-fix,29,0,8,10,3135,27340,6475,10,1,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile-symflower-fix,30,0,7,10,3142,24154,4799,10,3,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile-symflower-fix,30,0,9,10,4042,31338,7233,10,1,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile-symflower-fix,31,0,8,10,3055,26313,5074,10,3,10,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,ruby,ruby/transpile,transpile-symflower-fix,80,0,7,10,2990,34427,6039,10,3,10,50 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests,594,510,15,23,16321,267081,16619,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests,705,620,16,23,15144,238040,15450,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests,817,730,19,23,15955,243490,16301,23,22,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests,837,750,18,23,17820,280852,18123,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests,839,750,20,23,16980,278989,17262,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests-symflower-fix,777,690,18,23,15144,238716,15450,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests-symflower-fix,818,730,19,23,16321,268263,16619,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests-symflower-fix,879,790,20,23,17820,281532,18123,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests-symflower-fix,900,810,21,23,16980,279280,17262,23,23,23,0 +openrouter/mistralai/mistral-large,golang,golang/light,write-tests-symflower-fix,909,820,21,23,15955,244069,16301,23,22,23,0 +openrouter/mistralai/mistral-large,golang,golang/mistakes,code-repair,210,0,5,5,672,15465,734,5,5,5,190 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,14,10,1,1,74,1381,88,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,14,10,1,1,74,1482,88,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,14,10,1,1,79,2036,93,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,3,0,0,1,79,2260,89,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,3,0,0,1,99,1592,113,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1381,88,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1482,88,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,2036,93,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,99,1826,113,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,79,2260,89,1,1,1,0 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile,529,0,9,10,3061,31192,3169,10,10,10,490 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile,580,0,10,10,2814,43287,2922,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile,580,0,10,10,2897,37022,3001,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile,580,0,10,10,2957,28695,3061,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile,580,0,10,10,2977,37350,3081,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2814,43287,2922,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2897,37022,3001,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2957,28695,3061,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2977,37350,3081,10,10,10,540 +openrouter/mistralai/mistral-large,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3061,31560,3169,10,10,10,540 +openrouter/mistralai/mistral-large,java,java/light,write-tests,2781,2690,22,23,25442,193773,25718,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests,2921,2830,22,23,23816,341413,24092,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests,3002,2910,23,23,24793,228065,25069,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests,3092,3000,23,23,25527,226156,25803,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests,3092,3000,23,23,26710,324227,26986,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests-symflower-fix,2781,2690,22,23,25442,193773,25718,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests-symflower-fix,2921,2830,22,23,23816,341413,24092,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests-symflower-fix,3002,2910,23,23,24793,228065,25069,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests-symflower-fix,3092,3000,23,23,25527,226156,25803,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/light,write-tests-symflower-fix,3092,3000,23,23,26710,324227,26986,23,23,23,0 +openrouter/mistralai/mistral-large,java,java/mistakes,code-repair,160,0,5,5,1090,15946,1150,5,5,5,140 +openrouter/mistralai/mistral-large,java,java/plain,write-tests,14,10,1,1,144,3706,156,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests,14,10,1,1,188,2069,200,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests,14,10,1,1,258,2354,270,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests,14,10,1,1,289,3147,301,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests,14,10,1,1,306,3106,318,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests-symflower-fix,14,10,1,1,144,3706,156,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests-symflower-fix,14,10,1,1,188,2069,200,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests-symflower-fix,14,10,1,1,258,2354,270,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests-symflower-fix,14,10,1,1,289,3147,301,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/plain,write-tests-symflower-fix,14,10,1,1,306,3106,318,1,1,1,0 +openrouter/mistralai/mistral-large,java,java/transpile,transpile,439,0,9,10,3639,39635,3759,10,10,10,400 +openrouter/mistralai/mistral-large,java,java/transpile,transpile,480,0,10,10,3687,43415,3807,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile,480,0,10,10,3692,45268,3812,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile,480,0,10,10,3857,35848,3977,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile,480,0,10,10,3933,44895,4053,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile-symflower-fix,439,0,9,10,3639,39635,3759,10,10,10,400 +openrouter/mistralai/mistral-large,java,java/transpile,transpile-symflower-fix,480,0,10,10,3687,43415,3807,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile-symflower-fix,480,0,10,10,3692,45268,3812,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile-symflower-fix,480,0,10,10,3857,35848,3977,10,10,10,440 +openrouter/mistralai/mistral-large,java,java/transpile,transpile-symflower-fix,480,0,10,10,3933,44895,4053,10,10,10,440 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests,1706,1620,17,23,18496,163944,18772,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests,1708,1620,19,23,18827,161592,19103,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests,1938,1850,19,23,17831,167454,18107,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests,2180,2090,21,23,19412,167359,19688,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests,2229,2140,21,23,18694,147604,19036,23,22,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests-symflower-fix,1706,1620,17,23,18496,163944,18772,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests-symflower-fix,1708,1620,19,23,18827,161592,19103,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests-symflower-fix,1938,1850,19,23,17831,167454,18107,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests-symflower-fix,2180,2090,21,23,19412,167359,19688,23,23,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/light,write-tests-symflower-fix,2229,2140,21,23,18694,147604,19036,23,22,23,0 +openrouter/mistralai/mistral-large,ruby,ruby/mistakes,code-repair,170,0,5,5,620,7542,680,5,5,5,150 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests,14,10,1,1,147,1849,159,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests,14,10,1,1,234,2133,246,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests,14,10,1,1,240,3892,252,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests,14,10,1,1,241,2269,253,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests,14,10,1,1,317,2109,329,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,1849,159,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,234,2133,246,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,240,3892,252,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,241,2269,253,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,317,2109,329,1,1,1,0 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile,269,0,9,10,3142,30700,3262,10,10,10,230 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile,290,0,10,10,3299,29729,3419,10,10,10,250 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile,340,0,10,10,3128,28286,3248,10,10,10,300 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile,350,0,10,10,3161,29989,3281,10,10,10,310 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile,430,0,10,10,2998,36052,3118,10,10,10,390 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile-symflower-fix,269,0,9,10,3142,30700,3262,10,10,10,230 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile-symflower-fix,290,0,10,10,3299,29729,3419,10,10,10,250 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile-symflower-fix,340,0,10,10,3128,28286,3248,10,10,10,300 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,3161,29989,3281,10,10,10,310 +openrouter/mistralai/mistral-large,ruby,ruby/transpile,transpile-symflower-fix,430,0,10,10,2998,36052,3118,10,10,10,390 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,491,420,13,23,20708,269593,26912,23,12,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,535,460,13,23,19685,241741,23590,23,16,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,542,470,13,23,18944,267652,24721,23,13,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,606,530,15,23,20526,264497,23856,23,15,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,728,650,19,23,18239,235608,22825,23,13,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests-symflower-fix,696,620,17,23,18944,268803,24721,23,13,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests-symflower-fix,767,690,19,23,20708,272084,26912,23,12,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests-symflower-fix,800,720,19,23,20526,266082,23856,23,15,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests-symflower-fix,821,740,19,23,19685,243791,23590,23,16,23,0 +openrouter/mistralai/mistral-medium,golang,golang/light,write-tests-symflower-fix,871,790,22,23,18239,236445,22825,23,13,23,0 +openrouter/mistralai/mistral-medium,golang,golang/mistakes,code-repair,209,0,5,5,656,10116,876,5,4,5,190 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,13,10,1,1,74,2892,437,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,13,10,1,1,74,3624,531,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,13,10,1,1,74,4506,644,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,13,10,1,1,77,1925,273,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,2,0,0,1,145,3878,522,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,2892,437,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,3624,531,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,74,4506,644,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,77,1925,273,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,145,3878,522,1,0,1,0 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile,371,0,6,10,2870,106795,10507,10,5,10,340 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile,375,0,6,10,2756,36440,3274,10,9,10,340 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile,414,0,7,10,2930,46544,3994,10,7,10,380 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile,436,0,7,10,2858,36355,3495,10,9,10,400 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile,526,0,9,10,2754,40584,3962,10,7,10,490 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile-symflower-fix,473,0,8,10,2870,107413,10507,10,5,10,440 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile-symflower-fix,477,0,8,10,2756,37043,3274,10,9,10,440 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile-symflower-fix,538,0,9,10,2858,37480,3495,10,9,10,500 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile-symflower-fix,577,0,10,10,2754,40842,3962,10,7,10,540 +openrouter/mistralai/mistral-medium,golang,golang/transpile,transpile-symflower-fix,577,0,10,10,2930,47631,3994,10,7,10,540 +openrouter/mistralai/mistral-medium,java,java/light,write-tests,2149,2070,18,23,31541,288923,35750,23,15,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests,2268,2190,18,23,25350,240060,30568,23,14,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests,2437,2360,19,23,27867,286256,34167,23,12,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests,2685,2610,21,23,28766,283959,36381,23,8,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests,2958,2880,23,23,27621,360273,34335,23,9,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests-symflower-fix,2149,2070,18,23,31541,288923,35750,23,15,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests-symflower-fix,2268,2190,18,23,25350,240060,30568,23,14,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests-symflower-fix,2437,2360,19,23,27867,286256,34167,23,12,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests-symflower-fix,2685,2610,21,23,28766,283959,36381,23,8,23,0 +openrouter/mistralai/mistral-medium,java,java/light,write-tests-symflower-fix,2958,2880,23,23,27621,360273,34335,23,9,23,0 +openrouter/mistralai/mistral-medium,java,java/mistakes,code-repair,156,0,5,5,1212,18736,1829,5,1,5,140 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests,13,10,1,1,235,4619,685,1,0,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests,14,10,1,1,235,2153,247,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests,14,10,1,1,236,2182,248,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests,14,10,1,1,237,2206,249,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests,14,10,1,1,267,2295,279,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests-symflower-fix,13,10,1,1,235,4619,685,1,0,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests-symflower-fix,14,10,1,1,235,2153,247,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests-symflower-fix,14,10,1,1,236,2182,248,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests-symflower-fix,14,10,1,1,237,2206,249,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/plain,write-tests-symflower-fix,14,10,1,1,267,2295,279,1,1,1,0 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile,476,0,10,10,4099,58957,7083,10,6,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile,477,0,10,10,4077,50660,5754,10,7,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile,477,0,10,10,4108,50472,5417,10,7,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile,479,0,10,10,4146,45079,5374,10,9,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile,479,0,10,10,4150,44928,4395,10,9,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile-symflower-fix,476,0,10,10,4099,58957,7083,10,6,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile-symflower-fix,477,0,10,10,4077,50660,5754,10,7,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile-symflower-fix,477,0,10,10,4108,50472,5417,10,7,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile-symflower-fix,479,0,10,10,4146,45079,5374,10,9,10,440 +openrouter/mistralai/mistral-medium,java,java/transpile,transpile-symflower-fix,479,0,10,10,4150,44928,4395,10,9,10,440 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests,1488,1420,17,23,18236,234875,26534,23,5,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests,1734,1660,18,23,17250,219526,24574,23,10,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests,1794,1720,19,23,20101,240642,27735,23,9,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests,1861,1790,19,23,16857,224718,25652,23,6,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests,2203,2130,22,23,18787,223768,25215,23,5,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests-symflower-fix,1488,1420,17,23,18236,234875,26534,23,5,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests-symflower-fix,1734,1660,18,23,17250,219526,24574,23,10,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests-symflower-fix,1794,1720,19,23,20101,240642,27735,23,9,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests-symflower-fix,1861,1790,19,23,16857,224718,25652,23,6,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/light,write-tests-symflower-fix,2203,2130,22,23,18787,223768,25215,23,5,23,0 +openrouter/mistralai/mistral-medium,ruby,ruby/mistakes,code-repair,149,0,5,5,614,11297,1127,5,4,5,130 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests,13,10,1,1,225,5130,737,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests,13,10,1,1,225,5476,844,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests,13,10,1,1,251,6083,885,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests,2,0,0,1,285,6110,866,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests,2,0,0,1,460,5102,718,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,225,5130,737,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,225,5476,844,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,251,6083,885,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,285,6110,866,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,460,5102,718,1,0,1,0 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile,117,0,10,10,3586,44865,4796,10,7,10,80 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile,135,0,10,10,3397,47680,5454,10,5,10,100 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile,136,0,10,10,3434,52674,6083,10,6,10,100 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile,184,0,10,10,3622,50580,6005,10,4,10,150 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile,223,0,10,10,3255,51145,6424,10,3,10,190 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile-symflower-fix,117,0,10,10,3586,44865,4796,10,7,10,80 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile-symflower-fix,135,0,10,10,3397,47680,5454,10,5,10,100 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile-symflower-fix,136,0,10,10,3434,52674,6083,10,6,10,100 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile-symflower-fix,184,0,10,10,3622,50580,6005,10,4,10,150 +openrouter/mistralai/mistral-medium,ruby,ruby/transpile,transpile-symflower-fix,223,0,10,10,3255,51145,6424,10,3,10,190 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests,204,140,4,23,15298,136204,16597,23,17,20,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests,224,150,6,23,17475,249178,17959,23,22,23,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests,244,170,6,23,16646,125895,17626,23,22,23,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests,274,200,9,23,17098,116496,17682,23,20,22,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests,314,240,7,23,28800,196135,29027,23,22,22,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests-symflower-fix,266,200,6,23,15298,136885,16597,23,17,20,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests-symflower-fix,275,200,10,23,17098,116691,17682,23,20,22,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests-symflower-fix,316,240,8,23,17475,249533,17959,23,22,23,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests-symflower-fix,388,310,10,23,16646,127518,17626,23,22,23,0 +openrouter/mistralai/mistral-nemo,golang,golang/light,write-tests-symflower-fix,477,400,10,23,28800,197635,29027,23,22,22,0 +openrouter/mistralai/mistral-nemo,golang,golang/mistakes,code-repair,138,0,3,5,666,9594,716,5,5,5,120 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests,14,10,1,1,102,1453,110,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests,2,0,0,1,97,1000,192,1,0,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests,3,0,0,1,102,1365,112,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests,3,0,0,1,239,2190,249,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests,3,0,0,1,79,751,93,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,102,1453,110,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,97,1000,192,1,0,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,102,1365,112,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,239,2190,249,1,1,1,0 +openrouter/mistralai/mistral-nemo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,79,751,93,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests,1101,1030,8,23,20864,105970,21587,22,20,21,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests,1175,1100,9,23,21030,105721,21531,23,21,22,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests,1186,1110,9,23,19562,116310,20683,23,21,23,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests,1366,1290,11,23,31233,117480,32045,23,20,22,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests,1509,1430,11,23,21893,125816,22434,23,22,23,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests-symflower-fix,1101,1030,8,23,20864,105970,21587,22,20,21,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests-symflower-fix,1175,1100,9,23,21030,105721,21531,23,21,22,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests-symflower-fix,1186,1110,9,23,19562,116310,20683,23,21,23,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests-symflower-fix,1366,1290,11,23,31233,117480,32045,23,20,22,0 +openrouter/mistralai/mistral-nemo,java,java/light,write-tests-symflower-fix,1509,1430,11,23,21893,125816,22434,23,22,23,0 +openrouter/mistralai/mistral-nemo,java,java/mistakes,code-repair,139,0,4,5,996,7845,1056,5,5,5,120 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests,14,10,1,1,230,1964,246,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests,14,10,1,1,242,681,258,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests,14,10,1,1,341,1007,357,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests,3,0,0,1,268,2824,280,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests,3,0,0,1,289,1755,305,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests-symflower-fix,14,10,1,1,230,1964,246,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests-symflower-fix,14,10,1,1,242,681,258,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests-symflower-fix,14,10,1,1,341,1007,357,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests-symflower-fix,3,0,0,1,268,2824,280,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/plain,write-tests-symflower-fix,3,0,0,1,289,1755,305,1,1,1,0 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile,378,0,8,10,3781,26389,3897,10,10,10,340 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile,408,0,8,10,3506,23706,3618,10,10,10,370 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile,429,0,9,10,3409,22583,3509,10,10,10,390 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile,437,0,9,10,3696,28457,3791,10,9,9,400 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile,439,0,9,10,3415,26851,3527,10,10,10,400 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile-symflower-fix,378,0,8,10,3781,26389,3897,10,10,10,340 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile-symflower-fix,408,0,8,10,3506,23706,3618,10,10,10,370 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile-symflower-fix,429,0,9,10,3409,22583,3509,10,10,10,390 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile-symflower-fix,437,0,9,10,3696,28457,3791,10,9,9,400 +openrouter/mistralai/mistral-nemo,java,java/transpile,transpile-symflower-fix,439,0,9,10,3415,26851,3527,10,10,10,400 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests,1275,1190,19,23,14584,95095,15126,23,21,22,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests,1496,1410,20,23,20203,120034,21570,23,20,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests,1649,1570,19,23,17659,99725,19633,23,16,21,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests,1667,1580,19,23,15691,107583,16094,23,22,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests,1716,1630,20,23,16962,125382,17679,23,20,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests-symflower-fix,1275,1190,19,23,14584,95095,15126,23,21,22,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests-symflower-fix,1496,1410,20,23,20203,120034,21570,23,20,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests-symflower-fix,1649,1570,19,23,17659,99725,19633,23,16,21,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests-symflower-fix,1667,1580,19,23,15691,107583,16094,23,22,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/light,write-tests-symflower-fix,1716,1630,20,23,16962,125382,17679,23,20,23,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/mistakes,code-repair,170,0,5,5,539,7921,587,5,5,5,150 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests,13,10,1,1,141,1843,295,1,0,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests,13,10,1,1,193,2520,391,1,0,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests,14,10,1,1,172,928,180,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests,3,0,0,1,283,2339,291,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests,4,0,1,1,118,1652,126,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,141,1843,295,1,0,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,193,2520,391,1,0,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,172,928,180,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,283,2339,291,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,118,1652,126,1,1,1,0 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile,167,0,7,10,2669,20998,2785,10,10,10,130 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile,186,0,7,10,3076,31121,3529,10,9,10,150 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile,218,0,9,10,2280,24030,3033,10,9,10,180 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile,257,0,7,10,2820,22226,2928,10,10,10,220 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile,276,0,7,10,2914,32088,3617,10,9,10,240 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile-symflower-fix,167,0,7,10,2669,20998,2785,10,10,10,130 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile-symflower-fix,186,0,7,10,3076,31121,3529,10,9,10,150 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile-symflower-fix,218,0,9,10,2280,24030,3033,10,9,10,180 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile-symflower-fix,257,0,7,10,2820,22226,2928,10,10,10,220 +openrouter/mistralai/mistral-nemo,ruby,ruby/transpile,transpile-symflower-fix,276,0,7,10,2914,32088,3617,10,9,10,240 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests,267,190,8,23,16406,136006,16675,23,23,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests,335,260,8,23,19917,157224,20399,23,21,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests,347,270,8,23,17178,138780,17447,23,23,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests,375,300,9,23,17313,144753,18047,23,20,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests,399,320,11,23,16212,131289,16541,23,22,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests-symflower-fix,298,220,9,23,16406,136193,16675,23,23,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests-symflower-fix,398,320,9,23,17178,139003,17447,23,23,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests-symflower-fix,542,460,14,23,16212,132311,16541,23,22,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests-symflower-fix,591,510,14,23,19917,159392,20399,23,21,23,0 +openrouter/mistralai/mistral-small,golang,golang/light,write-tests-symflower-fix,599,520,13,23,17313,146256,18047,23,20,23,0 +openrouter/mistralai/mistral-small,golang,golang/mistakes,code-repair,210,0,5,5,652,6235,714,5,5,5,190 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,14,10,1,1,119,949,129,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,14,10,1,1,74,797,88,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,3,0,0,1,118,1088,128,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,3,0,0,1,139,2115,149,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,3,0,0,1,168,1495,182,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,118,1699,128,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,119,949,129,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,797,88,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,139,2115,149,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,168,1495,182,1,1,1,0 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile,438,0,8,10,3277,31385,3379,10,10,10,400 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile,478,0,8,10,2971,47455,3075,10,10,10,440 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile,499,0,9,10,2863,26619,2975,10,10,10,460 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile,560,0,10,10,2821,31729,2929,10,10,10,520 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile,560,0,10,10,3005,33098,3109,10,10,10,520 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile-symflower-fix,499,0,9,10,3277,31826,3379,10,10,10,460 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile-symflower-fix,539,0,9,10,2971,47790,3075,10,10,10,500 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2821,31729,2929,10,10,10,520 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2863,26802,2975,10,10,10,520 +openrouter/mistralai/mistral-small,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,3005,33098,3109,10,10,10,520 +openrouter/mistralai/mistral-small,java,java/light,write-tests,2236,2150,18,23,21717,130964,22175,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests,2528,2440,20,23,24254,147877,24679,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests,2699,2610,21,23,24565,295191,25337,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests,2759,2670,21,23,23184,137757,23606,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests,2848,2760,21,23,25239,147180,25712,23,21,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests-symflower-fix,2236,2150,18,23,21717,130964,22175,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests-symflower-fix,2528,2440,20,23,24254,147877,24679,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests-symflower-fix,2699,2610,21,23,24565,295191,25337,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests-symflower-fix,2759,2670,21,23,23184,137757,23606,23,22,23,0 +openrouter/mistralai/mistral-small,java,java/light,write-tests-symflower-fix,2848,2760,21,23,25239,147180,25712,23,21,23,0 +openrouter/mistralai/mistral-small,java,java/mistakes,code-repair,160,0,5,5,1245,9430,1305,5,5,5,140 +openrouter/mistralai/mistral-small,java,java/plain,write-tests,14,10,1,1,187,1371,199,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests,14,10,1,1,187,1385,199,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests,14,10,1,1,230,1609,242,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests,14,10,1,1,277,2158,289,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests,14,10,1,1,299,1808,311,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1371,199,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1385,199,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests-symflower-fix,14,10,1,1,230,1609,242,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests-symflower-fix,14,10,1,1,277,2158,289,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/plain,write-tests-symflower-fix,14,10,1,1,299,1808,311,1,1,1,0 +openrouter/mistralai/mistral-small,java,java/transpile,transpile,479,0,10,10,3970,32499,4114,10,9,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile,479,0,10,10,4006,35844,4196,10,9,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile,480,0,10,10,3813,31670,3933,10,10,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile,480,0,10,10,3946,31599,4066,10,10,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile,480,0,10,10,4070,35843,4190,10,10,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile-symflower-fix,479,0,10,10,3970,32499,4114,10,9,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile-symflower-fix,479,0,10,10,4006,35844,4196,10,9,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile-symflower-fix,480,0,10,10,3813,31670,3933,10,10,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile-symflower-fix,480,0,10,10,3946,31599,4066,10,10,10,440 +openrouter/mistralai/mistral-small,java,java/transpile,transpile-symflower-fix,480,0,10,10,4070,35843,4190,10,10,10,440 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests,3,0,0,1,129,1131,141,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests,3,0,0,1,132,1527,144,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests,3,0,0,1,161,1310,173,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests,3,0,0,1,162,1379,174,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests,3,0,0,1,197,1376,209,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,129,1131,141,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,132,1527,144,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,161,1310,173,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,162,1379,174,1,1,1,0 +openrouter/mistralai/mistral-small,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,197,1376,209,1,1,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,131,70,2,23,21857,173892,25600,23,13,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,139,80,2,23,19243,110592,23536,23,11,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,158,100,2,23,18462,100434,23787,23,10,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,182,120,3,23,20680,109218,23806,23,13,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,221,160,5,23,18843,114888,25818,23,10,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests-symflower-fix,369,300,10,23,21857,175758,25600,23,13,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests-symflower-fix,448,380,12,23,18843,117063,25818,23,10,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests-symflower-fix,449,380,13,23,18462,104031,23787,23,10,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests-symflower-fix,509,440,12,23,19243,113957,23536,23,11,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests-symflower-fix,523,450,14,23,20680,113608,23806,23,13,23,0 +openrouter/mistralai/mistral-tiny,golang,golang/mistakes,code-repair,209,0,5,5,961,5803,1299,5,4,5,190 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,13,10,1,1,340,1829,613,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,2,0,0,1,132,1789,630,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,2,0,0,1,214,2082,477,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,2,0,0,1,279,2356,760,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,3,0,0,1,181,807,191,1,1,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,340,1829,613,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,132,1789,630,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,214,2082,477,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,279,2356,760,1,0,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,181,807,191,1,1,1,0 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile,178,0,3,10,3070,69183,5631,10,5,10,150 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile,240,0,4,10,3000,29569,4512,10,6,10,210 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile,262,0,5,10,3166,19680,4241,10,7,10,230 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile,332,0,6,10,3795,22628,4912,10,6,10,300 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile,333,0,6,10,3170,20603,4377,10,7,10,300 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile-symflower-fix,495,0,9,10,3795,23577,4912,10,6,10,460 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile-symflower-fix,527,0,10,10,3170,21600,4377,10,7,10,490 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile-symflower-fix,537,0,10,10,3166,21372,4241,10,7,10,500 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile-symflower-fix,555,0,10,10,3070,71559,5631,10,5,10,520 +openrouter/mistralai/mistral-tiny,golang,golang/transpile,transpile-symflower-fix,556,0,10,10,3000,31507,4512,10,6,10,520 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests,536,480,4,23,34912,121105,42232,23,6,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests,576,520,5,23,31197,116586,39260,23,5,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests,699,640,6,23,38964,135522,45581,23,7,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests,989,930,7,23,39691,191692,48200,23,6,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests,998,940,9,23,28536,119043,36120,23,3,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests-symflower-fix,536,480,4,23,34912,121105,42232,23,6,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests-symflower-fix,576,520,5,23,31197,116586,39260,23,5,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests-symflower-fix,699,640,6,23,38964,135522,45581,23,7,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests-symflower-fix,989,930,7,23,39691,191692,48200,23,6,23,0 +openrouter/mistralai/mistral-tiny,java,java/light,write-tests-symflower-fix,998,940,9,23,28536,119043,36120,23,3,23,0 +openrouter/mistralai/mistral-tiny,java,java/mistakes,code-repair,156,0,5,5,1228,7502,2129,5,1,5,140 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,13,10,1,1,250,2031,710,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,2,0,0,1,1077,4500,1701,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,2,0,0,1,2447,7854,2899,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,2,0,0,1,350,3378,1129,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,2,0,0,1,946,3481,1184,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests-symflower-fix,13,10,1,1,250,2031,710,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests-symflower-fix,2,0,0,1,1077,4500,1701,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests-symflower-fix,2,0,0,1,2447,7854,2899,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests-symflower-fix,2,0,0,1,350,3378,1129,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/plain,write-tests-symflower-fix,2,0,0,1,946,3481,1184,1,0,1,0 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile,387,0,9,10,4358,19973,4595,10,8,10,350 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile,436,0,9,10,4190,18360,4976,10,7,10,400 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile,470,0,10,10,4181,19514,4301,10,10,10,430 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile,478,0,10,10,4260,23717,4493,10,8,10,440 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile,480,0,10,10,4078,21009,4198,10,10,10,440 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile-symflower-fix,387,0,9,10,4358,19973,4595,10,8,10,350 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile-symflower-fix,436,0,9,10,4190,18360,4976,10,7,10,400 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile-symflower-fix,470,0,10,10,4181,19514,4301,10,10,10,430 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile-symflower-fix,478,0,10,10,4260,23717,4493,10,8,10,440 +openrouter/mistralai/mistral-tiny,java,java/transpile,transpile-symflower-fix,480,0,10,10,4078,21009,4198,10,10,10,440 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests,2,0,0,1,168,4652,1468,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests,2,0,0,1,214,2028,611,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests,2,0,0,1,220,1748,545,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests,2,0,0,1,234,3453,1063,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests,2,0,0,1,296,2404,831,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,168,4652,1468,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,214,2028,611,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,220,1748,545,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,234,3453,1063,1,0,1,0 +openrouter/mistralai/mistral-tiny,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,296,2404,831,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests,525,450,14,23,18036,196873,21829,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests,562,480,14,23,22094,299151,22704,23,22,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests,576,500,15,23,19083,165903,22106,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests,726,640,18,23,19396,208568,19979,23,22,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests,880,800,21,23,18009,201286,21732,23,13,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests-symflower-fix,727,640,19,23,19396,208810,19979,23,22,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests-symflower-fix,749,670,18,23,18036,198397,21829,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests-symflower-fix,776,690,18,23,22094,300353,22704,23,22,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests-symflower-fix,831,750,20,23,19083,167096,22106,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/light,write-tests-symflower-fix,921,840,22,23,18009,201614,21732,23,13,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/mistakes,code-repair,177,0,4,5,643,10295,1364,5,3,5,160 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,13,10,1,1,211,3157,845,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,13,10,1,1,214,3731,878,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,14,10,1,1,228,2655,243,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,14,10,1,1,260,2875,275,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,2,0,0,1,340,5211,1103,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,211,3157,845,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,214,3731,878,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,228,2655,243,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,260,2875,275,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,340,5211,1103,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile,448,0,8,10,2933,21547,3054,10,10,10,410 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile,468,0,8,10,2954,26346,3075,10,10,10,430 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile,468,0,9,10,2965,28040,3333,10,9,10,430 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile,527,0,9,10,2917,22997,3103,10,8,10,490 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile,528,0,9,10,2987,20129,3317,10,9,10,490 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile-symflower-fix,519,0,10,10,2965,28276,3333,10,9,10,480 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile-symflower-fix,519,0,9,10,2954,26626,3075,10,10,10,480 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2933,22041,3054,10,10,10,520 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile-symflower-fix,578,0,10,10,2917,23253,3103,10,8,10,540 +openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/transpile,transpile-symflower-fix,579,0,10,10,2987,20381,3317,10,9,10,540 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,1816,1740,16,23,22728,160076,26240,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,2642,2560,20,23,26567,206101,29853,23,16,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,2696,2610,21,23,22381,181860,25108,23,19,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,2932,2850,22,23,22816,179048,27014,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,2994,2910,23,23,24024,202610,26540,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests-symflower-fix,1816,1740,16,23,22728,160076,26240,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests-symflower-fix,2642,2560,20,23,26567,206101,29853,23,16,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests-symflower-fix,2696,2610,21,23,22381,181860,25108,23,19,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests-symflower-fix,2932,2850,22,23,22816,179048,27014,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests-symflower-fix,2994,2910,23,23,24024,202610,26540,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/mistakes,code-repair,158,0,5,5,1149,11742,1538,5,3,5,140 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,13,10,1,1,187,2031,441,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,13,10,1,1,231,2290,412,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,13,10,1,1,380,2937,600,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,2,0,0,1,461,6272,992,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,3,0,0,1,487,2267,499,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests-symflower-fix,13,10,1,1,187,2031,441,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests-symflower-fix,13,10,1,1,231,2290,412,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests-symflower-fix,13,10,1,1,380,2937,600,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests-symflower-fix,2,0,0,1,461,6272,992,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,487,2267,499,1,1,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile,439,0,9,10,4039,26296,4168,10,10,10,400 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile,439,0,9,10,4100,26982,4230,10,10,10,400 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile,480,0,10,10,3626,29850,3756,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile,480,0,10,10,3647,29045,3776,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile,480,0,10,10,3869,26932,3999,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile-symflower-fix,439,0,9,10,4039,26296,4168,10,10,10,400 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile-symflower-fix,439,0,9,10,4100,26982,4230,10,10,10,400 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3626,29850,3756,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3647,29045,3776,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3869,26932,3999,10,10,10,440 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests,1292,1220,13,23,17752,159696,20484,23,14,22,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests,1362,1290,14,23,15901,132965,23896,23,12,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests,1395,1320,14,23,17429,145734,21414,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests,1439,1370,15,23,14100,106315,19204,23,8,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests,1535,1460,15,23,17812,155728,21287,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests-symflower-fix,1292,1220,13,23,17752,159696,20484,23,14,22,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests-symflower-fix,1362,1290,14,23,15901,132965,23896,23,12,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests-symflower-fix,1395,1320,14,23,17429,145734,21414,23,15,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests-symflower-fix,1439,1370,15,23,14100,106315,19204,23,8,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/light,write-tests-symflower-fix,1535,1460,15,23,17812,155728,21287,23,14,23,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/mistakes,code-repair,168,0,5,5,582,7714,1085,5,3,5,150 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests,13,10,1,1,226,3852,476,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests,13,10,1,1,290,2750,531,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests,2,0,0,1,220,4703,998,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests,2,0,0,1,225,2153,507,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests,2,0,0,1,291,6825,987,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,226,3852,476,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,290,2750,531,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,220,4703,998,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,225,2153,507,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,291,6825,987,1,0,1,0 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile,117,0,9,10,3188,29271,3847,10,8,10,80 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile,119,0,9,10,3070,28989,3198,10,10,10,80 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile,129,0,9,10,3077,24207,3206,10,10,10,90 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile,89,0,9,10,3281,34001,3410,10,10,10,50 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile,90,0,10,10,3366,29194,3495,10,10,10,50 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile-symflower-fix,117,0,9,10,3188,29271,3847,10,8,10,80 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile-symflower-fix,119,0,9,10,3070,28989,3198,10,10,10,80 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile-symflower-fix,129,0,9,10,3077,24207,3206,10,10,10,90 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile-symflower-fix,89,0,9,10,3281,34001,3410,10,10,10,50 +openrouter/mistralai/mixtral-8x22b-instruct,ruby,ruby/transpile,transpile-symflower-fix,90,0,10,10,3366,29194,3495,10,10,10,50 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,333,260,9,23,24552,185205,25735,23,20,21,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,381,310,11,23,21194,173002,22139,23,17,20,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,423,350,11,23,25509,220501,26120,23,19,20,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,478,400,12,23,22134,222958,22968,23,21,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,579,500,15,23,23798,150379,24737,23,19,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests-symflower-fix,478,400,14,23,24552,186435,25735,23,20,21,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests-symflower-fix,544,470,14,23,21194,174344,22139,23,17,20,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests-symflower-fix,558,480,16,23,25509,222121,26120,23,19,20,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests-symflower-fix,632,550,16,23,22134,225445,22968,23,21,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests-symflower-fix,640,560,16,23,23798,150732,24737,23,19,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/mistakes,code-repair,123,0,3,5,618,10121,1477,5,1,4,110 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,14,10,1,1,173,8754,183,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,14,10,1,1,198,1104,209,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,14,10,1,1,307,1981,318,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,3,0,0,1,340,1724,351,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,4,0,1,1,392,3349,403,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,173,8754,183,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,198,1104,209,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,307,1981,318,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,340,1724,351,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests-symflower-fix,4,0,1,1,392,3349,403,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile,289,0,5,10,2966,19554,3062,10,7,7,260 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile,352,0,6,10,2825,23377,2923,10,8,8,320 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile,402,0,7,10,2865,26961,3002,10,7,8,370 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile,508,0,9,10,2960,23295,3125,10,9,10,470 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile,529,0,9,10,2937,28032,3045,10,10,10,490 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile-symflower-fix,391,0,7,10,2966,20493,3062,10,7,7,360 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile-symflower-fix,414,0,8,10,2825,24240,2923,10,8,8,380 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile-symflower-fix,453,0,8,10,2865,27400,3002,10,7,8,420 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile-symflower-fix,559,0,10,10,2960,23611,3125,10,9,10,520 +openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2937,28471,3045,10,10,10,540 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,2256,2170,19,23,31827,198345,32113,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,2526,2440,20,23,34049,201036,35575,23,20,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,2544,2460,19,23,30708,159770,31226,23,20,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,2646,2560,21,23,33029,144532,34526,23,20,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,2737,2650,20,23,28546,279072,28836,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests-symflower-fix,2256,2170,19,23,31827,198345,32113,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests-symflower-fix,2526,2440,20,23,34049,201036,35575,23,20,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests-symflower-fix,2544,2460,19,23,30708,159770,31226,23,20,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests-symflower-fix,2646,2560,21,23,33029,144532,34526,23,20,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests-symflower-fix,2737,2650,20,23,28546,279072,28836,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/mistakes,code-repair,156,0,5,5,1107,17114,2381,5,1,5,140 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,1,0,0,1,288,1899,293,1,0,0,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,14,10,1,1,306,2019,319,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,14,10,1,1,335,2230,348,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,14,10,1,1,393,3277,406,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,3,0,0,1,459,36010,472,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests-symflower-fix,1,0,0,1,288,1899,293,1,0,0,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,306,2019,319,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,335,2230,348,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,393,3277,406,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,459,36010,472,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile,431,0,9,10,4257,38203,5093,10,2,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile,432,0,9,10,3973,32528,4482,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile,432,0,9,10,4080,31374,4641,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile,432,0,9,10,4102,32443,4821,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile,434,0,9,10,3956,24081,4316,10,5,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile-symflower-fix,431,0,9,10,4257,38203,5093,10,2,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile-symflower-fix,432,0,9,10,3973,32528,4482,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile-symflower-fix,432,0,9,10,4080,31374,4641,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile-symflower-fix,432,0,9,10,4102,32443,4821,10,3,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,java,java/transpile,transpile-symflower-fix,434,0,9,10,3956,24081,4316,10,5,10,400 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests,1327,1270,12,23,14864,106459,15422,16,14,15,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests,1340,1260,16,23,19896,164545,21705,23,19,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests,1463,1380,16,23,19014,126479,19303,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests,1848,1760,20,23,23446,235901,24291,23,22,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests,2129,2040,21,23,19698,143044,20305,23,22,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests-symflower-fix,1327,1270,12,23,14864,106459,15422,16,14,15,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests-symflower-fix,1340,1260,16,23,19896,164545,21705,23,19,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests-symflower-fix,1463,1380,16,23,19014,126479,19303,23,22,22,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests-symflower-fix,1848,1760,20,23,23446,235901,24291,23,22,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/light,write-tests-symflower-fix,2129,2040,21,23,19698,143044,20305,23,22,23,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/mistakes,code-repair,133,0,4,5,609,12832,1837,5,0,4,120 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,150,2096,163,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,150,2632,163,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,138,2280,150,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,240,3091,253,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,362,2068,375,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,150,2096,163,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,150,2632,163,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,138,2280,150,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,240,3091,253,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,362,2068,375,1,1,1,0 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile,136,0,8,10,3106,27702,3227,10,9,9,100 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile,138,0,8,10,3126,22491,3254,10,10,10,100 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile,170,0,6,10,2764,24716,2870,10,7,7,140 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile,188,0,8,10,3244,26261,3372,10,10,10,150 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile,70,0,4,10,1919,17896,1987,6,5,5,50 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,136,0,8,10,3106,27702,3227,10,9,9,100 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,138,0,8,10,3126,22491,3254,10,10,10,100 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,170,0,6,10,2764,24716,2870,10,7,7,140 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,188,0,8,10,3244,26261,3372,10,10,10,150 +openrouter/mistralai/mixtral-8x7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,70,0,4,10,1919,17896,1987,6,5,5,50 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,330,260,8,23,16615,442753,17702,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,401,330,11,23,14936,328774,15187,23,18,19,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,444,380,11,23,15045,321413,15246,23,15,15,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,485,410,12,23,14215,319795,14465,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,490,420,11,23,14894,310915,15123,23,18,18,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests-symflower-fix,401,330,11,23,14936,328774,15187,23,18,19,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests-symflower-fix,465,400,12,23,15045,321866,15246,23,15,15,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests-symflower-fix,503,430,11,23,16615,443432,17702,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests-symflower-fix,526,450,13,23,14215,320041,14465,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests-symflower-fix,583,510,14,23,14894,312098,15123,23,18,18,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/mistakes,code-repair,175,0,4,5,674,18903,716,5,3,3,160 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,1,0,0,1,301,5177,304,1,0,0,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,14,10,1,1,79,3041,89,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,3,0,0,1,102,2284,112,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,3,0,0,1,127,3058,137,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,3,0,0,1,360,6491,374,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,301,5177,304,1,0,0,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,102,2476,112,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,127,3252,137,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,3041,89,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,360,6491,374,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile,230,0,4,10,3402,81676,3447,10,3,3,210 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile,318,0,6,10,3788,92002,3802,10,1,1,300 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile,338,0,6,10,2891,71765,2913,10,1,1,320 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile,363,0,7,10,2947,65034,2985,10,3,3,340 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile,452,0,8,10,3058,66750,3086,10,2,2,430 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile-symflower-fix,440,0,8,10,3788,92660,3802,10,1,1,420 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile-symflower-fix,454,0,8,10,3402,82605,3447,10,3,3,430 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile-symflower-fix,465,0,9,10,2947,65700,2985,10,3,3,440 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile-symflower-fix,501,0,9,10,2891,72751,2913,10,1,1,480 +openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/transpile,transpile-symflower-fix,534,0,10,10,3058,67273,3086,10,2,2,510 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,1122,1050,9,23,23710,342169,23960,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,1217,1140,12,23,19757,285398,20009,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,1592,1510,13,23,20610,327763,20886,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,1713,1630,16,23,20929,347699,21197,23,22,22,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,963,890,8,23,20237,309638,20492,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests-symflower-fix,1122,1050,9,23,23710,342169,23960,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests-symflower-fix,1217,1140,12,23,19757,285398,20009,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests-symflower-fix,1592,1510,13,23,20610,327763,20886,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests-symflower-fix,1713,1630,16,23,20929,347699,21197,23,22,22,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests-symflower-fix,963,890,8,23,20237,309638,20492,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/mistakes,code-repair,137,0,4,5,2054,36699,2102,5,4,4,120 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,14,10,1,1,267,3712,279,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,3,0,0,1,121,2694,133,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,3,0,0,1,224,3053,236,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,3,0,0,1,301,4475,313,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,3,0,0,1,384,7147,396,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests-symflower-fix,14,10,1,1,267,3712,279,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests-symflower-fix,3,0,0,1,121,2694,133,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests-symflower-fix,3,0,0,1,224,3053,236,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests-symflower-fix,3,0,0,1,301,4475,313,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests-symflower-fix,3,0,0,1,384,7147,396,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile,356,0,8,10,4278,72887,4333,10,4,4,330 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile,364,0,8,10,4309,84691,4355,10,3,3,340 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile,370,0,8,10,4516,77221,4594,10,6,6,340 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile,416,0,9,10,4508,74947,4830,10,3,4,390 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile,429,0,9,10,4036,71153,4096,10,5,5,400 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile-symflower-fix,356,0,8,10,4278,72887,4333,10,4,4,330 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile-symflower-fix,364,0,8,10,4309,84691,4355,10,3,3,340 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile-symflower-fix,370,0,8,10,4516,77221,4594,10,6,6,340 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile-symflower-fix,416,0,9,10,4508,74947,4830,10,3,4,390 +openrouter/neversleep/llama-3-lumimaid-70b,java,java/transpile,transpile-symflower-fix,429,0,9,10,4036,71153,4096,10,5,5,400 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests,1091,1030,10,23,13199,260995,13379,23,14,14,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests,1579,1500,16,23,13990,264406,14233,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests,604,550,9,23,14045,251813,14192,23,11,11,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests,616,550,9,23,13628,269984,13835,23,17,17,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests,881,820,12,23,14768,265959,14933,23,13,13,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests-symflower-fix,1091,1030,10,23,13199,260995,13379,23,14,14,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests-symflower-fix,1579,1500,16,23,13990,264406,14233,23,20,20,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests-symflower-fix,604,550,9,23,14045,251813,14192,23,11,11,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests-symflower-fix,616,550,9,23,13628,269984,13835,23,17,17,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/light,write-tests-symflower-fix,881,820,12,23,14768,265959,14933,23,13,13,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/mistakes,code-repair,110,0,5,5,645,22443,705,5,5,5,90 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests,14,10,1,1,154,2992,166,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests,14,10,1,1,162,4063,174,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests,14,10,1,1,170,5826,182,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests,14,10,1,1,171,3590,183,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests,4,0,1,1,136,4101,148,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,154,2992,166,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,162,4063,174,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,170,5826,182,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,171,3590,183,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,136,4101,148,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile,126,0,8,10,3375,81657,3427,10,4,4,100 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile,136,0,8,10,3023,64691,3075,10,4,4,110 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile,193,0,6,10,2846,62950,2930,10,3,4,170 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile,207,0,7,10,3180,76330,3243,10,5,5,180 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile,238,0,8,10,2984,68857,3044,10,5,5,210 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile-symflower-fix,126,0,8,10,3375,81657,3427,10,4,4,100 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile-symflower-fix,136,0,8,10,3023,64691,3075,10,4,4,110 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile-symflower-fix,193,0,6,10,2846,62950,2930,10,3,4,170 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile-symflower-fix,207,0,7,10,3180,76330,3243,10,5,5,180 +openrouter/neversleep/llama-3-lumimaid-70b,ruby,ruby/transpile,transpile-symflower-fix,238,0,8,10,2984,68857,3044,10,5,5,210 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,1,0,0,1,217,4669,217,1,0,0,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,3,0,0,1,136,3127,146,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,3,0,0,1,288,1201,298,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,3,0,0,1,559,1884,570,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,3,0,0,1,85,2891,95,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,217,4669,217,1,0,0,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,136,3127,146,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,288,1201,298,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,559,1884,570,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,85,2891,95,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,1405,1330,10,23,23039,177315,23298,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,1574,1500,12,23,25913,232820,26880,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,297,230,2,23,24868,157784,25122,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,355,290,3,23,23858,144401,24362,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,695,630,7,23,25123,151969,26057,23,17,18,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests-symflower-fix,1405,1330,10,23,23039,177315,23298,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests-symflower-fix,1574,1500,12,23,25913,232820,26880,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests-symflower-fix,297,230,2,23,24868,157784,25122,23,21,21,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests-symflower-fix,355,290,3,23,23858,144401,24362,23,19,20,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests-symflower-fix,695,630,7,23,25123,151969,26057,23,17,18,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/mistakes,code-repair,138,0,4,5,991,18847,1295,5,4,5,120 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,3,0,0,1,211,6187,223,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,3,0,0,1,330,1184,342,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,3,0,0,1,424,1277,432,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,3,0,0,1,807,5773,819,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,4,0,1,1,255,3753,267,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,211,6187,223,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,330,1184,342,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,424,1277,432,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,807,5773,819,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests-symflower-fix,4,0,1,1,255,3753,267,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile,373,0,8,10,3770,19271,4079,10,7,8,340 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile,376,0,8,10,3910,25142,4017,10,9,9,340 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile,396,0,8,10,3982,34198,4089,10,9,9,360 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile,407,0,9,10,4038,35647,4145,10,9,9,370 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile,467,0,10,10,3780,29128,3965,10,8,9,430 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile-symflower-fix,373,0,8,10,3770,19271,4079,10,7,8,340 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile-symflower-fix,376,0,8,10,3910,25142,4017,10,9,9,340 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile-symflower-fix,396,0,8,10,3982,34198,4089,10,9,9,360 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile-symflower-fix,407,0,9,10,4038,35647,4145,10,9,9,370 +openrouter/neversleep/llama-3-lumimaid-8b,java,java/transpile,transpile-symflower-fix,467,0,10,10,3780,29128,3965,10,8,9,430 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests,180,110,4,23,25542,181839,26942,23,21,22,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests,281,210,6,23,19798,136103,27403,23,19,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests,554,480,5,23,19170,115276,19467,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests,661,580,12,23,18638,117295,18902,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests,726,650,9,23,19744,103579,20093,23,21,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests-symflower-fix,180,110,4,23,25542,181839,26942,23,21,22,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests-symflower-fix,281,210,6,23,19798,136103,27403,23,19,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests-symflower-fix,554,480,5,23,19170,115276,19467,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests-symflower-fix,661,580,12,23,18638,117295,18902,23,23,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/light,write-tests-symflower-fix,726,650,9,23,19744,103579,20093,23,21,23,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/mistakes,code-repair,109,0,4,5,669,9557,729,5,5,5,90 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests,13,10,1,1,146,3315,321,1,0,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests,3,0,0,1,189,2605,201,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests,3,0,0,1,211,1332,223,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests,3,0,0,1,327,1164,339,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests,3,0,0,1,334,1215,346,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,146,3315,321,1,0,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,189,2605,201,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,211,1332,223,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,327,1164,339,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,334,1215,346,1,1,1,0 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile,239,0,9,10,2600,27900,2720,10,10,10,200 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile,257,0,7,10,2883,23629,3003,10,10,10,220 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile,337,0,7,10,2772,26319,2884,10,10,10,300 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile,368,0,8,10,2863,29367,2975,10,10,10,330 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile,369,0,9,10,2756,35949,2864,10,10,10,330 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile-symflower-fix,239,0,9,10,2600,27900,2720,10,10,10,200 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile-symflower-fix,257,0,7,10,2883,23629,3003,10,10,10,220 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile-symflower-fix,337,0,7,10,2772,26319,2884,10,10,10,300 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile-symflower-fix,368,0,8,10,2863,29367,2975,10,10,10,330 +openrouter/neversleep/llama-3-lumimaid-8b,ruby,ruby/transpile,transpile-symflower-fix,369,0,9,10,2756,35949,2864,10,10,10,330 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,2,0,0,1,671,2685,822,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,3,0,0,1,215,1010,229,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,3,0,0,1,226,1125,240,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,3,0,0,1,273,1215,287,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,3,0,0,1,517,2014,527,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,671,2685,822,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,215,1010,229,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,226,1125,240,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,273,1215,287,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,517,2014,527,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,1341,1270,10,23,18957,65484,21437,23,15,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,1400,1320,14,23,20502,67248,21927,23,20,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,1403,1330,11,23,20945,68654,23604,23,16,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,1777,1700,15,23,20993,67421,23317,23,16,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,930,860,9,23,23951,81036,29453,23,15,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests-symflower-fix,1341,1270,10,23,18957,65484,21437,23,15,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests-symflower-fix,1400,1320,14,23,20502,67248,21927,23,20,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests-symflower-fix,1403,1330,11,23,20945,68654,23604,23,16,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests-symflower-fix,1777,1700,15,23,20993,67421,23317,23,16,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests-symflower-fix,930,860,9,23,23951,81036,29453,23,15,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/mistakes,code-repair,129,0,4,5,1145,5353,1205,5,5,5,110 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,14,10,1,1,273,1181,289,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,2,0,0,1,236,2153,548,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,3,0,0,1,224,1119,240,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,3,0,0,1,243,1188,255,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,3,0,0,1,407,1715,419,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests-symflower-fix,14,10,1,1,273,1181,289,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests-symflower-fix,2,0,0,1,236,2153,548,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,224,1119,240,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,243,1188,255,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,407,1715,419,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile,376,0,8,10,3805,24827,7459,10,8,10,340 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile,407,0,9,10,3651,18935,4299,10,8,10,370 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile,416,0,9,10,3964,18493,4956,10,7,10,380 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile,425,0,9,10,3396,16936,3970,10,6,10,390 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile,435,0,10,10,3861,19844,4896,10,5,10,400 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile-symflower-fix,376,0,8,10,3805,24827,7459,10,8,10,340 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile-symflower-fix,407,0,9,10,3651,18935,4299,10,8,10,370 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile-symflower-fix,416,0,9,10,3964,18493,4956,10,7,10,380 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile-symflower-fix,425,0,9,10,3396,16936,3970,10,6,10,390 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/transpile,transpile-symflower-fix,435,0,10,10,3861,19844,4896,10,5,10,400 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests,1121,1040,18,23,13153,60530,17241,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests,1192,1110,19,23,13917,62694,16967,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests,1211,1130,18,23,16208,68381,19312,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests,1291,1210,18,23,13746,65105,18343,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests,1327,1240,21,23,12570,54528,13878,23,20,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1121,1040,18,23,13153,60530,17241,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1192,1110,19,23,13917,62694,16967,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1211,1130,18,23,16208,68381,19312,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1291,1210,18,23,13746,65105,18343,23,17,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1327,1240,21,23,12570,54528,13878,23,20,23,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/mistakes,code-repair,166,0,5,5,640,11060,1563,5,1,5,150 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests,13,10,1,1,147,1896,263,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests,13,10,1,1,209,1648,444,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests,14,10,1,1,193,1070,205,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests,14,10,1,1,194,1132,206,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests,3,0,0,1,281,1850,293,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,147,1896,263,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,209,1648,444,1,0,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,193,1070,205,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,194,1132,206,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,281,1850,293,1,1,1,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile,262,0,7,10,3892,21327,5172,10,5,10,230 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile,276,0,10,10,2975,19309,4376,10,6,10,240 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile,282,0,8,10,3081,20007,4357,10,5,9,250 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile,334,0,9,10,3077,22310,4871,10,5,10,300 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile,387,0,10,10,2919,17621,3551,10,7,10,350 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,262,0,7,10,3892,21327,5172,10,5,10,230 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,276,0,10,10,2975,19309,4376,10,6,10,240 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,282,0,8,10,3081,20007,4357,10,5,9,250 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,334,0,9,10,3077,22310,4871,10,5,10,300 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,387,0,10,10,2919,17621,3551,10,7,10,350 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests,2,0,0,1,125,3642,333,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests,2,0,0,1,178,8059,740,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests,2,0,0,1,246,6637,669,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests,2,0,0,1,687,7416,710,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests,3,0,0,1,261,3416,271,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,125,3642,333,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,178,8059,740,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,246,6637,669,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,687,7416,710,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,261,3416,271,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests,1359,1280,13,23,27379,239033,28826,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests,1427,1350,11,23,24819,222248,26845,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests,1566,1490,12,23,24267,230302,27616,23,18,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests,1707,1630,14,23,24671,211115,26861,23,17,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests,1760,1680,15,23,23247,206639,24823,23,19,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests-symflower-fix,1359,1280,13,23,27379,239033,28826,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests-symflower-fix,1427,1350,11,23,24819,222248,26845,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests-symflower-fix,1566,1490,12,23,24267,230302,27616,23,18,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests-symflower-fix,1707,1630,14,23,24671,211115,26861,23,17,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/light,write-tests-symflower-fix,1760,1680,15,23,23247,206639,24823,23,19,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/mistakes,code-repair,158,0,5,5,1192,20367,1977,5,3,5,140 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests,14,10,1,1,163,4463,179,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests,14,10,1,1,222,6856,238,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests,14,10,1,1,273,3333,285,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests,3,0,0,1,233,2919,249,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests,3,0,0,1,591,5071,603,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests-symflower-fix,14,10,1,1,163,4463,179,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests-symflower-fix,14,10,1,1,222,6856,238,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests-symflower-fix,14,10,1,1,273,3333,285,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,233,2919,249,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,591,5071,603,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile,419,0,9,10,3413,45971,3533,10,10,10,380 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile,426,0,9,10,3886,48590,4385,10,7,10,390 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile,428,0,9,10,3605,43907,3760,10,9,10,390 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile,468,0,10,10,3602,45711,3977,10,8,10,430 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile,469,0,10,10,3658,48254,4184,10,9,10,430 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile-symflower-fix,419,0,9,10,3413,45971,3533,10,10,10,380 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile-symflower-fix,426,0,9,10,3886,48590,4385,10,7,10,390 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile-symflower-fix,428,0,9,10,3605,43907,3760,10,9,10,390 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile-symflower-fix,468,0,10,10,3602,45711,3977,10,8,10,430 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,java,java/transpile,transpile-symflower-fix,469,0,10,10,3658,48254,4184,10,9,10,430 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests,1422,1340,16,23,17567,188870,18752,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests,843,760,16,23,21230,226517,21493,23,22,22,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests,880,800,14,23,16031,185886,19345,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests,883,800,16,23,15485,162775,16595,23,21,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests,960,880,13,23,17113,177104,18101,23,21,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,1422,1340,16,23,17567,188870,18752,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,843,760,16,23,21230,226517,21493,23,22,22,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,880,800,14,23,16031,185886,19345,23,20,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,883,800,16,23,15485,162775,16595,23,21,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/light,write-tests-symflower-fix,960,880,13,23,17113,177104,18101,23,21,23,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/mistakes,code-repair,166,0,5,5,587,19130,1657,5,1,5,150 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests,14,10,1,1,187,2835,199,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests,14,10,1,1,224,3091,236,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests,2,0,0,1,197,5611,635,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests,3,0,0,1,194,2923,206,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests,3,0,0,1,289,3643,301,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,187,2835,199,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,224,3091,236,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,197,5611,635,1,0,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,194,2923,206,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,289,3643,301,1,1,1,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile,297,0,9,10,2926,45106,3250,10,8,10,260 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile,298,0,10,10,2884,44533,3222,10,8,10,260 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile,373,0,9,10,2735,52692,4521,10,4,10,340 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile,388,0,9,10,2877,44253,3093,10,9,10,350 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile,418,0,10,10,2903,44208,3262,10,8,10,380 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,297,0,9,10,2926,45106,3250,10,8,10,260 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,298,0,10,10,2884,44533,3222,10,8,10,260 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,373,0,9,10,2735,52692,4521,10,4,10,340 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,388,0,9,10,2877,44253,3093,10,9,10,350 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,ruby,ruby/transpile,transpile-symflower-fix,418,0,10,10,2903,44208,3262,10,8,10,380 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,2,0,0,1,162,2035,972,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,2,0,0,1,445,2820,602,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,3,0,0,1,131,2509,145,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,3,0,0,1,324,1617,338,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,3,0,0,1,563,3416,577,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,162,2035,972,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,445,2820,602,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,131,2509,145,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,324,1617,338,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,563,3416,577,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile,341,0,6,10,3111,17270,4266,10,5,10,310 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile,398,0,7,10,3570,19415,5273,10,3,8,370 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile,460,0,8,10,3051,25525,6668,10,3,9,430 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile,470,0,8,10,2944,19486,5722,10,2,10,440 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile,501,0,9,10,3294,18678,4371,10,4,8,470 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile-symflower-fix,449,0,8,10,3570,19735,5273,10,3,8,420 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile-symflower-fix,501,0,9,10,3294,18678,4371,10,4,8,470 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile-symflower-fix,514,0,9,10,3111,18182,4266,10,5,10,480 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile-symflower-fix,521,0,9,10,3051,25843,6668,10,3,9,490 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/transpile,transpile-symflower-fix,572,0,10,10,2944,20195,5722,10,2,10,540 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,1486,1420,14,23,38044,128191,44601,23,7,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,1628,1560,13,23,57989,541180,126599,23,10,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,1752,1680,16,23,32446,133769,35514,23,11,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,2131,2060,18,23,35266,127323,40527,23,8,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,2135,2060,17,23,34168,151863,39705,23,13,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests-symflower-fix,1486,1420,14,23,38044,128191,44601,23,7,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests-symflower-fix,1628,1560,13,23,57989,541180,126599,23,10,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests-symflower-fix,1752,1680,16,23,32446,133769,35514,23,11,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests-symflower-fix,2131,2060,18,23,35266,127323,40527,23,8,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests-symflower-fix,2135,2060,17,23,34168,151863,39705,23,13,22,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/mistakes,code-repair,155,0,5,5,1182,57191,12693,5,0,5,140 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,13,10,1,1,230,2162,680,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,13,10,1,1,262,2487,543,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,2,0,0,1,211,1428,582,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,3,0,0,1,224,1430,236,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,3,0,0,1,241,1373,253,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests-symflower-fix,13,10,1,1,230,2162,680,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests-symflower-fix,13,10,1,1,262,2487,543,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests-symflower-fix,2,0,0,1,211,1428,582,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests-symflower-fix,3,0,0,1,224,1430,236,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests-symflower-fix,3,0,0,1,241,1373,253,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile,455,0,10,10,3804,315032,98802,10,5,10,420 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile,463,0,10,10,3996,27568,8415,10,3,10,430 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile,463,0,10,10,4298,19063,5038,10,3,10,430 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile,471,0,10,10,4078,21002,7023,10,1,10,440 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile,474,0,10,10,4136,18382,5143,10,4,10,440 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile-symflower-fix,455,0,10,10,3804,315032,98802,10,5,10,420 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile-symflower-fix,463,0,10,10,3996,27568,8415,10,3,10,430 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile-symflower-fix,463,0,10,10,4298,19063,5038,10,3,10,430 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile-symflower-fix,471,0,10,10,4078,21002,7023,10,1,10,440 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/transpile,transpile-symflower-fix,474,0,10,10,4136,18382,5143,10,4,10,440 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests,1088,1020,15,23,21646,100952,27881,23,7,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests,747,680,12,23,31468,154601,36714,23,9,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests,833,760,14,23,22713,107799,28147,23,13,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests,908,840,13,23,21119,112792,30441,23,9,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests,963,890,17,23,49305,226167,53840,23,10,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests-symflower-fix,1088,1020,15,23,21646,100952,27881,23,7,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests-symflower-fix,747,680,12,23,31468,154601,36714,23,9,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests-symflower-fix,833,760,14,23,22713,107799,28147,23,13,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests-symflower-fix,908,840,13,23,21119,112792,30441,23,9,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/light,write-tests-symflower-fix,963,890,17,23,49305,226167,53840,23,10,23,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/mistakes,code-repair,116,0,5,5,621,7564,1909,5,1,5,100 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests,14,10,1,1,146,1602,158,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests,14,10,1,1,149,901,161,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests,14,10,1,1,185,1983,197,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests,2,0,0,1,149,1293,470,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests,4,0,1,1,116,823,128,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,146,1602,158,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,149,901,161,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,185,1983,197,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,149,1293,470,1,0,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,116,823,128,1,1,1,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile,198,0,8,10,2983,34966,9435,10,0,10,170 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile,269,0,8,10,3047,29960,6832,10,1,10,240 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile,320,0,8,10,2868,23485,4906,10,2,10,290 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile,329,0,9,10,2927,24710,5309,10,1,9,300 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile,341,0,7,10,2875,21290,4951,10,4,10,310 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile-symflower-fix,198,0,8,10,2983,34966,9435,10,0,10,170 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile-symflower-fix,269,0,8,10,3047,29960,6832,10,1,10,240 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile-symflower-fix,320,0,8,10,2868,23485,4906,10,2,10,290 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile-symflower-fix,329,0,9,10,2927,24710,5309,10,1,9,300 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,ruby,ruby/transpile,transpile-symflower-fix,341,0,7,10,2875,21290,4951,10,4,10,310 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,208,140,5,23,26462,265064,28633,23,18,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,237,170,5,23,38817,326660,39476,23,18,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,309,240,6,23,33163,398534,33761,23,19,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,355,280,8,23,22547,175230,22777,23,22,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,437,360,11,23,21693,263118,21990,23,21,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests-symflower-fix,270,200,7,23,26462,265717,28633,23,18,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests-symflower-fix,330,260,8,23,38817,328910,39476,23,18,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests-symflower-fix,402,330,9,23,33163,399588,33761,23,19,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests-symflower-fix,458,380,11,23,22547,176564,22777,23,22,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests-symflower-fix,601,520,15,23,21693,264528,21990,23,21,22,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/mistakes,code-repair,210,0,5,5,648,6398,718,5,5,5,190 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,14,10,1,1,101,1483,111,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,14,10,1,1,74,1234,84,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,14,10,1,1,79,1141,94,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,2,0,0,1,703,5052,745,1,0,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,3,0,0,1,126,1986,140,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,101,1483,111,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1234,84,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,79,1141,94,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,703,5052,745,1,0,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,126,1986,140,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile,177,0,4,10,3125,30357,3790,10,1,2,160 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile,249,0,6,10,3343,30871,3507,10,5,8,220 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile,278,0,5,10,3214,27603,3460,10,6,7,250 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile,308,0,5,10,3218,31575,4049,10,5,8,280 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile,459,0,8,10,2668,68436,9529,10,5,6,430 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile-symflower-fix,269,0,6,10,3125,30861,3790,10,1,2,250 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile-symflower-fix,310,0,7,10,3343,31237,3507,10,5,8,280 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile-symflower-fix,329,0,6,10,3214,27948,3460,10,6,7,300 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile-symflower-fix,370,0,7,10,3218,32296,4049,10,5,8,340 +openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/transpile,transpile-symflower-fix,459,0,8,10,2668,68436,9529,10,5,6,430 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,312,250,4,23,45125,348974,45701,23,17,18,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,340,270,3,23,37134,342310,38951,23,21,23,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,558,490,4,23,28824,208768,29415,23,20,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,724,660,6,23,30871,164445,31159,23,17,18,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,900,830,7,23,36952,287198,37402,23,20,20,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests-symflower-fix,312,250,4,23,45125,348974,45701,23,17,18,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests-symflower-fix,340,270,3,23,37134,342310,38951,23,21,23,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests-symflower-fix,558,490,4,23,28824,208768,29415,23,20,21,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests-symflower-fix,724,660,6,23,30871,164445,31159,23,17,18,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests-symflower-fix,900,830,7,23,36952,287198,37402,23,20,20,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/mistakes,code-repair,128,0,4,5,1075,10556,1254,5,4,5,110 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,12,10,1,1,270,2047,270,1,0,0,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,13,10,1,1,137,2859,385,1,0,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,3,0,0,1,222,1890,234,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,3,0,0,1,250,1947,262,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,3,0,0,1,280,2603,292,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests-symflower-fix,12,10,1,1,270,2047,270,1,0,0,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests-symflower-fix,13,10,1,1,137,2859,385,1,0,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests-symflower-fix,3,0,0,1,222,1890,234,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests-symflower-fix,3,0,0,1,250,1947,262,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests-symflower-fix,3,0,0,1,280,2603,292,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile,340,0,7,10,3715,27462,3932,10,5,8,310 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile,353,0,8,10,3811,29272,3941,10,7,8,320 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile,356,0,7,10,4941,31530,5048,10,4,5,330 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile,418,0,9,10,4161,28742,4290,10,9,10,380 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile,456,0,10,10,4095,27213,4193,10,8,8,420 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile-symflower-fix,340,0,7,10,3715,27462,3932,10,5,8,310 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile-symflower-fix,353,0,8,10,3811,29272,3941,10,7,8,320 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile-symflower-fix,356,0,7,10,4941,31530,5048,10,4,5,330 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile-symflower-fix,418,0,9,10,4161,28742,4290,10,9,10,380 +openrouter/nousresearch/nous-hermes-yi-34b,java,java/transpile,transpile-symflower-fix,456,0,10,10,4095,27213,4193,10,8,8,420 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests,1,0,0,1,161,2211,161,1,0,0,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests,3,0,0,1,128,1719,140,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests,3,0,0,1,142,1634,154,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests,3,0,0,1,144,2138,156,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests,3,0,0,1,165,2039,177,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests-symflower-fix,1,0,0,1,161,2211,161,1,0,0,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,128,1719,140,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,142,1634,154,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,144,2138,156,1,1,1,0 +openrouter/nousresearch/nous-hermes-yi-34b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,165,2039,177,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests,654,570,15,23,15628,124123,15951,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests,655,570,16,23,15191,122668,15506,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests,696,610,17,23,15737,130744,16060,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests,727,640,18,23,14457,113269,14779,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests,798,710,19,23,16457,136073,16779,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests-symflower-fix,685,600,16,23,15628,124733,15951,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests-symflower-fix,798,710,19,23,15737,131677,16060,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests-symflower-fix,798,710,19,23,16457,136073,16779,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests-symflower-fix,849,760,20,23,14457,114260,14779,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/light,write-tests-symflower-fix,849,760,20,23,15191,123806,15506,23,23,23,0 +openrouter/openai/gpt-4o,golang,golang/mistakes,code-repair,210,0,5,5,638,9700,708,5,5,5,190 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests,14,10,1,1,74,1121,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests,14,10,1,1,74,1416,84,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests,14,10,1,1,74,1598,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests,14,10,1,1,74,1679,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1121,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1416,84,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1598,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1679,88,1,1,1,0 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile,560,0,10,10,2952,20377,3060,10,10,10,520 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile,560,0,10,10,3022,25662,3142,10,10,10,520 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile,580,0,10,10,2845,20262,2945,10,10,10,540 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile,580,0,10,10,2876,20347,2988,10,10,10,540 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile,580,0,10,10,2933,18710,3049,10,10,10,540 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2952,20377,3060,10,10,10,520 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,3022,25662,3142,10,10,10,520 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2845,20262,2945,10,10,10,540 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2876,20347,2988,10,10,10,540 +openrouter/openai/gpt-4o,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2933,18710,3049,10,10,10,540 +openrouter/openai/gpt-4o,java,java/light,write-tests,2952,2860,23,23,24547,134124,24823,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests,3072,2980,23,23,25092,144730,25368,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests,3072,2980,23,23,25851,166779,26127,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests,3082,2990,23,23,25306,136896,25582,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests,3092,3000,23,23,24259,148201,24535,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests-symflower-fix,2952,2860,23,23,24547,134124,24823,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests-symflower-fix,3072,2980,23,23,25092,144730,25368,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests-symflower-fix,3072,2980,23,23,25851,166779,26127,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests-symflower-fix,3082,2990,23,23,25306,136896,25582,23,23,23,0 +openrouter/openai/gpt-4o,java,java/light,write-tests-symflower-fix,3092,3000,23,23,24259,148201,24535,23,23,23,0 +openrouter/openai/gpt-4o,java,java/mistakes,code-repair,160,0,5,5,943,10034,1003,5,5,5,140 +openrouter/openai/gpt-4o,java,java/plain,write-tests,14,10,1,1,137,1210,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests,14,10,1,1,137,1253,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests,14,10,1,1,137,1955,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests,14,10,1,1,137,2120,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests,14,10,1,1,213,2589,225,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1210,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1253,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1955,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,2120,149,1,1,1,0 +openrouter/openai/gpt-4o,java,java/plain,write-tests-symflower-fix,14,10,1,1,213,2589,225,1,1,1,0 +openrouter/openai/gpt-4o,java,java/transpile,transpile,480,0,10,10,3511,21681,3631,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile,480,0,10,10,3584,20373,3704,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile,480,0,10,10,3682,21969,3802,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile,480,0,10,10,3797,27157,3917,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile,480,0,10,10,4092,20935,4212,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile-symflower-fix,480,0,10,10,3511,21681,3631,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile-symflower-fix,480,0,10,10,3584,20373,3704,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile-symflower-fix,480,0,10,10,3682,21969,3802,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile-symflower-fix,480,0,10,10,3797,27157,3917,10,10,10,440 +openrouter/openai/gpt-4o,java,java/transpile,transpile-symflower-fix,480,0,10,10,4092,20935,4212,10,10,10,440 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests,645,560,16,23,16706,128668,16936,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests,768,680,19,23,17162,118523,17392,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests,789,700,20,23,20588,114556,20818,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests,880,790,21,23,15947,108786,16177,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests,920,830,21,23,16121,126765,16351,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests-symflower-fix,860,770,21,23,17162,119167,17392,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests-symflower-fix,931,840,22,23,16706,131262,16936,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests-symflower-fix,962,870,23,23,20588,115983,20818,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests-symflower-fix,992,900,23,23,15947,109620,16177,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/light,write-tests-symflower-fix,992,900,23,23,16121,127351,16351,23,23,23,0 +openrouter/openai/gpt-4o-mini,golang,golang/mistakes,code-repair,210,0,5,5,638,7751,708,5,5,5,190 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests,14,10,1,1,107,1117,117,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests,14,10,1,1,121,1917,131,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests,14,10,1,1,125,1168,135,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests,14,10,1,1,125,1507,135,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests,14,10,1,1,74,1085,84,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,107,1117,117,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,121,1917,131,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,125,1168,135,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,125,1507,135,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1085,84,1,1,1,0 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile,580,0,10,10,2725,20076,2825,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile,580,0,10,10,2783,21322,2883,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile,580,0,10,10,2784,21704,2884,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile,580,0,10,10,2867,20262,2967,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile,580,0,10,10,2929,26359,3033,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2725,20076,2825,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2783,21322,2883,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2784,21704,2884,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2867,20262,2967,10,10,10,540 +openrouter/openai/gpt-4o-mini,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2929,26359,3033,10,10,10,540 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests,2981,2890,22,23,31772,154682,32048,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests,3032,2940,23,23,29685,124054,29961,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests,3052,2960,23,23,30114,146717,30390,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests,3072,2980,23,23,30979,140776,31255,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests,3092,3000,23,23,31195,138608,31471,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests-symflower-fix,2981,2890,22,23,31772,154682,32048,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests-symflower-fix,3032,2940,23,23,29685,124054,29961,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests-symflower-fix,3052,2960,23,23,30114,146717,30390,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests-symflower-fix,3072,2980,23,23,30979,140776,31255,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/light,write-tests-symflower-fix,3092,3000,23,23,31195,138608,31471,23,23,23,0 +openrouter/openai/gpt-4o-mini,java,java/mistakes,code-repair,160,0,5,5,943,9047,1003,5,5,5,140 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests,14,10,1,1,137,1735,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests,14,10,1,1,137,1930,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests,14,10,1,1,137,3106,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests,14,10,1,1,175,2672,187,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests,14,10,1,1,185,1588,197,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1735,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,1930,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,3106,149,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests-symflower-fix,14,10,1,1,175,2672,187,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/plain,write-tests-symflower-fix,14,10,1,1,185,1588,197,1,1,1,0 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile,480,0,10,10,3391,24301,3511,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile,480,0,10,10,3414,26076,3534,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile,480,0,10,10,3421,24219,3541,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile,480,0,10,10,3423,24714,3543,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile,480,0,10,10,3547,29279,3667,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile-symflower-fix,480,0,10,10,3391,24301,3511,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile-symflower-fix,480,0,10,10,3414,26076,3534,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile-symflower-fix,480,0,10,10,3421,24219,3541,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile-symflower-fix,480,0,10,10,3423,24714,3543,10,10,10,440 +openrouter/openai/gpt-4o-mini,java,java/transpile,transpile-symflower-fix,480,0,10,10,3547,29279,3667,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests,2312,2220,23,23,21212,92484,21488,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests,2342,2250,23,23,21531,86844,21807,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests,2352,2260,23,23,22925,127687,23201,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests,2372,2280,23,23,21122,96994,21398,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests,2382,2290,23,23,21711,90467,21987,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests-symflower-fix,2312,2220,23,23,21212,92484,21488,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests-symflower-fix,2342,2250,23,23,21531,86844,21807,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests-symflower-fix,2352,2260,23,23,22925,127687,23201,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests-symflower-fix,2372,2280,23,23,21122,96994,21398,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/light,write-tests-symflower-fix,2382,2290,23,23,21711,90467,21987,23,23,23,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/mistakes,code-repair,170,0,5,5,619,9109,679,5,5,5,150 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests,14,10,1,1,140,1059,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests,14,10,1,1,140,1171,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests,14,10,1,1,140,1272,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests,14,10,1,1,147,1302,159,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests,14,10,1,1,147,1772,159,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1059,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1171,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1272,152,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,1302,159,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,1772,159,1,1,1,0 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile,480,0,10,10,2807,31316,2927,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile,480,0,10,10,2816,21696,2936,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile,480,0,10,10,2821,22337,2941,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile,480,0,10,10,2827,22096,2947,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile,480,0,10,10,2828,30716,2948,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2807,31316,2927,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2816,21696,2936,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2821,22337,2941,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2827,22096,2947,10,10,10,440 +openrouter/openai/gpt-4o-mini,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2828,30716,2948,10,10,10,440 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests,2312,2220,23,23,19016,98936,19293,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests,2362,2270,23,23,18117,96980,18394,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests,2412,2320,23,23,17570,94846,17846,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests,2412,2320,23,23,18752,94260,19028,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests,2432,2340,23,23,18638,92426,18915,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests-symflower-fix,2312,2220,23,23,19016,98936,19293,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests-symflower-fix,2362,2270,23,23,18117,96980,18394,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests-symflower-fix,2412,2320,23,23,17570,94846,17846,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests-symflower-fix,2412,2320,23,23,18752,94260,19028,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/light,write-tests-symflower-fix,2432,2340,23,23,18638,92426,18915,23,23,23,0 +openrouter/openai/gpt-4o,ruby,ruby/mistakes,code-repair,170,0,5,5,619,7471,679,5,5,5,150 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests,14,10,1,1,140,1136,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests,14,10,1,1,140,1261,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests,14,10,1,1,140,1318,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests,14,10,1,1,140,1963,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests,14,10,1,1,140,3090,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1136,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1261,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1318,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1963,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,3090,152,1,1,1,0 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile,389,0,9,10,3000,24308,3120,10,10,10,350 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile,470,0,10,10,3088,26145,3208,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile,470,0,10,10,3094,26912,3214,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile,470,0,10,10,3127,25082,3247,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile,480,0,10,10,3058,24547,3178,10,10,10,440 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile-symflower-fix,389,0,9,10,3000,24308,3120,10,10,10,350 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,3088,26145,3208,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,3094,26912,3214,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile-symflower-fix,470,0,10,10,3127,25082,3247,10,10,10,430 +openrouter/openai/gpt-4o,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,3058,24547,3178,10,10,10,440 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,656,570,17,23,19447,285725,19715,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,818,730,19,23,18039,258666,18317,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,869,780,20,23,18491,248418,18753,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,890,800,21,23,17993,284816,18243,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,900,810,21,23,18287,301462,18546,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests-symflower-fix,972,880,23,23,19447,287713,19715,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests-symflower-fix,982,890,23,23,18039,260495,18317,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests-symflower-fix,982,890,23,23,18491,249404,18753,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests-symflower-fix,992,900,23,23,17993,286413,18243,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests-symflower-fix,992,900,23,23,18287,302560,18546,23,23,23,0 +openrouter/openai/gpt-4-turbo,golang,golang/mistakes,code-repair,210,0,5,5,638,14361,704,5,5,5,190 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,14,10,1,1,120,1576,130,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,14,10,1,1,120,2266,134,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,14,10,1,1,122,1739,136,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,14,10,1,1,128,2658,138,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,14,10,1,1,74,2259,84,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,120,1576,130,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,120,2266,134,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,122,1739,136,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,128,2658,138,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,2259,84,1,1,1,0 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile,540,0,10,10,2848,47149,2960,10,10,10,500 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile,560,0,10,10,2677,49942,2797,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile,560,0,10,10,2765,50617,2877,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile,560,0,10,10,2779,45699,2887,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile,580,0,10,10,2720,51589,2824,10,10,10,540 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2848,47149,2960,10,10,10,500 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2677,49942,2797,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2765,50617,2877,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2779,45699,2887,10,10,10,520 +openrouter/openai/gpt-4-turbo,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2720,51589,2824,10,10,10,540 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests,3082,2990,23,23,25244,262573,25521,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests,3102,3010,23,23,27792,278123,28068,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests,3112,3020,23,23,26210,266368,26486,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests,3122,3030,23,23,27198,325120,27474,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests,3122,3030,23,23,27935,309987,28211,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests-symflower-fix,3082,2990,23,23,25244,262573,25521,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests-symflower-fix,3102,3010,23,23,27792,278123,28068,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests-symflower-fix,3112,3020,23,23,26210,266368,26486,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests-symflower-fix,3122,3030,23,23,27198,325120,27474,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/light,write-tests-symflower-fix,3122,3030,23,23,27935,309987,28211,23,23,23,0 +openrouter/openai/gpt-4-turbo,java,java/mistakes,code-repair,160,0,5,5,1051,21517,1111,5,5,5,140 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,14,10,1,1,136,1883,148,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,14,10,1,1,137,2586,149,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,14,10,1,1,213,2143,225,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,14,10,1,1,213,3107,225,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,14,10,1,1,219,2893,231,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests-symflower-fix,14,10,1,1,136,1883,148,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests-symflower-fix,14,10,1,1,137,2586,149,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests-symflower-fix,14,10,1,1,213,2143,225,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests-symflower-fix,14,10,1,1,213,3107,225,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/plain,write-tests-symflower-fix,14,10,1,1,219,2893,231,1,1,1,0 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile,480,0,10,10,3895,57660,4015,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile,480,0,10,10,4005,64679,4125,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile,480,0,10,10,4072,54930,4192,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile,480,0,10,10,4170,60265,4290,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile,480,0,10,10,4187,61026,4307,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile-symflower-fix,480,0,10,10,3895,57660,4015,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile-symflower-fix,480,0,10,10,4005,64679,4125,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile-symflower-fix,480,0,10,10,4072,54930,4192,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile-symflower-fix,480,0,10,10,4170,60265,4290,10,10,10,440 +openrouter/openai/gpt-4-turbo,java,java/transpile,transpile-symflower-fix,480,0,10,10,4187,61026,4307,10,10,10,440 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests,2131,2040,22,23,18404,249405,18680,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests,2352,2260,23,23,17520,224341,17796,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests,2362,2270,23,23,17460,232262,17736,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests,2372,2280,23,23,16893,223348,17169,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests,2382,2290,23,23,17523,218066,17799,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests-symflower-fix,2131,2040,22,23,18404,249405,18680,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests-symflower-fix,2352,2260,23,23,17520,224341,17796,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests-symflower-fix,2362,2270,23,23,17460,232262,17736,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests-symflower-fix,2372,2280,23,23,16893,223348,17169,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/light,write-tests-symflower-fix,2382,2290,23,23,17523,218066,17799,23,23,23,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/mistakes,code-repair,170,0,5,5,620,12703,680,5,5,5,150 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests,14,10,1,1,168,4895,180,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests,14,10,1,1,181,2457,193,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests,14,10,1,1,181,2788,193,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests,14,10,1,1,188,2592,200,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests,4,0,1,1,108,2060,120,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,168,4895,180,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,181,2457,193,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,181,2788,193,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,188,2592,200,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/plain,write-tests-symflower-fix,4,0,1,1,108,2060,120,1,1,1,0 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile,160,0,10,10,3501,57621,3621,10,10,10,120 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile,200,0,10,10,3372,59999,3492,10,10,10,160 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile,230,0,10,10,3659,58045,3779,10,10,10,190 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile,240,0,10,10,3356,57630,3476,10,10,10,200 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile,380,0,10,10,3137,52782,3257,10,10,10,340 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile-symflower-fix,160,0,10,10,3501,57621,3621,10,10,10,120 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile-symflower-fix,200,0,10,10,3372,59999,3492,10,10,10,160 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile-symflower-fix,230,0,10,10,3659,58045,3779,10,10,10,190 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile-symflower-fix,240,0,10,10,3356,57630,3476,10,10,10,200 +openrouter/openai/gpt-4-turbo,ruby,ruby/transpile,transpile-symflower-fix,380,0,10,10,3137,52782,3257,10,10,10,340 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests,230,170,5,23,15635,105119,18208,22,11,22,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests,231,170,7,23,17689,121332,22611,23,9,22,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests,246,180,7,23,15681,97970,18259,23,13,23,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests,277,210,8,23,16410,106207,19772,23,13,23,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests,357,290,7,23,14238,91222,17555,23,14,23,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests-symflower-fix,380,310,11,23,16410,107080,19772,23,13,23,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests-symflower-fix,384,320,10,23,17689,122522,22611,23,9,22,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests-symflower-fix,415,350,10,23,15635,107248,18208,22,11,22,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests-symflower-fix,483,410,14,23,15681,101399,18259,23,13,23,0 +openrouter/openchat/openchat-8b,golang,golang/light,write-tests-symflower-fix,500,430,10,23,14238,92359,17555,23,14,23,0 +openrouter/openchat/openchat-8b,golang,golang/mistakes,code-repair,127,0,3,5,555,8171,1469,5,4,5,110 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,14,10,1,1,122,1170,136,1,1,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,2,0,0,1,135,1539,194,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,2,0,0,1,14,2441,532,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,2,0,0,1,229,1848,319,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,2,0,0,1,5,1922,370,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,122,1170,136,1,1,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,135,1539,194,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,14,2441,532,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,229,1848,319,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,5,1922,370,1,0,1,0 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile,174,0,4,10,2595,18079,3152,9,3,8,150 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile,289,0,5,10,2727,19500,3062,10,5,9,260 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile,391,0,7,10,2985,20447,3276,10,5,9,360 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile,392,0,7,10,2989,20710,3309,10,5,10,360 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile,462,0,8,10,3108,21598,3875,10,4,10,430 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile-symflower-fix,337,0,7,10,2595,19094,3152,9,3,8,310 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile-symflower-fix,452,0,8,10,2727,20494,3062,10,5,9,420 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile-symflower-fix,513,0,9,10,2985,21067,3276,10,5,9,480 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile-symflower-fix,514,0,9,10,2989,21323,3309,10,5,10,480 +openrouter/openchat/openchat-8b,golang,golang/transpile,transpile-symflower-fix,574,0,10,10,3108,22317,3875,10,4,10,540 +openrouter/openchat/openchat-8b,java,java/plain,write-tests,2,0,0,1,201,1494,272,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests,2,0,0,1,365,5012,1388,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests,2,0,0,1,894,3515,990,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests,3,0,0,1,469,2044,481,1,1,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests,3,0,0,1,579,2128,625,1,1,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests-symflower-fix,2,0,0,1,201,1494,272,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests-symflower-fix,2,0,0,1,365,5012,1388,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests-symflower-fix,2,0,0,1,894,3515,990,1,0,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,469,2044,481,1,1,1,0 +openrouter/openchat/openchat-8b,java,java/plain,write-tests-symflower-fix,3,0,0,1,579,2128,625,1,1,1,0 +openrouter/openchat/openchat-8b,java,java/transpile,transpile,358,0,8,10,3824,21142,4240,9,2,9,330 +openrouter/openchat/openchat-8b,java,java/transpile,transpile,428,0,9,10,3795,22669,4979,10,0,9,400 +openrouter/openchat/openchat-8b,java,java/transpile,transpile,433,0,9,10,3951,23263,4960,10,4,10,400 +openrouter/openchat/openchat-8b,java,java/transpile,transpile,461,0,10,10,4075,25509,4723,10,1,10,430 +openrouter/openchat/openchat-8b,java,java/transpile,transpile,475,0,10,10,3680,22968,4173,10,5,10,440 +openrouter/openchat/openchat-8b,java,java/transpile,transpile-symflower-fix,358,0,8,10,3824,21142,4240,9,2,9,330 +openrouter/openchat/openchat-8b,java,java/transpile,transpile-symflower-fix,428,0,9,10,3795,22669,4979,10,0,9,400 +openrouter/openchat/openchat-8b,java,java/transpile,transpile-symflower-fix,433,0,9,10,3951,23263,4960,10,4,10,400 +openrouter/openchat/openchat-8b,java,java/transpile,transpile-symflower-fix,461,0,10,10,4075,25509,4723,10,1,10,430 +openrouter/openchat/openchat-8b,java,java/transpile,transpile-symflower-fix,475,0,10,10,3680,22968,4173,10,5,10,440 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests,2,0,0,1,158,1821,339,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests,2,0,0,1,209,2768,662,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests,2,0,0,1,239,2070,389,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests,2,0,0,1,252,2800,662,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests,3,0,0,1,255,1617,275,1,1,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,158,1821,339,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,209,2768,662,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,239,2070,389,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,252,2800,662,1,0,1,0 +openrouter/openchat/openchat-8b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,255,1617,275,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,759,670,20,23,15198,131126,15392,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,769,680,20,23,15853,156501,16049,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,769,680,20,23,16552,142328,16740,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,820,730,21,23,16594,162665,16786,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,861,770,22,23,15333,145256,15519,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests-symflower-fix,820,730,21,23,16552,142504,16740,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests-symflower-fix,861,770,22,23,15333,145256,15519,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests-symflower-fix,861,770,22,23,15853,157101,16049,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests-symflower-fix,881,790,22,23,15198,131834,15392,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests-symflower-fix,881,790,22,23,16594,163097,16786,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/mistakes,code-repair,210,0,5,5,648,6610,688,5,5,5,190 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,14,10,1,1,74,1038,84,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,14,10,1,1,74,1085,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,14,10,1,1,74,1164,84,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,14,10,1,1,74,1264,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,14,10,1,1,74,726,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1038,84,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1085,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1164,84,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,1264,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,74,726,82,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile,468,0,8,10,2829,28581,2911,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile,509,0,9,10,3086,30122,3166,10,10,10,470 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile,519,0,9,10,2842,29311,2922,10,10,10,480 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile,519,0,9,10,2923,28048,3003,10,10,10,480 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile,580,0,10,10,3041,30574,3121,10,10,10,540 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,3086,30607,3166,10,10,10,500 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2829,29223,2911,10,10,10,540 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2842,29673,2922,10,10,10,540 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2923,28297,3003,10,10,10,540 +openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,3041,30574,3121,10,10,10,540 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,2429,2340,20,23,23533,161563,23717,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,2639,2550,20,23,23965,155654,24149,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,2710,2620,21,23,23603,146098,23795,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,2840,2750,21,23,24141,148420,24325,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,2851,2760,22,23,23862,174937,24046,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests-symflower-fix,2429,2340,20,23,23533,161563,23717,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests-symflower-fix,2639,2550,20,23,23965,155654,24149,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests-symflower-fix,2710,2620,21,23,23603,146098,23795,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests-symflower-fix,2840,2750,21,23,24141,148420,24325,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests-symflower-fix,2851,2760,22,23,23862,174937,24046,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/mistakes,code-repair,126,0,4,5,816,8059,848,4,4,4,110 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,13,10,1,1,199,3161,415,1,0,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,14,10,1,1,198,1738,206,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,14,10,1,1,224,1996,232,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,14,10,1,1,280,1772,288,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,14,10,1,1,296,1996,304,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests-symflower-fix,13,10,1,1,199,3161,415,1,0,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,198,1738,206,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,224,1996,232,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,280,1772,288,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,296,1996,304,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile,416,0,9,10,3423,30336,3499,9,9,9,380 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile,439,0,9,10,3873,32024,3953,10,10,10,400 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile,470,0,10,10,3753,29430,3837,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile,470,0,10,10,3862,33485,3942,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile,470,0,10,10,3889,35735,3969,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile-symflower-fix,416,0,9,10,3423,30336,3499,9,9,9,380 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile-symflower-fix,439,0,9,10,3873,32024,3953,10,10,10,400 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile-symflower-fix,470,0,10,10,3753,29430,3837,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile-symflower-fix,470,0,10,10,3862,33485,3942,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/transpile,transpile-symflower-fix,470,0,10,10,3889,35735,3969,10,10,10,430 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests,1898,1810,19,23,15539,341560,15723,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests,2060,1970,21,23,14866,286604,15050,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests,2099,2010,20,23,16180,362825,16364,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests,2312,2220,23,23,15709,286658,15893,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests,2352,2260,23,23,16756,246229,16944,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests-symflower-fix,1898,1810,19,23,15539,341560,15723,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests-symflower-fix,2060,1970,21,23,14866,286604,15050,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests-symflower-fix,2099,2010,20,23,16180,362825,16364,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests-symflower-fix,2312,2220,23,23,15709,286658,15893,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/light,write-tests-symflower-fix,2352,2260,23,23,16756,246229,16944,23,23,23,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/mistakes,code-repair,170,0,5,5,619,12930,659,5,5,5,150 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests,14,10,1,1,155,5381,163,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests,14,10,1,1,207,6920,215,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests,14,10,1,1,222,3066,230,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests,14,10,1,1,255,6461,263,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests,3,0,0,1,155,7009,163,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,155,5381,163,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,207,6920,215,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,222,3066,230,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,255,6461,263,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,155,7009,163,1,1,1,0 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile,200,0,10,10,3467,81422,3547,10,10,10,160 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile,230,0,10,10,3430,67292,3510,10,10,10,190 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile,279,0,9,10,3315,63805,3395,10,10,10,240 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile,280,0,10,10,3334,90186,3418,10,10,10,240 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile,330,0,10,10,3145,63417,3225,10,10,10,290 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,200,0,10,10,3467,81422,3547,10,10,10,160 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,230,0,10,10,3430,67292,3510,10,10,10,190 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,279,0,9,10,3315,63805,3395,10,10,10,240 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,280,0,10,10,3334,90186,3418,10,10,10,240 +openrouter/perplexity/llama-3-sonar-large-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,330,0,10,10,3145,63417,3225,10,10,10,290 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests,135,80,3,23,22837,66520,26560,23,6,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests,139,80,3,23,51091,231300,54311,23,10,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests,207,150,3,23,40081,125545,43638,23,8,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests,226,170,5,23,24722,78514,29336,23,5,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests,50,0,1,23,78549,384004,81956,23,4,22,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests-symflower-fix,208,150,6,23,22837,67774,26560,23,6,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests-symflower-fix,258,200,4,23,40081,125856,43638,23,8,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests-symflower-fix,275,220,6,23,78549,386056,81956,23,4,22,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests-symflower-fix,319,260,8,23,24722,80034,29336,23,5,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/light,write-tests-symflower-fix,343,280,7,23,51091,232664,54311,23,10,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/mistakes,code-repair,209,0,5,5,712,4512,908,5,4,5,190 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,2,0,0,1,140,819,158,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,2,0,0,1,143,611,212,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,2,0,0,1,143,966,230,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,2,0,0,1,92,831,168,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,3,0,1,1,248,866,335,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,140,819,158,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,143,611,212,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,143,966,230,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,92,831,168,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests-symflower-fix,3,0,1,1,248,866,335,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,2,0,0,1,1306,1805,1390,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,2,0,0,1,212,1117,380,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,2,0,0,1,304,1303,530,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,2,0,0,1,720,1846,958,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,3,0,0,1,1096,1819,1108,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,1306,1805,1390,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,212,1117,380,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,304,1303,530,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,720,1846,958,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests-symflower-fix,3,0,0,1,1096,1819,1108,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile,376,0,8,10,3942,11260,4306,10,8,10,340 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile,416,0,9,10,3710,12136,4464,10,7,10,380 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile,427,0,9,10,3892,11977,4280,10,8,10,390 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile,468,0,10,10,3945,11616,4126,10,8,10,430 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile,469,0,10,10,3801,12029,4101,10,9,10,430 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile-symflower-fix,376,0,8,10,3942,11260,4306,10,8,10,340 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile-symflower-fix,416,0,9,10,3710,12136,4464,10,7,10,380 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile-symflower-fix,427,0,9,10,3892,11977,4280,10,8,10,390 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile-symflower-fix,468,0,10,10,3945,11616,4126,10,8,10,430 +openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/transpile,transpile-symflower-fix,469,0,10,10,3801,12029,4101,10,9,10,430 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests,363,300,5,23,21218,59877,23772,23,12,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests,378,310,6,23,18509,55437,19588,23,17,22,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests,382,320,5,23,20124,57710,22404,23,11,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests,451,390,5,23,18177,54331,21095,23,10,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests,480,420,5,23,21257,59869,23139,21,13,21,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests-symflower-fix,363,300,5,23,21218,59877,23772,23,12,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests-symflower-fix,378,310,6,23,18509,55437,19588,23,17,22,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests-symflower-fix,382,320,5,23,20124,57710,22404,23,11,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests-symflower-fix,451,390,5,23,18177,54331,21095,23,10,23,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/light,write-tests-symflower-fix,480,420,5,23,21257,59869,23139,21,13,21,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/mistakes,code-repair,108,0,5,5,602,6359,898,5,3,5,90 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests,14,10,1,1,153,661,165,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests,2,0,0,1,132,1055,236,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests,2,0,0,1,172,1124,378,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests,2,0,0,1,639,1719,771,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests,3,0,0,1,169,686,185,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,153,661,165,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,132,1055,236,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,172,1124,378,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,639,1719,771,1,0,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,169,686,185,1,1,1,0 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile,317,0,8,10,2462,12236,2724,10,9,10,280 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile,385,0,9,10,2706,12355,3705,10,6,10,350 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile,420,0,10,10,2661,11395,2781,10,10,10,380 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile,438,0,10,10,2619,11905,3123,10,8,10,400 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile,449,0,10,10,2782,11383,3013,10,9,10,410 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,317,0,8,10,2462,12236,2724,10,9,10,280 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,385,0,9,10,2706,12355,3705,10,6,10,350 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,420,0,10,10,2661,11395,2781,10,10,10,380 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,438,0,10,10,2619,11905,3123,10,8,10,400 +openrouter/perplexity/llama-3-sonar-small-32k-chat,ruby,ruby/transpile,transpile-symflower-fix,449,0,10,10,2782,11383,3013,10,9,10,410 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,298,230,7,23,17572,162354,21907,23,15,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,459,380,13,23,16312,138612,17632,23,20,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,472,400,14,23,16932,160198,21306,23,12,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,542,470,12,23,15801,150080,19663,23,14,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,552,480,15,23,17091,173234,22874,23,11,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests-symflower-fix,512,440,11,23,17572,163537,21907,23,15,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests-symflower-fix,513,440,15,23,16932,160645,21306,23,12,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests-symflower-fix,612,530,16,23,16312,139894,17632,23,20,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests-symflower-fix,675,600,18,23,17091,174776,22874,23,11,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests-symflower-fix,696,620,16,23,15801,151540,19663,23,14,23,0 +openrouter/qwen/qwen-110b-chat,golang,golang/mistakes,code-repair,210,0,5,5,648,8022,718,5,5,5,190 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,13,10,1,1,79,4663,776,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,14,10,1,1,806,4699,817,1,1,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,2,0,0,1,273,4582,797,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,2,0,0,1,547,6184,1042,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,2,0,0,1,638,5909,1054,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,79,4663,776,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,806,4699,817,1,1,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,273,4582,797,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,547,6184,1042,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,638,5909,1054,1,0,1,0 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile,325,0,6,10,2841,26119,3135,10,9,10,290 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile,346,0,6,10,2904,28401,3044,10,10,10,310 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile,357,0,7,10,2851,26648,2971,10,10,10,320 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile,418,0,8,10,2762,25738,2898,10,10,10,380 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile,438,0,8,10,2833,26149,2969,10,10,10,400 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile-symflower-fix,418,0,8,10,2851,27142,2971,10,10,10,380 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile-symflower-fix,498,0,9,10,2841,27074,3135,10,9,10,460 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile-symflower-fix,540,0,10,10,2762,26311,2898,10,10,10,500 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile-symflower-fix,560,0,10,10,2833,26791,2969,10,10,10,520 +openrouter/qwen/qwen-110b-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2904,29678,3044,10,10,10,540 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,2,0,0,1,315,6658,1141,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,2,0,0,1,509,4506,911,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,2,0,0,1,601,4690,896,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,3,0,0,1,221,1721,233,1,1,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,3,0,0,1,366,2219,378,1,1,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,315,6658,1141,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,509,4506,911,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests-symflower-fix,2,0,0,1,601,4690,896,1,0,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests-symflower-fix,3,0,0,1,221,1721,233,1,1,1,0 +openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests-symflower-fix,3,0,0,1,366,2219,378,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests,801,720,15,23,21091,148350,23021,23,20,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests,839,760,13,23,20118,140850,21230,23,20,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests,912,830,15,23,20920,135957,21603,23,21,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests,915,830,16,23,21987,148284,22263,23,23,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests,915,830,18,23,21425,129225,22397,23,21,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests-symflower-fix,801,720,15,23,21091,148350,23021,23,20,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests-symflower-fix,839,760,13,23,20118,140850,21230,23,20,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests-symflower-fix,912,830,15,23,20920,135957,21603,23,21,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests-symflower-fix,915,830,16,23,21987,148284,22263,23,23,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/light,write-tests-symflower-fix,915,830,18,23,21425,129225,22397,23,21,23,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/mistakes,code-repair,170,0,5,5,631,6847,691,5,5,5,150 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests,14,10,1,1,147,1671,159,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests,14,10,1,1,148,1178,160,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests,14,10,1,1,168,1551,180,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests,2,0,0,1,236,3669,615,1,0,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests,3,0,0,1,140,1220,152,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,147,1671,159,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,148,1178,160,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,168,1551,180,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,236,3669,615,1,0,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,140,1220,152,1,1,1,0 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile,310,0,10,10,2574,23898,2694,10,10,10,270 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile,330,0,10,10,2709,29230,2829,10,10,10,290 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile,379,0,10,10,2552,24469,2885,10,9,10,340 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile,420,0,10,10,2483,23602,2603,10,10,10,380 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile,480,0,10,10,2592,23718,2712,10,10,10,440 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile-symflower-fix,310,0,10,10,2574,23898,2694,10,10,10,270 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile-symflower-fix,330,0,10,10,2709,29230,2829,10,10,10,290 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile-symflower-fix,379,0,10,10,2552,24469,2885,10,9,10,340 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile-symflower-fix,420,0,10,10,2483,23602,2603,10,10,10,380 +openrouter/qwen/qwen-110b-chat,ruby,ruby/transpile,transpile-symflower-fix,480,0,10,10,2592,23718,2712,10,10,10,440 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,3,0,0,1,130,1477,144,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,3,0,0,1,144,1617,158,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,3,0,0,1,154,2700,168,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,3,0,0,1,155,1512,169,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,3,0,0,1,308,2445,322,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,155,2103,169,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,130,1477,144,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,144,1617,158,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,154,2700,168,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests-symflower-fix,4,0,1,1,308,2794,322,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,2217,2130,18,23,20912,153913,21188,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,2368,2280,19,23,21604,227290,21880,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,2440,2350,21,23,20923,173750,21199,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,2478,2390,19,23,21495,183821,21771,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,2506,2420,19,23,18441,164733,18708,23,22,22,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests-symflower-fix,2217,2130,18,23,20912,153913,21188,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests-symflower-fix,2368,2280,19,23,21604,227290,21880,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests-symflower-fix,2440,2350,21,23,20923,173750,21199,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests-symflower-fix,2478,2390,19,23,21495,183821,21771,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests-symflower-fix,2506,2420,19,23,18441,164733,18708,23,22,22,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/mistakes,code-repair,160,0,5,5,1068,11064,1128,5,5,5,140 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,14,10,1,1,208,63326,220,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,14,10,1,1,213,17895,225,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,14,10,1,1,291,1754,303,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,14,10,1,1,336,3254,348,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,14,10,1,1,343,2276,355,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,208,63326,220,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,213,17895,225,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,291,1754,303,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,336,3254,348,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,343,2276,355,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile,418,0,9,10,4013,73332,4539,10,9,10,380 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile,429,0,9,10,3801,38254,3921,10,10,10,390 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile,429,0,9,10,3914,32726,4034,10,10,10,390 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile,470,0,10,10,3732,33191,3852,10,10,10,430 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile,480,0,10,10,3830,54771,3950,10,10,10,440 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile-symflower-fix,418,0,9,10,4013,73332,4539,10,9,10,380 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile-symflower-fix,429,0,9,10,3801,38254,3921,10,10,10,390 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile-symflower-fix,429,0,9,10,3914,32726,4034,10,10,10,390 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile-symflower-fix,470,0,10,10,3732,33191,3852,10,10,10,430 +openrouter/qwen/qwen-2-72b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,3830,54771,3950,10,10,10,440 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests,1645,1560,18,23,15745,150689,16452,23,21,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests,1755,1670,18,23,17471,239014,19177,23,21,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests,1880,1790,21,23,16535,148472,16816,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests,1885,1800,19,23,17311,199049,18390,23,20,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests,2090,2000,21,23,16765,212909,17041,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests-symflower-fix,1645,1560,18,23,15745,150689,16452,23,21,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests-symflower-fix,1755,1670,18,23,17471,239014,19177,23,21,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests-symflower-fix,1880,1790,21,23,16535,148472,16816,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests-symflower-fix,1885,1800,19,23,17311,199049,18390,23,20,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/light,write-tests-symflower-fix,2090,2000,21,23,16765,212909,17041,23,23,23,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/mistakes,code-repair,170,0,5,5,619,8609,679,5,5,5,150 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,126,2453,138,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,134,2036,146,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,134,2487,146,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,179,1392,191,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,254,6975,267,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,126,2453,138,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,134,2036,146,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,134,2487,146,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,179,1392,191,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,254,6975,267,1,1,1,0 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile,165,0,5,10,3034,37127,3154,10,10,10,130 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile,246,0,8,10,2949,42308,3976,10,8,10,210 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile,256,0,6,10,2883,27863,3003,10,10,10,220 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile,298,0,8,10,3218,30676,3338,10,10,10,260 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile,350,0,10,10,2957,64423,3077,10,10,10,310 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile-symflower-fix,165,0,5,10,3034,37127,3154,10,10,10,130 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile-symflower-fix,246,0,8,10,2949,42308,3976,10,8,10,210 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile-symflower-fix,256,0,6,10,2883,27863,3003,10,10,10,220 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile-symflower-fix,298,0,8,10,3218,30676,3338,10,10,10,260 +openrouter/qwen/qwen-2-72b-instruct,ruby,ruby/transpile,transpile-symflower-fix,350,0,10,10,2957,64423,3077,10,10,10,310 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests,3,0,0,1,108,1504,122,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests,3,0,0,1,137,1198,151,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests,3,0,0,1,182,2401,196,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests,3,0,0,1,427,2381,441,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests,3,0,0,1,587,2414,601,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,108,1504,122,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,137,1198,151,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,182,2401,196,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,427,2381,441,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,587,2414,601,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests,1198,1120,10,23,29069,117845,30484,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests,1600,1520,12,23,23414,99464,24403,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests,270,200,2,23,23898,113237,25097,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests,693,620,5,23,24978,103955,26007,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests,697,620,8,23,22864,101134,23121,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests-symflower-fix,1198,1120,10,23,29069,117845,30484,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests-symflower-fix,1600,1520,12,23,23414,99464,24403,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests-symflower-fix,270,200,2,23,23898,113237,25097,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests-symflower-fix,693,620,5,23,24978,103955,26007,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/light,write-tests-symflower-fix,697,620,8,23,22864,101134,23121,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/mistakes,code-repair,139,0,4,5,1216,7184,1276,5,5,5,120 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests,14,10,1,1,202,1347,210,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests,3,0,0,1,1107,3584,1115,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests,3,0,0,1,236,1854,248,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests,3,0,0,1,245,1160,257,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests,3,0,0,1,349,2394,361,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,202,1347,210,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,1107,3584,1115,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,236,1854,248,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,245,1160,257,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/plain,write-tests-symflower-fix,3,0,0,1,349,2394,361,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile,439,0,9,10,4258,22834,4378,10,10,10,400 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile,439,0,9,10,4338,20083,4458,10,10,10,400 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile,480,0,10,10,4132,25364,4256,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile,480,0,10,10,4211,19725,4335,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile,480,0,10,10,4301,21865,4421,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile-symflower-fix,439,0,9,10,4258,22834,4378,10,10,10,400 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile-symflower-fix,439,0,9,10,4338,20083,4458,10,10,10,400 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,4132,25364,4256,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,4211,19725,4335,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,java,java/transpile,transpile-symflower-fix,480,0,10,10,4301,21865,4421,10,10,10,440 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests,246,170,7,23,25158,110302,25434,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests,275,200,7,23,24695,107818,24984,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests,459,380,10,23,20890,100406,21166,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests,656,580,7,23,22970,112241,23246,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests,828,750,9,23,22531,100061,22809,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests-symflower-fix,246,170,7,23,25158,110302,25434,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests-symflower-fix,275,200,7,23,24695,107818,24984,23,22,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests-symflower-fix,459,380,10,23,20890,100406,21166,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests-symflower-fix,656,580,7,23,22970,112241,23246,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/light,write-tests-symflower-fix,828,750,9,23,22531,100061,22809,23,23,23,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/mistakes,code-repair,139,0,4,5,626,6810,682,5,5,5,120 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,206,1412,218,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests,14,10,1,1,488,2330,500,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests,2,0,0,1,297,2819,580,1,0,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,189,1149,201,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests,3,0,0,1,352,2043,364,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,206,1412,218,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,488,2330,500,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,297,2819,580,1,0,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,189,1149,201,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,352,2043,364,1,1,1,0 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile,164,0,4,10,2644,19044,2764,10,10,10,130 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile,215,0,5,10,2731,19853,2847,10,10,10,180 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile,245,0,5,10,2622,19603,2743,10,10,10,210 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile,255,0,6,10,2598,19170,2720,10,9,10,220 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile,276,0,6,10,2529,19009,2649,10,10,10,240 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,164,0,4,10,2644,19044,2764,10,10,10,130 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,215,0,5,10,2731,19853,2847,10,10,10,180 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,245,0,5,10,2622,19603,2743,10,10,10,210 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,255,0,6,10,2598,19170,2720,10,9,10,220 +openrouter/qwen/qwen-2-7b-instruct,ruby,ruby/transpile,transpile-symflower-fix,276,0,6,10,2529,19009,2649,10,10,10,240 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,403,330,11,23,19028,175701,20960,23,16,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,446,380,11,23,19402,201234,23037,23,9,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,545,470,15,23,17513,197283,19788,23,14,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,567,490,16,23,18452,166379,20444,23,15,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,606,530,16,23,17809,189042,20134,23,14,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests-symflower-fix,673,600,18,23,19402,204102,23037,23,9,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests-symflower-fix,739,660,19,23,17809,190309,20134,23,14,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests-symflower-fix,740,660,18,23,19028,177241,20960,23,16,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests-symflower-fix,760,680,20,23,17513,198486,19788,23,14,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests-symflower-fix,761,680,20,23,18452,167468,20444,23,15,23,0 +openrouter/qwen/qwen-72b-chat,golang,golang/mistakes,code-repair,210,0,5,5,667,9129,737,5,5,5,190 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,13,10,1,1,141,2533,252,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,13,10,1,1,166,3590,490,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,13,10,1,1,186,2501,332,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,13,10,1,1,221,3590,451,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,14,10,1,1,427,3412,441,1,1,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,141,2533,252,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,166,3590,490,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,186,2501,332,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,221,3590,451,1,0,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests-symflower-fix,14,10,1,1,427,3412,441,1,1,1,0 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile,174,0,4,10,2934,25807,3042,10,10,10,140 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile,213,0,4,10,3088,29528,3464,10,9,10,180 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile,254,0,4,10,2843,26302,2951,10,10,10,220 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile,254,0,5,10,2838,28408,3203,10,9,10,220 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile,295,0,5,10,2939,25324,3047,10,10,10,260 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile-symflower-fix,498,0,9,10,3088,31004,3464,10,9,10,460 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile-symflower-fix,500,0,10,10,2934,27952,3042,10,10,10,460 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile-symflower-fix,539,0,10,10,2838,30274,3203,10,9,10,500 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2843,27922,2951,10,10,10,540 +openrouter/qwen/qwen-72b-chat,golang,golang/transpile,transpile-symflower-fix,580,0,10,10,2939,27048,3047,10,10,10,540 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,2051,1970,17,23,24805,167390,26456,23,18,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,2094,2010,17,23,25313,172530,25974,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,2275,2190,18,23,24746,186293,25718,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,2304,2220,18,23,24086,170949,24924,23,20,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,2365,2280,18,23,23511,156355,24266,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests-symflower-fix,2051,1970,17,23,24805,167390,26456,23,18,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests-symflower-fix,2094,2010,17,23,25313,172530,25974,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests-symflower-fix,2275,2190,18,23,24746,186293,25718,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests-symflower-fix,2304,2220,18,23,24086,170949,24924,23,20,23,0 +openrouter/qwen/qwen-72b-chat,java,java/light,write-tests-symflower-fix,2365,2280,18,23,23511,156355,24266,23,21,23,0 +openrouter/qwen/qwen-72b-chat,java,java/mistakes,code-repair,158,0,5,5,1064,13650,1350,5,3,5,140 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,14,10,1,1,150,1398,162,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,14,10,1,1,187,1889,199,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,14,10,1,1,222,1787,234,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,14,10,1,1,376,5921,388,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,14,10,1,1,388,2997,400,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,150,1398,162,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,187,1889,199,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,222,1787,234,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,376,5921,388,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests-symflower-fix,14,10,1,1,388,2997,400,1,1,1,0 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile,429,0,9,10,4025,26822,4145,10,10,10,390 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile,429,0,9,10,4166,25790,4286,10,10,10,390 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile,480,0,10,10,4101,28238,4221,10,10,10,440 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile,480,0,10,10,4146,30544,4266,10,10,10,440 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile,480,0,10,10,4185,30766,4305,10,10,10,440 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile-symflower-fix,429,0,9,10,4025,26822,4145,10,10,10,390 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile-symflower-fix,429,0,9,10,4166,25790,4286,10,10,10,390 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,4101,28238,4221,10,10,10,440 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,4146,30544,4266,10,10,10,440 +openrouter/qwen/qwen-72b-chat,java,java/transpile,transpile-symflower-fix,480,0,10,10,4185,30766,4305,10,10,10,440 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests,1557,1480,20,23,17269,119952,19525,22,13,22,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests,1715,1630,22,23,17434,123314,18915,23,17,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests,1824,1750,20,23,13672,96686,15064,20,14,20,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests,2048,1970,22,23,17654,129072,21531,23,10,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests,2108,2030,21,23,18714,132167,21353,23,11,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests-symflower-fix,1557,1480,20,23,17269,119952,19525,22,13,22,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests-symflower-fix,1715,1630,22,23,17434,123314,18915,23,17,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests-symflower-fix,1824,1750,20,23,13672,96686,15064,20,14,20,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests-symflower-fix,2048,1970,22,23,17654,129072,21531,23,10,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/light,write-tests-symflower-fix,2108,2030,21,23,18714,132167,21353,23,11,23,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/mistakes,code-repair,169,0,5,5,624,7637,793,5,4,5,150 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests,13,10,1,1,153,2675,344,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests,13,10,1,1,176,2357,374,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests,13,10,1,1,189,3407,617,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests,13,10,1,1,190,2788,455,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests,13,10,1,1,218,4331,821,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,153,2675,344,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,176,2357,374,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,189,3407,617,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,190,2788,455,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,218,4331,821,1,0,1,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile,118,0,8,10,2915,23570,3039,10,10,10,80 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile,129,0,9,10,2972,25516,3092,10,10,10,90 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile,140,0,10,10,2733,25267,2853,10,10,10,100 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile,169,0,9,10,2851,25485,2975,10,10,10,130 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile,27,0,6,10,2022,18894,2110,7,7,7,0 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile-symflower-fix,118,0,8,10,2915,23570,3039,10,10,10,80 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile-symflower-fix,129,0,9,10,2972,25516,3092,10,10,10,90 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile-symflower-fix,140,0,10,10,2733,25267,2853,10,10,10,100 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile-symflower-fix,169,0,9,10,2851,25485,2975,10,10,10,130 +openrouter/qwen/qwen-72b-chat,ruby,ruby/transpile,transpile-symflower-fix,27,0,6,10,2022,18894,2110,7,7,7,0 +openrouter/recursal/eagle-7b,golang,golang/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/eagle-7b,golang,golang/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/eagle-7b,java,java/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/eagle-7b,java,java/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/eagle-7b,ruby,ruby/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/eagle-7b,ruby,ruby/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,golang,golang/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,golang,golang/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,java,java/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,java,java/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,ruby,ruby/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,ruby,ruby/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,1,0,0,1,1884,29819,1888,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,2,0,0,1,463,6357,556,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,2,0,0,1,511,8000,598,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,2,0,0,1,918,13296,955,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,3,0,0,1,118,2330,130,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests-symflower-fix,1,0,0,1,1884,29819,1888,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,463,6357,556,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,511,8000,598,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,918,13296,955,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,118,2330,130,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,1,0,0,1,51,3058,52,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,1,0,0,1,73,1684,74,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,2,0,0,1,175,3393,269,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,3,0,0,1,147,2488,160,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,3,0,0,1,207,3502,220,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests-symflower-fix,1,0,0,1,51,3058,52,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests-symflower-fix,1,0,0,1,73,1684,74,1,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests-symflower-fix,2,0,0,1,175,3393,269,1,0,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests-symflower-fix,3,0,0,1,147,2488,160,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests-symflower-fix,3,0,0,1,207,3502,220,1,1,1,0 +openrouter/rwkv/rwkv-5-world-3b,ruby,ruby/plain,write-tests,0,0,0,1,0,0,0,0,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,ruby,ruby/plain,write-tests-symflower-fix,0,0,0,1,0,0,0,0,0,0,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,3,0,0,1,103,1036,117,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,3,0,0,1,326,1902,340,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,3,0,0,1,386,2253,400,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,3,0,0,1,681,2790,695,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,3,0,0,1,94,1631,108,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,103,1036,117,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,326,1902,340,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,386,2253,400,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,681,2790,695,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,94,1631,108,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,1271,1190,13,23,17823,71056,18232,23,22,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,1398,1320,12,23,21119,73670,22090,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,1802,1720,17,23,24432,91152,25946,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,1982,1900,16,23,24928,96346,25465,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,2092,2010,17,23,22558,82873,24711,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests-symflower-fix,1271,1190,13,23,17823,71056,18232,23,22,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests-symflower-fix,1398,1320,12,23,21119,73670,22090,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests-symflower-fix,1802,1720,17,23,24432,91152,25946,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests-symflower-fix,1982,1900,16,23,24928,96346,25465,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests-symflower-fix,2092,2010,17,23,22558,82873,24711,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/mistakes,code-repair,96,0,3,5,828,7490,1792,5,3,5,80 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,14,10,1,1,196,1034,208,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,14,10,1,1,223,1131,235,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,14,10,1,1,258,1086,270,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,14,10,1,1,295,1535,307,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,3,0,0,1,235,1391,247,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests-symflower-fix,14,10,1,1,196,1034,208,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests-symflower-fix,14,10,1,1,223,1131,235,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests-symflower-fix,14,10,1,1,258,1086,270,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests-symflower-fix,14,10,1,1,295,1535,307,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests-symflower-fix,3,0,0,1,235,1391,247,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile,370,0,8,10,4043,23671,5330,10,3,9,340 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile,404,0,9,10,4193,19972,4723,10,6,9,370 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile,423,0,9,10,3612,20830,4720,10,5,9,390 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile,465,0,10,10,3926,22899,4480,10,6,9,430 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile,468,0,10,10,4061,20251,4282,10,8,10,430 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile-symflower-fix,370,0,8,10,4043,23671,5330,10,3,9,340 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile-symflower-fix,404,0,9,10,4193,19972,4723,10,6,9,370 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile-symflower-fix,423,0,9,10,3612,20830,4720,10,5,9,390 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile-symflower-fix,465,0,10,10,3926,22899,4480,10,6,9,430 +openrouter/teknium/openhermes-2.5-mistral-7b,java,java/transpile,transpile-symflower-fix,468,0,10,10,4061,20251,4282,10,8,10,430 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests,1115,1040,11,23,14859,74061,16286,23,18,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests,1268,1190,14,23,15168,74862,16730,23,19,22,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests,497,420,9,23,16124,70625,16524,23,22,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests,634,560,9,23,13887,66705,15199,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests,896,820,10,23,14659,74100,16148,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests-symflower-fix,1115,1040,11,23,14859,74061,16286,23,18,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests-symflower-fix,1268,1190,14,23,15168,74862,16730,23,19,22,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests-symflower-fix,497,420,9,23,16124,70625,16524,23,22,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests-symflower-fix,634,560,9,23,13887,66705,15199,23,19,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/light,write-tests-symflower-fix,896,820,10,23,14659,74100,16148,23,20,23,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/mistakes,code-repair,108,0,4,5,659,5724,794,5,4,5,90 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests,1,0,0,1,253,2950,253,1,0,0,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests,14,10,1,1,140,1105,152,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests,3,0,0,1,238,3992,250,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests,3,0,0,1,316,1618,328,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests,3,0,1,1,13,1844,381,1,0,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests-symflower-fix,1,0,0,1,253,2950,253,1,0,0,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests-symflower-fix,14,10,1,1,140,1105,152,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,238,3992,250,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,316,1618,328,1,1,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,1,1,13,1844,381,1,0,1,0 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile,114,0,5,10,3187,18986,3456,10,9,10,80 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile,123,0,3,10,2969,19041,3089,10,10,10,90 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile,83,0,4,10,2879,19552,3218,10,9,10,50 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile,84,0,4,10,3142,18557,3262,10,10,10,50 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile,90,0,5,10,3031,24241,3614,10,7,8,60 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile-symflower-fix,114,0,5,10,3187,18986,3456,10,9,10,80 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile-symflower-fix,123,0,3,10,2969,19041,3089,10,10,10,90 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile-symflower-fix,83,0,4,10,2879,19552,3218,10,9,10,50 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile-symflower-fix,84,0,4,10,3142,18557,3262,10,10,10,50 +openrouter/teknium/openhermes-2.5-mistral-7b,ruby,ruby/transpile,transpile-symflower-fix,90,0,5,10,3031,24241,3614,10,7,8,60 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,189,120,4,23,14902,76789,18620,23,19,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,216,150,5,23,23631,101453,25934,23,17,21,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,250,180,5,23,19266,90413,21490,23,19,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,308,240,9,23,16076,93011,21466,23,14,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,403,330,10,23,15857,89819,19999,23,17,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests-symflower-fix,263,190,8,23,14902,77954,18620,23,19,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests-symflower-fix,267,200,6,23,23631,101710,25934,23,17,21,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests-symflower-fix,309,240,10,23,16076,93223,21466,23,14,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests-symflower-fix,403,330,8,23,19266,91692,21490,23,19,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests-symflower-fix,557,480,14,23,15857,91329,19999,23,17,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/mistakes,code-repair,210,0,5,5,664,6031,735,5,5,5,190 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,2,0,0,1,534,2453,688,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,3,0,0,1,157,914,172,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,3,0,0,1,200,1156,215,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,3,0,0,1,530,2352,545,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,4,0,1,1,130,1184,145,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,534,2453,688,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,157,914,172,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,200,1156,215,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,530,2352,545,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests-symflower-fix,4,0,1,1,130,1184,145,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile,334,0,7,10,2894,17048,3318,10,7,10,300 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile,353,0,6,10,2949,17022,3445,10,8,9,320 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile,396,0,7,10,3027,19639,4064,10,9,10,360 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile,397,0,7,10,2903,16946,3013,10,10,10,360 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile,416,0,7,10,2996,18730,3554,10,9,10,380 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile-symflower-fix,385,0,8,10,2894,17541,3318,10,7,10,350 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile-symflower-fix,404,0,7,10,2949,17312,3445,10,8,9,370 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile-symflower-fix,416,0,7,10,2996,18730,3554,10,9,10,380 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile-symflower-fix,427,0,8,10,3027,19934,4064,10,9,10,390 +openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/transpile,transpile-symflower-fix,448,0,8,10,2903,17147,3013,10,10,10,410 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,1052,980,10,23,16522,67415,18710,23,17,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,1233,1160,13,23,16749,66856,17843,23,17,20,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,1295,1220,12,23,17506,78009,21668,23,17,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,723,650,7,23,17058,70230,19520,23,20,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,998,930,10,23,19855,85179,23581,23,13,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests-symflower-fix,1052,980,10,23,16522,67415,18710,23,17,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests-symflower-fix,1233,1160,13,23,16749,66856,17843,23,17,20,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests-symflower-fix,1295,1220,12,23,17506,78009,21668,23,17,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests-symflower-fix,723,650,7,23,17058,70230,19520,23,20,23,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests-symflower-fix,998,930,10,23,19855,85179,23581,23,13,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/mistakes,code-repair,127,0,4,5,956,7414,1362,5,3,5,110 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,1,0,0,1,257,1351,258,1,0,0,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,2,0,0,1,417,3258,1171,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,2,0,0,1,901,3309,1078,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,3,0,0,1,258,1303,281,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,3,0,1,1,240,2046,499,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests-symflower-fix,1,0,0,1,257,1351,258,1,0,0,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,417,3258,1171,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,901,3309,1078,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests-symflower-fix,3,0,0,1,258,1303,281,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests-symflower-fix,3,0,1,1,240,2046,499,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests,355,290,8,23,9705,55011,10846,22,16,19,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests,398,330,6,23,10670,55497,11620,22,18,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests,434,370,9,23,13376,65804,15741,23,13,19,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests,511,440,9,23,10822,53565,11724,23,18,21,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests,713,640,9,23,9456,55264,11590,23,19,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests-symflower-fix,355,290,8,23,9705,55011,10846,22,16,19,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests-symflower-fix,398,330,6,23,10670,55497,11620,22,18,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests-symflower-fix,434,370,9,23,13376,65804,15741,23,13,19,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests-symflower-fix,511,440,9,23,10822,53565,11724,23,18,21,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/light,write-tests-symflower-fix,713,640,9,23,9456,55264,11590,23,19,22,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/mistakes,code-repair,109,0,4,5,681,5270,835,5,5,5,90 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests,13,10,1,1,185,1622,343,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests,13,10,1,1,1989,6614,2301,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests,3,0,0,1,143,1139,156,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests,3,0,0,1,208,1424,221,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests,3,0,1,1,141,1385,326,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,185,1622,343,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,1989,6614,2301,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,143,1139,156,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,0,1,208,1424,221,1,1,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/plain,write-tests-symflower-fix,3,0,1,1,141,1385,326,1,0,1,0 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile,116,0,7,10,2551,17600,3253,10,9,10,80 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile,135,0,8,10,2676,19181,3984,10,7,10,100 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile,172,0,6,10,2335,13799,2442,10,8,8,140 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile,43,0,7,10,2452,15308,2562,10,8,8,10 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile,54,0,7,10,2335,16360,3575,10,7,10,20 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile-symflower-fix,116,0,7,10,2551,17600,3253,10,9,10,80 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile-symflower-fix,135,0,8,10,2676,19181,3984,10,7,10,100 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile-symflower-fix,172,0,6,10,2335,13799,2442,10,8,8,140 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile-symflower-fix,43,0,7,10,2452,15308,2562,10,8,8,10 +openrouter/togethercomputer/stripedhyena-nous-7b,ruby,ruby/transpile,transpile-symflower-fix,54,0,7,10,2335,16360,3575,10,7,10,20 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests,253,190,5,23,22807,113045,30166,23,12,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests,330,270,8,23,13730,87934,23058,23,6,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests,334,270,9,23,21367,118680,30181,23,10,22,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests,401,340,9,23,22674,153311,45718,23,7,22,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests,529,460,13,23,16536,97837,25380,23,10,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests-symflower-fix,304,240,6,23,22807,113410,30166,23,12,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests-symflower-fix,387,320,12,23,21367,120202,30181,23,10,22,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests-symflower-fix,453,390,11,23,22674,153783,45718,23,7,22,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests-symflower-fix,484,420,12,23,13730,89803,23058,23,6,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/light,write-tests-symflower-fix,529,460,13,23,16536,97837,25380,23,10,23,0 +openrouter/undi95/toppy-m-7b,golang,golang/mistakes,code-repair,166,0,4,5,850,8209,1708,5,2,5,150 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,13,10,1,1,156,2358,559,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,2,0,0,1,198,2325,513,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,2,0,0,1,494,30787,14146,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,2,0,0,1,84,1785,392,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,3,0,0,1,121,919,135,1,1,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,156,2358,559,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,198,2325,513,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,494,30787,14146,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,84,1785,392,1,0,1,0 +openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,121,919,135,1,1,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,2,0,0,1,1570,6969,2150,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,2,0,0,1,405,2227,638,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,2,0,0,1,527,3759,982,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,2,0,0,1,812,2795,1165,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,3,0,0,1,198,1105,210,1,1,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,1570,6969,2150,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,405,2227,638,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,527,3759,982,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests-symflower-fix,2,0,0,1,812,2795,1165,1,0,1,0 +openrouter/undi95/toppy-m-7b,java,java/plain,write-tests-symflower-fix,3,0,0,1,198,1105,210,1,1,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests,123,70,1,23,14085,70376,21936,23,6,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests,385,330,2,23,15532,92981,30514,23,7,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests,47,0,0,23,19428,83694,28916,23,2,22,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests,49,0,1,23,14040,74055,24654,23,2,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests,63,10,1,23,13630,72023,22849,23,6,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests-symflower-fix,123,70,1,23,14085,70376,21936,23,6,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests-symflower-fix,385,330,2,23,15532,92981,30514,23,7,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests-symflower-fix,47,0,0,23,19428,83694,28916,23,2,22,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests-symflower-fix,49,0,1,23,14040,74055,24654,23,2,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/light,write-tests-symflower-fix,63,10,1,23,13630,72023,22849,23,6,23,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/mistakes,code-repair,136,0,5,5,659,6597,1661,5,1,5,120 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests,13,10,1,1,156,5777,1923,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests,2,0,0,1,138,1629,473,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests,2,0,0,1,138,2591,683,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests,2,0,0,1,151,2102,882,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests,2,0,0,1,211,3103,905,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests-symflower-fix,13,10,1,1,156,5777,1923,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,138,1629,473,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,138,2591,683,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,151,2102,882,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,211,3103,905,1,0,1,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile,29,0,8,10,3279,24894,8538,10,1,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile,30,0,9,10,2564,40802,20661,10,1,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile,31,0,9,10,3999,25771,7990,10,2,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile,33,0,10,10,2891,25058,7240,10,3,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile,33,0,9,10,3094,24348,7516,10,4,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile-symflower-fix,29,0,8,10,3279,24894,8538,10,1,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile-symflower-fix,30,0,9,10,2564,40802,20661,10,1,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile-symflower-fix,31,0,9,10,3999,25771,7990,10,2,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile-symflower-fix,33,0,10,10,2891,25058,7240,10,3,10,0 +openrouter/undi95/toppy-m-7b,ruby,ruby/transpile,transpile-symflower-fix,33,0,9,10,3094,24348,7516,10,4,10,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,141,100,4,23,14518,561408,18282,23,2,12,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,170,130,3,23,15216,560810,18469,23,2,12,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,85,50,2,23,15512,570482,18370,23,1,9,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,85,50,3,23,15293,568781,17544,23,1,8,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,98,60,2,23,14880,554643,17416,23,3,10,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests-symflower-fix,190,150,7,23,15512,572013,18370,23,1,9,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests-symflower-fix,209,170,7,23,15293,569622,17544,23,1,8,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests-symflower-fix,285,240,9,23,14880,555892,17416,23,3,10,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests-symflower-fix,317,270,10,23,14518,562837,18282,23,2,12,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests-symflower-fix,367,320,10,23,15216,563013,18469,23,2,12,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/mistakes,code-repair,51,0,1,5,844,51674,2428,5,0,5,40 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,13,10,1,1,448,13229,527,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,13,10,1,1,80,9492,474,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,13,10,1,1,96,14318,721,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,2,0,0,1,186,10827,477,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,2,0,0,1,250,12809,638,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,186,11241,477,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,448,13229,527,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,80,9492,474,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests-symflower-fix,13,10,1,1,96,14318,721,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests-symflower-fix,2,0,0,1,250,12809,638,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile,290,0,5,10,3586,125362,4755,10,5,10,260 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile,343,0,7,10,4330,135865,5097,10,6,10,310 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile,344,0,6,10,3198,113253,3683,10,8,10,310 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile,465,0,8,10,3211,106391,3617,10,7,10,430 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile,474,0,8,10,3978,121954,4462,10,6,10,440 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile-symflower-fix,516,0,10,10,4330,136969,5097,10,6,10,480 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile-symflower-fix,517,0,9,10,3198,114415,3683,10,8,10,480 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile-symflower-fix,525,0,9,10,3978,122387,4462,10,6,10,490 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile-symflower-fix,557,0,10,10,3211,106908,3617,10,7,10,520 +openrouter/xwin-lm/xwin-lm-70b,golang,golang/transpile,transpile-symflower-fix,575,0,10,10,3586,127299,4755,10,5,10,540 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,126,90,1,23,20738,576687,25228,23,0,12,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,318,280,1,23,21320,526046,25365,23,0,14,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,32,0,0,23,22533,564653,25621,23,0,9,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,34,0,0,23,23507,586651,26695,23,1,10,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,36,0,0,23,20928,533434,25486,23,1,12,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests-symflower-fix,126,90,1,23,20738,576687,25228,23,0,12,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests-symflower-fix,318,280,1,23,21320,526046,25365,23,0,14,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests-symflower-fix,32,0,0,23,22533,564653,25621,23,0,9,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests-symflower-fix,34,0,0,23,23507,586651,26695,23,1,10,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests-symflower-fix,36,0,0,23,20928,533434,25486,23,1,12,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/mistakes,code-repair,155,0,5,5,1271,51966,2683,5,0,5,140 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,13,10,1,1,134,24719,1251,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,2,0,0,1,1082,30533,1269,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,2,0,0,1,137,10901,543,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,2,0,0,1,200,12580,620,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,2,0,0,1,219,10846,492,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests-symflower-fix,13,10,1,1,134,24719,1251,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests-symflower-fix,2,0,0,1,1082,30533,1269,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests-symflower-fix,2,0,0,1,137,10901,543,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests-symflower-fix,2,0,0,1,200,12580,620,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests-symflower-fix,2,0,0,1,219,10846,492,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile,427,0,9,10,4199,108354,4686,10,8,10,390 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile,454,0,10,10,4187,105972,4876,10,4,10,420 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile,459,0,10,10,4016,102773,4317,10,9,10,420 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile,468,0,10,10,3995,109596,4393,10,8,10,430 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile,468,0,10,10,4140,104756,4864,10,8,10,430 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile-symflower-fix,427,0,9,10,4199,108354,4686,10,8,10,390 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile-symflower-fix,454,0,10,10,4187,105972,4876,10,4,10,420 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile-symflower-fix,459,0,10,10,4016,102773,4317,10,9,10,420 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile-symflower-fix,468,0,10,10,3995,109596,4393,10,8,10,430 +openrouter/xwin-lm/xwin-lm-70b,java,java/transpile,transpile-symflower-fix,468,0,10,10,4140,104756,4864,10,8,10,430 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests,1,0,0,1,320,8490,321,1,0,0,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests,2,0,0,1,156,9768,399,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests,2,0,0,1,166,13081,713,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests,2,0,0,1,174,15958,718,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests,2,0,0,1,228,24502,1195,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests-symflower-fix,1,0,0,1,320,8490,321,1,0,0,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,156,9768,399,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,166,13081,713,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,174,15958,718,1,0,1,0 +openrouter/xwin-lm/xwin-lm-70b,ruby,ruby/plain,write-tests-symflower-fix,2,0,0,1,228,24502,1195,1,0,1,0 diff --git a/docs/reports/v0.6/max-score.txt b/docs/reports/v0.6/max-score.txt new file mode 100644 index 00000000..41c8a756 --- /dev/null +++ b/docs/reports/v0.6/max-score.txt @@ -0,0 +1,141 @@ +WRITE TESTS +java: +Loaded 123 log files (total of 1813290 lines) +Coverage + - src/main/java/com/eval/BalancedBrackets.java:14 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/BinarySearch.java:15 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/CallLoopConditionsOftenEnough.java:13 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/CascadingIfElse.java:8 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/CollatzCalculator.java:13 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/ConditionsAnd.java:5 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/EqulibriumIndices.java:14 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/ForLoop.java:9 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/JacobiSymbol.java:26 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/KlarnerRadoSequence.java:18 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/Knapsack.java:21 ("openrouter/deepseek/deepseek-coder") + - src/main/java/com/eval/LargestProperDivisor.java:13 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/MagicSquareOdd.java:28 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/MatchBytes.java:13 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/PascalsTriangle.java:14 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/PhoneNumber.java:16 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/Plain.java:1 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/SimpleIfElse.java:5 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/SimpleSwitchWithReturn.java:6 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/Sort.java:6 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/TypeArrayAccess.java:5 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/TypeArrayConsecutiveAccess.java:9 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/TypeArrayMultidimensionalArrayLength.java:8 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/ValidDate.java:25 ("openrouter/google/gemma-2-9b-it") +∑ = 305 +ruby: +Loaded 128 log files (total of 938269 lines) +Coverage + - lib/balanced_brackets.rb:11 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/binary_search.rb:12 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/call_loop_conditions_often_enough.rb:8 ("openrouter/google/gemma-2-27b-it") + - lib/cascading_if_else.rb:6 ("openrouter/google/gemma-2-27b-it") + - lib/collatz_calculator.rb:9 ("openrouter/deepseek/deepseek-coder") + - lib/conditions_and.rb:4 ("openrouter/google/gemma-2-27b-it") + - lib/equilibrium_indices.rb:11 ("openrouter/google/gemma-2-27b-it") + - lib/for_loop.rb:7 ("openrouter/google/gemma-2-27b-it") + - lib/jacobi_symbol.rb:20 ("openrouter/meta-llama/llama-3.1-70b-instruct") + - lib/klarner_rado_sequence.rb:14 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/knapsack.rb:21 ("openrouter/google/gemma-2-27b-it") + - lib/largest_proper_divisor.rb:11 ("openrouter/deepseek/deepseek-coder") + - lib/magic_square_odd.rb:23 ("openrouter/google/gemma-2-27b-it") + - lib/match_bytes.rb:10 ("openrouter/openai/gpt-4o") + - lib/pascals_triangle.rb:11 ("openrouter/google/gemma-2-27b-it") + - lib/phone_number.rb:12 ("openrouter/deepseek/deepseek-coder") + - lib/plain.rb:1 ("openrouter/google/gemma-2-27b-it") + - lib/simple_if_else.rb:4 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/simple_switch_with_return.rb:4 ("openrouter/google/gemma-2-27b-it") + - lib/sort.rb:5 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/type_array_access.rb:4 ("openrouter/nousresearch/hermes-2-theta-llama-3-8b") + - lib/type_array_consecutive_access.rb:7 ("openrouter/google/gemma-2-27b-it") + - lib/type_array_multidimensional_array_length.rb:6 ("openrouter/google/gemma-2-27b-it") + - lib/validate_date.rb:17 ("openrouter/meta-llama/llama-3.1-70b-instruct") +∑ = 238 +golang: +Loaded 117 log files (total of 2151460 lines) +Coverage + - balancedBrackets.go:5 ("openrouter/google/gemma-2-9b-it") + - binarySearch.go:4 ("openrouter/google/gemma-2-9b-it") + - callLoopConditionsOftenEnough.go:4 ("openrouter/google/gemma-2-9b-it") + - cascadingIfElse.go:3 ("openrouter/google/gemma-2-9b-it") + - collatzCalculator.go:6 ("openrouter/google/gemma-2-9b-it") + - conditionsAnd.go:2 ("openrouter/google/gemma-2-9b-it") + - equilibriumIndices.go:4 ("openrouter/google/gemma-2-9b-it") + - forLoop.go:3 ("openrouter/google/gemma-2-9b-it") + - jacobiSymbol.go:6 ("openrouter/google/gemma-2-27b-it") + - klarnerRadoSequence.go:3 ("openrouter/google/gemma-2-9b-it") + - knapsack.go:5 ("openrouter/google/gemma-2-9b-it") + - largestProperDivisor.go:4 ("openrouter/google/gemma-2-9b-it") + - magicSquareOdd.go:8 ("openrouter/google/gemma-2-9b-it") + - matchBytes.go:2 ("openrouter/google/gemma-2-9b-it") + - pascalsTriangle.go:4 ("openrouter/google/gemma-2-9b-it") + - phoneNumber.go:5 ("openrouter/google/gemma-2-9b-it") + - plain.go:1 ("openrouter/google/gemma-2-9b-it") + - simpleIfElse.go:2 ("openrouter/google/gemma-2-9b-it") + - simpleSwitchWithReturn.go:3 ("openrouter/google/gemma-2-9b-it") + - sort.go:2 ("openrouter/google/gemma-2-9b-it") + - typeArrayAccess.go:2 ("openrouter/google/gemma-2-27b-it") + - typeArrayConsecutiveAccess.go:3 ("openrouter/google/gemma-2-9b-it") + - typeArrayMultidimensionalArrayLength.go:3 ("openrouter/google/gemma-2-9b-it") + - validateDate.go:7 ("openrouter/google/gemma-2-9b-it") +∑ = 91 +TRANSPILE +java: +Loaded 56 log files (total of 291778 lines) +Tests Passing + - BalancedBrackets:5 ("openrouter/google/gemma-2-9b-it") + - BinarySearch:5 ("openrouter/google/gemma-2-9b-it") + - CascadingIfElse:3 ("openrouter/google/gemma-2-9b-it") + - IsSorted:5 ("openrouter/google/gemma-2-27b-it") + - PascalsTriangle:4 ("openrouter/google/gemma-2-9b-it") +∑ = 22 +ruby: +Loaded 5 log files (total of 28694 lines) +Tests Passing + - ../lib/balanced_brackets:5 ("openrouter/cohere/command-r-08-2024") + - ../lib/binary_search:5 ("openrouter/cohere/command-r-08-2024") + - ../lib/cascading_if_else:3 ("openrouter/cohere/command-r-08-2024") + - ../lib/pascals_triangle:4 ("openrouter/cohere/command-r-08-2024") + - ../lib/sort:5 ("openrouter/cohere/command-r-08-2024") +∑ = 22 +golang: +Loaded 50 log files (total of 237312 lines) +Tests Passing + - balancedBrackets:6 ("openrouter/google/gemma-2-27b-it") + - binarySearch:6 ("openrouter/google/gemma-2-9b-it") + - cascadingIfElse:4 ("openrouter/google/gemma-2-27b-it") + - isSorted:6 ("openrouter/google/gemma-2-9b-it") + - pascalsTriangle:5 ("openrouter/google/gemma-2-9b-it") +∑ = 27 +CODE REPAIR +java: +Loaded 56 log files (total of 92147 lines) +Tests Passing + - src/main/java/com/eval/ImportMissing.java:2 ("openrouter/google/gemma-2-9b-it") + - src/main/java/com/eval/OpeningBracketMissing.java:3 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/SyntaxErrorTypeMissing.java:3 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/TypeUnknown.java:3 ("openrouter/google/gemma-2-27b-it") + - src/main/java/com/eval/VariableUnknown.java:3 ("openrouter/google/gemma-2-9b-it") +∑ = 14 +ruby: +Loaded 5 log files (total of 14438 lines) +Tests Passing + - lib/argument_missing.rb:3 ("openrouter/cohere/command-r-08-2024") + - lib/end_keyword_missing.rb:3 ("openrouter/cohere/command-r-08-2024") + - lib/import_missing.rb:3 ("openrouter/cohere/command-r-08-2024") + - lib/type_unknown.rb:3 ("openrouter/cohere/command-r-08-2024") + - lib/variable_unknown.rb:3 ("openrouter/cohere/command-r-08-2024") +∑ = 15 +golang: +Loaded 49 log files (total of 29932 lines) +Tests Passing + - importMissing.go:3 ("openrouter/google/gemma-2-9b-it") + - openingBracketMissing.go:4 ("openrouter/google/gemma-2-9b-it") + - syntaxErrorTypeMissing.go:4 ("openrouter/google/gemma-2-9b-it") + - typeUnknown.go:4 ("openrouter/google/gemma-2-27b-it") + - variableUnknown.go:4 ("openrouter/google/gemma-2-9b-it") +∑ = 19 diff --git a/docs/reports/v0.6/meta.csv b/docs/reports/v0.6/meta.csv new file mode 100644 index 00000000..e54e8abe --- /dev/null +++ b/docs/reports/v0.6/meta.csv @@ -0,0 +1,176 @@ +model_id,model_name,completion,image,prompt,request +openrouter/01-ai/yi-1.5-34b-chat,Yi 1.5 34B Chat,0.00000072,0,0.00000072,0 +openrouter/01-ai/yi-34b-chat,Yi 34B Chat,0.00000072,0,0.00000072,0 +openrouter/aetherwiing/mn-starcannon-12b,Mistral Nemo 12B Starcannon,0.000002,0,0.000002,0 +openrouter/ai21/jamba-1-5-large,AI21: Jamba 1.5 Large,0.000008,0,0.000002,0 +openrouter/ai21/jamba-1-5-mini,AI21: Jamba 1.5 Mini,0.0000004,0,0.0000002,0 +openrouter/ai21/jamba-instruct,AI21: Jamba Instruct,0.0000007,0,0.0000005,0 +openrouter/alpindale/goliath-120b,Goliath 120B,0.000009375,0,0.000009375,0 +openrouter/alpindale/magnum-72b,Magnum 72B,0.0000045,0,0.00000375,0 +openrouter/anthropic/claude-1,Anthropic: Claude v1,0.000024,0,0.000008,0 +openrouter/anthropic/claude-1.2,Anthropic: Claude v1.2,0.000024,0,0.000008,0 +openrouter/anthropic/claude-2,Anthropic: Claude v2,0.000024,0,0.000008,0 +openrouter/anthropic/claude-2.0,Anthropic: Claude v2.0,0.000024,0,0.000008,0 +openrouter/anthropic/claude-2.0:beta,Anthropic: Claude v2.0 (self-moderated),0.000024,0,0.000008,0 +openrouter/anthropic/claude-2.1,Anthropic: Claude v2.1,0.000024,0,0.000008,0 +openrouter/anthropic/claude-2.1:beta,Anthropic: Claude v2.1 (self-moderated),0.000024,0,0.000008,0 +openrouter/anthropic/claude-2:beta,Anthropic: Claude v2 (self-moderated),0.000024,0,0.000008,0 +openrouter/anthropic/claude-3-haiku,Anthropic: Claude 3 Haiku,0.00000125,0.0004,0.00000025,0 +openrouter/anthropic/claude-3-haiku:beta,Anthropic: Claude 3 Haiku (self-moderated),0.00000125,0.0004,0.00000025,0 +openrouter/anthropic/claude-3-opus,Anthropic: Claude 3 Opus,0.000075,0.024,0.000015,0 +openrouter/anthropic/claude-3-opus:beta,Anthropic: Claude 3 Opus (self-moderated),0.000075,0.024,0.000015,0 +openrouter/anthropic/claude-3-sonnet,Anthropic: Claude 3 Sonnet,0.000015,0.0048,0.000003,0 +openrouter/anthropic/claude-3-sonnet:beta,Anthropic: Claude 3 Sonnet (self-moderated),0.000015,0.0048,0.000003,0 +openrouter/anthropic/claude-3.5-sonnet,Anthropic: Claude 3.5 Sonnet,0.000015,0.0048,0.000003,0 +openrouter/anthropic/claude-3.5-sonnet:beta,Anthropic: Claude 3.5 Sonnet (self-moderated),0.000015,0.0048,0.000003,0 +openrouter/anthropic/claude-instant-1,Anthropic: Claude Instant v1,0.0000024,0,0.0000008,0 +openrouter/anthropic/claude-instant-1.0,Anthropic: Claude Instant v1.0,0.0000024,0,0.0000008,0 +openrouter/anthropic/claude-instant-1.1,Anthropic: Claude Instant v1.1,0.0000024,0,0.0000008,0 +openrouter/anthropic/claude-instant-1:beta,Anthropic: Claude Instant v1 (self-moderated),0.0000024,0,0.0000008,0 +openrouter/austism/chronos-hermes-13b,Chronos Hermes 13B v2,0.00000013,0,0.00000013,0 +openrouter/cognitivecomputations/dolphin-llama-3-70b,Dolphin Llama 3 70B 🐬,0.0000004,0,0.00000035,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x22b,Dolphin 2.9.2 Mixtral 8x22B 🐬,0.0000009,0,0.0000009,0 +openrouter/cognitivecomputations/dolphin-mixtral-8x7b,Dolphin 2.6 Mixtral 8x7B 🐬,0.0000005,0,0.0000005,0 +openrouter/cohere/command,Cohere: Command,0.000002,0,0.000001,0 +openrouter/cohere/command-r,Cohere: Command R,0.0000015,0,0.0000005,0 +openrouter/cohere/command-r-03-2024,Cohere: Command R (03-2024),0.0000015,0,0.0000005,0 +openrouter/cohere/command-r-08-2024,Cohere: Command R (08-2024),0.0000006,0,0.00000015,0 +openrouter/cohere/command-r-plus,Cohere: Command R+,0.000015,0,0.000003,0 +openrouter/cohere/command-r-plus-04-2024,Cohere: Command R+ (04-2024),0.000015,0,0.000003,0 +openrouter/cohere/command-r-plus-08-2024,Cohere: Command R+ (08-2024),0.00001,0,0.0000025,0 +openrouter/databricks/dbrx-instruct,Databricks: DBRX 132B Instruct,0.00000108,0,0.00000108,0 +openrouter/deepseek/deepseek-chat,DeepSeek-V2 Chat,0.00000028,0,0.00000014,0 +openrouter/deepseek/deepseek-coder,DeepSeek-Coder-V2,0.00000028,0,0.00000014,0 +openrouter/google/gemini-flash-1.5,Google: Gemini Flash 1.5,0.00000015,0.00004,0.0000000375,0 +openrouter/google/gemini-flash-1.5-exp,Google: Gemini Flash 1.5 Experimental,0,0,0,0 +openrouter/google/gemini-flash-8b-1.5-exp,Google: Gemini Flash 8B 1.5 Experimental,0,0,0,0 +openrouter/google/gemini-pro,Google: Gemini Pro 1.0,0.000000375,0.0025,0.000000125,0 +openrouter/google/gemini-pro-1.5,Google: Gemini Pro 1.5,0.0000075,0.00263,0.0000025,0 +openrouter/google/gemini-pro-1.5-exp,Google: Gemini Pro 1.5 Experimental,0,0,0,0 +openrouter/google/gemini-pro-vision,Google: Gemini Pro Vision 1.0,0.000000375,0.0025,0.000000125,0 +openrouter/google/gemma-2-27b-it,Google: Gemma 2 27B,0.00000027,0,0.00000027,0 +openrouter/google/gemma-2-9b-it,Google: Gemma 2 9B,0.00000006,0,0.00000006,0 +openrouter/google/gemma-2-9b-it:free,Google: Gemma 2 9B (free),0,0,0,0 +openrouter/google/palm-2-chat-bison,Google: PaLM 2 Chat,0.0000005,0,0.00000025,0 +openrouter/google/palm-2-chat-bison-32k,Google: PaLM 2 Chat 32k,0.0000005,0,0.00000025,0 +openrouter/google/palm-2-codechat-bison,Google: PaLM 2 Code Chat,0.0000005,0,0.00000025,0 +openrouter/google/palm-2-codechat-bison-32k,Google: PaLM 2 Code Chat 32k,0.0000005,0,0.00000025,0 +openrouter/gryphe/mythomax-l2-13b,MythoMax 13B,0.0000001,0,0.0000001,0 +openrouter/gryphe/mythomax-l2-13b:extended,MythoMax 13B (extended),0.000001125,0,0.000001125,0 +openrouter/gryphe/mythomax-l2-13b:nitro,MythoMax 13B (nitro),0.0000002,0,0.0000002,0 +openrouter/gryphe/mythomist-7b,MythoMist 7B,0.000000375,0,0.000000375,0 +openrouter/gryphe/mythomist-7b:free,MythoMist 7B (free),0,0,0,0 +openrouter/huggingfaceh4/zephyr-7b-beta:free,Hugging Face: Zephyr 7B (free),0,0,0,0 +openrouter/jondurbin/airoboros-l2-70b,Airoboros 70B,0.0000005,0,0.0000005,0 +openrouter/lizpreciatior/lzlv-70b-fp16-hf,lzlv 70B,0.0000004,0,0.00000035,0 +openrouter/mancer/weaver,Mancer: Weaver (alpha),0.00000225,0,0.000001875,0 +openrouter/meta-llama/llama-2-13b-chat,Meta: Llama v2 13B Chat,0.00000027,0,0.00000027,0 +openrouter/meta-llama/llama-3-70b-instruct,Meta: Llama 3 70B Instruct,0.0000004,0,0.00000035,0 +openrouter/meta-llama/llama-3-70b-instruct:nitro,Meta: Llama 3 70B Instruct (nitro),0.000000792,0,0.000000792,0 +openrouter/meta-llama/llama-3-8b-instruct,Meta: Llama 3 8B Instruct,0.000000055,0,0.000000055,0 +openrouter/meta-llama/llama-3-8b-instruct:extended,Meta: Llama 3 8B Instruct (extended),0.000001125,0,0.0000001875,0 +openrouter/meta-llama/llama-3-8b-instruct:free,Meta: Llama 3 8B Instruct (free),0,0,0,0 +openrouter/meta-llama/llama-3-8b-instruct:nitro,Meta: Llama 3 8B Instruct (nitro),0.000000162,0,0.000000162,0 +openrouter/meta-llama/llama-3.1-405b,Meta: Llama 3.1 405B (base),0.000002,0,0.000002,0 +openrouter/meta-llama/llama-3.1-405b-instruct,Meta: Llama 3.1 405B Instruct,0.0000025,0,0.0000025,0 +openrouter/meta-llama/llama-3.1-70b-instruct,Meta: Llama 3.1 70B Instruct,0.0000004,0,0.00000035,0 +openrouter/meta-llama/llama-3.1-8b-instruct,Meta: Llama 3.1 8B Instruct,0.000000055,0,0.000000055,0 +openrouter/meta-llama/llama-3.1-8b-instruct:free,Meta: Llama 3.1 8B Instruct (free),0,0,0,0 +openrouter/meta-llama/llama-guard-2-8b,Meta: LlamaGuard 2 8B,0.00000018,0,0.00000018,0 +openrouter/microsoft/phi-3-medium-128k-instruct,Phi-3 Medium 128K Instruct,0.000001,0,0.000001,0 +openrouter/microsoft/phi-3-medium-128k-instruct:free,Phi-3 Medium 128K Instruct (free),0,0,0,0 +openrouter/microsoft/phi-3-medium-4k-instruct,Phi-3 Medium 4K Instruct,0.00000014,0,0.00000014,0 +openrouter/microsoft/phi-3-mini-128k-instruct,Phi-3 Mini 128K Instruct,0.0000001,0,0.0000001,0 +openrouter/microsoft/phi-3-mini-128k-instruct:free,Phi-3 Mini 128K Instruct (free),0,0,0,0 +openrouter/microsoft/phi-3.5-mini-128k-instruct,Phi-3.5 Mini 128K Instruct,0.0000001,0,0.0000001,0 +openrouter/microsoft/wizardlm-2-7b,WizardLM-2 7B,0.000000055,0,0.000000055,0 +openrouter/microsoft/wizardlm-2-8x22b,WizardLM-2 8x22B,0.0000005,0,0.0000005,0 +openrouter/mistralai/codestral-mamba,Mistral: Codestral Mamba,0.00000025,0,0.00000025,0 +openrouter/mistralai/mistral-7b-instruct,Mistral: Mistral 7B Instruct,0.000000055,0,0.000000055,0 +openrouter/mistralai/mistral-7b-instruct-v0.1,Mistral: Mistral 7B Instruct v0.1,0.000000055,0,0.000000055,0 +openrouter/mistralai/mistral-7b-instruct-v0.2,Mistral: Mistral 7B Instruct v0.2,0.000000055,0,0.000000055,0 +openrouter/mistralai/mistral-7b-instruct-v0.3,Mistral: Mistral 7B Instruct v0.3,0.000000055,0,0.000000055,0 +openrouter/mistralai/mistral-7b-instruct:free,Mistral: Mistral 7B Instruct (free),0,0,0,0 +openrouter/mistralai/mistral-7b-instruct:nitro,Mistral: Mistral 7B Instruct (nitro),0.00000007,0,0.00000007,0 +openrouter/mistralai/mistral-large,Mistral Large,0.000009,0,0.000003,0 +openrouter/mistralai/mistral-medium,Mistral Medium,0.0000081,0,0.0000027,0 +openrouter/mistralai/mistral-nemo,Mistral: Mistral Nemo,0.00000017,0,0.00000017,0 +openrouter/mistralai/mistral-small,Mistral Small,0.000006,0,0.000002,0 +openrouter/mistralai/mistral-tiny,Mistral Tiny,0.00000025,0,0.00000025,0 +openrouter/mistralai/mixtral-8x22b-instruct,Mistral: Mixtral 8x22B Instruct,0.00000065,0,0.00000065,0 +openrouter/mistralai/mixtral-8x7b,Mixtral 8x7B (base),0.00000054,0,0.00000054,0 +openrouter/mistralai/mixtral-8x7b-instruct,Mixtral 8x7B Instruct,0.00000024,0,0.00000024,0 +openrouter/mistralai/mixtral-8x7b-instruct:nitro,Mixtral 8x7B Instruct (nitro),0.00000054,0,0.00000054,0 +openrouter/neversleep/llama-3-lumimaid-70b,Llama 3 Lumimaid 70B,0.0000045,0,0.000003375,0 +openrouter/neversleep/llama-3-lumimaid-8b,Llama 3 Lumimaid 8B,0.000001125,0,0.0000001875,0 +openrouter/neversleep/llama-3-lumimaid-8b:extended,Llama 3 Lumimaid 8B (extended),0.000001125,0,0.0000001875,0 +openrouter/neversleep/noromaid-20b,Noromaid 20B,0.00000225,0,0.0000015,0 +openrouter/nothingiisreal/mn-celeste-12b,Mistral Nemo 12B Celeste,0.0000015,0,0.0000015,0 +openrouter/nousresearch/hermes-2-pro-llama-3-8b,NousResearch: Hermes 2 Pro - Llama-3 8B,0.00000014,0,0.00000014,0 +openrouter/nousresearch/hermes-2-theta-llama-3-8b,Nous: Hermes 2 Theta 8B,0.000001125,0,0.0000001875,0 +openrouter/nousresearch/hermes-3-llama-3.1-405b,Nous: Hermes 3 405B Instruct,0,0,0,0 +openrouter/nousresearch/hermes-3-llama-3.1-405b:extended,Nous: Hermes 3 405B Instruct (extended),0,0,0,0 +openrouter/nousresearch/hermes-3-llama-3.1-70b,Nous: Hermes 3 70B Instruct,0.0000004,0,0.0000004,0 +openrouter/nousresearch/nous-capybara-7b:free,Nous: Capybara 7B (free),0,0,0,0 +openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,Nous: Hermes 2 Mixtral 8x7B DPO,0.00000045,0,0.00000045,0 +openrouter/nousresearch/nous-hermes-llama2-13b,Nous: Hermes 13B,0.00000017,0,0.00000017,0 +openrouter/nousresearch/nous-hermes-yi-34b,Nous: Hermes 2 Yi 34B,0.00000072,0,0.00000072,0 +openrouter/openai/chatgpt-4o-latest,OpenAI: ChatGPT-4o,0.000015,0.007225,0.000005,0 +openrouter/openai/gpt-3.5-turbo,OpenAI: GPT-3.5 Turbo,0.0000015,0,0.0000005,0 +openrouter/openai/gpt-3.5-turbo-0125,OpenAI: GPT-3.5 Turbo 16k,0.0000015,0,0.0000005,0 +openrouter/openai/gpt-3.5-turbo-0301,OpenAI: GPT-3.5 Turbo (older v0301),0.000002,0,0.000001,0 +openrouter/openai/gpt-3.5-turbo-0613,OpenAI: GPT-3.5 Turbo (older v0613),0.000002,0,0.000001,0 +openrouter/openai/gpt-3.5-turbo-1106,OpenAI: GPT-3.5 Turbo 16k (older v1106),0.000002,0,0.000001,0 +openrouter/openai/gpt-3.5-turbo-16k,OpenAI: GPT-3.5 Turbo 16k,0.000004,0,0.000003,0 +openrouter/openai/gpt-3.5-turbo-instruct,OpenAI: GPT-3.5 Turbo Instruct,0.000002,0,0.0000015,0 +openrouter/openai/gpt-4,OpenAI: GPT-4,0.00006,0,0.00003,0 +openrouter/openai/gpt-4-0314,OpenAI: GPT-4 (older v0314),0.00006,0,0.00003,0 +openrouter/openai/gpt-4-1106-preview,OpenAI: GPT-4 Turbo (older v1106),0.00003,0,0.00001,0 +openrouter/openai/gpt-4-32k,OpenAI: GPT-4 32k,0.00012,0,0.00006,0 +openrouter/openai/gpt-4-32k-0314,OpenAI: GPT-4 32k (older v0314),0.00012,0,0.00006,0 +openrouter/openai/gpt-4-turbo,OpenAI: GPT-4 Turbo,0.00003,0.01445,0.00001,0 +openrouter/openai/gpt-4-turbo-preview,OpenAI: GPT-4 Turbo Preview,0.00003,0,0.00001,0 +openrouter/openai/gpt-4-vision-preview,OpenAI: GPT-4 Vision,0.00003,0.01445,0.00001,0 +openrouter/openai/gpt-4o,OpenAI: GPT-4o,0.000015,0.007225,0.000005,0 +openrouter/openai/gpt-4o-2024-05-13,OpenAI: GPT-4o (2024-05-13),0.000015,0.007225,0.000005,0 +openrouter/openai/gpt-4o-2024-08-06,OpenAI: GPT-4o (2024-08-06),0.00001,0.0036125,0.0000025,0 +openrouter/openai/gpt-4o-mini,OpenAI: GPT-4o-mini,0.0000006,0.007225,0.00000015,0 +openrouter/openai/gpt-4o-mini-2024-07-18,OpenAI: GPT-4o-mini (2024-07-18),0.0000006,0.007225,0.00000015,0 +openrouter/openai/gpt-4o:extended,OpenAI: GPT-4o (extended),0.000018,0.007225,0.000006,0 +openrouter/openchat/openchat-7b,OpenChat 3.5 7B,0.000000055,0,0.000000055,0 +openrouter/openchat/openchat-7b:free,OpenChat 3.5 7B (free),0,0,0,0 +openrouter/openchat/openchat-8b,OpenChat 3.6 8B,0.000000055,0,0.000000055,0 +openrouter/openrouter/auto,Auto (best for prompt),-1,-1,-1,-1 +openrouter/openrouter/flavor-of-the-week,Flavor of The Week,-1,-1,-1,-1 +openrouter/perplexity/llama-3-sonar-large-32k-chat,Perplexity: Llama3 Sonar 70B,0.000001,0,0.000001,0 +openrouter/perplexity/llama-3-sonar-large-32k-online,Perplexity: Llama3 Sonar 70B Online,0.000001,0,0.000001,0.005 +openrouter/perplexity/llama-3-sonar-small-32k-chat,Perplexity: Llama3 Sonar 8B,0.0000002,0,0.0000002,0 +openrouter/perplexity/llama-3-sonar-small-32k-online,Perplexity: Llama3 Sonar 8B Online,0.0000002,0,0.0000002,0.005 +openrouter/perplexity/llama-3.1-sonar-huge-128k-online,Perplexity: Llama 3.1 Sonar 405B Online,0.000005,0,0.000005,0.005 +openrouter/perplexity/llama-3.1-sonar-large-128k-chat,Perplexity: Llama 3.1 Sonar 70B,0.000001,0,0.000001,0 +openrouter/perplexity/llama-3.1-sonar-large-128k-online,Perplexity: Llama 3.1 Sonar 70B Online,0.000001,0,0.000001,0.005 +openrouter/perplexity/llama-3.1-sonar-small-128k-chat,Perplexity: Llama 3.1 Sonar 8B,0.0000002,0,0.0000002,0 +openrouter/perplexity/llama-3.1-sonar-small-128k-online,Perplexity: Llama 3.1 Sonar 8B Online,0.0000002,0,0.0000002,0.005 +openrouter/pygmalionai/mythalion-13b,Pygmalion: Mythalion 13B,0.000001125,0,0.000001125,0 +openrouter/qwen/qwen-110b-chat,Qwen 1.5 110B Chat,0.00000162,0,0.00000162,0 +openrouter/qwen/qwen-2-72b-instruct,Qwen 2 72B Instruct,0.0000004,0,0.00000035,0 +openrouter/qwen/qwen-2-7b-instruct,Qwen 2 7B Instruct,0.000000055,0,0.000000055,0 +openrouter/qwen/qwen-2-7b-instruct:free,Qwen 2 7B Instruct (free),0,0,0,0 +openrouter/qwen/qwen-72b-chat,Qwen 1.5 72B Chat,0.00000081,0,0.00000081,0 +openrouter/recursal/eagle-7b,RWKV v5: Eagle 7B,0,0,0,0 +openrouter/recursal/rwkv-5-3b-ai-town,RWKV v5 3B AI Town,0,0,0,0 +openrouter/rwkv/rwkv-5-world-3b,RWKV v5 World 3B,0,0,0,0 +openrouter/sao10k/fimbulvetr-11b-v2,Fimbulvetr 11B v2,0.0000015,0,0.000000375,0 +openrouter/sao10k/l3-euryale-70b,Llama 3 Euryale 70B v2.1,0.0000004,0,0.00000035,0 +openrouter/sao10k/l3-lunaris-8b,Llama 3 8B Lunaris,0.000002,0,0.000002,0 +openrouter/sao10k/l3-stheno-8b,Llama 3 Stheno 8B v3.3 32K,0.0000015,0,0.00000025,0 +openrouter/sao10k/l3.1-euryale-70b,Llama 3.1 Euryale 70B v2.2,0.0000015,0,0.0000015,0 +openrouter/sophosympatheia/midnight-rose-70b,Midnight Rose 70B,0.0000008,0,0.0000008,0 +openrouter/teknium/openhermes-2.5-mistral-7b,OpenHermes 2.5 Mistral 7B,0.00000017,0,0.00000017,0 +openrouter/togethercomputer/stripedhyena-nous-7b,StripedHyena Nous 7B,0.00000018,0,0.00000018,0 +openrouter/undi95/remm-slerp-l2-13b,ReMM SLERP 13B,0.000001125,0,0.000001125,0 +openrouter/undi95/remm-slerp-l2-13b:extended,ReMM SLERP 13B (extended),0.000001125,0,0.000001125,0 +openrouter/undi95/toppy-m-7b,Toppy M 7B,0.00000007,0,0.00000007,0 +openrouter/undi95/toppy-m-7b:free,Toppy M 7B (free),0,0,0,0 +openrouter/undi95/toppy-m-7b:nitro,Toppy M 7B (nitro),0.00000007,0,0.00000007,0 +openrouter/xwin-lm/xwin-lm-70b,Xwin 70B,0.00000375,0,0.00000375,0 diff --git a/scripts/evaluation/v0.6-openrouter.sh b/scripts/evaluation/v0.6-openrouter.sh new file mode 100755 index 00000000..4753759a --- /dev/null +++ b/scripts/evaluation/v0.6-openrouter.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +REVISION="197ece8" +RESULTS="./docs/reports/v0.6" + +make install + +eval-dev-quality evaluate \ + --runtime docker \ + --runtime-image ghcr.io/symflower/eval-dev-quality:$REVISION \ + --result-path $RESULTS \ + --runs 5 \ + --parallel 20 \ + --model openrouter/01-ai/yi-34b-chat \ + --model openrouter/ai21/jamba-instruct \ + --model openrouter/alpindale/goliath-120b \ + --model openrouter/alpindale/magnum-72b \ + --model openrouter/anthropic/claude-3-haiku \ + --model openrouter/anthropic/claude-3-opus \ + --model openrouter/anthropic/claude-3-sonnet \ + --model openrouter/anthropic/claude-3.5-sonnet \ + --model openrouter/austism/chronos-hermes-13b \ + --model openrouter/cognitivecomputations/dolphin-llama-3-70b \ + --model openrouter/cognitivecomputations/dolphin-mixtral-8x22b \ + --model openrouter/cognitivecomputations/dolphin-mixtral-8x7b \ + --model openrouter/cohere/command-r-03-2024 \ + --model openrouter/cohere/command-r-08-2024 \ + --model openrouter/cohere/command-r-plus-04-2024 \ + --model openrouter/cohere/command-r-plus-08-2024 \ + --model openrouter/databricks/dbrx-instruct \ + --model openrouter/deepseek/deepseek-chat \ + --model openrouter/deepseek/deepseek-coder \ + --model openrouter/google/gemini-flash-1.5 \ + --model openrouter/google/gemini-pro-1.5 \ + --model openrouter/google/gemma-2-27b-it \ + --model openrouter/google/gemma-2-9b-it \ + --model openrouter/google/palm-2-chat-bison \ + --model openrouter/google/palm-2-codechat-bison \ + --model openrouter/meta-llama/llama-3-70b-instruct \ + --model openrouter/meta-llama/llama-3-8b-instruct \ + --model openrouter/meta-llama/llama-3.1-405b-instruct \ + --model openrouter/meta-llama/llama-3.1-70b-instruct \ + --model openrouter/meta-llama/llama-3.1-8b-instruct \ + --model openrouter/microsoft/phi-3-medium-128k-instruct \ + --model openrouter/microsoft/phi-3-medium-4k-instruct \ + --model openrouter/microsoft/phi-3-mini-128k-instruct \ + --model openrouter/microsoft/wizardlm-2-7b \ + --model openrouter/microsoft/wizardlm-2-8x22b \ + --model openrouter/mistralai/codestral-mamba \ + --model openrouter/mistralai/mistral-7b-instruct-v0.3 \ + --model openrouter/mistralai/mistral-large \ + --model openrouter/mistralai/mistral-medium \ + --model openrouter/mistralai/mistral-nemo \ + --model openrouter/mistralai/mistral-small \ + --model openrouter/mistralai/mistral-tiny \ + --model openrouter/mistralai/mixtral-8x22b-instruct \ + --model openrouter/mistralai/mixtral-8x7b-instruct \ + --model openrouter/neversleep/llama-3-lumimaid-70b \ + --model openrouter/neversleep/llama-3-lumimaid-8b \ + --model openrouter/nousresearch/hermes-2-pro-llama-3-8b \ + --model openrouter/nousresearch/hermes-2-theta-llama-3-8b \ + --model openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo \ + --model openrouter/nousresearch/nous-hermes-yi-34b \ + --model openrouter/openai/gpt-4-turbo \ + --model openrouter/openai/gpt-4o \ + --model openrouter/openai/gpt-4o-mini \ + --model openrouter/openchat/openchat-8b \ + --model openrouter/perplexity/llama-3-sonar-large-32k-chat \ + --model openrouter/perplexity/llama-3-sonar-small-32k-chat \ + --model openrouter/qwen/qwen-110b-chat \ + --model openrouter/qwen/qwen-2-72b-instruct \ + --model openrouter/qwen/qwen-2-7b-instruct \ + --model openrouter/qwen/qwen-72b-chat \ + --model openrouter/recursal/eagle-7b \ + --model openrouter/recursal/rwkv-5-3b-ai-town \ + --model openrouter/rwkv/rwkv-5-world-3b \ + --model openrouter/teknium/openhermes-2.5-mistral-7b \ + --model openrouter/togethercomputer/stripedhyena-nous-7b \ + --model openrouter/undi95/toppy-m-7b \ + --model openrouter/xwin-lm/xwin-lm-70b