diff --git a/generate_test.py b/generate_test.py new file mode 100644 index 0000000..8529aa9 --- /dev/null +++ b/generate_test.py @@ -0,0 +1,21 @@ +import json +from scorer import QualityScorer + +samples = [ + {"type": "json", "content": '{"name": "test"}'}, + {"type": "json", "content": '{"title": "Full Analysis", "body": "Long body text...", "meta": {"a": 1, "b": 2, "c": 3}, "extra": [1,2,3,4,5,6,7,8,9,0]}'}, + {"type": "markdown", "content": "# Header\nContent"}, + {"type": "markdown", "content": "# Big Report\n## Section 1\nDetailed content about agents and revenue models for 2026."} +] * 5 # Duplicate to get 20 + +scorer = QualityScorer() +results = [] +for i, s in enumerate(samples): + results.append({ + "submission_id": i + 1, + "result": scorer.score_submission(s["content"], s["type"]) + }) + +with open("scorecards.json", "w") as f: + json.dump(results, f, indent=2) +print(f"Generated {len(results)} scorecards.") diff --git a/scorecards.json b/scorecards.json new file mode 100644 index 0000000..051345f --- /dev/null +++ b/scorecards.json @@ -0,0 +1,367 @@ +[ + { + "submission_id": 1, + "result": { + "weighted_score": 0.4813, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.12, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 2, + "result": { + "weighted_score": 0.6687, + "quality_rating": "Fair", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.88, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 3, + "result": { + "weighted_score": 0.33, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 0.5, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Weak Markdown structure.", + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 4, + "result": { + "weighted_score": 0.43, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 5, + "result": { + "weighted_score": 0.4813, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.12, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 6, + "result": { + "weighted_score": 0.6687, + "quality_rating": "Fair", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.88, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 7, + "result": { + "weighted_score": 0.33, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 0.5, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Weak Markdown structure.", + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 8, + "result": { + "weighted_score": 0.43, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 9, + "result": { + "weighted_score": 0.4813, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.12, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 10, + "result": { + "weighted_score": 0.6687, + "quality_rating": "Fair", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.88, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 11, + "result": { + "weighted_score": 0.33, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 0.5, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Weak Markdown structure.", + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 12, + "result": { + "weighted_score": 0.43, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 13, + "result": { + "weighted_score": 0.4813, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.12, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 14, + "result": { + "weighted_score": 0.6687, + "quality_rating": "Fair", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.88, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 15, + "result": { + "weighted_score": 0.33, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 0.5, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Weak Markdown structure.", + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 16, + "result": { + "weighted_score": 0.43, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 17, + "result": { + "weighted_score": 0.4813, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.12, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 18, + "result": { + "weighted_score": 0.6687, + "quality_rating": "Fair", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.88, + "clarity": 1.0, + "validity": 1.0 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 19, + "result": { + "weighted_score": 0.33, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 0.5, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Weak Markdown structure.", + "Content is too brief." + ], + "pass_threshold": false + } + }, + { + "submission_id": 20, + "result": { + "weighted_score": 0.43, + "quality_rating": "Poor", + "scores": { + "completeness": 0.0, + "format_compliance": 1.0, + "coverage": 0.0, + "clarity": 1.0, + "validity": 0.8 + }, + "feedback": [ + "Content is too brief." + ], + "pass_threshold": false + } + } +] \ No newline at end of file diff --git a/scorer.py b/scorer.py new file mode 100644 index 0000000..641effc --- /dev/null +++ b/scorer.py @@ -0,0 +1,76 @@ +import json +import re +import math + +class QualityScorer: + def __init__(self, weights=None): + self.weights = weights or { + "completeness": 0.30, + "format_compliance": 0.20, + "coverage": 0.25, + "clarity": 0.15, + "validity": 0.10 + } + self.pass_threshold = 0.70 + + def score_submission(self, content, submission_type="json"): + scores = { + "completeness": 0.0, + "format_compliance": 0.0, + "coverage": 0.0, + "clarity": 0.0, + "validity": 0.0 + } + feedback = [] + + # 1. Format Compliance + if submission_type == "json": + try: + data = json.loads(content) + scores["format_compliance"] = 1.0 + scores["validity"] = 1.0 + except: + scores["format_compliance"] = 0.2 + feedback.append("Invalid JSON format detected.") + elif submission_type == "markdown": + if content.startswith("#"): scores["format_compliance"] += 0.5 + if "##" in content: scores["format_compliance"] += 0.5 + if scores["format_compliance"] < 1.0: feedback.append("Weak Markdown structure.") + + # 2. Completeness (Length and structure density) + words = content.split() + if len(words) > 200: scores["completeness"] = 1.0 + elif len(words) > 50: scores["completeness"] = 0.6 + else: feedback.append("Content is too brief.") + + # 3. Coverage (Diversity of keys/headers) + if submission_type == "json": + keys = re.findall(r'"([^"]+)"\s*:', content) + if len(set(keys)) > 8: scores["coverage"] = 1.0 + else: scores["coverage"] = len(set(keys)) / 8.0 + + # 4. Clarity (Readability heuristic) + avg_word_len = sum(len(w) for w in words) / max(len(words), 1) + if 4 < avg_word_len < 10: scores["clarity"] = 1.0 + else: scores["clarity"] = 0.5 + + # 5. Validity (Placeholder for logic check) + if scores["validity"] == 0: scores["validity"] = 0.8 # Assume valid if readable + + # Calculate weighted score + weighted_score = sum(scores[dim] * self.weights[dim] for dim in scores) + + quality_rating = "Excellent" if weighted_score > 0.9 else "Good" if weighted_score > 0.7 else "Fair" if weighted_score > 0.5 else "Poor" + + return { + "weighted_score": round(weighted_score, 4), + "quality_rating": quality_rating, + "scores": {k: round(v, 2) for k, v in scores.items()}, + "feedback": feedback, + "pass_threshold": weighted_score >= self.pass_threshold + } + +if __name__ == "__main__": + scorer = QualityScorer() + sample = '{"title": "Research Report", "content": "This is a detailed analysis of AI agent revenue streams...", "metadata": {"author": "Pioneer", "date": "2026-02-28"}, "tags": ["finance", "ai"]}' + print(json.dumps(scorer.score_submission(sample, "json"), indent=2))