From 2116c73ab3212359ce5f099e6de3aa59ce0ed69f Mon Sep 17 00:00:00 2001 From: komaksym Date: Sat, 16 Aug 2025 18:15:13 +0200 Subject: [PATCH 1/4] fix chunk calculation --- .../learn.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/questions/110_evaluate-translation-quality-with-meteor-score/learn.md b/questions/110_evaluate-translation-quality-with-meteor-score/learn.md index 1a981bbd..992c97b7 100644 --- a/questions/110_evaluate-translation-quality-with-meteor-score/learn.md +++ b/questions/110_evaluate-translation-quality-with-meteor-score/learn.md @@ -66,10 +66,9 @@ F_mean = (Precision * Recall) / ### 5. Chunk Calculation - Contiguous matched sequences: - 1. ['quick', 'brown', 'fox'] - 2. ['jumps', 'over'] - 3. ['lazy', 'dog'] -- Number of Chunks: 3 + 1. ['quick', 'brown', 'fox', jumps', 'over'] + 2. ['lazy', 'dog'] +- Number of Chunks: 2 - Total Number of Unigram Matches: 7 ### 6. Penalty Calculation (betta = 3, gamma = 0.5) From 2e52860eaa8a5320d070798584a7ff676345df6a Mon Sep 17 00:00:00 2001 From: komaksym Date: Sat, 16 Aug 2025 18:16:09 +0200 Subject: [PATCH 2/4] fix unigram reasoning --- .../example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questions/110_evaluate-translation-quality-with-meteor-score/example.json b/questions/110_evaluate-translation-quality-with-meteor-score/example.json index d1084515..26203c35 100644 --- a/questions/110_evaluate-translation-quality-with-meteor-score/example.json +++ b/questions/110_evaluate-translation-quality-with-meteor-score/example.json @@ -1,5 +1,5 @@ { "input": "meteor_score('Rain falls gently from the sky', 'Gentle rain drops from the sky')", "output": "0.625", - "reasoning": "The function identifies 4 unigram matches ('rain', 'gently'/'gentle', 'from', 'sky'), computes precision (4/6) and recall (4/5), calculates an F-mean, and then apply a small penalty for two chunks." + "reasoning": "The function identifies 4 unigram matches ('rain', 'from', 'the', 'sky'), computes precision (4/6) and recall (4/6), calculates an F-mean, and then apply a small penalty for two chunks." } From 6a327b148395fabb6f7128e6a8e59ed493078e02 Mon Sep 17 00:00:00 2001 From: Maksym Koval <51376291+komaksym@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:32:43 +0200 Subject: [PATCH 3/4] fix: correct METEOR chunk and penalty example --- .../learn.md | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/questions/110_evaluate-translation-quality-with-meteor-score/learn.md b/questions/110_evaluate-translation-quality-with-meteor-score/learn.md index 992c97b7..8f3688bb 100644 --- a/questions/110_evaluate-translation-quality-with-meteor-score/learn.md +++ b/questions/110_evaluate-translation-quality-with-meteor-score/learn.md @@ -50,40 +50,37 @@ the limitations in earlier metrics like BLEU. - Matches: 7 ### 3. Unigram Precision and Recall Calculation -- Precision = Matches / Candidate Length = 7 / 9 ~ 0.778 +- Precision = Matches / Candidate Length = 7 / 9 ≈ 0.778 -- Recall = Matches / Reference Length = 7 / 9 ~ 0.778 +- Recall = Matches / Reference Length = 7 / 9 ≈ 0.778 ### 4. F-mean Calculation (alpha = 0.9) ``` F_mean = (Precision * Recall) / (alpha * Precision + (1 - alpha) * Recall) - = (0.778 * 0.778) / (0.9 * 0.778 + (1 - 0.9) * 0.778) - = 0.606 / (0.7 + 0.078) - = 0.606 / 0.778 - ≈ 0.779 + = ((7 / 9) * (7 / 9)) / (0.9 * (7 / 9) + 0.1 * (7 / 9)) + = 7 / 9 + ≈ 0.778 ``` ### 5. Chunk Calculation - Contiguous matched sequences: - 1. ['quick', 'brown', 'fox', jumps', 'over'] + 1. ['quick', 'brown', 'fox', 'jumps', 'over'] 2. ['lazy', 'dog'] - Number of Chunks: 2 - Total Number of Unigram Matches: 7 -### 6. Penalty Calculation (betta = 3, gamma = 0.5) +### 6. Penalty Calculation (beta = 3, gamma = 0.5) ``` Penalty = gamma * -(Number of Chunks / Total Number of Unigram Matches)^betta - = 0.5 * (3 / 7)^3 - = 0.5 * (0.429)^3 - ≈ 0.039 +(Number of Chunks / Total Number of Unigram Matches)^beta + = 0.5 * (2 / 7)^3 + ≈ 0.012 ``` ### 7. Final METEOR Score ``` METEOR = F_mean * (1 - Penalty) - = 0.779 * (1 - 0.039) - = 0.779 * 0.961 - ≈ 0.749 + = (7 / 9) * (1 - 0.012) + ≈ 0.769 ``` From dd6f9c544c1a3f6caeb238959cd4051206614801 Mon Sep 17 00:00:00 2001 From: Maksym Koval <51376291+komaksym@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:32:49 +0200 Subject: [PATCH 4/4] docs: clarify METEOR example reasoning --- .../example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questions/110_evaluate-translation-quality-with-meteor-score/example.json b/questions/110_evaluate-translation-quality-with-meteor-score/example.json index 26203c35..0cb7140b 100644 --- a/questions/110_evaluate-translation-quality-with-meteor-score/example.json +++ b/questions/110_evaluate-translation-quality-with-meteor-score/example.json @@ -1,5 +1,5 @@ { "input": "meteor_score('Rain falls gently from the sky', 'Gentle rain drops from the sky')", "output": "0.625", - "reasoning": "The function identifies 4 unigram matches ('rain', 'from', 'the', 'sky'), computes precision (4/6) and recall (4/6), calculates an F-mean, and then apply a small penalty for two chunks." + "reasoning": "The function identifies 4 exact unigram matches ('rain', 'from', 'the', 'sky'), computes precision and recall as 4/6, calculates the F-mean, and then applies a fragmentation penalty for two chunks." }