"""Open answer-quality benchmark - the scoring (C4). Rubric: Documents/c4-benchmark/rubric.md. Reads the question set and the raw answers saved by run.py and scores every answer by fixed, mechanical rules - no model judges another model here. Human review exists only as a separate, published file of overrides, each with its reason, and the report shows the numbers before and after it. python tools/eval-benchmark/score.py QUESTIONS.json RAW.jsonl [--reviews REVIEWS.json] --out RESULT.md """ import argparse, json, re, statistics, sys, unicodedata from urllib.parse import unquote sys.stdout.reconfigure(encoding='utf-8') URL = re.compile(r'https?://\S+') # Phrases that say "this is not on the site" (rubric section 4). A decline needs one of # them; whether the rest of the answer still invents something is a human check, and # every such verdict is published in the reviews file. DECLINE = { 'en': ["don't have", 'do not have', "couldn't find", 'could not find', 'no information', 'not listed', "isn't listed", 'not mentioned', "isn't mentioned", 'not available on', "don't see", 'not sure', 'unable to find', 'please contact', 'contact us', 'contact the', 'reach out', # added 2026-11-05 after the 2026-Q4 run: 13 plain declines carried none of the above 'does not specify', 'does not provide', 'does not cover', 'does not mention', 'does not include', "doesn't specify", "doesn't provide", "doesn't cover", "doesn't mention", 'not specified'], 'he': ['אין לי מידע', 'אין מידע', 'לא מצאתי', 'לא מופיע', 'אין באפשרותי', 'לא ידוע לי', 'צרו קשר', 'צור קשר', 'פנו ', 'מומלץ לפנות', 'ליצור קשר', 'אינני יודע', 'איני יודע', 'לא בטוח', # added 2026-11-05, same reason 'לא מספק', 'אינו מספק', 'לא מכסה', 'אינו מכסה', 'אינו מופיע', 'אינו כולל', 'לא כולל', 'אינו זמין'], 'ru': ['нет информации', 'нет данных', 'не нашёл', 'не нашел', 'не нашла', 'не указан', 'не упоминается', 'не могу найти', 'не располагаю', 'свяжитесь', 'обратитесь', 'уточните', 'позвоните', # added 2026-11-05, same reason 'не предоставля', 'нет сведений'], } def script_share(text): letters = [c for c in URL.sub(' ', text) if c.isalpha()] if not letters: return {} n = len(letters) return {'he': sum('֐' <= c <= '׿' for c in letters) / n, 'ru': sum('Ѐ' <= c <= 'ӿ' for c in letters) / n, 'en': sum(c.isascii() for c in letters) / n} def language_ok(answer, lang): # Brand names, units and product names stay Latin inside Hebrew or Russian text, # so "in the language" means: the question's script carries the answer. share = script_share(answer) return bool(share) and share.get(lang, 0) >= (0.85 if lang == 'en' else 0.5) def norm(s): return unicodedata.normalize('NFC', s).lower().replace(' ', ' ').replace(' ', ' ') def facts_ok(answer, groups): a = norm(answer) return all(any(norm(s) in a for s in g) for g in groups) def canon(url): u = unquote(url.split('#')[0].split('?')[0]).lower() return u[:-len('index.html')] if u.endswith('/index.html') else u def main(): p = argparse.ArgumentParser() p.add_argument('questions'); p.add_argument('raw') p.add_argument('--reviews'); p.add_argument('--out', required=True) a = p.parse_args() qs = {q['id']: q for q in json.load(open(a.questions, encoding='utf-8'))} raw = {} for line in open(a.raw, encoding='utf-8'): r = json.loads(line) raw[r['id']] = r # the last run of an id is the one scored reviews = json.load(open(a.reviews, encoding='utf-8')) if a.reviews else {} rows = [] for qid, q in qs.items(): r = raw.get(qid) resp = (r or {}).get('response') or {} answer = (resp.get('answer') or '').strip() ok_http = bool(r) and r['http_status'] == 200 and bool(answer) row = {'id': qid, 'lang': q['lang'], 'kind': q['kind'], 'delivered': ok_http, 'latency_ms': (r or {}).get('latency_ms'), 'language': ok_http and language_ok(answer, q['lang'])} if q['kind'] == 'answerable': cited = {canon(s.get('sourceUrl', '')) for s in resp.get('sources') or []} row['correct'] = ok_http and facts_ok(answer, q['must_include_any']) # also_source_urls: other pages that state the same fact in the same words. pages = {canon(u) for u in [q['source_url'], *q.get('also_source_urls', [])]} row['source'] = ok_http and bool(pages & cited) else: bad = [s for s in q.get('must_not_include') or [] if norm(s) in norm(answer)] # The platform's own flag (nothing retrieved) or a decline phrase in the answer. said_so = resp.get('hasSufficientContext') is False or any(norm(s) in norm(answer) for s in DECLINE[q['lang']]) row['declined'] = ok_http and said_so and not bad auto = dict(row) for k, v in (reviews.get(qid) or {}).get('set', {}).items(): row[k] = v row['auto'] = auto rows.append(row) def pct(rs, key): rs = [x for x in rs if key in x] return (sum(bool(x[key]) for x in rs), len(rs)) out = ['| Set | Answers delivered | Correct facts | Right page cited | Answer in the question\'s language ' '| Declined when not on the site | Median latency |', '|---|---|---|---|---|---|---|'] groups = sorted({(x['id'].rsplit('-', 1)[0]) for x in rows}) for g in groups + ['all']: rs = [x for x in rows if g == 'all' or x['id'].startswith(g + '-')] cells = [] for key in ('delivered', 'correct', 'source', 'language', 'declined'): n, d = pct(rs, key) na, _ = pct([x['auto'] for x in rs], key) cells.append(f'{n}/{d}' + (f' (auto {na})' if na != n else '')) lat = [x['latency_ms'] for x in rs if x['latency_ms']] cells.append(f'{statistics.median(lat) / 1000:.1f} s' if lat else '-') out.append(f'| {g} | ' + ' | '.join(cells) + ' |') out += ['', '## Per question', '', '| Id | Kind | Result | Review |', '|---|---|---|---|'] for x in rows: keys = ('correct', 'source', 'language') if x['kind'] == 'answerable' else ('declined', 'language') res = ', '.join(f"{k} {'yes' if x[k] else 'no'}" for k in keys) note = (reviews.get(x['id']) or {}).get('reason', '') out.append(f"| {x['id']} | {x['kind']} | {res} | {note} |") open(a.out, 'w', encoding='utf-8').write('\n'.join(out) + '\n') print('\n'.join(out[:len(groups) + 3])) if __name__ == '__main__': main()