#!/usr/bin/env python3
"""Nine local synthetic extraction calls. No keys, real records, or cloud model API."""
import datetime, json, pathlib, statistics, time, urllib.request
ROOT = pathlib.Path(__file__).resolve().parent
BASE = 'http://127.0.0.1:11434'
MODEL = 'hermes3:8b'
OPTIONS = {'temperature': 0, 'seed': 42, 'num_ctx': 4096, 'num_thread': 24, 'num_predict': 128}
SCHEMA = {'type': 'object', 'properties': {'reference': {'type': 'string'}, 'quantity': {'type': ['integer', 'null']}, 'due_date': {'type': ['string', 'null']}, 'needs_review': {'type': 'boolean'}}, 'required': ['reference', 'quantity', 'due_date', 'needs_review'], 'additionalProperties': False}
CASES = [
 ('Job BIKE_01 needs two tires. Collection requested for 2026-10-09. The note is complete.', {'reference':'BIKE_01','quantity':2,'due_date':'2026-10-09','needs_review':False}),
 ('Job CATER_02 is a buffet inquiry. Guest count not supplied. No date supplied. Needs clarification.', {'reference':'CATER_02','quantity':None,'due_date':None,'needs_review':True}),
 ('Job CLEAN_03 needs 4 rooms cleaned. The old deadline was 2026-10-06. The confirmed replacement deadline is 2026-10-08. The note is complete.', {'reference':'CLEAN_03','quantity':4,'due_date':'2026-10-08','needs_review':False})
]
def call(path, data=None):
 req = urllib.request.Request(BASE+path, data=None if data is None else json.dumps(data).encode(), headers={'Content-Type':'application/json'})
 with urllib.request.urlopen(req, timeout=180) as response: return json.load(response)
if (ROOT/'quantized-benchmark-results.json').exists(): raise SystemExit('Existing results retained. Use a new directory for another run.')
metadata = {'checked_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'model':MODEL,'options':OPTIONS,'runtime':call('/api/version'),'model_metadata':next(x for x in call('/api/tags')['models'] if x['name']==MODEL),'resident_models_before':call('/api/ps'),'scope':'3 synthetic cases repeated 3 times; not a production accuracy study; direct local Ollama only'}
rows=[]
for repeat in range(3):
 for index,(text,expected) in enumerate(CASES):
  start=time.perf_counter()
  result=call('/api/chat', {'model':MODEL,'stream':False,'format':SCHEMA,'keep_alive':'5m','options':OPTIONS,'messages':[{'role':'system','content':'Extract only the reference, requested quantity, confirmed due date in YYYY-MM-DD, and whether missing quantity or date requires review. Use null for absent quantity or date. Prefer an explicitly confirmed replacement date to an old date. Do not invent information. Return JSON.'},{'role':'user','content':text}]})
  elapsed=time.perf_counter()-start
  try: answer=json.loads(result['message']['content']); valid=True
  except (KeyError,ValueError): answer={}; valid=False
  duration=result.get('eval_duration',0)
  row={'repeat':repeat+1,'case':index+1,'wall_seconds':round(elapsed,3),'load_seconds':round(result.get('load_duration',0)/1e9,3),'prompt_tokens':result.get('prompt_eval_count'),'output_tokens':result.get('eval_count'),'generation_tokens_per_second':round(result.get('eval_count',0)*1e9/duration,3) if duration else None,'parseable_json':valid,'matched_fields':sum(answer.get(k)==v for k,v in expected.items()),'field_count':4,'exact_match':answer==expected,'expected':expected,'actual':answer}
  rows.append(row); print(json.dumps({k:v for k,v in row.items() if k not in ['expected','actual']}),flush=True)
  (ROOT/'quantized-benchmark-progress.json').write_text(json.dumps({'metadata':metadata,'runs':rows},indent=2))
report={'metadata':metadata,'runs':rows,'summary':{'requests':len(rows),'distinct_synthetic_cases':len(CASES),'exact_matches':sum(r['exact_match'] for r in rows),'matched_fields':sum(r['matched_fields'] for r in rows),'total_fields':len(rows)*4,'median_wall_seconds':round(statistics.median(r['wall_seconds'] for r in rows),3),'median_generation_tokens_per_second':round(statistics.median(r['generation_tokens_per_second'] for r in rows),3),'min_wall_seconds':min(r['wall_seconds'] for r in rows),'max_wall_seconds':max(r['wall_seconds'] for r in rows),'external_model_api_used':False},'resident_models_after':call('/api/ps')}
(ROOT/'quantized-benchmark-results.json').write_text(json.dumps(report,indent=2))
print(json.dumps(report['summary']),flush=True)