{
  "metadata": {
    "checked_utc": "2026-10-04T20:53:32.371745+00:00",
    "model": "hermes3:8b",
    "options": {
      "temperature": 0,
      "seed": 42,
      "num_ctx": 4096,
      "num_thread": 24,
      "num_predict": 128
    },
    "runtime": {
      "version": "0.35.1"
    },
    "model_metadata": {
      "name": "hermes3:8b",
      "model": "hermes3:8b",
      "modified_at": "2026-10-04T15:27:12.605589006Z",
      "size": 4661227243,
      "digest": "4f6b83f30b62bc3d0cf9be09266db222805ee815c8fd7d8b38f863f655be78b7",
      "details": {
        "parent_model": "",
        "format": "gguf",
        "family": "llama",
        "families": [
          "llama"
        ],
        "parameter_size": "8.0B",
        "quantization_level": "Q4_0",
        "context_length": 131072,
        "embedding_length": 4096
      },
      "capabilities": [
        "completion",
        "tools"
      ]
    },
    "resident_models_before": {
      "models": []
    },
    "scope": "3 synthetic cases repeated 3 times; not a production accuracy study; direct local Ollama only"
  },
  "runs": [
    {
      "repeat": 1,
      "case": 1,
      "wall_seconds": 12.462,
      "load_seconds": 7.489,
      "prompt_tokens": 91,
      "output_tokens": 40,
      "generation_tokens_per_second": 12.161,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      },
      "actual": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      }
    },
    {
      "repeat": 1,
      "case": 2,
      "wall_seconds": 2.939,
      "load_seconds": 0.002,
      "prompt_tokens": 88,
      "output_tokens": 33,
      "generation_tokens_per_second": 13.431,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      },
      "actual": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      }
    },
    {
      "repeat": 1,
      "case": 3,
      "wall_seconds": 3.85,
      "load_seconds": 0.002,
      "prompt_tokens": 106,
      "output_tokens": 40,
      "generation_tokens_per_second": 12.324,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      },
      "actual": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      }
    },
    {
      "repeat": 2,
      "case": 1,
      "wall_seconds": 3.011,
      "load_seconds": 0.002,
      "prompt_tokens": 91,
      "output_tokens": 40,
      "generation_tokens_per_second": 13.738,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      },
      "actual": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      }
    },
    {
      "repeat": 2,
      "case": 2,
      "wall_seconds": 2.716,
      "load_seconds": 0.002,
      "prompt_tokens": 88,
      "output_tokens": 33,
      "generation_tokens_per_second": 14.264,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      },
      "actual": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      }
    },
    {
      "repeat": 2,
      "case": 3,
      "wall_seconds": 3.469,
      "load_seconds": 0.002,
      "prompt_tokens": 106,
      "output_tokens": 40,
      "generation_tokens_per_second": 13.875,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      },
      "actual": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      }
    },
    {
      "repeat": 3,
      "case": 1,
      "wall_seconds": 2.924,
      "load_seconds": 0.003,
      "prompt_tokens": 91,
      "output_tokens": 40,
      "generation_tokens_per_second": 14.105,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      },
      "actual": {
        "reference": "BIKE_01",
        "quantity": 2,
        "due_date": "2026-10-09",
        "needs_review": false
      }
    },
    {
      "repeat": 3,
      "case": 2,
      "wall_seconds": 2.727,
      "load_seconds": 0.002,
      "prompt_tokens": 88,
      "output_tokens": 33,
      "generation_tokens_per_second": 14.167,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      },
      "actual": {
        "reference": "CATER_02",
        "quantity": null,
        "due_date": null,
        "needs_review": true
      }
    },
    {
      "repeat": 3,
      "case": 3,
      "wall_seconds": 3.524,
      "load_seconds": 0.002,
      "prompt_tokens": 106,
      "output_tokens": 40,
      "generation_tokens_per_second": 13.679,
      "parseable_json": true,
      "matched_fields": 4,
      "field_count": 4,
      "exact_match": true,
      "expected": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      },
      "actual": {
        "reference": "CLEAN_03",
        "quantity": 4,
        "due_date": "2026-10-08",
        "needs_review": false
      }
    }
  ],
  "summary": {
    "requests": 9,
    "distinct_synthetic_cases": 3,
    "exact_matches": 9,
    "matched_fields": 36,
    "total_fields": 36,
    "median_wall_seconds": 3.011,
    "median_generation_tokens_per_second": 13.738,
    "min_wall_seconds": 2.716,
    "max_wall_seconds": 12.462,
    "external_model_api_used": false
  },
  "resident_models_after": {
    "models": [
      {
        "name": "hermes3:8b",
        "model": "hermes3:8b",
        "size": 5316594891,
        "digest": "4f6b83f30b62bc3d0cf9be09266db222805ee815c8fd7d8b38f863f655be78b7",
        "details": {
          "parent_model": "",
          "format": "gguf",
          "family": "llama",
          "families": [
            "llama"
          ],
          "parameter_size": "8.0B",
          "quantization_level": "Q4_0"
        },
        "expires_at": "2026-10-04T20:59:10.01284022Z",
        "size_vram": 0,
        "context_length": 4096
      }
    ]
  }
}