{
  "date_utc": "2026-10-04T21:16:42.145029+00:00",
  "scope": "Twelve synthetic short routing requests, not a general accuracy or concurrency benchmark",
  "settings": {
    "context": 2048,
    "max_output_tokens": 48,
    "temperature": 0,
    "seed": 42,
    "single_request_at_a_time": true,
    "threads_tested": [
      12,
      24
    ]
  },
  "models": {
    "qwen2.5:1.5b": {
      "parent_model": "",
      "format": "gguf",
      "family": "qwen2",
      "families": [
        "qwen2"
      ],
      "parameter_size": "1.5B",
      "quantization_level": "Q4_K_M"
    },
    "hermes3:8b": {
      "parent_model": "",
      "format": "gguf",
      "family": "llama",
      "families": [
        "llama"
      ],
      "parameter_size": "8.0B",
      "quantization_level": "Q4_0"
    }
  },
  "warmups": [
    {
      "model": "qwen2.5:1.5b",
      "threads": 12,
      "wall_seconds": 2.622,
      "load_seconds": 2.406
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 24,
      "wall_seconds": 2.41,
      "load_seconds": 2.214
    },
    {
      "model": "hermes3:8b",
      "threads": 12,
      "wall_seconds": 7.926,
      "load_seconds": 7.297
    },
    {
      "model": "hermes3:8b",
      "threads": 24,
      "wall_seconds": 8.104,
      "load_seconds": 7.534
    }
  ],
  "results": [
    {
      "model": "qwen2.5:1.5b",
      "threads": 12,
      "case": 1,
      "expected": "maintenance",
      "category": "maintenance",
      "correct": true,
      "wall_seconds": 0.444,
      "output_tokens": 6,
      "prompt_tokens": 54,
      "load_seconds": 0.002,
      "output_tokens_per_second": 32.2
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 12,
      "case": 2,
      "expected": "scheduling",
      "category": "scheduling",
      "correct": true,
      "wall_seconds": 0.416,
      "output_tokens": 11,
      "prompt_tokens": 48,
      "load_seconds": 0.002,
      "output_tokens_per_second": 34.8
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 12,
      "case": 3,
      "expected": "billing",
      "category": "billing",
      "correct": true,
      "wall_seconds": 0.308,
      "output_tokens": 6,
      "prompt_tokens": 50,
      "load_seconds": 0.002,
      "output_tokens_per_second": 36.11
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 24,
      "case": 1,
      "expected": "maintenance",
      "category": "maintenance",
      "correct": true,
      "wall_seconds": 0.397,
      "output_tokens": 6,
      "prompt_tokens": 54,
      "load_seconds": 0.004,
      "output_tokens_per_second": 37.68
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 24,
      "case": 2,
      "expected": "scheduling",
      "category": "scheduling",
      "correct": true,
      "wall_seconds": 0.365,
      "output_tokens": 11,
      "prompt_tokens": 48,
      "load_seconds": 0.003,
      "output_tokens_per_second": 39.16
    },
    {
      "model": "qwen2.5:1.5b",
      "threads": 24,
      "case": 3,
      "expected": "billing",
      "category": "billing",
      "correct": true,
      "wall_seconds": 0.296,
      "output_tokens": 6,
      "prompt_tokens": 50,
      "load_seconds": 0.002,
      "output_tokens_per_second": 37.48
    },
    {
      "model": "hermes3:8b",
      "threads": 12,
      "case": 1,
      "expected": "maintenance",
      "category": "maintenance",
      "correct": true,
      "wall_seconds": 2.074,
      "output_tokens": 10,
      "prompt_tokens": 55,
      "load_seconds": 0.002,
      "output_tokens_per_second": 11.41
    },
    {
      "model": "hermes3:8b",
      "threads": 12,
      "case": 2,
      "expected": "scheduling",
      "category": "scheduling",
      "correct": true,
      "wall_seconds": 1.394,
      "output_tokens": 11,
      "prompt_tokens": 49,
      "load_seconds": 0.003,
      "output_tokens_per_second": 10.77
    },
    {
      "model": "hermes3:8b",
      "threads": 12,
      "case": 3,
      "expected": "billing",
      "category": "billing",
      "correct": true,
      "wall_seconds": 1.351,
      "output_tokens": 10,
      "prompt_tokens": 51,
      "load_seconds": 0.002,
      "output_tokens_per_second": 12.2
    },
    {
      "model": "hermes3:8b",
      "threads": 24,
      "case": 1,
      "expected": "maintenance",
      "category": "maintenance",
      "correct": true,
      "wall_seconds": 1.655,
      "output_tokens": 10,
      "prompt_tokens": 55,
      "load_seconds": 0.002,
      "output_tokens_per_second": 12.64
    },
    {
      "model": "hermes3:8b",
      "threads": 24,
      "case": 2,
      "expected": "scheduling",
      "category": "scheduling",
      "correct": true,
      "wall_seconds": 1.046,
      "output_tokens": 11,
      "prompt_tokens": 49,
      "load_seconds": 0.003,
      "output_tokens_per_second": 13.33
    },
    {
      "model": "hermes3:8b",
      "threads": 24,
      "case": 3,
      "expected": "billing",
      "category": "billing",
      "correct": true,
      "wall_seconds": 1.018,
      "output_tokens": 10,
      "prompt_tokens": 51,
      "load_seconds": 0.002,
      "output_tokens_per_second": 13.54
    }
  ]
}