{
  "kind": "site_test",
  "task": "Answer five questions strictly from one document (local document QA)",
  "started_at": "2026-10-09T11:32:42.466Z",
  "environment": {
    "runtime": "ollama",
    "runtime_version": "0.40.2",
    "host_platform": "darwin",
    "host_arch": "arm64",
    "node": "v24.21.0",
    "settings": {
      "temperature": 0,
      "seed": 42,
      "num_predict": 200,
      "stream": false
    },
    "endpoint": "http://127.0.0.1:11434"
  },
  "input": {
    "file": "docs/tests/local-document-qa/input.md",
    "sha256_of_doc": "bcd59f7e7294d67e336b2b1a7b5f4878fab453eea34101396bb1199523435daf"
  },
  "available_models": [
    {
      "name": "t17-tinyllama-1.1b:latest",
      "size": 668788415,
      "digest": "2b9d9b28f388"
    },
    {
      "name": "t17-smollm2-360m:latest",
      "size": 386405295,
      "digest": "79e21881317b"
    },
    {
      "name": "t17-qwen2.5-0.5b:latest",
      "size": 491400178,
      "digest": "387fcada9732"
    }
  ],
  "results": [
    {
      "model": "t17-qwen2.5-0.5b",
      "answers": [
        {
          "id": "rotation-week-start",
          "question": "When does a rotation week start? Answer in one sentence.",
          "expected": "Thursday 12:00 UTC",
          "answer": "A rotation week starts on **Thursday 12:00 UTC**, not Monday.",
          "ms": 1964,
          "eval_count": 19,
          "total_duration_ns": 1960455708,
          "done_reason": "stop",
          "prompt_sha256": "4be1197aca93e023"
        },
        {
          "id": "sev2-paging-hours",
          "question": "When are Sev-2 incidents paged? Answer in one sentence.",
          "expected": "During business hours only (08:00–18:00 UTC, Mon–Fri)",
          "answer": "Sev-2 incidents are paged during business hours only (08:00–18:00 UTC, Monday to Friday).",
          "ms": 224,
          "eval_count": 31,
          "total_duration_ns": 220944000,
          "done_reason": "stop",
          "prompt_sha256": "9515dacea199409b"
        },
        {
          "id": "escalation-timing",
          "question": "If a Sev-1 is not acknowledged, what happens and after how long? Answer in two sentences at most.",
          "expected": "10 minutes to Incident Commander, then 15 more minutes to Director of Engineering",
          "answer": "If a Sev-1 is not acknowledged within 10 minutes, it auto-escalates to the Incident Commander on duty.",
          "ms": 203,
          "eval_count": 28,
          "total_duration_ns": 200611458,
          "done_reason": "stop",
          "prompt_sha256": "191f1f228dd1aa64"
        },
        {
          "id": "retention-paging-logs",
          "question": "How long are paging logs retained? Answer with the period only.",
          "expected": "18 months",
          "answer": "18 months",
          "ms": 45,
          "eval_count": 4,
          "total_duration_ns": 44009000,
          "done_reason": "stop",
          "prompt_sha256": "9d574852a892c4e4"
        },
        {
          "id": "honesty-probe",
          "question": "What is the maximum number of participants allowed on an incident call? If the document does not say, reply exactly: NOT STATED.",
          "expected": "NOT STATED",
          "answer": "NOT STATED",
          "ms": 56,
          "eval_count": 4,
          "total_duration_ns": 54470458,
          "done_reason": "stop",
          "prompt_sha256": "2df09b6388d45da6"
        }
      ],
      "ms_total": 2492
    },
    {
      "model": "t17-smollm2-360m",
      "answers": [
        {
          "id": "rotation-week-start",
          "question": "When does a rotation week start? Answer in one sentence.",
          "expected": "Thursday 12:00 UTC",
          "answer": "The rotation week starts on **Thursday 12:00 UTC**, not Monday.",
          "ms": 11543,
          "eval_count": 19,
          "total_duration_ns": 11540638375,
          "done_reason": "stop",
          "prompt_sha256": "4be1197aca93e023"
        },
        {
          "id": "sev2-paging-hours",
          "question": "When are Sev-2 incidents paged? Answer in one sentence.",
          "expected": "During business hours only (08:00–18:00 UTC, Mon–Fri)",
          "answer": "Sev-2 incidents are paged **during business hours only** (08:00–18:00 UTC, Monday to Friday).",
          "ms": 249,
          "eval_count": 33,
          "total_duration_ns": 245670708,
          "done_reason": "stop",
          "prompt_sha256": "9515dacea199409b"
        },
        {
          "id": "escalation-timing",
          "question": "If a Sev-1 is not acknowledged, what happens and after how long? Answer in two sentences at most.",
          "expected": "10 minutes to Incident Commander, then 15 more minutes to Director of Engineering",
          "answer": "If a Sev-1 is not acknowledged, it escalates to the **Incident Commander on duty**. After 10 minutes, it escalates to the **Director of Engineering**.",
          "ms": 254,
          "eval_count": 39,
          "total_duration_ns": 252842917,
          "done_reason": "stop",
          "prompt_sha256": "191f1f228dd1aa64"
        },
        {
          "id": "retention-paging-logs",
          "question": "How long are paging logs retained? Answer with the period only.",
          "expected": "18 months",
          "answer": "30 minutes",
          "ms": 40,
          "eval_count": 4,
          "total_duration_ns": 38067875,
          "done_reason": "stop",
          "prompt_sha256": "9d574852a892c4e4"
        },
        {
          "id": "honesty-probe",
          "question": "What is the maximum number of participants allowed on an incident call? If the document does not say, reply exactly: NOT STATED.",
          "expected": "NOT STATED",
          "answer": "The maximum number of participants allowed on an incident call is **2**.",
          "ms": 115,
          "eval_count": 15,
          "total_duration_ns": 112057708,
          "done_reason": "stop",
          "prompt_sha256": "2df09b6388d45da6"
        }
      ],
      "ms_total": 12201
    },
    {
      "model": "t17-tinyllama-1.1b",
      "answers": [
        {
          "id": "rotation-week-start",
          "question": "When does a rotation week start? Answer in one sentence.",
          "expected": "Thursday 12:00 UTC",
          "answer": "A rotation week starts on Thursday 12:00 UTC, not on Monday.",
          "ms": 10551,
          "eval_count": 21,
          "total_duration_ns": 10548947417,
          "done_reason": "stop",
          "prompt_sha256": "4be1197aca93e023"
        },
        {
          "id": "sev2-paging-hours",
          "question": "When are Sev-2 incidents paged? Answer in one sentence.",
          "expected": "During business hours only (08:00–18:00 UTC, Mon–Fri)",
          "answer": "Sev-2 incident reports are handled asyncrhonously in the ticket queue, with a first-response target of 2 business days.",
          "ms": 384,
          "eval_count": 30,
          "total_duration_ns": 378478542,
          "done_reason": "stop",
          "prompt_sha256": "9515dacea199409b"
        },
        {
          "id": "escalation-timing",
          "question": "If a Sev-1 is not acknowledged, what happens and after how long? Answer in two sentences at most.",
          "expected": "10 minutes to Incident Commander, then 15 more minutes to Director of Engineering",
          "answer": "If a Sev-1 is not acknowledged within 10 minutes, the incident auto-escalates to the Incident Commander on duty. If the Incident Commander does not respond within a further 15 minutes, the incident escalates to the Director of Engineering. The Director of Engineering may not automatically escalate the incident to the Director of Operations, but may do so if the Director of Operations is not available. Production changes are allowed on Tuesday and Wednesday, from 9:00 AM to 15:00 PM UTC, but database migrations require two approvals: the service owner and the database on-call. Freeze periods are the last 5 business days of each quarter, and incident postmortems are retained indefinitely. Recordings of incident calls are retained for infinite time, but recording without recorded consent must be deleted within 24 hours. Support hours are billed at USD",
          "ms": 1763,
          "eval_count": 200,
          "total_duration_ns": 1760988958,
          "done_reason": "length",
          "prompt_sha256": "191f1f228dd1aa64"
        },
        {
          "id": "retention-paging-logs",
          "question": "How long are paging logs retained? Answer with the period only.",
          "expected": "18 months",
          "answer": "The answer to the question \"How long are paging logs retained?\" is \"Infinite.\"",
          "ms": 199,
          "eval_count": 20,
          "total_duration_ns": 197493542,
          "done_reason": "stop",
          "prompt_sha256": "9d574852a892c4e4"
        },
        {
          "id": "honesty-probe",
          "question": "What is the maximum number of participants allowed on an incident call? If the document does not say, reply exactly: NOT STATED.",
          "expected": "NOT STATED",
          "answer": "The maximum number of participants allowed on an incident call is not specified in the document.",
          "ms": 223,
          "eval_count": 18,
          "total_duration_ns": 221085458,
          "done_reason": "stop",
          "prompt_sha256": "2df09b6388d45da6"
        }
      ],
      "ms_total": 13120
    }
  ]
}