{
  "date": "2026-05-04",
  "status": "kg_aware_retrieval_eval",
  "related_issue": "BK-73",
  "pack": "docs/verification/sota-conversational-agent-pack-2026-05-03.json",
  "criteria_scored": [
    "top_k_hit_rate",
    "mean_reciprocal_rank",
    "source_graph_trace_expected_hit_coverage",
    "candidate_support_expected_hit_coverage",
    "canonical_source_restraint"
  ],
  "role": "support",
  "baseline_ref": "docs/verification/generated/hybrid-retrieval-eval-2026-05-03.json",
  "baseline": {
    "mode": "hybrid",
    "top_k_hit_rate": 1.0,
    "mean_reciprocal_rank": 1.0,
    "cases": [
      {
        "case_id": "SOTA-001",
        "kind": "local_fact",
        "query": "Kuran'ın indiği, İncil'in, Tevrat'ın, Zebur'un yazdırıldığı yer burasıdır",
        "score": {
          "expected_blocks": [
            "fas01:p4:b1"
          ],
          "retrieved_blocks": [
            "fas01:p4:b1",
            "fas02:p5:b1",
            "fas22:p3:b5",
            "fas52:p8:b6",
            "fas04:p1:b1"
          ],
          "hit": true,
          "hits": [
            "fas01:p4:b1"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-002",
        "kind": "native_term_definition",
        "query": "Bilgi Kitabı nedir ve kendisini nasıl tanımlar?",
        "score": {
          "expected_blocks": [
            "fas15:p12:b8",
            "fas18:p3:b2",
            "fas20:p6:b3"
          ],
          "retrieved_blocks": [
            "fas18:p3:b2",
            "fas22:p3:b5",
            "fas28:p1:b1",
            "fas15:p12:b8",
            "fas32:p5:b4"
          ],
          "hit": true,
          "hits": [
            "fas18:p3:b2",
            "fas15:p12:b8"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-003",
        "kind": "typed_tension",
        "query": "Kitaptaki çelişkiler neden konulmuştur?",
        "score": {
          "expected_blocks": [
            "fas18:p3:b2",
            "fas13:p8:b2",
            "fas20:p6:b3"
          ],
          "retrieved_blocks": [
            "fas18:p3:b2",
            "ek7:p3:b2",
            "fas13:p8:b2",
            "fas15:p1:b1",
            "fas52:p8:b6"
          ],
          "hit": true,
          "hits": [
            "fas18:p3:b2",
            "fas13:p8:b2"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-004",
        "kind": "question_gated_access",
        "query": "Yüce Mekanizma'dan neden soru sormadan cevap alınamaz?",
        "score": {
          "expected_blocks": [
            "ek1:p1:b1",
            "ek1:p1:b2"
          ],
          "retrieved_blocks": [
            "ek1:p1:b2",
            "fas42:p8:b9",
            "ek2:p1:b1",
            "ek7:p3:b2",
            "ek3:p2:b1"
          ],
          "hit": true,
          "hits": [
            "ek1:p1:b2"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-005",
        "kind": "cross_information_displacement",
        "query": "Kros Bilgi Tatbikatı nedir, bilgiler neden yer değiştirir?",
        "score": {
          "expected_blocks": [
            "ek5:p11:b5"
          ],
          "retrieved_blocks": [
            "ek5:p11:b5",
            "fas54:p2:b2",
            "ek7:p3:b2",
            "ek2:p5:b4",
            "fas53:p3:b2"
          ],
          "hit": true,
          "hits": [
            "ek5:p11:b5"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-006",
        "kind": "term_role_trace",
        "query": "Gen Şifresi, Arşiv, Görev ve Birleşik İnsanlık nasıl bağlanır?",
        "score": {
          "expected_blocks": [
            "fas45:p1:b1"
          ],
          "retrieved_blocks": [
            "fas45:p1:b1",
            "fas42:p8:b9",
            "fas53:p3:b2",
            "fas51:p4:b3",
            "fas51:p10:b5"
          ],
          "hit": true,
          "hits": [
            "fas45:p1:b1"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-007",
        "kind": "under_grounded",
        "query": "Kitap bugünkü siyasi seçimler hakkında kimi destekliyor?",
        "score": {
          "expected_blocks": [],
          "retrieved_blocks": [
            "fas28:p8:b2",
            "fas32:p5:b4",
            "fas32:p4:b3",
            "fas08:p4:b2",
            "fas22:p2:b3"
          ],
          "hit": null,
          "hits": [],
          "reciprocal_rank": 0.0
        }
      }
    ]
  },
  "kg_aware": {
    "mode": "kg_aware_support",
    "top_k_hit_rate": 1.0,
    "mean_reciprocal_rank": 1.0,
    "cases": [
      {
        "case_id": "SOTA-001",
        "kind": "local_fact",
        "query": "Kuran'ın indiği, İncil'in, Tevrat'ın, Zebur'un yazdırıldığı yer burasıdır",
        "score": {
          "expected_blocks": [
            "fas01:p4:b1"
          ],
          "retrieved_blocks": [
            "fas01:p4:b1",
            "fas02:p5:b1",
            "fas22:p3:b5",
            "fas52:p8:b6",
            "fas04:p1:b1"
          ],
          "hit": true,
          "hits": [
            "fas01:p4:b1"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-002",
        "kind": "native_term_definition",
        "query": "Bilgi Kitabı nedir ve kendisini nasıl tanımlar?",
        "score": {
          "expected_blocks": [
            "fas15:p12:b8",
            "fas18:p3:b2",
            "fas20:p6:b3"
          ],
          "retrieved_blocks": [
            "fas18:p3:b2",
            "fas22:p3:b5",
            "fas28:p1:b1",
            "fas15:p12:b8",
            "fas32:p5:b4"
          ],
          "hit": true,
          "hits": [
            "fas18:p3:b2",
            "fas15:p12:b8"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-003",
        "kind": "typed_tension",
        "query": "Kitaptaki çelişkiler neden konulmuştur?",
        "score": {
          "expected_blocks": [
            "fas18:p3:b2",
            "fas13:p8:b2",
            "fas20:p6:b3"
          ],
          "retrieved_blocks": [
            "fas18:p3:b2",
            "ek7:p3:b2",
            "fas13:p8:b2",
            "fas15:p1:b1",
            "fas52:p8:b6"
          ],
          "hit": true,
          "hits": [
            "fas18:p3:b2",
            "fas13:p8:b2"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-004",
        "kind": "question_gated_access",
        "query": "Yüce Mekanizma'dan neden soru sormadan cevap alınamaz?",
        "score": {
          "expected_blocks": [
            "ek1:p1:b1",
            "ek1:p1:b2"
          ],
          "retrieved_blocks": [
            "ek1:p1:b2",
            "fas42:p8:b9",
            "ek2:p1:b1",
            "ek7:p3:b2",
            "ek3:p2:b1"
          ],
          "hit": true,
          "hits": [
            "ek1:p1:b2"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-005",
        "kind": "cross_information_displacement",
        "query": "Kros Bilgi Tatbikatı nedir, bilgiler neden yer değiştirir?",
        "score": {
          "expected_blocks": [
            "ek5:p11:b5"
          ],
          "retrieved_blocks": [
            "ek5:p11:b5",
            "fas54:p2:b2",
            "ek7:p3:b2",
            "ek2:p5:b4",
            "fas53:p3:b2"
          ],
          "hit": true,
          "hits": [
            "ek5:p11:b5"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-006",
        "kind": "term_role_trace",
        "query": "Gen Şifresi, Arşiv, Görev ve Birleşik İnsanlık nasıl bağlanır?",
        "score": {
          "expected_blocks": [
            "fas45:p1:b1"
          ],
          "retrieved_blocks": [
            "fas45:p1:b1",
            "fas42:p8:b9",
            "fas53:p3:b2",
            "fas51:p4:b3",
            "fas51:p10:b5"
          ],
          "hit": true,
          "hits": [
            "fas45:p1:b1"
          ],
          "reciprocal_rank": 1.0
        }
      },
      {
        "case_id": "SOTA-007",
        "kind": "under_grounded",
        "query": "Kitap bugünkü siyasi seçimler hakkında kimi destekliyor?",
        "score": {
          "expected_blocks": [],
          "retrieved_blocks": [
            "fas28:p8:b2",
            "fas32:p5:b4",
            "fas32:p4:b3",
            "fas08:p4:b2",
            "fas22:p2:b3"
          ],
          "hit": null,
          "hits": [],
          "reciprocal_rank": 0.0
        }
      }
    ]
  },
  "trace_summary": {
    "source_graph_trace_expected_hit_coverage": 1.0,
    "candidate_support_expected_hit_coverage": 0.8333,
    "cases": [
      {
        "case_id": "SOTA-001",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": false
      },
      {
        "case_id": "SOTA-002",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": true
      },
      {
        "case_id": "SOTA-003",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": true
      },
      {
        "case_id": "SOTA-004",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": true
      },
      {
        "case_id": "SOTA-005",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": true
      },
      {
        "case_id": "SOTA-006",
        "source_graph_trace_on_expected_hit": true,
        "candidate_support_on_expected_hit": true
      }
    ]
  },
  "promotion_rule": "Keep only if KG-aware retrieval does not regress hybrid metrics and keeps unreviewed candidates out of canonical answer truth.",
  "disposition": "keep_as_support",
  "notes": [
    "The current hybrid baseline is already perfect on the frozen pack, so BK-73 is not allowed to claim score uplift by narrative.",
    "Source graph context is canonical_source metadata; BK-72 model extraction remains unreviewed candidate support."
  ]
}