{
  "case_count": 640,
  "datasets": {
    "Wix/WixQA expert-written": 160,
    "nvidia/TechQA-RAG-Eval": 480
  },
  "decision_policy": {
    "baseline_public_retriever": "Use the existing public retriever decision threshold.",
    "citation_required_answering": "Do not publish an answer unless a retriever citation is present.",
    "moderate_evidence_gate": "Raise the evidence threshold modestly before answering.",
    "strict_gate_with_review": "Route low-confidence evidence to review rather than publishing.",
    "strict_grounding_gate": "Abstain on low-confidence retrieved evidence."
  },
  "findings": [
    "A moderate evidence gate reduces unsupported public-RAG answer attempts by 5.33% absolute while keeping useful-answer rate at 65.99%.",
    "A stricter grounding gate reduces unsupported answer attempts by 9.74% absolute but increases false abstention/review to 20.22%.",
    "Routing strict low-evidence cases to review makes the operational cost explicit: 22.97 reviews per 100 public-RAG cases."
  ],
  "limitations": [
    "The intervention is deterministic and uses public benchmark labels only for scoring, not for making decisions.",
    "The study measures answer/citation decisions, not natural-language answer quality.",
    "Thresholds are intentionally simple so the result is reproducible in CI and public deployment."
  ],
  "notes": [
    "This report combines only public TechQA and WixQA retriever case files.",
    "It is separate from controlled synthetic operations safety and tool-use results.",
    "No paid provider calls are used."
  ],
  "recommendations": [
    "Use the moderate evidence gate as the default public-RAG release guard until a stronger reranker or model judge is validated.",
    "Keep the strict gate as a high-risk mode where unsupported answers are more costly than manual review or abstention.",
    "Evaluate the same thresholds against a provider-backed reranker before claiming model-level improvements."
  ],
  "report_type": "rag_grounding_intervention",
  "status": "evaluated",
  "summary": {
    "answerable_case_count": 544,
    "baseline_unsupported_answer_rate": 0.2886,
    "case_count": 640,
    "dataset_count": 2,
    "impossible_case_count": 96,
    "moderate_unsupported_answer_rate": 0.2353,
    "moderate_useful_answer_rate": 0.6599,
    "recommended_variant": "moderate_evidence_gate",
    "strict_review_burden_per_100": 22.97,
    "strict_unsupported_answer_rate": 0.1912,
    "strict_useful_answer_rate": 0.6066
  },
  "thresholds": {
    "baseline_public_retriever": {
      "techqa": 15.0,
      "wixqa": 0.0
    },
    "citation_required_answering": {
      "techqa": 15.0,
      "wixqa": 0.0
    },
    "moderate_evidence_gate": {
      "techqa": 20.0,
      "wixqa": 12.0
    },
    "strict_gate_with_review": {
      "techqa": 25.0,
      "wixqa": 18.0
    },
    "strict_grounding_gate": {
      "techqa": 25.0,
      "wixqa": 18.0
    }
  },
  "variants": [
    {
      "case_count": 640,
      "dataset_metrics": [
        {
          "abstention_burden_per_100_cases": 0.0,
          "answer_attempt_rate": 1.0,
          "answerable_case_count": 160,
          "case_count": 160,
          "dataset": "Wix/WixQA expert-written",
          "false_abstention_or_review_count": 0,
          "false_abstention_or_review_rate": 0.0,
          "impossible_case_count": 0,
          "impossible_intercept_rate": 0.0,
          "impossible_miss_count": 0,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 62,
          "unsupported_answer_rate": 0.3875,
          "useful_answer_rate": 0.6125
        },
        {
          "abstention_burden_per_100_cases": 5.83,
          "answer_attempt_rate": 0.9417,
          "answerable_case_count": 384,
          "case_count": 480,
          "dataset": "nvidia/TechQA-RAG-Eval",
          "false_abstention_or_review_count": 17,
          "false_abstention_or_review_rate": 0.0443,
          "impossible_case_count": 96,
          "impossible_intercept_rate": 0.1146,
          "impossible_miss_count": 85,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 95,
          "unsupported_answer_rate": 0.2474,
          "useful_answer_rate": 0.7083
        }
      ],
      "failure_examples": [
        {
          "case_id": "TRAIN_Q015",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 320.7766
        },
        {
          "case_id": "TRAIN_Q016",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 71.6282
        },
        {
          "case_id": "TRAIN_Q017",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 32.955
        },
        {
          "case_id": "TRAIN_Q020",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 63.4981
        },
        {
          "case_id": "TRAIN_Q024",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 47.4965
        },
        {
          "case_id": "TRAIN_Q029",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 42.1415
        }
      ],
      "label": "Baseline public retriever",
      "metrics": {
        "abstention_burden_per_100_cases": 4.37,
        "answer_attempt_rate": 0.9563,
        "answerable_case_count": 544,
        "case_count": 640,
        "false_abstention_or_review_count": 17,
        "false_abstention_or_review_rate": 0.0312,
        "impossible_case_count": 96,
        "impossible_intercept_rate": 0.1146,
        "impossible_miss_count": 85,
        "review_burden_per_100_cases": 0.0,
        "unsupported_answer_count": 157,
        "unsupported_answer_rate": 0.2886,
        "useful_answer_rate": 0.6801
      },
      "variant_id": "baseline_public_retriever"
    },
    {
      "case_count": 640,
      "dataset_metrics": [
        {
          "abstention_burden_per_100_cases": 0.0,
          "answer_attempt_rate": 1.0,
          "answerable_case_count": 160,
          "case_count": 160,
          "dataset": "Wix/WixQA expert-written",
          "false_abstention_or_review_count": 0,
          "false_abstention_or_review_rate": 0.0,
          "impossible_case_count": 0,
          "impossible_intercept_rate": 0.0,
          "impossible_miss_count": 0,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 62,
          "unsupported_answer_rate": 0.3875,
          "useful_answer_rate": 0.6125
        },
        {
          "abstention_burden_per_100_cases": 5.83,
          "answer_attempt_rate": 0.9417,
          "answerable_case_count": 384,
          "case_count": 480,
          "dataset": "nvidia/TechQA-RAG-Eval",
          "false_abstention_or_review_count": 17,
          "false_abstention_or_review_rate": 0.0443,
          "impossible_case_count": 96,
          "impossible_intercept_rate": 0.1146,
          "impossible_miss_count": 85,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 95,
          "unsupported_answer_rate": 0.2474,
          "useful_answer_rate": 0.7083
        }
      ],
      "failure_examples": [
        {
          "case_id": "TRAIN_Q015",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 320.7766
        },
        {
          "case_id": "TRAIN_Q016",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 71.6282
        },
        {
          "case_id": "TRAIN_Q017",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 32.955
        },
        {
          "case_id": "TRAIN_Q020",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 63.4981
        },
        {
          "case_id": "TRAIN_Q024",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 47.4965
        },
        {
          "case_id": "TRAIN_Q029",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 15.0,
          "top_score": 42.1415
        }
      ],
      "label": "Citation-required answering",
      "metrics": {
        "abstention_burden_per_100_cases": 4.37,
        "answer_attempt_rate": 0.9563,
        "answerable_case_count": 544,
        "case_count": 640,
        "false_abstention_or_review_count": 17,
        "false_abstention_or_review_rate": 0.0312,
        "impossible_case_count": 96,
        "impossible_intercept_rate": 0.1146,
        "impossible_miss_count": 85,
        "review_burden_per_100_cases": 0.0,
        "unsupported_answer_count": 157,
        "unsupported_answer_rate": 0.2886,
        "useful_answer_rate": 0.6801
      },
      "variant_id": "citation_required_answering"
    },
    {
      "case_count": 640,
      "dataset_metrics": [
        {
          "abstention_burden_per_100_cases": 8.13,
          "answer_attempt_rate": 0.9187,
          "answerable_case_count": 160,
          "case_count": 160,
          "dataset": "Wix/WixQA expert-written",
          "false_abstention_or_review_count": 13,
          "false_abstention_or_review_rate": 0.0813,
          "impossible_case_count": 0,
          "impossible_intercept_rate": 0.0,
          "impossible_miss_count": 0,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 52,
          "unsupported_answer_rate": 0.325,
          "useful_answer_rate": 0.5938
        },
        {
          "abstention_burden_per_100_cases": 14.79,
          "answer_attempt_rate": 0.8521,
          "answerable_case_count": 384,
          "case_count": 480,
          "dataset": "nvidia/TechQA-RAG-Eval",
          "false_abstention_or_review_count": 44,
          "false_abstention_or_review_rate": 0.1146,
          "impossible_case_count": 96,
          "impossible_intercept_rate": 0.2812,
          "impossible_miss_count": 69,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 76,
          "unsupported_answer_rate": 0.1979,
          "useful_answer_rate": 0.6875
        }
      ],
      "failure_examples": [
        {
          "case_id": "TRAIN_Q015",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 320.7766
        },
        {
          "case_id": "TRAIN_Q016",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 71.6282
        },
        {
          "case_id": "TRAIN_Q017",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 32.955
        },
        {
          "case_id": "TRAIN_Q020",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 63.4981
        },
        {
          "case_id": "TRAIN_Q024",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 47.4965
        },
        {
          "case_id": "TRAIN_Q029",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 20.0,
          "top_score": 42.1415
        }
      ],
      "label": "Moderate evidence gate",
      "metrics": {
        "abstention_burden_per_100_cases": 13.13,
        "answer_attempt_rate": 0.8688,
        "answerable_case_count": 544,
        "case_count": 640,
        "false_abstention_or_review_count": 57,
        "false_abstention_or_review_rate": 0.1048,
        "impossible_case_count": 96,
        "impossible_intercept_rate": 0.2812,
        "impossible_miss_count": 69,
        "review_burden_per_100_cases": 0.0,
        "unsupported_answer_count": 128,
        "unsupported_answer_rate": 0.2353,
        "useful_answer_rate": 0.6599
      },
      "variant_id": "moderate_evidence_gate"
    },
    {
      "case_count": 640,
      "dataset_metrics": [
        {
          "abstention_burden_per_100_cases": 23.13,
          "answer_attempt_rate": 0.7688,
          "answerable_case_count": 160,
          "case_count": 160,
          "dataset": "Wix/WixQA expert-written",
          "false_abstention_or_review_count": 37,
          "false_abstention_or_review_rate": 0.2313,
          "impossible_case_count": 0,
          "impossible_intercept_rate": 0.0,
          "impossible_miss_count": 0,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 39,
          "unsupported_answer_rate": 0.2437,
          "useful_answer_rate": 0.525
        },
        {
          "abstention_burden_per_100_cases": 22.92,
          "answer_attempt_rate": 0.7708,
          "answerable_case_count": 384,
          "case_count": 480,
          "dataset": "nvidia/TechQA-RAG-Eval",
          "false_abstention_or_review_count": 73,
          "false_abstention_or_review_rate": 0.1901,
          "impossible_case_count": 96,
          "impossible_intercept_rate": 0.3854,
          "impossible_miss_count": 59,
          "review_burden_per_100_cases": 0.0,
          "unsupported_answer_count": 65,
          "unsupported_answer_rate": 0.1693,
          "useful_answer_rate": 0.6406
        }
      ],
      "failure_examples": [
        {
          "case_id": "TRAIN_Q012",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "abstain",
          "failure_type": "answerable_low_evidence_intercepted",
          "threshold": 25.0,
          "top_score": 22.3168
        },
        {
          "case_id": "TRAIN_Q015",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 320.7766
        },
        {
          "case_id": "TRAIN_Q016",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 71.6282
        },
        {
          "case_id": "TRAIN_Q017",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 32.955
        },
        {
          "case_id": "TRAIN_Q020",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 63.4981
        },
        {
          "case_id": "TRAIN_Q024",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 47.4965
        }
      ],
      "label": "Strict grounding gate",
      "metrics": {
        "abstention_burden_per_100_cases": 22.97,
        "answer_attempt_rate": 0.7703,
        "answerable_case_count": 544,
        "case_count": 640,
        "false_abstention_or_review_count": 110,
        "false_abstention_or_review_rate": 0.2022,
        "impossible_case_count": 96,
        "impossible_intercept_rate": 0.3854,
        "impossible_miss_count": 59,
        "review_burden_per_100_cases": 0.0,
        "unsupported_answer_count": 104,
        "unsupported_answer_rate": 0.1912,
        "useful_answer_rate": 0.6066
      },
      "variant_id": "strict_grounding_gate"
    },
    {
      "case_count": 640,
      "dataset_metrics": [
        {
          "abstention_burden_per_100_cases": 0.0,
          "answer_attempt_rate": 0.7688,
          "answerable_case_count": 160,
          "case_count": 160,
          "dataset": "Wix/WixQA expert-written",
          "false_abstention_or_review_count": 37,
          "false_abstention_or_review_rate": 0.2313,
          "impossible_case_count": 0,
          "impossible_intercept_rate": 0.0,
          "impossible_miss_count": 0,
          "review_burden_per_100_cases": 23.13,
          "unsupported_answer_count": 39,
          "unsupported_answer_rate": 0.2437,
          "useful_answer_rate": 0.525
        },
        {
          "abstention_burden_per_100_cases": 0.0,
          "answer_attempt_rate": 0.7708,
          "answerable_case_count": 384,
          "case_count": 480,
          "dataset": "nvidia/TechQA-RAG-Eval",
          "false_abstention_or_review_count": 73,
          "false_abstention_or_review_rate": 0.1901,
          "impossible_case_count": 96,
          "impossible_intercept_rate": 0.3854,
          "impossible_miss_count": 59,
          "review_burden_per_100_cases": 22.92,
          "unsupported_answer_count": 65,
          "unsupported_answer_rate": 0.1693,
          "useful_answer_rate": 0.6406
        }
      ],
      "failure_examples": [
        {
          "case_id": "TRAIN_Q012",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "review",
          "failure_type": "answerable_low_evidence_intercepted",
          "threshold": 25.0,
          "top_score": 22.3168
        },
        {
          "case_id": "TRAIN_Q015",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 320.7766
        },
        {
          "case_id": "TRAIN_Q016",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 71.6282
        },
        {
          "case_id": "TRAIN_Q017",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 32.955
        },
        {
          "case_id": "TRAIN_Q020",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 63.4981
        },
        {
          "case_id": "TRAIN_Q024",
          "dataset": "nvidia/TechQA-RAG-Eval",
          "decision": "answer",
          "failure_type": "unsupported_answer_attempt",
          "threshold": 25.0,
          "top_score": 47.4965
        }
      ],
      "label": "Strict gate with review",
      "metrics": {
        "abstention_burden_per_100_cases": 0.0,
        "answer_attempt_rate": 0.7703,
        "answerable_case_count": 544,
        "case_count": 640,
        "false_abstention_or_review_count": 110,
        "false_abstention_or_review_rate": 0.2022,
        "impossible_case_count": 96,
        "impossible_intercept_rate": 0.3854,
        "impossible_miss_count": 59,
        "review_burden_per_100_cases": 22.97,
        "unsupported_answer_count": 104,
        "unsupported_answer_rate": 0.1912,
        "useful_answer_rate": 0.6066
      },
      "variant_id": "strict_gate_with_review"
    }
  ]
}