{
  "findings": [
    "Across public RAG tracks, top-3 reranking could lift weighted top-1 citation accuracy from 69.12% to 79.78%.",
    "The current compact public samples contain 58 rerankable cases and 110 residual retrieval misses.",
    "Reranking is worth testing, but retriever recall still needs separate work because reranking cannot recover documents outside the candidate set."
  ],
  "notes": [
    "This is a reranking opportunity analysis, not an oracle production reranker. It measures the maximum top-1 citation lift available when the expected public document is already in the top 3.",
    "Residual retrieval gap measures cases a reranker cannot fix because the expected document was not retrieved into the candidate set."
  ],
  "recommendations": [
    "Add a query-document reranker over the top-3 or top-5 retrieved public documents and compare it against this opportunity ceiling.",
    "Track reranking lift separately from retrieval-hit@3 so ranking gains are not confused with candidate-generation gains.",
    "Prioritize candidate-generation changes alongside reranking because the residual retrieval gap is larger than the rerankable top-1 gap."
  ],
  "report_type": "public_rag_reranking_opportunity",
  "status": "evaluated",
  "summary": {
    "current_top1_correct_count": 376,
    "current_weighted_top1_citation_accuracy": 0.6912,
    "evaluated_track_count": 2,
    "largest_rerankable_track": "Wix/WixQA expert-written (+16.25%)",
    "largest_residual_gap_track": "Wix/WixQA expert-written (+22.50%)",
    "not_retrieved_case_count": 110,
    "oracle_top3_rerank_ceiling": 0.7978,
    "possible_weighted_top1_lift": 0.1066,
    "rerankable_case_count": 58,
    "residual_retrieval_gap": 0.2022,
    "total_case_count": 544
  },
  "tracks": [
    {
      "benchmark_track": "external_public_rag",
      "case_count": 384,
      "current_top1_citation_accuracy": 0.724,
      "current_top1_correct_count": 278,
      "dataset": "nvidia/TechQA-RAG-Eval",
      "not_retrieved_case_count": 74,
      "not_retrieved_case_examples": [
        {
          "case_id": "TRAIN_Q016",
          "expected_citation_ids": [
            "swg21469413.txt"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "swg21067352.txt",
            "swg21902654.txt",
            "swg21981870.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q017",
          "expected_citation_ids": [
            "swg27019359.txt"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "swg21690673.txt",
            "swg21982008.txt",
            "swg21695094.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q020",
          "expected_citation_ids": [
            "swg24035040.txt"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "swg21263677.txt",
            "swg21651101.txt",
            "swg21689914.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q024",
          "expected_citation_ids": [
            "swg21592093.txt"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "swg24043574.txt",
            "swg21507587.txt",
            "swg21477587.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q029",
          "expected_citation_ids": [
            "swg21964202.txt"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "swg21515420.txt",
            "swg21700865.txt",
            "swg22002975.txt"
          ]
        }
      ],
      "oracle_top3_rerank_ceiling": 0.8073,
      "possible_top1_lift": 0.0833,
      "rerankable_case_count": 32,
      "rerankable_case_examples": [
        {
          "case_id": "TRAIN_Q015",
          "expected_citation_ids": [
            "swg21998452.txt"
          ],
          "expected_rank": 3,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "swg22010832.txt",
            "swg22011696.txt",
            "swg21998452.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q043",
          "expected_citation_ids": [
            "swg21657023.txt"
          ],
          "expected_rank": 2,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "swg21377984.txt",
            "swg21657023.txt",
            "swg24043263.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q067",
          "expected_citation_ids": [
            "swg21451229.txt"
          ],
          "expected_rank": 3,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "swg21690658.txt",
            "swg21611073.txt",
            "swg21451229.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q069",
          "expected_citation_ids": [
            "swg21632275.txt"
          ],
          "expected_rank": 3,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "swg21498634.txt",
            "swg21399255.txt",
            "swg21632275.txt"
          ]
        },
        {
          "case_id": "TRAIN_Q071",
          "expected_citation_ids": [
            "swg21962862.txt"
          ],
          "expected_rank": 2,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "swg21974546.txt",
            "swg21962862.txt",
            "swg21977398.txt"
          ]
        }
      ],
      "residual_retrieval_gap": 0.1927,
      "status": "evaluated"
    },
    {
      "benchmark_track": "external_public_enterprise_rag",
      "case_count": 160,
      "current_top1_citation_accuracy": 0.6125,
      "current_top1_correct_count": 98,
      "dataset": "Wix/WixQA expert-written",
      "not_retrieved_case_count": 36,
      "not_retrieved_case_examples": [
        {
          "case_id": "WIXQA-EXPERT-0005",
          "expected_citation_ids": [
            "8cc75fbcc571336d1ef1768e7727bd9d1e6c1333f22eeb5366232b9a1a066418"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "fff4ccf96f0bf0a2c3b82d0734f384299bd9a621212087f9268941acd38f8d36",
            "3407d197c71652bee58835ba148794a8be7e70cf9e25c599044b823c82db29fd",
            "6ac11c077e31c980c3de33be613fa8af2af6f78ed5da96ede5644a7195a1e866"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0025",
          "expected_citation_ids": [
            "49d9e88fadbf11fa4e685c847590078ff9394c2fe7566094f504f53ca4aca465"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "73962a6c69a291d8a5d108b0a2116dd29341275b37c155084612574eb649c8d7",
            "50d02c7ce566adb08742f7cc810bfceab55f9a7c43fcd7fe73f932c2cd8c88b1",
            "7a49619f637ed0508bca64c0995e2726276d8faba2907d97350e0f01384709af"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0026",
          "expected_citation_ids": [
            "42494683352ccb36b16ef52dc0b32ff1aab64f5f3a6fe90827c33e0252a59098"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "89e39fdcca5874eb36d1ac6e359aad2cd93b601cac1cf7b8743449ff7893ead8",
            "5641574289f2768293891471cfcc69b935623ef2723abd28fcb9dddf430ce5e8",
            "fd36cb06f2de70a197a60c3d2e28ec709fb884cad11153f4bb6086f82755bd5a"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0036",
          "expected_citation_ids": [
            "b4717557702f3383d3d4d7bf071c2ab5558b34fc62860d2879f9d48f86ec2de8"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "e4b87cd9f38d652e0a018619be5ab7f6065f07e1bd2556d629fb34e693feff91",
            "13b7554664bd39da1825c1d396cefbb0afd2d9989c1a15659dd158b0a46b916c",
            "0b6d939e9e0d8599e895cad834a336ccec3f5e4c149645def2ea4d4d6091b356"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0045",
          "expected_citation_ids": [
            "674a8ee961c87e71e35bc7f786673c780ab0b522230ff5806aed8dac13989744"
          ],
          "expected_rank": null,
          "failure_reasons": [
            "expected_document_not_retrieved_at_3"
          ],
          "retrieved_citation_ids": [
            "4c851bd1efd1009bb75a387d406fb7914e906fda9016127529184916bd23aaf9",
            "a513a215c1798f329293330cb5f5e975a18978954b5ee27d84441e029f900d51",
            "33dfa4eb3abc1c459f4b8f24cb14757f39c1b9c7c13f598cf904b9d395cc6970"
          ]
        }
      ],
      "oracle_top3_rerank_ceiling": 0.775,
      "possible_top1_lift": 0.1625,
      "rerankable_case_count": 26,
      "rerankable_case_examples": [
        {
          "case_id": "WIXQA-EXPERT-0001",
          "expected_citation_ids": [
            "49d9e88fadbf11fa4e685c847590078ff9394c2fe7566094f504f53ca4aca465"
          ],
          "expected_rank": 2,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "51aca93604e8d423b3a87c0126335c5864f343205d7441aa23171523bb26c7aa",
            "49d9e88fadbf11fa4e685c847590078ff9394c2fe7566094f504f53ca4aca465",
            "88f6b5848f6f662d7fe9d69f2708f00c75067894320261c58578ad0ed732baa9"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0004",
          "expected_citation_ids": [
            "a513a215c1798f329293330cb5f5e975a18978954b5ee27d84441e029f900d51",
            "b9ba8c318ee75f49eaa89c1d5b8d461fa2b4c27167381b869aeb31027d7187f9"
          ],
          "expected_rank": 3,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "9b71e0c1c1101d626c050e2209b382430c36eb91644c25dc9016802164ea769f",
            "040613a5d53923efc2955eb8cfaf4573d7543dd0236bf90be63e40613359e4cf",
            "a513a215c1798f329293330cb5f5e975a18978954b5ee27d84441e029f900d51"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0019",
          "expected_citation_ids": [
            "8c2e56296ddb7b441238ff06cfa9b38204aca9e27371386689b1fa4643cdfbff"
          ],
          "expected_rank": 3,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "bb528ecfbf28534c702f7e30e97e1709741bb421299f376de6cb2bc8d8226fc4",
            "e9366441d1e037a1470c9bad57455dbadc24db0ca931791edd0f3b026ea6a58f",
            "8c2e56296ddb7b441238ff06cfa9b38204aca9e27371386689b1fa4643cdfbff"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0024",
          "expected_citation_ids": [
            "49d9e88fadbf11fa4e685c847590078ff9394c2fe7566094f504f53ca4aca465"
          ],
          "expected_rank": 2,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "50d02c7ce566adb08742f7cc810bfceab55f9a7c43fcd7fe73f932c2cd8c88b1",
            "49d9e88fadbf11fa4e685c847590078ff9394c2fe7566094f504f53ca4aca465",
            "ad7a6f1f5851bf2c4bc738d6223bb1546e78c7b7f3317d721b67181cd675c5a8"
          ]
        },
        {
          "case_id": "WIXQA-EXPERT-0028",
          "expected_citation_ids": [
            "13ae20110f373b7614a4594a0b47cd190eaecba13c74138551ee24f682b43f08",
            "0d6bacf2646098a55162336b12fc8eeb56c6618febc7ad94a9c7d5d9bb71ba3f"
          ],
          "expected_rank": 2,
          "failure_reasons": [
            "expected_document_retrieved_but_not_top1"
          ],
          "retrieved_citation_ids": [
            "f346781bc6cf23c95342ce003927de9d985a99eeb7561c69f03e65da7c10f2fb",
            "0d6bacf2646098a55162336b12fc8eeb56c6618febc7ad94a9c7d5d9bb71ba3f",
            "2df694686b70b35ed389ff590354cfaf19aafc07659b12b02078f2ebcf015ac8"
          ]
        }
      ],
      "residual_retrieval_gap": 0.225,
      "status": "evaluated"
    }
  ]
}