{ "benchmark": "Sovereign Engineering Memory : Pillar 1 (LongMemEval Retrieval @ $0 + Reader Cost Curve)", "dataset_name": "LongMemEval_Hardened_48Q_214M", "total_memories_ingested": 214, "supersessions_applied": 8, "memory_links_applied": 9, "total_questions": 48, "answerable_questions": 32, "unanswerable_questions": 16, "llm_calls_in_tier1": 0, "memory_query_cost_usd": 0.0, "wall_clock_seconds": 24.31, "metrics": { "Recall@5": 0.9661, "Recall@10": 0.974, "NDCG@10": 0.9746, "abstention_accuracy": 0.5, "superseded_fact_suppression_rate": 1.0, "temporal_recall_pack_expired_leaks": 0, "superseded_recall_pack_leaks": 0, "snippet_token_savings_pct": 34.39, "recall_pack_snippet_savings_pct": 30.67, "avg_snippet_tokens_per_query": 551.8, "avg_full_tokens_per_query": 841.0, "p50_ms": 34.11, "p95_ms": 52.53, "mean_ms": 38.7, "recall_pack_latency_ms": 21.57 }, "category_breakdown": { "knowledge_update": { "question_count": 9, "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "abstention_accuracy": null, "mean_latency_ms": 36.27 }, "temporal_reasoning": { "question_count": 7, "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "abstention_accuracy": null, "mean_latency_ms": 31.78 }, "multi_session_synthesis": { "question_count": 16, "recall_at_5": 0.9323, "recall_at_10": 0.9479, "ndcg_at_10": 0.9492, "abstention_accuracy": null, "mean_latency_ms": 38.47 }, "abstention": { "question_count": 8, "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "abstention_accuracy": 1.0, "mean_latency_ms": 44.33 }, "abstention_in_domain": { "question_count": 8, "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "abstention_accuracy": 0.0, "mean_latency_ms": 42.3 } }, "reader_stratified_cost_curve": [ { "configuration": "Brutha Tier-1 Pure Retrieval (No LLM Reader)", "memory_layer_llm_calls_per_query": 0, "memory_layer_cost_per_1k_queries_usd": 0.0, "reader_model": "None (Direct MCP Snippet + Recall Signal)", "avg_context_tokens_injected": 551.8, "reader_cost_per_1k_queries_usd": 0.0, "total_cost_per_1k_queries_usd": 0.0, "retrieval_recall_at_5": 0.9661, "abstention_accuracy": 0.5 }, { "configuration": "Brutha + Local Open-Weights Reader (Zero-Egress Sovereign)", "memory_layer_llm_calls_per_query": 0, "memory_layer_cost_per_1k_queries_usd": 0.0, "reader_model": "Qwen-2.5-7B-Instruct / Llama-3.1-8B (Local)", "avg_context_tokens_injected": 551.8, "reader_cost_per_1k_queries_usd": 0.0, "total_cost_per_1k_queries_usd": 0.0, "retrieval_recall_at_5": 0.9661, "abstention_accuracy": 0.5 }, { "configuration": "Brutha (Snippet Mode) + Cheap Cloud Reader", "memory_layer_llm_calls_per_query": 0, "memory_layer_cost_per_1k_queries_usd": 0.0, "reader_model": "Claude Haiku / GPT-4o-mini ($0.25/$1.25 per 1M)", "avg_context_tokens_injected": 551.8, "reader_cost_per_1k_queries_usd": 0.27, "total_cost_per_1k_queries_usd": 0.27, "retrieval_recall_at_5": 0.9661, "abstention_accuracy": 0.5 }, { "configuration": "Brutha (Snippet Mode) + Frontier Cloud Reader", "memory_layer_llm_calls_per_query": 0, "memory_layer_cost_per_1k_queries_usd": 0.0, "reader_model": "Claude Sonnet / GPT-4o ($3.00/$15.00 per 1M)", "avg_context_tokens_injected": 551.8, "reader_cost_per_1k_queries_usd": 3.22, "total_cost_per_1k_queries_usd": 3.22, "retrieval_recall_at_5": 0.9661, "abstention_accuracy": 0.5 }, { "configuration": "Brutha (Full Content Mode) + Frontier Cloud Reader", "memory_layer_llm_calls_per_query": 0, "memory_layer_cost_per_1k_queries_usd": 0.0, "reader_model": "Claude Sonnet / GPT-4o ($3.00/$15.00 per 1M)", "avg_context_tokens_injected": 841.0, "reader_cost_per_1k_queries_usd": 4.08, "total_cost_per_1k_queries_usd": 4.08, "retrieval_recall_at_5": 0.9661, "abstention_accuracy": 0.5 }, { "configuration": "Competitor Reference: Honcho Dialectic / Synthesis Endpoint", "memory_layer_llm_calls_per_query": "1-3 (Internal LLM synthesis)", "memory_layer_cost_per_1k_queries_usd": 25.0, "reader_model": "Frontier Agent Reader", "avg_context_tokens_injected": 1200.0, "reader_cost_per_1k_queries_usd": 5.16, "total_cost_per_1k_queries_usd": 30.16, "retrieval_recall_at_5": null, "abstention_accuracy": null }, { "configuration": "Competitor Reference: Hindsight Agentic Reflect Loop", "memory_layer_llm_calls_per_query": "3-8 (Multi-step LLM reflect)", "memory_layer_cost_per_1k_queries_usd": 65.0, "reader_model": "Frontier Agent Reader", "avg_context_tokens_injected": 1800.0, "reader_cost_per_1k_queries_usd": 6.96, "total_cost_per_1k_queries_usd": 71.96, "retrieval_recall_at_5": null, "abstention_accuracy": null } ], "questions": [ { "id": "LME-01", "category": "knowledge_update", "unanswerable": false, "question": "Which PostgreSQL connection pooler and port does the billing-ledger service currently use in production?", "expected_keys": [ "ku_pg_pool_new" ], "retrieved_keys_top5": [ "ku_pg_pool_new", "ms_paterson_incident_fix", "dist_db_sandbox-runner", "dist_db_audit-compliance", "dist_db_notification-mailer" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8932, "latency_ms": 48.97, "snippet_chars": 2686, "full_chars": 3274, "snippet_tokens_est": 671.5, "full_tokens_est": 818.5, "reference_answer": "PgCat on port 6433 (which replaced PgBouncer on port 6432)." }, { "id": "LME-02", "category": "knowledge_update", "unanswerable": false, "question": "How does Brutha execute hybrid dense and lexical vector search in the current architecture?", "expected_keys": [ "ku_vector_new" ], "retrieved_keys_top5": [ "ku_vector_new", "tv_slo_active", "hard_vector_search_floor", "ms_embed_mitigation", "ms_enterprise_visibility" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8472, "latency_ms": 41.34, "snippet_chars": 2349, "full_chars": 4129, "snippet_tokens_est": 587.2, "full_tokens_est": 1032.2, "reference_answer": "In-process Rust EmbeddedIndex combining in-process dense vector + lexical BM25 hybrid search, replacing Qdrant." }, { "id": "LME-03", "category": "knowledge_update", "unanswerable": false, "question": "What JWT signing algorithm and access token TTL are enforced by the API Gateway token policy?", "expected_keys": [ "ku_auth_new" ], "retrieved_keys_top5": [ "ku_auth_new", "dist_auth_feature-store-ml", "dist_auth_analytics-ingest", "dist_auth_notification-mailer", "dist_auth_search-indexer-legacy" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7881, "latency_ms": 31.9, "snippet_chars": 2074, "full_chars": 2414, "snippet_tokens_est": 518.5, "full_tokens_est": 603.5, "reference_answer": "Asymmetric EdDSA (Ed25519) JWTs with a 15-minute access token TTL and JWKS discovery." }, { "id": "LME-04", "category": "knowledge_update", "unanswerable": false, "question": "Which message broker technology backs the asynchronous webhook ingestion queue?", "expected_keys": [ "ku_queue_new" ], "retrieved_keys_top5": [ "ku_queue_new", "haystack_eng_009", "haystack_eng_021", "haystack_eng_033", "haystack_eng_057" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8268, "latency_ms": 30.07, "snippet_chars": 2258, "full_chars": 2982, "snippet_tokens_est": 564.5, "full_tokens_est": 745.5, "reference_answer": "NATS JetStream work-queue streams with Raft R3 storage and Nats-Msg-Id deduplication." }, { "id": "LME-05", "category": "knowledge_update", "unanswerable": false, "question": "How is Route 1 NLI contradiction detection configured to prevent sibling false positives?", "expected_keys": [ "ku_cde_new" ], "retrieved_keys_top5": [ "ku_cde_new", "hard_Cross_repo_route_stitching", "haystack_eng_029", "ku_vector_new", "haystack_eng_101" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8116, "latency_ms": 32.54, "snippet_chars": 2342, "full_chars": 3402, "snippet_tokens_est": 585.5, "full_tokens_est": 850.5, "reference_answer": "Dual-model NLI corroboration at a 0.80 confidence floor, cutting sibling FP rate to 3.3%." }, { "id": "LME-06", "category": "knowledge_update", "unanswerable": false, "question": "How are portable Linux and macOS release binaries built and packaged in CI?", "expected_keys": [ "ku_ci_new" ], "retrieved_keys_top5": [ "ku_ci_new", "haystack_eng_012", "haystack_eng_024", "haystack_eng_084", "haystack_eng_036" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7362, "latency_ms": 29.91, "snippet_chars": 2388, "full_chars": 3267, "snippet_tokens_est": 597.0, "full_tokens_est": 816.8, "reference_answer": "GitHub Actions CI using cargo-zigbuild targeting glibc 2.31 with bundled ONNX Runtime libraries and SHA-256 manifests." }, { "id": "LME-07", "category": "knowledge_update", "unanswerable": false, "question": "What caching datastore backs user rate-limit counters and GraphQL response fragments?", "expected_keys": [ "ku_cache_new" ], "retrieved_keys_top5": [ "ku_cache_new", "dist_cache_audit-compliance", "dist_cache_identity-sso", "dist_cache_analytics-ingest", "dist_cache_notification-mailer" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.84, "latency_ms": 30.3, "snippet_chars": 1819, "full_chars": 2384, "snippet_tokens_est": 454.8, "full_tokens_est": 596.0, "reference_answer": "DragonflyDB in Valkey/Redis-compatible cluster mode with mTLS and Lua atomic token-bucket rate limiting." }, { "id": "LME-08", "category": "knowledge_update", "unanswerable": false, "question": "Where are OpenTelemetry distributed traces exported and stored in the current observability stack?", "expected_keys": [ "ku_obs_new" ], "retrieved_keys_top5": [ "ku_obs_new", "tv_slo_active", "dist_obs_audit-compliance", "dist_obs_feature-store-ml", "dist_obs_identity-sso" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8077, "latency_ms": 31.01, "snippet_chars": 1735, "full_chars": 2291, "snippet_tokens_est": 433.8, "full_tokens_est": 572.8, "reference_answer": "Grafana Tempo backed by S3 object storage with TraceQL indexing and 30-day retention via OTLP/gRPC collectors." }, { "id": "LME-09", "category": "temporal_reasoning", "unanswerable": false, "question": "Who is the active primary on-call incident commander for the storage-engine squad in 2026-2027?", "expected_keys": [ "tv_oncall_active" ], "retrieved_keys_top5": [ "tv_oncall_active", "dist_oncall_identity-sso", "dist_oncall_artifact-registry", "dist_oncall_notification-mailer", "dist_oncall_feature-store-ml" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7773, "latency_ms": 32.01, "snippet_chars": 1944, "full_chars": 2210, "snippet_tokens_est": 486.0, "full_tokens_est": 552.5, "reference_answer": "Elena Rostova (@erostova) under PAGER-2026-ACTIVE, with Devon Brooks as secondary backup." }, { "id": "LME-10", "category": "temporal_reasoning", "unanswerable": false, "question": "What is the active Q4 2026 release governance policy for production deployments and canary checks?", "expected_keys": [ "tv_freeze_active" ], "retrieved_keys_top5": [ "tv_freeze_active", "dist_slo_audit-compliance", "tv_cert_active", "dist_slo_artifact-registry", "tv_quota_active" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8495, "latency_ms": 30.36, "snippet_chars": 1544, "full_chars": 2644, "snippet_tokens_est": 386.0, "full_tokens_est": 661.0, "reference_answer": "POLICY-2026-Q4: Continuous deployment Mon-Thu 08:00-18:00 UTC with 20-minute 5% canary verification and Friday deploys restricted to security patches." }, { "id": "LME-11", "category": "temporal_reasoning", "unanswerable": false, "question": "Which mTLS intermediate CA certificate serial and key algorithm are currently active in the service mesh?", "expected_keys": [ "tv_cert_active" ], "retrieved_keys_top5": [ "tv_cert_active", "dist_auth_notification-mailer", "dist_auth_audit-compliance", "dist_auth_artifact-registry", "ku_auth_new" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8517, "latency_ms": 30.03, "snippet_chars": 1877, "full_chars": 2485, "snippet_tokens_est": 469.2, "full_tokens_est": 621.2, "reference_answer": "Serial 0x9C3E72 issued by Vault PKI root v2 using ECDSA P-384 keys, valid through August 1, 2027." }, { "id": "LME-12", "category": "temporal_reasoning", "unanswerable": false, "question": "What are the current GA Enterprise tenant ingestion rate limits and maximum indexed chunk quotas?", "expected_keys": [ "tv_quota_active" ], "retrieved_keys_top5": [ "tv_quota_active", "ku_cache_new", "dist_cache_analytics-ingest", "tv_slo_active", "dist_cache_audit-compliance" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.9284, "latency_ms": 34.88, "snippet_chars": 1726, "full_chars": 2531, "snippet_tokens_est": 431.5, "full_tokens_est": 632.8, "reference_answer": "2,500 requests per minute per team schema and up to 2,000,000 indexed chunks per EmbeddedIndex generation." }, { "id": "LME-13", "category": "temporal_reasoning", "unanswerable": false, "question": "Which active runtime flag controls opportunistic background catch-up index rebuilds on truth-store drift?", "expected_keys": [ "tv_flag_active" ], "retrieved_keys_top5": [ "tv_flag_active", "dist_queue_feature-store-ml", "dist_cache_feature-store-ml", "dist_queue_search-indexer-legacy", "dist_cache_identity-sso" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8502, "latency_ms": 30.59, "snippet_chars": 1784, "full_chars": 2180, "snippet_tokens_est": 446.0, "full_tokens_est": 545.0, "reference_answer": "BRUTHA_AUTO_HEAL_ENABLED, with a 30-second cooldown and max 5 consecutive failures." }, { "id": "LME-14", "category": "temporal_reasoning", "unanswerable": false, "question": "What is the current production P95 and P50 hybrid search latency SLO for the Rust EmbeddedIndex?", "expected_keys": [ "tv_slo_active" ], "retrieved_keys_top5": [ "tv_slo_active", "dist_slo_search-indexer-legacy", "ku_vector_new", "dist_slo_artifact-registry", "dist_slo_identity-sso" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.9121, "latency_ms": 31.79, "snippet_chars": 1556, "full_chars": 2084, "snippet_tokens_est": 389.0, "full_tokens_est": 521.0, "reference_answer": "P95 <= 25ms and P50 <= 8ms at $0.00 external API cost with 99.95% availability." }, { "id": "LME-15", "category": "multi_session_synthesis", "unanswerable": false, "question": "How does Brutha's Code Knowledge Graph track symbol drift across generations and notify subscribed Characters via the outbox?", "expected_keys": [ "ms_code_graph_schema", "ms_code_graph_drift", "ms_outbox_subscriptions" ], "retrieved_keys_top5": [ "ms_outbox_subscriptions", "ms_code_graph_drift", "ms_code_graph_schema", "ms_enterprise_visibility", "haystack_eng_003" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8225, "latency_ms": 43.2, "snippet_chars": 2597, "full_chars": 4145, "snippet_tokens_est": 649.2, "full_tokens_est": 1036.2, "reference_answer": "Symbols in code_node carry signature/body fingerprints; the diff engine classifies modifications/moves and updates memory_code_link states, while matching character_code_subscription rules enqueue context packs into code_event_outbox." }, { "id": "LME-16", "category": "multi_session_synthesis", "unanswerable": false, "question": "How do SessionWatch Level 1 and Level 3 work together from MCP stdio startup through stdin EOF shutdown?", "expected_keys": [ "ms_session_watch_l1", "ms_session_watch_l3" ], "retrieved_keys_top5": [ "ms_session_watch_l3", "ms_session_watch_l1", "haystack_eng_018", "haystack_eng_030", "tv_flag_active" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7921, "latency_ms": 37.58, "snippet_chars": 2129, "full_chars": 3149, "snippet_tokens_est": 532.2, "full_tokens_est": 787.2, "reference_answer": "Level 1 logs PID/start in session_lifecycle_log and tracks tool counters; on stdin EOF, Level 3 holds a grace period for end_session_review before finalize_session records the final review outcome." }, { "id": "LME-17", "category": "multi_session_synthesis", "unanswerable": false, "question": "How does Brutha Enterprise enforce schema-per-team isolation while still supporting federated cross-team project search?", "expected_keys": [ "ms_enterprise_tenancy", "ms_enterprise_visibility" ], "retrieved_keys_top5": [ "ms_enterprise_tenancy", "ms_enterprise_visibility", "hard_Cross_repo_route_stitching", "haystack_eng_008", "haystack_eng_068" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8854, "latency_ms": 31.05, "snippet_chars": 2349, "full_chars": 3501, "snippet_tokens_est": 587.2, "full_tokens_est": 875.2, "reference_answer": "Each team has an isolated team_ Postgres schema with private/shared memory visibility, while explicit grants into enterprise.cross_team_project allow federated cross_team_search." }, { "id": "LME-18", "category": "multi_session_synthesis", "unanswerable": false, "question": "How does Lever 1 snippet-then-expand save tokens and how does memory_expand keep the token_savings counter honest?", "expected_keys": [ "ms_token_savings_lever1", "ms_token_savings_netting" ], "retrieved_keys_top5": [ "ms_token_savings_lever1", "ms_token_savings_netting", "haystack_eng_021", "haystack_eng_057", "haystack_eng_093" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8555, "latency_ms": 31.8, "snippet_chars": 2372, "full_chars": 3390, "snippet_tokens_est": 593.0, "full_tokens_est": 847.5, "reference_answer": "Search and recall serve 240-char snippets and bank avoided characters, while memory_expand subtracts the full character count of expanded memories so token_savings reflects net characters avoided." }, { "id": "LME-19", "category": "multi_session_synthesis", "unanswerable": false, "question": "Why did bulk embedding crash 16GB Apple Silicon hosts and what three configuration knobs bound ONNX and Rayon memory use?", "expected_keys": [ "ms_embed_concurrency", "ms_embed_mitigation" ], "retrieved_keys_top5": [ "ms_embed_mitigation", "ms_embed_concurrency", "tv_slo_active", "dist_cache_identity-sso", "dist_cache_audit-compliance" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8605, "latency_ms": 31.64, "snippet_chars": 1892, "full_chars": 2757, "snippet_tokens_est": 473.0, "full_tokens_est": 689.2, "reference_answer": "Uncapped Rayon par_chunks(256) x available_parallelism() ONNX threads exhausted memory; fixed by BRUTHA_EMBED_CONCURRENCY=4, BRUTHA_EMBED_INTRA_THREADS=4, and BRUTHA_EMBED_BATCH_SIZE=64." }, { "id": "LME-20", "category": "multi_session_synthesis", "unanswerable": false, "question": "How does the Code Graph source vault filter secrets before compression and what three retention modes does code_index_repo support?", "expected_keys": [ "ms_vault_secret_filter", "ms_vault_retention_modes" ], "retrieved_keys_top5": [ "ms_vault_retention_modes", "ms_vault_secret_filter", "haystack_eng_047", "haystack_eng_011", "haystack_eng_023" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8328, "latency_ms": 32.51, "snippet_chars": 2344, "full_chars": 3051, "snippet_tokens_est": 586.0, "full_tokens_est": 762.8, "reference_answer": "filter_blob blocks .env/credential files, binaries, >1MiB files, and secret patterns before zstd storage; retention modes are metadata_only, referenced_blobs, and full_snapshot." }, { "id": "LME-21", "category": "multi_session_synthesis", "unanswerable": false, "question": "What anchor states can a memory_code_link transition into when a symbol's body_fingerprint moves or changes across files?", "expected_keys": [ "ms_code_graph_drift", "ms_code_graph_schema" ], "retrieved_keys_top5": [ "ms_code_graph_drift", "ms_code_graph_schema", "ms_outbox_subscriptions", "ms_vault_retention_modes", "ms_vault_secret_filter" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8083, "latency_ms": 31.95, "snippet_chars": 2106, "full_chars": 4384, "snippet_tokens_est": 526.5, "full_tokens_est": 1096.0, "reference_answer": "Anchor states transition between current, changed, moved, ambiguous, and missing based on signature_fingerprint and body_fingerprint comparisons." }, { "id": "LME-22", "category": "multi_session_synthesis", "unanswerable": false, "question": "Which tables store the persistent code graph nodes and edges, and how are source blobs selectively retained?", "expected_keys": [ "ms_code_graph_schema", "ms_vault_retention_modes" ], "retrieved_keys_top5": [ "ms_code_graph_schema", "ms_vault_retention_modes", "ms_vault_secret_filter", "haystack_eng_003", "haystack_eng_039" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8136, "latency_ms": 49.49, "snippet_chars": 2755, "full_chars": 4263, "snippet_tokens_est": 688.8, "full_tokens_est": 1065.8, "reference_answer": "code_repo, code_generation, code_node, code_edge, code_blob, and memory_code_link store the graph, with blobs retained via metadata_only, referenced_blobs, or full_snapshot." }, { "id": "LME-23", "category": "abstention", "unanswerable": true, "question": "What is the synthesis temperature and crystal lattice annealing duration for the ytterbium-doped perovskite solar cell fabrication line?", "expected_keys": [], "retrieved_keys_top5": [ "tv_slo_active", "ku_cache_new", "dist_slo_analytics-ingest", "ku_auth_new", "ku_obs_new" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.5969, "latency_ms": 55.73, "snippet_chars": 2042, "full_chars": 3842, "snippet_tokens_est": 510.5, "full_tokens_est": 960.5, "reference_answer": "ABSTAIN (unanswerable: no memory in the workspace discusses perovskite solar cell fabrication or ytterbium annealing)." }, { "id": "LME-24", "category": "abstention", "unanswerable": true, "question": "Which equine veterinary dosage of phenylbutazone was prescribed for the thoroughbred stallion at Churchill Downs in May 1998?", "expected_keys": [], "retrieved_keys_top5": [ "ku_queue_new", "tv_oncall_active", "ku_cache_new", "ku_pg_pool_new", "tv_slo_active" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.523, "latency_ms": 32.0, "snippet_chars": 1615, "full_chars": 3336, "snippet_tokens_est": 403.8, "full_tokens_est": 834.0, "reference_answer": "ABSTAIN (unanswerable: no memory records veterinary equine prescriptions or horse racing)." }, { "id": "LME-25", "category": "abstention", "unanswerable": true, "question": "What were the pelagic trawl catch quotas for Atlantic mackerel in ICES Subdivision IVa negotiated by the North Sea Fisheries Council?", "expected_keys": [], "retrieved_keys_top5": [ "tv_quota_active", "tv_slo_active", "tv_flag_active", "ku_auth_new", "ku_pg_pool_new" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.5504, "latency_ms": 43.5, "snippet_chars": 2357, "full_chars": 5155, "snippet_tokens_est": 589.2, "full_tokens_est": 1288.8, "reference_answer": "ABSTAIN (unanswerable: no memory covers commercial fishing quotas or marine biology)." }, { "id": "LME-26", "category": "abstention", "unanswerable": true, "question": "How many carats was the cushion-cut Padparadscha sapphire auctioned at Christie's Geneva magnificent jewels sale?", "expected_keys": [], "retrieved_keys_top5": [ "tv_slo_active", "tv_oncall_active", "ku_vector_new", "tv_cert_active", "tv_quota_active" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.4431, "latency_ms": 48.51, "snippet_chars": 2361, "full_chars": 4401, "snippet_tokens_est": 590.2, "full_tokens_est": 1100.2, "reference_answer": "ABSTAIN (unanswerable: no memory contains gemstone auction records)." }, { "id": "LME-27", "category": "abstention", "unanswerable": true, "question": "What orbital inclination and apogee altitude were targeted by the propellant depot rendezvous burn during the Lunar Gateway NRHO insertion?", "expected_keys": [], "retrieved_keys_top5": [ "ku_auth_new", "hard_sse_keepalive_drop", "tv_cert_active", "ku_pg_pool_new", "tv_slo_active" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.5765, "latency_ms": 45.6, "snippet_chars": 2339, "full_chars": 4933, "snippet_tokens_est": 584.8, "full_tokens_est": 1233.2, "reference_answer": "ABSTAIN (unanswerable: no memory contains astrodynamics or lunar orbital mechanics)." }, { "id": "LME-28", "category": "abstention", "unanswerable": true, "question": "Which baroque lute continuo realization did Nikolaus Harnoncourt select for the 1979 recording of Monteverdi's L'Orfeo?", "expected_keys": [], "retrieved_keys_top5": [ "ku_cde_new", "tv_flag_active", "dist_cache_identity-sso", "ms_code_graph_drift", "dist_cache_audit-compliance" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.424, "latency_ms": 50.63, "snippet_chars": 2008, "full_chars": 3082, "snippet_tokens_est": 502.0, "full_tokens_est": 770.5, "reference_answer": "ABSTAIN (unanswerable: no memory covers baroque musicology or opera recordings)." }, { "id": "LME-29", "category": "abstention", "unanswerable": true, "question": "What malolactic fermentation strain and French oak toast level were used for the 2019 C\u00f4te-R\u00f4tie Syrah vintage?", "expected_keys": [], "retrieved_keys_top5": [ "tv_slo_active", "tv_oncall_active", "dist_cache_audit-compliance", "dist_cache_search-indexer-legacy", "dist_cache_analytics-ingest" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.5132, "latency_ms": 33.33, "snippet_chars": 2099, "full_chars": 2701, "snippet_tokens_est": 524.8, "full_tokens_est": 675.2, "reference_answer": "ABSTAIN (unanswerable: no memory discusses viticulture or winemaking)." }, { "id": "LME-30", "category": "abstention", "unanswerable": true, "question": "Which stratigraphic horizon in the Morrison Formation yielded the articulated Diplodocus caudal vertebrae specimen?", "expected_keys": [], "retrieved_keys_top5": [ "tv_cert_active", "hard_Cross_repo_route_stitching", "tv_slo_active", "ku_cde_new", "ms_vault_secret_filter" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": true, "recall_confidence": "low", "best_cosine": 0.5277, "latency_ms": 45.38, "snippet_chars": 2348, "full_chars": 4963, "snippet_tokens_est": 587.0, "full_tokens_est": 1240.8, "reference_answer": "ABSTAIN (unanswerable: no memory discusses paleontology or dinosaur fossils)." }, { "id": "LME-31", "category": "multi_session_synthesis", "unanswerable": false, "question": "What caused SQLite lock contention in Incident INC-409, how was DbSqlite remediated, and what were the Chaos Run #88 soak test results?", "expected_keys": [ "ms_ Paterson_incident_root", "ms_paterson_incident_fix", "ms_paterson_incident_verification" ], "retrieved_keys_top5": [ "ms_paterson_incident_verification", "ms_paterson_incident_fix", "ms_ Paterson_incident_root", "hard_wal_checkpoint_starvation", "ku_cache_new" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8532, "latency_ms": 31.49, "snippet_chars": 1968, "full_chars": 2412, "snippet_tokens_est": 492.0, "full_tokens_est": 603.0, "reference_answer": "INC-409 was caused by token_savings_daily upserts contending with decay sweeps under 64 parallel clients; remediated with busy_timeout=5000ms, synchronous=NORMAL, and a write-coalescing mutex, achieving 0 lock timeouts and 4.1ms p99 in Chaos Run #88." }, { "id": "LME-32", "category": "multi_session_synthesis", "unanswerable": false, "question": "Trace the complete end-to-end flow from a developer modifying a source file in a git repo, through code_index_repo secret filtering and symbol diffing, to a subscribed Character resolving a code_memory_proposal into dice_correction_log.", "expected_keys": [ "ms_vault_secret_filter", "ms_code_graph_schema", "ms_code_graph_drift", "ms_outbox_subscriptions" ], "retrieved_keys_top5": [ "ms_outbox_subscriptions", "ms_code_graph_drift", "ms_code_graph_schema", "ms_vault_secret_filter", "ms_vault_retention_modes" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7468, "latency_ms": 40.31, "snippet_chars": 2341, "full_chars": 4112, "snippet_tokens_est": 585.2, "full_tokens_est": 1028.0, "reference_answer": "filter_blob screens secrets before zstd storage; code_node fingerprints symbols; the diff engine updates memory_code_link states (current/changed/moved/ambiguous/missing) and enqueues matching character_code_subscription events into code_event_outbox for code_memory_proposal resolution into dice_correction_log." }, { "id": "LME-33", "category": "abstention_in_domain", "unanswerable": true, "question": "What is the configured Kafka partition count and consumer group rebalance timeout for the billing-ledger service?", "expected_keys": [], "retrieved_keys_top5": [ "ku_pg_pool_new", "ms_paterson_incident_fix", "ku_queue_new", "hard_sse_keepalive_drop", "dist_queue_analytics-ingest" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7267, "latency_ms": 43.76, "snippet_chars": 2598, "full_chars": 3399, "snippet_tokens_est": 649.5, "full_tokens_est": 849.8, "reference_answer": "ABSTAIN (unanswerable: billing-ledger uses PgCat/PostgreSQL, and no Kafka partition count for billing-ledger is recorded)." }, { "id": "LME-34", "category": "abstention_in_domain", "unanswerable": true, "question": "Which post-quantum Kyber-768 key encapsulation parameter set is used by Vault PKI root v3 in the service mesh?", "expected_keys": [], "retrieved_keys_top5": [ "tv_cert_active", "ms_vault_secret_filter", "dist_auth_sandbox-runner", "dist_auth_artifact-registry", "ku_auth_new" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.738, "latency_ms": 51.41, "snippet_chars": 2636, "full_chars": 4608, "snippet_tokens_est": 659.0, "full_tokens_est": 1152.0, "reference_answer": "ABSTAIN (unanswerable: only Vault PKI root v1 RSA-2048 and v2 ECDSA P-384 are recorded; no root v3 or Kyber-768 exists)." }, { "id": "LME-35", "category": "abstention_in_domain", "unanswerable": true, "question": "What is the AWS DynamoDB provisioned write capacity unit (WCU) ceiling for Brutha Enterprise team schemas?", "expected_keys": [], "retrieved_keys_top5": [ "ms_enterprise_tenancy", "ms_enterprise_visibility", "tv_quota_active", "ms_paterson_incident_fix", "ms_embed_mitigation" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7269, "latency_ms": 53.13, "snippet_chars": 2872, "full_chars": 5234, "snippet_tokens_est": 718.0, "full_tokens_est": 1308.5, "reference_answer": "ABSTAIN (unanswerable: Brutha Enterprise uses PostgreSQL schemas, not AWS DynamoDB)." }, { "id": "LME-36", "category": "abstention_in_domain", "unanswerable": true, "question": "Who is the primary on-call incident commander for the mobile-ios-client squad in Q4 2026?", "expected_keys": [], "retrieved_keys_top5": [ "tv_oncall_active", "dist_oncall_notification-mailer", "dist_oncall_analytics-ingest", "dist_oncall_identity-sso", "dist_oncall_feature-store-ml" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7514, "latency_ms": 31.19, "snippet_chars": 1944, "full_chars": 2210, "snippet_tokens_est": 486.0, "full_tokens_est": 552.5, "reference_answer": "ABSTAIN (unanswerable: no on-call rotation for a mobile-ios-client squad exists in memory)." }, { "id": "LME-37", "category": "abstention_in_domain", "unanswerable": true, "question": "What GPU tensor-parallelism degree is configured for running Llama-3.1-405B inside brutha-mcp?", "expected_keys": [], "retrieved_keys_top5": [ "haystack_eng_060", "haystack_eng_072", "ku_ci_new", "haystack_eng_036", "haystack_eng_084" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7213, "latency_ms": 31.94, "snippet_chars": 2155, "full_chars": 2754, "snippet_tokens_est": 538.8, "full_tokens_est": 688.5, "reference_answer": "ABSTAIN (unanswerable: brutha-mcp runs zero LLMs and uses CPU ONNX embeddings/cross-encoders)." }, { "id": "LME-38", "category": "abstention_in_domain", "unanswerable": true, "question": "What is the weekly Stripe invoice webhook retry schedule configured in the billing-ledger Go worker?", "expected_keys": [], "retrieved_keys_top5": [ "ku_pg_pool_new", "ms_paterson_incident_fix", "ku_queue_new", "tv_quota_active", "tv_slo_active" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.6904, "latency_ms": 47.61, "snippet_chars": 2214, "full_chars": 3439, "snippet_tokens_est": 553.5, "full_tokens_est": 859.8, "reference_answer": "ABSTAIN (unanswerable: billing-ledger only has PostgreSQL connection pooler memories; no Stripe webhook schedule is stored)." }, { "id": "LME-39", "category": "abstention_in_domain", "unanswerable": true, "question": "Which Consul service-discovery health check interval is set for the DragonflyDB rate-limit cluster?", "expected_keys": [], "retrieved_keys_top5": [ "ku_cache_new", "dist_db_audit-compliance", "dist_db_feature-store-ml", "dist_db_analytics-ingest", "dist_db_notification-mailer" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7888, "latency_ms": 29.48, "snippet_chars": 2299, "full_chars": 3354, "snippet_tokens_est": 574.8, "full_tokens_est": 838.5, "reference_answer": "ABSTAIN (unanswerable: no Consul service-discovery configuration is recorded for DragonflyDB)." }, { "id": "LME-40", "category": "abstention_in_domain", "unanswerable": true, "question": "What is the maximum WebAssembly plugin memory page limit enforced by the Brutha MCP stdio server?", "expected_keys": [], "retrieved_keys_top5": [ "ku_cache_new", "hard_wal_checkpoint_starvation", "ms_token_savings_lever1", "ms_embed_concurrency", "ms_session_watch_l1" ], "recall_at_5": null, "recall_at_10": null, "ndcg_at_10": null, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7192, "latency_ms": 49.87, "snippet_chars": 2390, "full_chars": 4553, "snippet_tokens_est": 597.5, "full_tokens_est": 1138.2, "reference_answer": "ABSTAIN (unanswerable: Brutha MCP does not have a WebAssembly plugin runtime)." }, { "id": "LME-41", "category": "multi_session_synthesis", "unanswerable": false, "question": "Compare all three database and storage stability remediations across the workspace: how were billing-ledger connection pooling, INC-409 concurrent write lock contention, and unbounded 4.2 GB write-ahead log file growth each resolved?", "expected_keys": [ "ku_pg_pool_new", "ms_paterson_incident_fix", "hard_wal_checkpoint_starvation" ], "retrieved_keys_top5": [ "ms_paterson_incident_fix", "hard_wal_checkpoint_starvation", "ms_paterson_incident_verification", "ms_embed_concurrency", "ku_pg_pool_new" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 0.9469, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7931, "latency_ms": 47.18, "snippet_chars": 2684, "full_chars": 4051, "snippet_tokens_est": 671.0, "full_tokens_est": 1012.8, "reference_answer": "billing-ledger moved to PgCat on port 6433; INC-409 added busy_timeout=5000ms, synchronous=NORMAL, and a write-coalescing mutex; 4.2 GB WAL growth was fixed with journal_size_limit=67108864 and RESTART checkpointing." }, { "id": "LME-42", "category": "multi_session_synthesis", "unanswerable": false, "question": "What three security mechanisms protect credentials and tokens across edge authentication, source vault indexing, and Enterprise PAT revocation when raw secrets are not stored?", "expected_keys": [ "ku_auth_new", "ms_vault_secret_filter", "hard_pat_prefix_revocation" ], "retrieved_keys_top5": [ "ku_auth_new", "hard_pat_prefix_revocation", "ms_vault_retention_modes", "ms_vault_secret_filter", "haystack_eng_043" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 0.9675, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7522, "latency_ms": 33.31, "snippet_chars": 2375, "full_chars": 3559, "snippet_tokens_est": 593.8, "full_tokens_est": 889.8, "reference_answer": "Edge auth uses asymmetric Ed25519 JWTs with 15-min TTL; filter_blob strips .env/PEM/high-entropy secrets before zstd storage; PAT revocation matches the 12-char token_prefix and sets revoked_at." }, { "id": "LME-43", "category": "multi_session_synthesis", "unanswerable": false, "question": "How does the Code Knowledge Graph handle macro-expanded SCIP symbols missing enclosing_range, stitch HTTP routes across separate repositories, and classify moved symbols in working-tree overlays?", "expected_keys": [ "hard_scip_enclosing_range_fallback", "hard_Cross_repo_route_stitching", "ms_code_graph_drift" ], "retrieved_keys_top5": [ "hard_scip_enclosing_range_fallback", "ms_code_graph_drift", "ms_code_graph_schema", "haystack_eng_014", "haystack_eng_038" ], "recall_at_5": 0.6667, "recall_at_10": 0.6667, "ndcg_at_10": 0.7654, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7734, "latency_ms": 44.44, "snippet_chars": 2585, "full_chars": 3769, "snippet_tokens_est": 646.2, "full_tokens_est": 942.2, "reference_answer": "SCIP falls back to the occurrence range span; link_cross_repo_edges stitches matching HTTP routes and manifest deps; the diff engine matches body_fingerprint at a new path as symbol_moved (moved anchor state)." }, { "id": "LME-44", "category": "multi_session_synthesis", "unanswerable": false, "question": "Across the retrieval and embedding stack, how are vector search candidate floors, 16GB Apple Silicon ONNX thread limits, and production P95 search latency SLOs configured?", "expected_keys": [ "hard_vector_search_floor", "ms_embed_mitigation", "tv_slo_active" ], "retrieved_keys_top5": [ "tv_slo_active", "hard_vector_search_floor", "ms_embed_mitigation", "ms_embed_concurrency", "ku_vector_new" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8199, "latency_ms": 36.6, "snippet_chars": 1850, "full_chars": 3003, "snippet_tokens_est": 462.5, "full_tokens_est": 750.8, "reference_answer": "candidate_floor is max(limit*4, 96); Apple Silicon caps are BRUTHA_EMBED_CONCURRENCY=4, BRUTHA_EMBED_INTRA_THREADS=4, BRUTHA_EMBED_BATCH_SIZE=64; production SLO is P95 <= 25ms and P50 <= 8ms." }, { "id": "LME-45", "category": "multi_session_synthesis", "unanswerable": false, "question": "When an operator imports a 5,000-file markdown vault and an HTTP client simultaneously queries the daemon while a background decay sweep runs, which mechanisms prevent re-embedding on interrupt, strip duplicate YAML headers during NLI dedup, and prevent SQLite snapshot lock errors?", "expected_keys": [ "haystack_eng_010", "haystack_eng_005", "ms_ Paterson_incident_root", "ms_paterson_incident_fix" ], "retrieved_keys_top5": [ "ms_ Paterson_incident_root", "haystack_eng_046", "haystack_eng_010", "haystack_eng_058", "haystack_eng_082" ], "recall_at_5": 0.5, "recall_at_10": 0.5, "ndcg_at_10": 0.5856, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.76, "latency_ms": 56.58, "snippet_chars": 2678, "full_chars": 3297, "snippet_tokens_est": 669.5, "full_tokens_est": 824.2, "reference_answer": "rebuild_checkpoint persists the last embedded row cursor; exclude_shared_leading_block strips >=20 char shared headers; busy_timeout=5000ms, synchronous=NORMAL, and DbSqlite write-coalescing mutex prevent SQLITE_BUSY_SNAPSHOT." }, { "id": "LME-46", "category": "multi_session_synthesis", "unanswerable": false, "question": "Compare the caching layer and message queue of the notification-mailer service against the main production API gateway cache and webhook ingestion queue.", "expected_keys": [ "dist_cache_notification-mailer", "dist_queue_notification-mailer", "ku_cache_new", "ku_queue_new" ], "retrieved_keys_top5": [ "ku_queue_new", "dist_cache_notification-mailer", "dist_db_notification-mailer", "ku_cache_new", "dist_slo_notification-mailer" ], "recall_at_5": 0.75, "recall_at_10": 1.0, "ndcg_at_10": 0.9223, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7813, "latency_ms": 36.35, "snippet_chars": 1931, "full_chars": 2627, "snippet_tokens_est": 482.8, "full_tokens_est": 656.8, "reference_answer": "notification-mailer uses KeyDB (128MB slabs) and AWS SQS (batch 32), whereas the main gateway/webhook stack uses DragonflyDB with mTLS and NATS JetStream with Raft R3." }, { "id": "LME-47", "category": "knowledge_update", "unanswerable": false, "question": "Across all services that use PostgreSQL connection poolers, which service migrated away from PgBouncer on port 6432 to PgCat on port 6433, and which sibling service still uses Odyssey on port 6434?", "expected_keys": [ "ku_pg_pool_new", "dist_db_notification-mailer" ], "retrieved_keys_top5": [ "ku_pg_pool_new", "dist_db_notification-mailer", "dist_db_audit-compliance", "dist_db_sandbox-runner", "ku_cache_new" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "high", "best_cosine": 0.8373, "latency_ms": 50.41, "snippet_chars": 2681, "full_chars": 3548, "snippet_tokens_est": 670.2, "full_tokens_est": 887.0, "reference_answer": "billing-ledger migrated from PgBouncer (6432) to PgCat (6433), while notification-mailer uses Odyssey on port 6434." }, { "id": "LME-48", "category": "temporal_reasoning", "unanswerable": false, "question": "Compare the active 2026-2027 primary on-call incident commanders for the storage-engine squad versus the audit-compliance squad.", "expected_keys": [ "tv_oncall_active", "dist_oncall_audit-compliance" ], "retrieved_keys_top5": [ "tv_oncall_active", "dist_oncall_audit-compliance", "dist_oncall_artifact-registry", "dist_oncall_identity-sso", "dist_oncall_notification-mailer" ], "recall_at_5": 1.0, "recall_at_10": 1.0, "ndcg_at_10": 1.0, "recall_weak": false, "recall_confidence": "medium", "best_cosine": 0.7143, "latency_ms": 32.79, "snippet_chars": 1944, "full_chars": 2210, "snippet_tokens_est": 486.0, "full_tokens_est": 552.5, "reference_answer": "Elena Rostova (@erostova) is primary on-call for storage-engine, while Jordan Patel (@jpatel-5) is primary on-call for audit-compliance." } ] }