{ "benchmark": "Sovereign Engineering Memory : Pillar 3b (106-Pair Real-Prose Contradiction Benchmark)", "dataset_name": "prose_contradiction_106", "dataset_source": "myzona-org/ergomem evals/prose/dataset.jsonl", "total_pairs": 106, "gold_contradictions": 31, "gold_non_contradictions": 75, "llm_calls": 0, "memory_save_cost_usd": 0.0, "wall_clock_seconds": 38.44, "metrics": { "tp": 30, "fp": 5, "fn": 1, "tn": 70, "precision": 0.8571, "recall": 0.9677, "f1": 0.9091, "accuracy": 0.9434, "false_positive_rate": 0.0667, "false_hard_blocks": 0, "duplicate_blocks_on_contradictions": 0, "p50_ms": 188.9, "p95_ms": 322.17, "mean_ms": 219.05 }, "family_breakdown": { "pos": { "total": 31, "tp": 30, "fp": 0, "fn": 1, "tn": 0, "hard_blocks": 0 }, "near-miss": { "total": 15, "tp": 0, "fp": 1, "fn": 0, "tn": 14, "hard_blocks": 0 }, "corpus-fp": { "total": 60, "tp": 0, "fp": 4, "fn": 0, "tn": 56, "hard_blocks": 0 } }, "tag_breakdown": { "pos/neg-flip": { "total": 6, "tp": 6, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/modality-flip": { "total": 3, "tp": 3, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/antonym": { "total": 6, "tp": 6, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/always-never": { "total": 2, "tp": 2, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/modality": { "total": 1, "tp": 1, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/neg-flip+reason": { "total": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/antonym+reason": { "total": 1, "tp": 1, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/coref": { "total": 3, "tp": 3, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/same-scope": { "total": 3, "tp": 3, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 }, "pos/semantic": { "total": 1, "tp": 0, "fp": 0, "fn": 1, "tn": 0, "hard_blocks": 0 }, "near-miss/scope-coexist": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "near-miss/time-coexist": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "near-miss/boilerplate": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "near-miss/homonym": { "total": 2, "tp": 0, "fp": 1, "fn": 0, "tn": 1, "hard_blocks": 0 }, "near-miss/refinement": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "near-miss/same-structure": { "total": 1, "tp": 0, "fp": 0, "fn": 0, "tn": 1, "hard_blocks": 0 }, "near-miss/stray-not": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "near-miss/compatible": { "total": 2, "tp": 0, "fp": 0, "fn": 0, "tn": 2, "hard_blocks": 0 }, "corpus-fp/flagged": { "total": 50, "tp": 0, "fp": 4, "fn": 0, "tn": 46, "hard_blocks": 0 }, "corpus-fp/warned": { "total": 10, "tp": 0, "fp": 0, "fn": 0, "tn": 10, "hard_blocks": 0 }, "pos/omission": { "total": 1, "tp": 1, "fp": 0, "fn": 0, "tn": 0, "hard_blocks": 0 } }, "competitor_head_to_head": [ { "system": "Brutha Write-Time Gate (Route 1 NLI Corroboration + Content-Overlap Gate)", "llm_calls_per_pair": 0, "cost_per_1k_pairs_usd": 0.0, "precision": 0.857, "recall": 0.968, "f1": 0.909, "false_hard_blocks": 0 }, { "system": "ergomem (LLM Normalizer claude-haiku + Structural + NLI after value-conflict fix)", "llm_calls_per_pair": 2, "cost_per_1k_pairs_usd": 0.8, "precision_band": "0.800-0.860", "recall_band": "0.770-0.840", "f1_band": "0.790-0.830", "false_hard_blocks": 0 }, { "system": "ergomem (LLM Normalizer claude-haiku + Structural + NLI pre-value-fix point)", "llm_calls_per_pair": 2, "cost_per_1k_pairs_usd": 0.8, "precision": 0.955, "recall": 0.677, "f1": 0.792, "false_hard_blocks": 0 }, { "system": "ergomem (LLM Normalizer claude-haiku + Structural-Only, No NLI)", "llm_calls_per_pair": 2, "cost_per_1k_pairs_usd": 0.8, "precision": 0.933, "recall": 0.467, "f1": 0.622, "false_hard_blocks": 0 }, { "system": "ergomem (LLM Normalizer claude-haiku + Ungated NLI pre-fix)", "llm_calls_per_pair": 2, "cost_per_1k_pairs_usd": 0.8, "precision": 0.42, "recall": 0.7, "f1": 0.525, "false_hard_blocks": 0 } ], "misclassified_pairs": [ { "pair_id": 27, "tag": "pos/semantic", "family": "pos", "gold_label": "contradiction", "predicted_label": "not_contradiction", "outcome": "FN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 278.18, "a": "The bot must reply to every message in the channel.", "b": "The bot must stay silent unless it is directly mentioned." }, { "pair_id": 38, "tag": "near-miss/homonym", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9785000085830688, "latency_ms": 266.14, "a": "Channel keys must use stable Slack channel IDs.", "b": "Room keys should be Matrix room IDs or aliases." }, { "pair_id": 47, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9592000246047974, "latency_ms": 326.17, "a": "If Slack ambient rooms do not trigger, verify the channel key is the Slack channel ID and the app has the required channels:history or groups:history scope for that room type.", "b": "If Telegram ambient rooms do not trigger at all, check BotFather privacy mode and verify the Gateway is receiving normal group messages." }, { "pair_id": 48, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.996399998664856, "latency_ms": 177.55, "a": "You should now see your bot in the Discord server.", "b": "Telegram exampleFor Telegram groups, the bot must be able to see normal group messages." }, { "pair_id": 49, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9350000023841858, "latency_ms": 205.41, "a": "To submit unmentioned always-on group chatter as quiet room context instead of user requests, use Ambient room events: The default is unmentionedInbound: \"userrequest\".", "b": "Ambient always-on room events, when enabled, stay strict and quiet unless the agent calls the message tool." }, { "pair_id": 79, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.89410001039505, "latency_ms": 281.21, "a": "\"always\": inject a concise notice on every run when truncation exists (recommended).", "b": "\"off\": never inject truncation notice text into the system prompt." } ], "pairs": [ { "pair_id": 1, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9980000257492065, "latency_ms": 2882.77, "a": "Production database backups must be encrypted at rest.", "b": "Production database backups must not be encrypted at rest." }, { "pair_id": 2, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9986000061035156, "latency_ms": 292.54, "a": "The /metrics endpoint must be exposed publicly.", "b": "The /metrics endpoint must never be exposed publicly." }, { "pair_id": 3, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9977999925613403, "latency_ms": 174.02, "a": "Webhook payloads should be validated before processing.", "b": "Webhook payloads do not need to be validated before processing." }, { "pair_id": 4, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9988999962806702, "latency_ms": 166.88, "a": "Inbound requests must be authenticated.", "b": "Inbound requests must not be authenticated." }, { "pair_id": 5, "tag": "pos/modality-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9689000248908997, "latency_ms": 411.38, "a": "The staging database may be seeded with synthetic data.", "b": "The staging database must not be seeded with synthetic data." }, { "pair_id": 6, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9994000196456909, "latency_ms": 269.21, "a": "Telemetry collection must be enabled for all production services.", "b": "Telemetry collection must be disabled for all production services." }, { "pair_id": 7, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9993000030517578, "latency_ms": 170.83, "a": "Guest accounts are allowed to access the staging dashboard.", "b": "Guest accounts are forbidden from accessing the staging dashboard." }, { "pair_id": 8, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9991000294685364, "latency_ms": 270.58, "a": "The legacy importer should be removed in the next release.", "b": "The legacy importer should be retained in the next release." }, { "pair_id": 9, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9991999864578247, "latency_ms": 269.13, "a": "Anonymous pairing requests are approved automatically.", "b": "Anonymous pairing requests are rejected automatically." }, { "pair_id": 10, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9991999864578247, "latency_ms": 163.3, "a": "Outbound email is permitted from the worker nodes.", "b": "Outbound email is blocked from the worker nodes." }, { "pair_id": 11, "tag": "pos/always-never", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9993000030517578, "latency_ms": 175.34, "a": "You must always restart the gateway after editing cron jobs.", "b": "You should never restart the gateway after editing cron jobs." }, { "pair_id": 12, "tag": "pos/modality", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9991000294685364, "latency_ms": 187.72, "a": "Schema migrations must run automatically on deploy.", "b": "Schema migrations must not run automatically on deploy; an operator triggers them." }, { "pair_id": 13, "tag": "pos/always-never", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9994999766349792, "latency_ms": 173.85, "a": "Secrets must always be loaded from the vault.", "b": "Secrets should never be loaded from the vault." }, { "pair_id": 14, "tag": "pos/neg-flip+reason", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9979000091552734, "latency_ms": 288.84, "a": "All customer exports must be encrypted, because they contain PII subject to GDPR.", "b": "Customer exports must not be encrypted, since the analytics team needs raw file access." }, { "pair_id": 15, "tag": "pos/neg-flip+reason", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9968000054359436, "latency_ms": 182.61, "a": "The cache must be cleared on startup to avoid serving stale entries after a deploy.", "b": "The cache must not be cleared on startup, because a warm cache is critical for first-request latency." }, { "pair_id": 16, "tag": "pos/antonym+reason", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.994700014591217, "latency_ms": 196.7, "a": "Feature flags should be evaluated server-side so clients cannot tamper with them.", "b": "Feature flags should be evaluated client-side, since the server has no per-user context at render time." }, { "pair_id": 17, "tag": "pos/neg-flip+reason", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9948999881744385, "latency_ms": 191.65, "a": "Background jobs must be idempotent, because the queue delivers at-least-once.", "b": "Background jobs do not need to be idempotent, since the queue guarantees exactly-once delivery." }, { "pair_id": 18, "tag": "pos/neg-flip+reason", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9900000095367432, "latency_ms": 186.85, "a": "Container images must be rebuilt from source on every release for provenance.", "b": "Container images must not be rebuilt on every release; we promote the exact tested artifact." }, { "pair_id": 19, "tag": "pos/coref", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9812999963760376, "latency_ms": 189.83, "a": "The scheduler reads its config only at boot, so it must be restarted after any change.", "b": "The scheduler watches its config file, so it must not be restarted when the config changes." }, { "pair_id": 20, "tag": "pos/coref", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9990000128746033, "latency_ms": 185.35, "a": "Redis is the source of truth for sessions, so it must never be flushed in production.", "b": "Redis only holds a disposable cache, so it may be flushed in production whenever needed." }, { "pair_id": 21, "tag": "pos/coref", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9991999864578247, "latency_ms": 182.43, "a": "The token is short-lived, so it should be refreshed on every request.", "b": "The token is long-lived, so it should not be refreshed on every request." }, { "pair_id": 22, "tag": "pos/same-scope", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9993000030517578, "latency_ms": 270.48, "a": "Debug logging must be disabled in prod.", "b": "Debug logging must be enabled in prod." }, { "pair_id": 23, "tag": "pos/same-scope", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9987000226974487, "latency_ms": 185.21, "a": "TLS 1.2 must be enabled for public endpoints in prod.", "b": "TLS 1.2 must not be enabled for public endpoints in prod." }, { "pair_id": 24, "tag": "pos/same-scope", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.998199999332428, "latency_ms": 173.6, "a": "In the EU region, user data must be stored locally.", "b": "In the EU region, user data must not be stored locally." }, { "pair_id": 25, "tag": "pos/modality-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9988999962806702, "latency_ms": 296.91, "a": "API keys may be stored in environment variables for local development.", "b": "API keys must not be stored in environment variables under any circumstances." }, { "pair_id": 26, "tag": "pos/antonym", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9988999962806702, "latency_ms": 277.33, "a": "Force-pushing to main is allowed for release managers.", "b": "Force-pushing to main is prohibited for everyone, including release managers." }, { "pair_id": 27, "tag": "pos/semantic", "family": "pos", "gold_label": "contradiction", "predicted_label": "not_contradiction", "outcome": "FN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 278.18, "a": "The bot must reply to every message in the channel.", "b": "The bot must stay silent unless it is directly mentioned." }, { "pair_id": 28, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9965999722480774, "latency_ms": 175.17, "a": "Sessions expire after 30 minutes of inactivity.", "b": "Sessions never expire due to inactivity." }, { "pair_id": 29, "tag": "pos/neg-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9983000159263611, "latency_ms": 263.22, "a": "New plugins must ship a config schema.", "b": "New plugins are not required to ship a config schema." }, { "pair_id": 30, "tag": "pos/modality-flip", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9979000091552734, "latency_ms": 267.04, "a": "The webhook URL must use HTTPS.", "b": "The webhook URL may use plain HTTP." }, { "pair_id": 31, "tag": "near-miss/scope-coexist", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 258.52, "a": "Verbose debug logging must not be enabled in prod, because it leaks request bodies.", "b": "Verbose debug logging may be enabled in dev so engineers can trace failures." }, { "pair_id": 32, "tag": "near-miss/scope-coexist", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 253.24, "a": "In prod, the admin API must be disabled.", "b": "In dev, the admin API may be enabled." }, { "pair_id": 33, "tag": "near-miss/time-coexist", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 418.04, "a": "The legacy v1 auth flow may be used until 2026-09-30 during migration.", "b": "The legacy v1 auth flow must not be used after 2026-09-30." }, { "pair_id": 34, "tag": "near-miss/time-coexist", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 282.63, "a": "Maintenance mode may be enabled during the deploy window.", "b": "Maintenance mode must not be enabled outside the deploy window." }, { "pair_id": 35, "tag": "near-miss/boilerplate", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 277.53, "a": "The Slack plugin ships as a bundled plugin and is enabled by default.", "b": "The Discord plugin ships as a bundled plugin and is enabled by default." }, { "pair_id": 36, "tag": "near-miss/boilerplate", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 180.3, "a": "Twitch ships as a bundled plugin, so no separate install is required.", "b": "Microsoft Teams ships as a bundled plugin, so no separate install is required." }, { "pair_id": 37, "tag": "near-miss/homonym", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 258.45, "a": "File paths must not contain null bytes.", "b": "File paths must be resolved relative to the workspace root before use." }, { "pair_id": 38, "tag": "near-miss/homonym", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9785000085830688, "latency_ms": 266.14, "a": "Channel keys must use stable Slack channel IDs.", "b": "Room keys should be Matrix room IDs or aliases." }, { "pair_id": 39, "tag": "near-miss/refinement", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 251.25, "a": "All public endpoints should be rate-limited.", "b": "Public write endpoints must be rate-limited." }, { "pair_id": 40, "tag": "near-miss/refinement", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "duplicate_warning", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 151.21, "a": "Logs should be retained for at least 30 days.", "b": "Audit logs must be retained for at least 90 days." }, { "pair_id": 41, "tag": "near-miss/same-structure", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "duplicate_warning", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 142.47, "a": "The cache must not be cleared on startup.", "b": "The buffer must not be flushed on startup." }, { "pair_id": 42, "tag": "near-miss/stray-not", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 125.61, "a": "Migrations do not run automatically on deploy.", "b": "Backups run nightly at 02:00 UTC." }, { "pair_id": 43, "tag": "near-miss/stray-not", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 258.32, "a": "The bot must not post in archived channels.", "b": "The bot should greet new members when they join." }, { "pair_id": 44, "tag": "near-miss/compatible", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 125.83, "a": "Secrets must be referenced via SecretRef objects.", "b": "Tokens are stored hashed, never in plaintext." }, { "pair_id": 45, "tag": "near-miss/compatible", "family": "near-miss", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 252.86, "a": "Pairing in DMs uses the approve flow.", "b": "Channel access uses an allowlist of stable IDs." }, { "pair_id": 46, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 291.38, "a": "Tasks do not replace sessions, cron jobs, or heartbeats - they are the activity ledger that records what detached work happened, when, and whether it succeeded.", "b": "Related Background Tasks : the detached work ledger that flows coordinate CLI: tasks : CLI command reference for acme tasks flow Automation Overview : all automation mechanisms at a glance Cron Jobs : scheduled jobs that may feed into flows" }, { "pair_id": 47, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9592000246047974, "latency_ms": 326.17, "a": "If Slack ambient rooms do not trigger, verify the channel key is the Slack channel ID and the app has the required channels:history or groups:history scope for that room type.", "b": "If Telegram ambient rooms do not trigger at all, check BotFather privacy mode and verify the Gateway is receiving normal group messages." }, { "pair_id": 48, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.996399998664856, "latency_ms": 177.55, "a": "You should now see your bot in the Discord server.", "b": "Telegram exampleFor Telegram groups, the bot must be able to see normal group messages." }, { "pair_id": 49, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9350000023841858, "latency_ms": 205.41, "a": "To submit unmentioned always-on group chatter as quiet room context instead of user requests, use Ambient room events: The default is unmentionedInbound: \"userrequest\".", "b": "Ambient always-on room events, when enabled, stay strict and quiet unless the agent calls the message tool." }, { "pair_id": 50, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 22.12, "a": "Acme-aware clients may prefer the structured metadata for display while preserving the fallback text for copy, search, notifications, and accessibility.", "b": "The structured metadata is additive and must not be required for basic Matrix interoperability." }, { "pair_id": 51, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 126.08, "a": "\"always\": reply inside a thread rooted at the triggering message; that conversation is routed through a matching thread-scoped session from the first trigger onward.dm.threadReplies overrides this for DMs only - for example, keep room threads isolated while keeping DMs flat.", "b": "Use it when the agent should listen, decide when a reply is useful, and avoid the old prompt pattern of answering NOREPLY." }, { "pair_id": 52, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 126.66, "a": "Keys should use stable Teams conversation IDs from Teams links, not mutable display names.", "b": "What to do: backed-up keys can still be restored, but local-only encrypted history may remain unavailable.matrix: failed restoring room keys from legacy encrypted-state backup: ..." }, { "pair_id": 53, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 200.95, "a": "Use a persistent dev tunnel so your URL stays the same across sessions: Alternatives: ngrok http 3978 or tailscale funnel 3978 (URLs may change each session).", "b": "Start a tunnel (Teams can't reach localhost)Install and authenticate the devtunnel CLI if you haven't already (getting started guide). --allow-anonymous is required because Teams cannot authenticate with devtunnels." }, { "pair_id": 54, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 264.14, "a": "Activation modes are mention and always. requireMention: true maps to mention; requireMention: false maps to always.", "b": "Behavior Activation modes: mention (default) or always. mention requires a ping (real WhatsApp @-mentions via mentionedJids, safe regex patterns, or the bot's E.164 anywhere in the text). always wakes the agent on every message but it should reply only when it can add meaningful value; otherwise it returns the exact silent token NOREPLY / noreply." }, { "pair_id": 55, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 217.43, "a": "Pairing in DMs uses acme pairing approve slack . channels.slack.groupPolicy controls channel handling: open allowlist disabled Channel allowlist lives under channels.slack.channels and must use stable Slack channel IDs (for example C12345678) as config keys.", "b": "Room allowlist keys (groups, legacy rooms) should be room IDs or aliases." }, { "pair_id": 56, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 128.15, "a": "The channel lookup is ID-first by default, so a name-based key will never route successfully and all messages in that channel will be silently blocked.", "b": "Why: in single-agent mode, DMs typically land in the main session key (agent:main:main), while groups always use non-main session keys (agent:main: :group: )." }, { "pair_id": 57, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 230.82, "a": "It must point to the SMS webhook URL and use POST.", "b": "Set the webhook URL to your gateway endpoint (HTTPS required): The gateway responds to LINE's webhook verification (GET) and acknowledges signed inbound events (POST) immediately after signature and payload validation; agent processing continues asynchronously." }, { "pair_id": 58, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 187.98, "a": "Set the allowlist to the ships whose group invites should be accepted automatically.", "b": "Allowlist entries must identify senders: handles or static sender access groups (accessGroup: )." }, { "pair_id": 59, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 203.75, "a": "Bundled plugin Twitch ships as a bundled plugin in current Acme releases, so normal packaged builds do not need a separate install.", "b": "Bundled pluginMicrosoft Teams ships as a bundled plugin in current Acme releases, so no separate install is required in the normal packaged build." }, { "pair_id": 60, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 195.67, "a": "Access control (DMs)channels.zalouser.dmPolicy supports: pairing | allowlist | open | disabled (default: pairing).channels.zalouser.allowFrom should use stable Zalo user IDs.", "b": "They do not create owners, approve pairing requests, or grant tool permissions by themselves. dmPolicy: \"open\" still requires \"\" in the effective DM allowlist." }, { "pair_id": 61, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 28.63, "a": "After a Discord provider restart, a connected but quiet account may be healthy while no Discord session row appears until the next inbound or outbound conversation event.", "b": "Cron wakeups, heartbeat runs, exec notifications, and gateway bookkeeping may update the session row for routing/status, but they do not extend sessionStartedAt or lastInteractionAt." }, { "pair_id": 62, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 306.5, "a": "If a required auth SecretRef is unresolved in this command path, daemon status --json reports rpc.authWarning when probe connectivity/auth fails; pass --token/--password explicitly or resolve the secret source first.", "b": "SecretRef assignment(s) could not be resolved: referenced provider/ref currently cannot resolve (missing env var, invalid file pointer, exec provider failure, or provider/source mismatch)." }, { "pair_id": 63, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 128.22, "a": "That policy is disabled by default and never applies to operator/browser clients or upgrade requests.", "b": "They do not create owners, approve pairing requests, or grant tool permissions by themselves. dmPolicy: \"open\" still requires \"\" in the effective DM allowlist." }, { "pair_id": 64, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 29.47, "a": "Local Codex app-server launches use isolated per-agent homes, so install the Codex plugin first if needed, then use acme migrate plan codex to inventory assets that should be promoted deliberately.", "b": "If acme channels status shows Matrix is configured but the plugin is missing after an update, run acme doctor --fix or acme plugins install @acme/matrix; do not install Matrix SDK packages into the root Acme package." }, { "pair_id": 65, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 196.93, "a": "If a required auth SecretRef is unresolved in this command path, gateway status --json reports rpc.authWarning when probe connectivity/auth fails; pass --token/--password explicitly or resolve the secret source first.", "b": "SecretRef assignment(s) could not be resolved: referenced provider/ref currently cannot resolve (missing env var, invalid file pointer, exec provider failure, or provider/source mismatch)." }, { "pair_id": 66, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 40.78, "a": "Probes are real requests (may consume tokens and trigger rate limits).", "b": "Diagnostic probes are non-mutating for first-time device auth: they reuse an existing cached device token when one exists, but they do not create a new CLI device identity or read-only device pairing record just to check status. gateway status resolves configured auth SecretRefs for probe auth when possible." }, { "pair_id": 67, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 197.1, "a": "Content is byte-literal; \" and \\ are not allowed inside quotes.", "b": "Docs content must stay generic: no personal device names, hostnames, or local paths; use placeholders like user@gateway-host." }, { "pair_id": 68, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 171.94, "a": "Native Acme plugins must ship acme.plugin.json with an inline JSON Schema (configSchema, even if empty).", "b": "Acme should not spawn a native daemon when apiMode: \"container\" is selected." }, { "pair_id": 69, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 127.67, "a": "With --remote, if effectively active remote credentials are configured as SecretRefs and you do not pass --token or --password, the command resolves them from the active gateway snapshot.", "b": "If a provider has credentials but Acme cannot resolve a probeable model candidate for it, models status --probe reports status: nomodel with reasonCode: nomodel." }, { "pair_id": 70, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 338.36, "a": "Flags: --text : required system event text. --mode : now or next-heartbeat (default). --session-key : optional; target a specific agent session instead of the agent's main session.", "b": "Common flags --channel : channel id/alias (required when multiple channels are configured; auto when only one is configured) --account : account id (default: channel default) --json: output JSON Notes directory is meant to help you find IDs you can paste into other commands (especially acme message send --target ...)." }, { "pair_id": 71, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 180.03, "a": "Do not force every channel into the same native transport behavior.", "b": "Default to safe channel behavior (mentions required unless configured otherwise)." }, { "pair_id": 72, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 203.25, "a": "Adapter surfaceThe public SDK target should be one subpath: Target shape: Send adapter: Receive adapter: Before preflight authorization, core must run the shared Acme echo predicate whenever origin.decode returns Acme-origin metadata.", "b": "Before bot authorization, receive must apply the shared Acme echo policy when the channel can decode message origin metadata: This drop is tag-based, not text-based." }, { "pair_id": 73, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 125.73, "a": "Generic durable delivery must not bypass those effects; either move them into Tlon render/send/finalize adapters or keep Tlon on the channel-owned path.", "b": "Non goals Do not force every existing channel onto durable message delivery in the first phase." }, { "pair_id": 74, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 266.95, "a": "Acme gateway failure output must stay visible to humans, but tagged bot-authored room echoes must be dropped before allowBots authorization.", "b": "The first required use is gateway failure output." }, { "pair_id": 75, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 24.62, "a": "Which channels have native origin metadata, which need persisted outbound registries, and which cannot offer reliable cross-bot echo suppression.", "b": "The channel must still be allowed by its normal groupPolicy, room allowlist, and sender allowlist." }, { "pair_id": 76, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 174.93, "a": "Durable final sends fail closed when the durable intent cannot be written, unless a caller explicitly selected a documented non-durable mode.", "b": "Durable final sends must still populate that cache, otherwise Acme can re-ingest its own final replies as inbound user messages." }, { "pair_id": 77, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 133.13, "a": "\"Model is not allowed\" (and why replies stop)If agents.defaults.models is set, it becomes the allowlist for /model and for session overrides.", "b": "Notes: video generate accepts --size, --aspect-ratio, --resolution, --duration, --audio, --watermark, and --timeout-ms and forwards them to the video-generation runtime. --model must be for video describe." }, { "pair_id": 78, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 221.94, "a": "Why one-off CLI commands do not show upThe CLI often connects for short, one-off commands.", "b": "Plugin-owned CLI commands are usually installed as root acme command groups, but plugins may also register nested commands under a core parent such as acme nodes." }, { "pair_id": 79, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "contradiction", "outcome": "FP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.89410001039505, "latency_ms": 281.21, "a": "\"always\": inject a concise notice on every run when truncation exists (recommended).", "b": "\"off\": never inject truncation notice text into the system prompt." }, { "pair_id": 80, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 264.3, "a": "CLI backends are text-first; tools are always disabled.", "b": "Generic CLI backends do not, so engines that require it are rejected before the CLI process starts." }, { "pair_id": 81, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 130.24, "a": "TTS (text-to-speech) auto controls the default auto-TTS mode: off, always, inbound, or tagged. /tts on|off can override local prefs, and /tts status shows the effective state. summaryModel overrides agents.defaults.model.primary for auto-summary. modelOverrides is enabled by default; modelOverrides.allowProvider defaults to false (opt-in).", "b": "Doctor reports cron jobs with explicit payload.model overrides, including provider namespace counts and mismatches against agents.defaults.model, so scheduled jobs that do not inherit the default model are visible during auth or billing investigations." }, { "pair_id": 82, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 225.54, "a": "Auth: required by default.", "b": "If the policy declares bearer auth, the workflow uses the fixed ACMETRUSTEDPACKAGETOKEN secret; URL-embedded credentials are still rejected. source=artifact downloads one .tgz from artifactrunid and artifactname; packagesha256 is optional but should be supplied for externally shared artifacts." }, { "pair_id": 83, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 23.42, "a": "\"restart\": always restart the gateway process on config change.", "b": "The gateway must stay online to record votes." }, { "pair_id": 84, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 129.99, "a": "Paths: resolved relative to the including file, but must stay inside the top-level config directory (dirname of acme.json).", "b": "LINE, Zalo, Nostr, and other existing assembled/helper paths may have reply-token handling, media proxying, sent-message caches, loading/status cleanup, or callback-only targets." }, { "pair_id": 85, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 24.73, "a": "Must match the relay base URL baked into the official/TestFlight iOS build, so registration and send traffic reach the same relay deployment.", "b": "Base URL should omit /v1 (Anthropic client appends it)." }, { "pair_id": 86, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 174.97, "a": "Tool-capable heartbeat runs may instead call heartbeatrespond with notify: false for no visible update, or notify: true plus notificationText for an alert.", "b": "Heartbeat runs are main-session turns - they do not create task records." }, { "pair_id": 87, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 130.35, "a": "Session key formats: see Sessions and Groups. last: deliver to the last used external channel. explicit channel: any configured channel or plugin id, for example discord, matrix, telegram, or whatsapp. none (default): run the heartbeat but do not deliver externally.", "b": "Why: in single-agent mode, DMs typically land in the main session key (agent:main:main), while groups always use non-main session keys (agent:main: :group: )." }, { "pair_id": 88, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 32.0, "a": "Auth is required by default.", "b": "If the policy declares bearer auth, the workflow uses the fixed ACMETRUSTEDPACKAGETOKEN secret; URL-embedded credentials are still rejected. source=artifact downloads one .tgz from artifactrunid and artifactname; packagesha256 is optional but should be supplied for externally shared artifacts." }, { "pair_id": 89, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 199.66, "a": "Do not include the provider prefix there.", "b": "If the caller already selected an explicit channel, the provider prefix must match that channel; cross-channel combinations such as WhatsApp delivery to telegram:123 fail before plugin-specific target normalization." }, { "pair_id": 90, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 24.69, "a": "It drops the three heaviest default tools (browser, cron, message) and defaults larger tool catalogs behind structured Tool Search controls, except for runs that must keep direct message delivery semantics.", "b": "Required tools enabled for the agent, such as lobster, browser, and llm-task." }, { "pair_id": 91, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 121.64, "a": "Most setups should use one Gateway because a single Gateway can handle multiple messaging connections and agents.", "b": "The gateway must stay online to record votes." }, { "pair_id": 92, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 166.65, "a": "They do not keep the message body, attachment contents, raw frame body, tokens, cookies, or secret values.", "b": "Never export contact lists, donor data, or financial records." }, { "pair_id": 93, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 320.72, "a": "Operators may call skills.status (operator.read) to fetch the visible skill inventory for an agent. agentId is optional; omit it to read the default agent workspace.", "b": "The response includes grouped tools and provenance metadata: source: core or plugin pluginId: plugin owner when source=\"plugin\" optional: whether a plugin tool is optional Operators may call tools.effective (operator.read) to fetch the runtime-effective tool inventory for a session. sessionKey is required." }, { "pair_id": 94, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 321.47, "a": "It is returned in hello-ok.auth.deviceToken and should be persisted by the client for future connects.", "b": "When no device token is issued, hello-ok.auth reports the negotiated permissions without token fields: Trusted same-process backend clients (client.id: \"gateway-client\", client.mode: \"backend\") may omit device on direct loopback connections when they authenticate with the shared gateway token/password." }, { "pair_id": 95, "tag": "corpus-fp/flagged", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 322.41, "a": "Clients should persist the primary hello-ok.auth.deviceToken after any successful connect.", "b": "When no device token is issued, hello-ok.auth reports the negotiated permissions without token fields: Trusted same-process backend clients (client.id: \"gateway-client\", client.mode: \"backend\") may omit device on direct loopback connections when they authenticate with the shared gateway token/password." }, { "pair_id": 96, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 213.99, "a": "Tasks do not replace sessions, cron jobs, or heartbeats - they are the activity ledger that records what detached work happened, when, and whether it succeeded.", "b": "Related Background Tasks : the detached work ledger that flows coordinate CLI: tasks : CLI command reference for acme tasks flow Automation Overview : all automation mechanisms at a glance Cron Jobs : scheduled jobs that may feed into flows" }, { "pair_id": 97, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 124.66, "a": "Acme-aware clients may prefer the structured metadata for display while preserving the fallback text for copy, search, notifications, and accessibility.", "b": "The structured metadata is additive and must not be required for basic Matrix interoperability." }, { "pair_id": 98, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 131.49, "a": "Keys should use stable Teams conversation IDs from Teams links, not mutable display names.", "b": "What to do: backed-up keys can still be restored, but local-only encrypted history may remain unavailable.matrix: failed restoring room keys from legacy encrypted-state backup: ..." }, { "pair_id": 99, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 234.18, "a": "Use a persistent dev tunnel so your URL stays the same across sessions: Alternatives: ngrok http 3978 or tailscale funnel 3978 (URLs may change each session).", "b": "Start a tunnel (Teams can't reach localhost)Install and authenticate the devtunnel CLI if you haven't already (getting started guide). --allow-anonymous is required because Teams cannot authenticate with devtunnels." }, { "pair_id": 100, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 29.76, "a": "The channel lookup is ID-first by default, so a name-based key will never route successfully and all messages in that channel will be silently blocked.", "b": "Why: in single-agent mode, DMs typically land in the main session key (agent:main:main), while groups always use non-main session keys (agent:main: :group: )." }, { "pair_id": 101, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 228.09, "a": "Access control (DMs)channels.zalouser.dmPolicy supports: pairing | allowlist | open | disabled (default: pairing).channels.zalouser.allowFrom should use stable Zalo user IDs.", "b": "They do not create owners, approve pairing requests, or grant tool permissions by themselves. dmPolicy: \"open\" still requires \"\" in the effective DM allowlist." }, { "pair_id": 102, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 130.83, "a": "Local Codex app-server launches use isolated per-agent homes, so install the Codex plugin first if needed, then use acme migrate plan codex to inventory assets that should be promoted deliberately.", "b": "If acme channels status shows Matrix is configured but the plugin is missing after an update, run acme doctor --fix or acme plugins install @acme/matrix; do not install Matrix SDK packages into the root Acme package." }, { "pair_id": 103, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 35.32, "a": "Probes are real requests (may consume tokens and trigger rate limits).", "b": "Diagnostic probes are non-mutating for first-time device auth: they reuse an existing cached device token when one exists, but they do not create a new CLI device identity or read-only device pairing record just to check status. gateway status resolves configured auth SecretRefs for probe auth when possible." }, { "pair_id": 104, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 185.44, "a": "Native Acme plugins must ship acme.plugin.json with an inline JSON Schema (configSchema, even if empty).", "b": "Acme should not spawn a native daemon when apiMode: \"container\" is selected." }, { "pair_id": 105, "tag": "corpus-fp/warned", "family": "corpus-fp", "gold_label": "not_contradiction", "predicted_label": "not_contradiction", "outcome": "TN", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": null, "latency_ms": 24.44, "a": "The summary must include: refs and SHAs tested transport and scenario id machine provider and machine id or lease id credential source without secret values baseline result candidate result whether the bug reproduced on baseline whether the candidate fixed it artifact paths sanitized setup or cleanup issuesScreenshots are evidence, not secrets.", "b": "Content is byte-literal; \" and \\ are not allowed inside quotes." }, { "pair_id": 106, "tag": "pos/omission", "family": "pos", "gold_label": "contradiction", "predicted_label": "contradiction", "outcome": "TP", "status_b": "saved", "hard_blocked": false, "top_contradiction_score": 0.9976000189781189, "latency_ms": 285.75, "a": "Skip smoke-testing the Ergo wiring before relying on it", "b": "Smoke-test the Ergo wiring before relying on it" } ] }