{ blocked, probes?, passed, served } } * (see Table::recent()), or null to treat all * evidence as current. * @return array * Worst first: warns (a real breakage) before infos (an unenforced wish). */ public static function detect( array $crawlers, array $policy, $days, $recent = null ) { $out = array(); $days = max( 1, (int) $days ); $ai_input = ! isset( $policy['ai_input'] ) || false !== $policy['ai_input']; $ai_train = ! isset( $policy['ai_train'] ) || false !== $policy['ai_train']; // ── The edge is turning away a crawler the owner allows ──────────────── // The threshold runs PER CRAWLER, the message per operator. Bot Fight Mode // challenges one fetcher (ChatGPT-User) while the same company's crawler // (GPTBot) passes — an operator-level share would dilute the real breakage // below the threshold and hide it. "Cloudflare is blocking OpenAI" is // still the sentence an owner acts on, so qualifying crawlers roll up. if ( $ai_input ) { $per_operator = array(); foreach ( $crawlers as $c ) { $token = strtolower( (string) $c['ua'] ); if ( self::blocking_is_owners_wish( $token, $policy ) ) { continue; } // A blocked probe is no reader turned away ({@see refused()}) — and // no reading attempt either, so it leaves the share's base too. $blocked = self::refused( $c ); $requests = (int) $c['requests'] - self::probes( $c ); $op = (string) $c['operator']; if ( '' === $op || $blocked < self::MIN_BLOCKED || $blocked < $requests * self::MIN_BLOCKED_SHARE ) { continue; } // Currency: the week made it significant, but only still-happening // blocking fires — otherwise the fix already landed and the warning // would just be nagging about history. if ( null !== $recent && ( ! isset( $recent[ $token ] ) || 0 === self::refused( $recent[ $token ] ) ) ) { continue; } if ( ! isset( $per_operator[ $op ] ) ) { $per_operator[ $op ] = array( 'blocked' => 0, 'requests' => 0, 'recent' => 0 ); } $per_operator[ $op ]['blocked'] += $blocked; $per_operator[ $op ]['requests'] += $requests; $per_operator[ $op ]['recent'] += null !== $recent ? self::refused( $recent[ $token ] ) : 0; } foreach ( $per_operator as $op => $sums ) { $out[] = array( 'id' => 'edge-blocks-' . self::operator_slug( $op ), 'level' => 'warn', 'title' => sprintf( /* translators: %s: AI company name (e.g. OpenAI). */ __( 'Cloudflare is blocking %s, but your policy says allow', 'agentimus' ), $op ), // ⛔ HOW MANY IN THE LAST DAY, NOT "SOME". The week makes a // conflict significant, the last day only keeps it current — and // on heera.it (2026-09-23) "including some in the last day" was // ONE request, a probe for /wp-config.php, under 84 blocks that // were a week old. The owner weighs the warning by that number. 'body' => null !== $recent && $days > 1 ? sprintf( /* translators: 1: blocked request count, 2: total request count, 3: AI company name, 4: the window, e.g. "in the last 7 days", 5: blocked request count in the last day. */ __( 'Cloudflare turned away %1$s of %2$s requests from %3$s crawlers %4$s — %5$s of them in the last day. Your site policy allows AI reading — so an assistant that tries to read a page gets an error, and your request log never shows it. Either allow it at Cloudflare, or accept that %3$s cannot read this site. Once the blocking stops, this warning clears by itself within a day.', 'agentimus' ), number_format_i18n( $sums['blocked'] ), number_format_i18n( $sums['requests'] ), $op, self::window( $days ), number_format_i18n( $sums['recent'] ) ) : sprintf( /* translators: 1: blocked request count, 2: total request count, 3: AI company name, 4: the window, e.g. "in the last 7 days". */ __( 'Cloudflare turned away %1$s of %2$s requests from %3$s crawlers %4$s. Your site policy allows AI reading — so an assistant that tries to read a page gets an error, and your request log never shows it. Either allow it at Cloudflare, or accept that %3$s cannot read this site. Once the blocking stops, this warning clears by itself within a day.', 'agentimus' ), number_format_i18n( $sums['blocked'] ), number_format_i18n( $sums['requests'] ), $op, self::window( $days ) ), 'link' => 'bots', // ⭐ THE NUMBERS, BESIDE THE PROSE THAT USES THEM. Every other // consumer — the findings row, the weekly digest — needs the // same two figures in a shorter sentence, and re-parsing them // back out of $body would make the copy above load-bearing. 'counts' => array( 'blocked' => (int) $sums['blocked'], 'requests' => (int) $sums['requests'], 'recent' => (int) $sums['recent'] ), ); } } // ── ai-train=no declared, but training crawlers get pages anyway ─────── // The robots line is advisory; the edge is where it could be enforced. // ⛔ Counted by PAGES SERVED, not by requests past the edge: on heera.it // (2026-09-24) "545 let through" was robots.txt reads and scanners in // trainers' names that the origin refused — not one page (Module::aggregate()). if ( ! $ai_train ) { $passes = 0; $recent_passes = 0; foreach ( $crawlers as $c ) { if ( self::counts_as_training( (string) $c['ua'], $policy ) ) { $passes += isset( $c['served'] ) ? (int) $c['served'] : 0; if ( null !== $recent && ! empty( $recent[ (string) $c['ua'] ]['served'] ) ) { $recent_passes += (int) $recent[ (string) $c['ua'] ]['served']; } } } // Same currency rule as above: once the edge starts enforcing (e.g. the // owner set Training to Block), passes stop — and so does this notice, // within a day, instead of reciting the week's history. if ( null !== $recent && 0 === $recent_passes ) { $passes = 0; } if ( $passes >= self::MIN_TRAINER_PASSES ) { $out[] = array( 'id' => 'train-not-enforced', 'level' => 'info', 'title' => __( 'Cloudflare is letting training crawlers through', 'agentimus' ), 'body' => null !== $recent && $days > 1 ? sprintf( /* translators: 1: request count, 2: the window, e.g. "in the last 7 days", 3: request count in the last day. */ __( 'Your site asks AI companies not to use your content for training (the ai-train=no line in robots.txt). Cloudflare is letting training crawlers through anyway — %1$s requests %2$s, %3$s of them in the last day. Asking is not stopping: polite crawlers obey, the rest ignore you. Cloudflare is the one who can stop them — block Training crawlers there, and this notice clears by itself within a day.', 'agentimus' ), number_format_i18n( $passes ), self::window( $days ), number_format_i18n( $recent_passes ) ) : sprintf( /* translators: 1: request count, 2: the window, e.g. "in the last 7 days". */ __( 'Your site asks AI companies not to use your content for training (the ai-train=no line in robots.txt). Cloudflare is letting training crawlers through anyway — %1$s requests %2$s. Asking is not stopping: polite crawlers obey, the rest ignore you. Cloudflare is the one who can stop them — block Training crawlers there, and this notice clears by itself within a day.', 'agentimus' ), number_format_i18n( $passes ), self::window( $days ) ), 'link' => 'ai-crawlers', 'counts' => array( 'passed' => (int) $passes, 'recent' => (int) $recent_passes ), ); } } return $out; } /** * The window a conflict's counts cover, as the phrase its sentence uses. * * ⛔ One day is "in the last day" — never "in the last 1 days" — and a * sentence covering one day leaves out its "…of them in the last day" * clause, which would only repeat the whole count. * * @param int $days Window length in days. * @return string e.g. "in the last 7 days". */ public static function window( $days ) { $days = max( 1, (int) $days ); if ( 1 === $days ) { return __( 'in the last day', 'agentimus' ); } return sprintf( /* translators: %s: number of days, always 2 or more. */ _n( 'in the last %s day', 'in the last %s days', $days, 'agentimus' ), number_format_i18n( $days ) ); } /** * Whether the edge blocking this crawler is what the owner ASKED for — so * the block is agreement, never evidence of a conflict. * * Two ways the owner asks: naming the crawler on their own block list, or * declaring ai-train=no, which makes the edge blocking any TRAINING crawler * (e.g. Cloudflare's Training policy set to Block) the owner's wish being * enforced — even for a trainer they never named. Reading fetchers stay * warn-eligible. * * ⛔⛔ ONE RULE FOR EVERY READER OF A WARN CONFLICT. {@see detect()} always * skipped these crawlers, but {@see onset()} and the spoof check did not, so * on heera.it (2026-09-23) "Cloudflare is blocking OpenAI" was dated from * the day GPTBot — which he blocks on purpose — was blocked 24 times, and * seven of the eight requests the spoof check sampled were GPTBot's. * * @param string $token Crawler token (any case). * @param array $policy The same declarations {@see detect()} takes. * @return bool */ public static function blocking_is_owners_wish( $token, array $policy ) { $token = strtolower( (string) $token ); // The owner's block list stores UA names as typed ("GPTBot"); crawler rows // carry lowercase tokens ("gptbot"). Compare case-insensitively or a // deliberate block would never be recognised as one. $owner_blocked = array_map( 'strtolower', array_map( 'strval', isset( $policy['blocked_agents'] ) ? (array) $policy['blocked_agents'] : array() ) ); if ( in_array( $token, $owner_blocked, true ) ) { return true; } $ai_train = ! isset( $policy['ai_train'] ) || false !== $policy['ai_train']; return ! $ai_train && in_array( $token, self::TRAINERS, true ); } /** * Whether a page served to this crawler goes against ai-train=no — a * training crawler the owner has NOT allowed. * * ⛔⛔ AN ALLOWED CRAWLER IS HIS DECISION, NOT HIS WISH IGNORED. On heera.it * (2026-10-01) "letting training crawlers through — 36" was 27 pages to * ShapBot, which he allowed in the review queue. Pages served to a crawler * he let in are that choice being honoured. Used by {@see detect()} and * {@see onset()} alike, so the count and its start date agree. * * @param string $token Crawler token (any case). * @param array $policy The same declarations {@see detect()} takes. * @return bool */ public static function counts_as_training( $token, array $policy ) { $token = strtolower( (string) $token ); if ( ! in_array( $token, self::TRAINERS, true ) ) { return false; } // ⛔ A trainer he blocks stays counted even if the queue allows it — the // block is his statement about training; the allow only keeps it from // being refused at the door (heera.it's ClaudeBot sits on both lists). $blocked = array_map( 'strtolower', array_map( 'strval', isset( $policy['blocked_agents'] ) ? (array) $policy['blocked_agents'] : array() ) ); if ( in_array( $token, $blocked, true ) ) { return true; } // Allow rules are stored as typed ("ShapBot"); crawler rows carry // lowercase tokens — the same case-folding blocking_is_owners_wish() does. $allowed = array_map( 'strtolower', array_map( 'strval', isset( $policy['allowed_agents'] ) ? (array) $policy['allowed_agents'] : array() ) ); return ! in_array( $token, $allowed, true ); } /** * The requests the edge turned away that were a READER being refused: the * blocked count less the scanner probes in it ({@see Module::is_probe_path()}). * A row stored before probes were counted has none, and counts as before. * * @param array $row Any totals row carrying `blocked` and optionally `probes`. * @return int */ public static function refused( array $row ) { return max( 0, ( isset( $row['blocked'] ) ? (int) $row['blocked'] : 0 ) - self::probes( $row ) ); } /** * The scanner probes within a row's blocked count, never more than it. * * @param array $row Any totals row. * @return int */ private static function probes( array $row ) { $blocked = isset( $row['blocked'] ) ? (int) $row['blocked'] : 0; return min( $blocked, isset( $row['probes'] ) ? max( 0, (int) $row['probes'] ) : 0 ); } /** * The slug an operator's warn conflict carries in its id * ("edge-blocks-"). One transform, shared with every reader that has * to recover the operator from the id — drifted copies would silently stop * matching and the readers would fail open. * * @param string $operator Operator display name (e.g. "OpenAI"). * @return string */ public static function operator_slug( $operator ) { return trim( strtolower( preg_replace( '/[^a-z0-9]+/i', '-', (string) $operator ) ), '-' ); } /** * When a conflict actually began — the first day of the unbroken run of days * on which its own condition held. * * ⛔⛔ WHY DERIVED RATHER THAN REMEMBERED. The stored first-seen stamp can * only date a conflict from the moment the plugin first looked at it, so a * site that installs this feature today is told a problem that started three * days ago "started today". The hours are already on disk; the beginning is * in them, and reading it is honest where remembering is not. * * The SAME thresholds the conflict itself uses ({@see detect()}), applied a * day at a time. A day of light blocking that never crossed the bar is not * part of the run — on the site this was built for, OpenAI's crawler sat at * 10-13% blocked for three weeks, which is noise, and then jumped to 98%, * which is the event. Dating the conflict from the noise would be as wrong as * dating it from the install. * * ⚠️ BOUNDED SAYS WE RAN OUT OF DATA, and the caller must not print it as a * start. When the run reaches the oldest day we kept, all we know is "at * least this long" — the beginning is older than our retention, or there * never was one. An absence has to name itself. * * Pure — no DB, no time() — so it unit-tests standalone, like detect(). * * @param array $daily Day => ua => { requests, blocked, probes?, passed, served }, oldest first ({@see Table::daily()}). * @param string $id The conflict id to date. * @param array $operators ua => operator name, for the per-operator rollup a warn conflict uses. * @param array $policy The declarations {@see detect()} takes — a crawler the owner * wants blocked is left out of the rollup here too. * @return array{at:string,bounded:bool} at = 'Y-m-d' (UTC) or '' when the condition holds on no day at all. */ public static function onset( array $daily, $id, array $operators = array(), array $policy = array() ) { if ( ! $daily ) { return array( 'at' => '', 'bounded' => false ); } ksort( $daily ); $days = array_keys( $daily ); $holds = static function ( $day ) use ( $daily, $id, $operators, $policy ) { $rows = (array) $daily[ $day ]; if ( 'train-not-enforced' === $id ) { $passed = 0; foreach ( $rows as $ua => $t ) { if ( self::counts_as_training( (string) $ua, $policy ) ) { $passed += isset( $t['served'] ) ? (int) $t['served'] : 0; } } // A day's share of the weekly bar: the run is about whether the // behaviour was happening, not whether one day alone would have // tripped a seven-day threshold. return $passed > 0; } // edge-blocks-: roll the day up per operator and apply the // same "enough, and mostly" test detect() applies to the window. $want = substr( (string) $id, strlen( 'edge-blocks-' ) ); $sums = array( 'blocked' => 0, 'requests' => 0 ); foreach ( $rows as $ua => $t ) { $op = isset( $operators[ $ua ] ) ? (string) $operators[ $ua ] : ''; if ( '' === $op || self::operator_slug( $op ) !== $want || self::blocking_is_owners_wish( (string) $ua, $policy ) ) { continue; } $sums['blocked'] += self::refused( $t ); $sums['requests'] += (int) $t['requests'] - self::probes( $t ); } return $sums['blocked'] >= self::MIN_BLOCKED && $sums['blocked'] >= $sums['requests'] * self::MIN_BLOCKED_SHARE; }; // Walk back from the newest day while the condition keeps holding. A day // we have no rows for BREAKS nothing and EXTENDS nothing: it is a gap in // the record, not evidence either way, so the run walks straight over it. $start = ''; $hit = false; for ( $i = count( $days ) - 1; $i >= 0; $i-- ) { $day = $days[ $i ]; // ⛔ A GAP IS NOT A CONTRADICTION. A day the poll never ran records // nothing, and reading that silence as "the conflict stopped" would // cut the run short and date a three-week problem to yesterday. // {@see Table::daily()} leaves such days out entirely, so this only // fires for a caller that zero-fills — but the rule belongs with the // walk, not with one caller's habits. if ( ! $daily[ $day ] ) { continue; } if ( $holds( $day ) ) { $start = $day; $hit = true; continue; } if ( $hit ) { // A day that positively contradicts the condition ends the run. return array( 'at' => $start, 'bounded' => false ); } } // Never contradicted: either it has held for as long as we have kept, or // it holds on no day at all. return array( 'at' => $start, 'bounded' => '' !== $start ); } }