$policy Policy array: `bots` (token => `allow`|`disallow`), * `disallow_paths`, `always_allow`, `content_signal`, * `sitemaps`, `site_name`, `policy_url`, `generated` * and `existing` (robots.txt built so far). * @return string robots.txt block, `\n` separated, with a trailing newline. */ public function generate( array $policy ): string { $blocks = [ $this->header( $policy ) ]; $groups = $this->generate_groups( $policy ); if ( '' !== $groups ) { $blocks[] = $groups; } $signal = self::content_signal_line( $this->sub_list( $policy, 'content_signal' ) ); if ( '' !== $signal ) { $blocks[] = 'Content-Signal: ' . $signal; } $sitemaps = $this->sitemap_lines( $this->sub_list( $policy, 'sitemaps' ), (string) ( $policy['existing'] ?? '' ) ); if ( [] !== $sitemaps ) { $blocks[] = implode( "\n", $sitemaps ); } return implode( "\n\n", $blocks ) . "\n"; } /** * Renders only the per-bot groups of the policy. * * The same directives `generate()` emits, without the header comment, the * `Content-Signal` line and the `Sitemap` lines: `Generators\AiTxtGenerator` * mirrors the policy into `/ai.txt` and wraps it in its own frame, so the * two files can never disagree about what a crawler may do. * * @param array $policy Policy array, as `generate()` takes it. * @return string Groups separated by a blank line, or an empty string when no bot is listed. */ public function generate_groups( array $policy ): string { $groups = []; foreach ( $this->sorted_tokens( $this->sub_list( $policy, 'bots' ) ) as $token ) { $groups[] = $this->bot_group( $token, $policy ); } return implode( "\n\n", $groups ); } /** * The comment block that opens the section. * * @param array $policy Policy array. * @return string Header comment lines. */ private function header( array $policy ): string { $rule = '# ' . str_repeat( '-', 60 ); $site_name = trim( (string) ( $policy['site_name'] ?? '' ) ); $title = '# AI crawler policy — generated by Acid AEO'; if ( '' !== $site_name ) { $title .= ' for ' . $site_name; } $meta = []; if ( '' !== trim( (string) ( $policy['policy_url'] ?? '' ) ) ) { $meta[] = 'Details: ' . trim( (string) $policy['policy_url'] ); } if ( '' !== trim( (string) ( $policy['generated'] ?? '' ) ) ) { $meta[] = 'Updated: ' . trim( (string) $policy['generated'] ); } $lines = [ $rule, $title ]; if ( [] !== $meta ) { $lines[] = '# ' . implode( ' · ', $meta ); } $lines[] = $rule; return implode( "\n", $lines ); } /** * The directives for one user agent. * * Allowed bots get the site's disallowed paths and then the whole site. * Disallowed bots are still pointed at the AI context files, so an engine * that refuses to crawl can still read the site's own summary of itself. * * @param string $token Registry user-agent token. * @param array $policy Policy array. * @return string Directive lines for the group. */ private function bot_group( string $token, array $policy ): string { $bots = $this->sub_list( $policy, 'bots' ); $lines = [ 'User-agent: ' . $token ]; if ( 'allow' === ( $bots[ $token ] ?? 'disallow' ) ) { foreach ( $this->paths( $policy, 'disallow_paths' ) as $path ) { $lines[] = 'Disallow: ' . $path; } $lines[] = 'Allow: /'; return implode( "\n", $lines ); } foreach ( $this->paths( $policy, 'always_allow' ) as $path ) { $lines[] = 'Allow: ' . $path; } $lines[] = 'Disallow: /'; return implode( "\n", $lines ); } /** * Orders the user agents by vendor name and then by token. * * @param array $bots Token => state. * @return array Sorted tokens. */ private function sorted_tokens( array $bots ): array { $registry = Registry::all(); $tokens = array_keys( $bots ); usort( $tokens, static function ( string $a, string $b ) use ( $registry ): int { $vendor = strcasecmp( (string) ( $registry[ $a ]['vendor'] ?? self::UNKNOWN_VENDOR ), (string) ( $registry[ $b ]['vendor'] ?? self::UNKNOWN_VENDOR ) ); return 0 !== $vendor ? $vendor : strcmp( $a, $b ); } ); return $tokens; } /** * Serializes a resolved Content-Signal into its wire value. * * The one place the signal is turned into text. Everything that publishes it * — the robots.txt block below, `Generators\AiTxtGenerator`, the HTTP header * and `Data\SiteData::content_signal_string()` — calls this, so a site can * never state one signal in one place and another somewhere else. * * Returns the field value only (`search=yes, ai-input=yes, ai-train=no`), not * a whole line: the field name differs between a robots.txt directive and an * HTTP header, so each caller prefixes its own. * * @param array $signal Wire-format signal, e.g. `search` => `yes`. * @return string Field value, or an empty string when there is no signal. */ public static function content_signal_line( array $signal ): string { $pairs = []; foreach ( $signal as $name => $value ) { $pairs[] = $name . '=' . $value; } return implode( ', ', $pairs ); } /** * Sitemap lines, but only when nobody else has named a sitemap. * * WordPress and every SEO plugin add their own `Sitemap:` line to * robots.txt, and they add the one they actually serve. This block is * written last, so a line that is already there is both older and better * informed than anything that could be worked out from here — a second line * could only repeat it or contradict it, and a crawler reading two would * fetch both. * * So the rule is not "skip this exact URL", it is "say nothing when the * question has been answered". What remains is the site where nothing * publishes a sitemap line, which is the case worth having this for. * * @param array $sitemaps Sitemap URLs. * @param string $existing robots.txt built so far. * @return array Sitemap lines. */ private function sitemap_lines( array $sitemaps, string $existing ): array { if ( 1 === preg_match( '/^\s*Sitemap\s*:\s*\S+/mi', $existing ) ) { return []; } $lines = []; foreach ( array_unique( $sitemaps ) as $sitemap ) { $url = trim( (string) $sitemap ); if ( '' === $url || false !== strpos( $existing, $url ) ) { continue; } $lines[] = 'Sitemap: ' . $url; } return $lines; } /** * Trimmed, de-duplicated, non-empty paths from one policy key. * * @param array $policy Policy array. * @param string $key Policy key holding a list of paths. * @return array Paths. */ private function paths( array $policy, string $key ): array { $paths = []; foreach ( $this->sub_list( $policy, $key ) as $path ) { $path = trim( (string) $path ); if ( '' === $path || in_array( $path, $paths, true ) ) { continue; } $paths[] = $path; } return $paths; } /** * Reads one policy key as an array. * * @param array $policy Policy array. * @param string $key Policy key. * @return array The value, or an empty array. */ private function sub_list( array $policy, string $key ): array { return isset( $policy[ $key ] ) && is_array( $policy[ $key ] ) ? $policy[ $key ] : []; } }