$data Collected site data. */ if ( ! defined( 'ABSPATH' ) ) { exit; } // phpcs:disable WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedVariableFound -- Template variables scoped to including method. $identity = $data['identity']; $permissions = $data['permissions']; $settings = $data['settings']; // Attribution header. // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- Pre-built internal string. echo AIDF_Generator::get_attribution( '#', 'robots-ai-txt' ); echo '# AI Crawler Directives for ' . esc_html( $identity['name'] ) . "\n"; echo '# Website: ' . esc_html( $identity['url'] ) . "\n"; echo '# Last Updated: ' . esc_html( wp_date( 'Y-m-d' ) ) . "\n"; echo "#\n"; echo "# This file provides supplementary AI-specific crawler guidance.\n"; echo "# Standard robots.txt remains the authoritative source for all crawlers.\n\n"; // Known AI crawlers (current user-agent tokens as of June 2026). $crawlers = array( 'GPTBot' => 'OpenAI (ChatGPT, model training)', 'ChatGPT-User' => 'OpenAI (ChatGPT live user visits)', 'OAI-SearchBot' => 'OpenAI (ChatGPT search indexing)', 'ClaudeBot' => 'Anthropic (Claude)', 'Claude-User' => 'Anthropic (Claude live user visits)', 'Claude-SearchBot' => 'Anthropic (Claude search indexing)', 'Google-Extended' => 'Google (Gemini AI training)', 'Googlebot' => 'Google (Search, including AI Overviews)', 'Bingbot' => 'Microsoft (Bing, Copilot)', 'PerplexityBot' => 'Perplexity AI', 'Perplexity-User' => 'Perplexity (live user visits)', 'Bytespider' => 'ByteDance (TikTok AI)', 'CCBot' => 'Common Crawl', 'Applebot' => 'Apple (Siri, Apple Intelligence)', 'Applebot-Extended' => 'Apple (AI training)', 'Meta-ExternalAgent' => 'Meta (AI training)', ); $training_crawlers = array( 'Google-Extended', 'CCBot', 'Bytespider', 'Meta-ExternalAgent', 'Applebot-Extended' ); /** * Echo one RFC 9309 stanza for a crawler. * * @param string $name Crawler user-agent token. * @param string $desc Operator description. * @param bool $allowed Whether the crawler is permitted. * @param string $note Optional extra comment line. */ $aidf_render_stanza = static function ( $name, $desc, $allowed, $note = '' ) { echo '# ' . esc_html( $desc ) . "\n"; if ( '' !== $note ) { echo '# ' . esc_html( $note ) . "\n"; } echo 'User-agent: ' . esc_html( $name ) . "\n"; echo $allowed ? "Allow: /\n\n" : "Disallow: /\n\n"; }; switch ( $permissions['crawler_policy'] ) { case 'allow_all': echo "# Policy: All AI crawlers are permitted.\n\n"; foreach ( $crawlers as $name => $desc ) { $aidf_render_stanza( $name, $desc, true ); } break; case 'allow_known': echo "# Policy: Known AI crawlers are permitted for content retrieval.\n"; echo "# Training-specific crawlers may be restricted based on training policy.\n\n"; foreach ( $crawlers as $name => $desc ) { $is_training = in_array( $name, $training_crawlers, true ); if ( $is_training && 'allow' !== $permissions['ai_training'] ) { $aidf_render_stanza( $name, $desc, false, 'Training crawler restricted per training policy' ); } else { $aidf_render_stanza( $name, $desc, true ); } } break; case 'restrict': echo "# Policy: AI crawler access is restricted. Only select crawlers are permitted.\n\n"; $allowed_crawlers = array( 'Googlebot', 'Bingbot', 'Applebot' ); foreach ( $crawlers as $name => $desc ) { $aidf_render_stanza( $name, $desc, in_array( $name, $allowed_crawlers, true ) ); } break; } // Sitemap reference (WordPress core sitemap, filterable). $sitemap_url = home_url( '/wp-sitemap.xml' ); /** * Filters the sitemap URL referenced from robots-ai.txt. * * @param string $sitemap_url Sitemap URL. */ $sitemap_url = apply_filters( 'aidf_sitemap_url', $sitemap_url ); if ( ! empty( $sitemap_url ) ) { echo "# Sitemap reference\n"; echo 'Sitemap: ' . esc_url( $sitemap_url ) . "\n\n"; } // Notes. echo "# Notes for AI systems:\n"; echo "# - This file supplements robots.txt; it does not replace it.\n"; $active = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array(); if ( in_array( 'ai-txt', $active, true ) ) { echo '# - See ' . esc_url( home_url( '/ai.txt' ) ) . " for content usage permissions.\n"; } if ( in_array( 'ai-json', $active, true ) ) { echo '# - See ' . esc_url( home_url( '/ai.json' ) ) . " for structured permissions.\n"; } echo '# - Authoritative crawler rules: ' . esc_url( home_url( '/robots.txt' ) ) . "\n";