$data Collected site data. */ if ( ! defined( 'ABSPATH' ) ) { exit; } // phpcs:disable WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedVariableFound -- Template variables scoped to including method. $identity = $data['identity']; $permissions = $data['permissions']; $settings = $data['settings']; // Attribution header. // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- Pre-built internal string. echo AIDF_Generator::get_attribution( '#', 'robots-ai-txt' ); echo '# AI Crawler Directives for ' . esc_html( $identity['name'] ) . "\n\n"; echo 'Website: [' . esc_html( $identity['url'] ) . '](' . esc_url( $identity['url'] ) . ')' . "\n"; echo 'Last Updated: ' . esc_html( wp_date( 'Y-m-d' ) ) . "\n\n"; echo "## Purpose\n\n"; echo "This file provides AI-specific crawler guidance that complements robots.txt.\n"; echo "It covers crawlers used by AI systems for training, retrieval, and indexing.\n\n"; // Known AI crawlers. $crawlers = array( 'GPTBot' => 'OpenAI (ChatGPT, plugins)', 'ChatGPT-User' => 'OpenAI (ChatGPT browsing)', 'ClaudeBot' => 'Anthropic (Claude)', 'Claude-Web' => 'Anthropic (Claude web search)', 'Google-Extended' => 'Google (Gemini AI training)', 'Googlebot' => 'Google (Search, including AI Overviews)', 'Bingbot' => 'Microsoft (Bing, Copilot)', 'PerplexityBot' => 'Perplexity AI', 'Bytespider' => 'ByteDance (TikTok AI)', 'CCBot' => 'Common Crawl', 'Applebot' => 'Apple (Siri, Apple Intelligence)', 'Meta-ExternalAgent' => 'Meta (AI training)', ); echo "## Crawler Access\n\n"; switch ( $permissions['crawler_policy'] ) { case 'allow_all': echo "Policy: All AI crawlers are permitted.\n\n"; foreach ( $crawlers as $name => $desc ) { echo esc_html( $name ) . ': Allow' . "\n"; echo ' # ' . esc_html( $desc ) . "\n\n"; } break; case 'allow_known': echo "Policy: Known AI crawlers are permitted for content retrieval.\n"; echo "Training-specific crawlers may be restricted based on training policy.\n\n"; foreach ( $crawlers as $name => $desc ) { $is_training = in_array( $name, array( 'Google-Extended', 'CCBot', 'Bytespider', 'Meta-ExternalAgent' ), true ); if ( $is_training && 'allow' !== $permissions['ai_training'] ) { echo esc_html( $name ) . ': Disallow' . "\n"; echo ' # ' . esc_html( $desc ) . ' — training crawler restricted per training policy' . "\n\n"; } else { echo esc_html( $name ) . ': Allow' . "\n"; echo ' # ' . esc_html( $desc ) . "\n\n"; } } break; case 'restrict': echo "Policy: AI crawler access is restricted. Only select crawlers are permitted.\n\n"; // Allow major search/AI assistants, block training-specific. $allowed = array( 'Googlebot', 'Bingbot', 'Applebot' ); foreach ( $crawlers as $name => $desc ) { if ( in_array( $name, $allowed, true ) ) { echo esc_html( $name ) . ': Allow' . "\n"; echo ' # ' . esc_html( $desc ) . "\n\n"; } else { echo esc_html( $name ) . ': Disallow' . "\n"; echo ' # ' . esc_html( $desc ) . "\n\n"; } } break; } // Note about robots.txt. echo "## Notes\n\n"; echo "- This file supplements robots.txt — it does not replace it.\n"; echo "- AI systems should also check ai.txt for content usage permissions.\n"; echo "- For structured permissions, see ai.json.\n"; $active = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array(); echo "\n## Related Files\n\n"; echo '- [robots.txt](' . esc_url( home_url( '/robots.txt' ) ) . ')' . "\n"; if ( in_array( 'ai-txt', $active, true ) ) { echo '- [ai.txt](' . esc_url( home_url( '/ai.txt' ) ) . ')' . "\n"; } if ( in_array( 'ai-json', $active, true ) ) { echo '- [ai.json](' . esc_url( home_url( '/ai.json' ) ) . ')' . "\n"; }