<?php
/**
 * AI-aware robots.txt management.
 *
 * Every known AI crawler gets its own group and, by default, an explicit
 * Allow. That looks redundant next to "User-agent: *", and it is not.
 *
 * A crawler reads exactly ONE group: the most specific one that names it
 * (RFC 9309). Naming a bot and saying nothing else would hand it a group
 * without the site's own Disallow rules, so every group below repeats the
 * rules of the "*" group. And a spoken Allow beats an inherited one: Apple
 * documents that Applebot follows the Googlebot rules when it is not named
 * itself, which is the kind of guesswork a discovery plugin should not leave
 * to the crawler.
 *
 * Each group also carries a Content-Signal line (contentsignals.org): it
 * tells a crawler what it may do with the content, not only whether it may
 * fetch it. The signal is derived from the two policy checkboxes, so it can
 * never contradict the Allow and Disallow lines next to it.
 *
 * Until 4.4.2 the default was the other way round. Thirteen training
 * crawlers, GPTBot and ClaudeBot among them, were blocked out of the box, on
 * a plugin whose whole job is to make a site visible to AI, and most site
 * owners never saw the checkbox. Blocking is still one click away.
 *
 * @since 3.0.0
 * @since 4.4.3 Explicit allow groups, Content Signals, rules copied from "*".
 */

if ( ! defined( 'ABSPATH' ) ) {
    exit;
}

class AIREWP_Robots_Txt {

    /**
     * The line that says this block came from here.
     *
     * It exists so the readiness check can ask the SERVED robots.txt whether
     * the block arrived, instead of asking the settings whether it was meant
     * to. Those are different questions: this plugin writes robots.txt through
     * the `robots_txt` filter, and that filter only runs when WordPress is the
     * one answering. A static robots.txt in the site root, or a plugin that
     * hands back its own, and the filter never fires while every setting still
     * says the feature is on.
     */
    public const BLOCK_MARKER = '# AI crawlers (managed by AI-Ready WP)';

    /**
     * Crawlers that fetch pages to train models.
     *
     * Every entry names where the user agent is documented. A name without a
     * source has no place here: that is how cohere-ai (Cohere states it runs
     * no crawler) and FacebookBot (no longer in Meta's crawler list) lived in
     * this list for months.
     */
    public const TRAINING_BOTS = [
        'GPTBot'             => 'OpenAI, platform.openai.com/docs/bots',
        'ClaudeBot'          => 'Anthropic, support.claude.com (web crawling)',
        'anthropic-ai'       => 'Anthropic, earlier agent name kept for stragglers',
        'Google-Extended'    => 'Google, control token that never fetches itself',
        'Applebot-Extended'  => 'Apple, control token that never fetches itself',
        'Meta-ExternalAgent' => 'Meta, developers.facebook.com web crawlers',
        'Amazonbot'          => 'Amazon, developer.amazon.com/amazonbot',
        'Bytespider'         => 'ByteDance',
        'CCBot'              => 'Common Crawl, commoncrawl.org/ccbot',
        'Diffbot'            => 'Diffbot, docs.diffbot.com',
        'Omgilibot'          => 'Webz.io',
        'ImagesiftBot'       => 'Imagesift, imagesift.com',
        'MistralAI-Training' => 'Mistral, docs.mistral.ai/robots',
    ];

    /**
     * Crawlers that fetch pages to answer a question or fill a search index.
     */
    public const RETRIEVAL_BOTS = [
        'OAI-SearchBot'         => 'OpenAI search index, platform.openai.com/docs/bots',
        'ChatGPT-User'          => 'ChatGPT, fetches a page a user asked about',
        'Claude-SearchBot'      => 'Anthropic search index, support.claude.com',
        'Claude-User'           => 'Claude, fetches a page a user asked about',
        'PerplexityBot'         => 'Perplexity index, docs.perplexity.ai/guides/bots',
        'Perplexity-User'       => 'Perplexity, fetches a page a user asked about',
        'GoogleOther'           => 'Google, generic crawler for research and development',
        'Google-CloudVertexBot' => 'Google, crawls for Vertex AI Agents when a site owner asks',
        'Applebot'              => 'Apple, Siri, Spotlight, Safari and Apple Intelligence',
        'DuckAssistBot'         => 'DuckDuckGo, fetches for DuckAssist answers',
        'Meta-ExternalFetcher'  => 'Meta AI, fetches a link a user asked about',
        'Meta-WebIndexer'       => 'Meta AI search index',
        'MistralAI-Index'       => 'Mistral search index, docs.mistral.ai/robots',
        'MistralAI-User'        => 'Mistral, fetches a page a user asked about',
    ];

    public function __construct() {
        $settings = AI_Ready_WP::get_settings();
        if ( ! empty( $settings['enable_robots_ai'] ) ) {
            add_filter( 'robots_txt', [ $this, 'modify_robots_txt' ], 100, 2 );
        }
    }

    /**
     * Add the AI crawler groups to the virtual robots.txt.
     */
    public function modify_robots_txt( string $output, bool $public ): string {
        if ( ! $public ) {
            return $output;
        }
        return self::build( $output, AI_Ready_WP::get_settings() );
    }

    /**
     * Append the AI groups to whatever robots.txt WordPress and the other
     * plugins have written so far.
     *
     * A pure function of its two arguments, so the tests can feed it any
     * prefix without touching global state.
     */
    public static function build( string $output, array $settings ): string {
        $block_training  = ! empty( $settings['robots_block_training'] );
        $block_retrieval = ! empty( $settings['robots_block_retrieval'] );

        $signal = self::content_signal( $block_training, $block_retrieval );
        $output = self::signal_everyone( $output, $signal );
        $shared = self::rules_for_everyone( $output );

        $lines   = [];
        $lines[] = '';
        $lines[] = self::BLOCK_MARKER;

        $groups = [
            [ self::TRAINING_BOTS, $block_training ],
            [ self::RETRIEVAL_BOTS, $block_retrieval ],
        ];
        foreach ( $groups as [ $bots, $blocked ] ) {
            foreach ( array_keys( $bots ) as $bot ) {
                // Someone already speaks for this crawler: a security plugin,
                // a hand-written rule. A second group for the same name would
                // be merged with theirs, and on a tie between Allow and
                // Disallow RFC 9309 prefers the Allow, so our default would
                // quietly override their block. Leaving the bot alone is the
                // only honest option.
                if ( self::already_addressed( $output, $bot ) ) {
                    continue;
                }
                $lines[] = 'User-agent: ' . $bot;
                if ( $blocked ) {
                    $lines[] = 'Disallow: /';
                } else {
                    $lines[] = 'Content-Signal: ' . $signal;
                    $lines[] = 'Allow: /';
                    foreach ( $shared as $rule ) {
                        $lines[] = $rule;
                    }
                }
                $lines[] = '';
            }
        }

        $lines[] = '# AI discovery';

        // The Sitemap line belongs to whoever answers /sitemap.xml. Only this
        // plugin's own SEO module counts here, and even then only when nobody
        // has advertised that URL yet: WordPress core adds a Sitemap line, most
        // SEO plugins add theirs, and on one user's site the same URL stood
        // there twice, once from The SEO Framework and once from us.
        $sitemap = home_url( '/sitemap.xml' );
        if ( AIREWP_SEO::serves_sitemap( $settings ) && ! self::has_sitemap_line( $output, $sitemap ) ) {
            $lines[] = 'Sitemap: ' . $sitemap;
        }

        // Both documents answer whether or not a file was written (agents.json
        // since 4.4.0, llms.txt since 4.4.4), so the feature toggle is the
        // fact that matters.
        if ( ! empty( $settings['enable_agents_json'] ) ) {
            $lines[] = '# agents.json: ' . home_url( '/.well-known/agents.json' );
        }
        if ( ! empty( $settings['enable_llms_txt'] ) ) {
            $lines[] = '# llms.txt: ' . home_url( '/llms.txt' );
        }

        return rtrim( $output, "\n" ) . "\n" . implode( "\n", $lines ) . "\n";
    }

    /**
     * The Content-Signal value that matches the policy checkboxes.
     *
     * search is always yes: nothing in this plugin ever hides a site from
     * search. ai-input follows the retrieval checkbox, ai-train the training
     * checkbox. The optional use field (a Cloudflare extension) says how far
     * an answer may go: full when training is allowed, reference when only
     * retrieval is, nothing when both are blocked.
     */
    public static function content_signal( bool $block_training, bool $block_retrieval ): string {
        $fields = [
            'search=yes',
            'ai-input=' . ( $block_retrieval ? 'no' : 'yes' ),
            'ai-train=' . ( $block_training ? 'no' : 'yes' ),
        ];
        if ( ! $block_training ) {
            $fields[] = 'use=full';
        } elseif ( ! $block_retrieval ) {
            $fields[] = 'use=reference';
        }
        return implode( ', ', $fields );
    }

    /**
     * Put the Content-Signal line into the "*" group as well.
     *
     * The named groups below only reach the crawlers that are named. Every
     * other crawler reads "*", and a preference that is not stated there is
     * not stated for them. A signal that is already in the group (Cloudflare's
     * managed robots.txt, another plugin) is left alone; two signals in one
     * group would be worse than one.
     */
    private static function signal_everyone( string $output, string $signal ): string {
        $lines = explode( "\n", $output );
        $count = count( $lines );
        $star  = null;

        foreach ( $lines as $i => $line ) {
            if ( preg_match( '/^\s*User-agent:\s*\*\s*$/i', $line ) ) {
                $star = $i;
                break;
            }
        }
        if ( null === $star ) {
            return $output;
        }

        // A group may open with several User-agent lines in a row; the signal
        // has to go after the last of them or it would split the group.
        $insert_at = $star + 1;
        while ( $insert_at < $count && preg_match( '/^\s*User-agent:/i', $lines[ $insert_at ] ) ) {
            $insert_at++;
        }

        for ( $i = $insert_at; $i < $count; $i++ ) {
            if ( preg_match( '/^\s*User-agent:/i', $lines[ $i ] ) ) {
                break;
            }
            if ( preg_match( '/^\s*Content-Signal:/i', $lines[ $i ] ) ) {
                return $output;
            }
        }

        array_splice( $lines, $insert_at, 0, [ 'Content-Signal: ' . $signal ] );
        return implode( "\n", $lines );
    }

    /**
     * The Allow and Disallow rules every crawler is supposed to follow.
     *
     * Collected from all groups that name "*", because a crawler that reads
     * its own named group never sees them otherwise. Blank lines and comments
     * do not end a group (RFC 9309); the next User-agent line does.
     *
     * @return string[] Rules in the form "Disallow: /wp-admin/".
     */
    public static function rules_for_everyone( string $output ): array {
        $rules       = [];
        $in_star     = false;
        $in_ua_block = false;

        foreach ( explode( "\n", $output ) as $raw ) {
            $line = trim( $raw );

            if ( preg_match( '/^User-agent:\s*(.*)$/i', $line, $m ) ) {
                if ( ! $in_ua_block ) {
                    $in_star = false;
                }
                $in_ua_block = true;
                if ( '*' === trim( $m[1] ) ) {
                    $in_star = true;
                }
                continue;
            }

            if ( '' === $line || '#' === $line[0] ) {
                continue;
            }

            $in_ua_block = false;
            if ( $in_star && preg_match( '/^(Allow|Disallow):\s*(.*)$/i', $line, $m ) ) {
                $rules[] = rtrim( ucfirst( strtolower( $m[1] ) ) . ': ' . trim( $m[2] ) );
            }
        }

        return array_values( array_unique( $rules ) );
    }

    /**
     * Whether robots.txt already has a group for this crawler.
     */
    private static function already_addressed( string $output, string $bot ): bool {
        return (bool) preg_match( '/^\s*User-agent:\s*' . preg_quote( $bot, '/' ) . '\s*$/im', $output );
    }

    /**
     * Whether this exact sitemap URL is already advertised.
     */
    private static function has_sitemap_line( string $output, string $url ): bool {
        return (bool) preg_match( '/^\s*Sitemap:\s*' . preg_quote( $url, '/' ) . '\s*$/im', $output );
    }

    /**
     * Get all known AI bot user-agent strings for analytics matching.
     */
    public static function get_all_bot_agents(): array {
        return array_merge(
            array_keys( self::TRAINING_BOTS ),
            array_keys( self::RETRIEVAL_BOTS )
        );
    }
}