Reading > "Discourage search engines" is on, * which hides the whole site from search and AI crawlers alike, with a * one-click fix and a way to dismiss it for staging sites. * * Settings live in schema_ai_settings: llms_txt_enabled (missing = on) and * allow_ai_crawlers (missing = off). Turning either off restores WordPress's * normal behaviour at once. * * @package Schema_AI * @since 1.0.36 */ if ( ! defined( 'ABSPATH' ) ) { exit; } class Schema_AI_AI_Access { const LLMS_TRANSIENT = 'schema_ai_llms_txt'; const LLMS_LAST_GOOD = 'schema_ai_llms_txt_last'; const ROBOTS_STATUS_CACHE = 'schema_ai_ai_crawler_status'; const HIDDEN_DISMISSED = 'schema_ai_hidden_notice_dismissed'; /** * The AI crawlers we check and, when switched on, allow. User-agent token => plain name. * * @return array */ public static function crawlers() { return array( 'GPTBot' => __( 'ChatGPT (training)', 'ai-schema-gen' ), 'OAI-SearchBot' => __( 'ChatGPT search', 'ai-schema-gen' ), 'ChatGPT-User' => __( 'ChatGPT (pages a user asks it to open)', 'ai-schema-gen' ), 'ClaudeBot' => __( 'Claude (training)', 'ai-schema-gen' ), 'Claude-SearchBot' => __( 'Claude search', 'ai-schema-gen' ), 'Claude-User' => __( 'Claude (pages a user asks it to open)', 'ai-schema-gen' ), 'PerplexityBot' => __( 'Perplexity search', 'ai-schema-gen' ), 'Perplexity-User' => __( 'Perplexity (pages a user asks it to open)', 'ai-schema-gen' ), 'Google-Extended' => __( 'Google Gemini (training)', 'ai-schema-gen' ), ); } /** * Hook everything up. Called once from Schema_AI_Gen::define_public_hooks(). */ public function register() { add_action( 'init', array( $this, 'maybe_serve_llms_txt' ), 1 ); add_filter( 'robots_txt', array( $this, 'filter_robots_txt' ), 99, 2 ); add_action( 'admin_notices', array( $this, 'hidden_site_notice' ) ); add_action( 'admin_post_schema_ai_make_visible', array( $this, 'handle_make_visible' ) ); add_action( 'admin_post_schema_ai_dismiss_hidden', array( $this, 'handle_dismiss_hidden' ) ); // A settings save may have switched either feature, so re-read fresh. add_action( 'update_option_schema_ai_settings', array( __CLASS__, 'clear_caches' ) ); } /** @return array */ private static function settings() { $settings = get_option( 'schema_ai_settings', array() ); return is_array( $settings ) ? $settings : array(); } /** @return bool */ public static function llms_enabled() { $settings = self::settings(); return ! array_key_exists( 'llms_txt_enabled', $settings ) || ! empty( $settings['llms_txt_enabled'] ); } /** @return bool */ public static function crawlers_allowed() { $settings = self::settings(); return ! empty( $settings['allow_ai_crawlers'] ); } /** @return bool True when a real file sits where we would serve one. */ public static function physical_file_exists( $name ) { return file_exists( trailingslashit( ABSPATH ) . $name ); } public static function clear_caches() { delete_transient( self::LLMS_TRANSIENT ); delete_transient( self::ROBOTS_STATUS_CACHE ); } // ── 1. llms.txt ───────────────────────────────────────────────────────── /** * Serve /llms.txt when that is what was asked for. Runs on init, before * WordPress decides the URL is a 404, and before any theme output. */ public function maybe_serve_llms_txt() { if ( is_admin() || ! self::llms_enabled() || empty( $_SERVER['REQUEST_URI'] ) ) { return; } $path = wp_parse_url( esc_url_raw( wp_unslash( $_SERVER['REQUEST_URI'] ) ), PHP_URL_PATH ); if ( ! is_string( $path ) || strtolower( $path ) !== strtolower( self::llms_path() ) ) { return; } $content = self::get_llms_content(); if ( '' === $content ) { return; // Nothing useful to publish yet: let WordPress answer 404 as before. } status_header( 200 ); header( 'Content-Type: text/plain; charset=utf-8' ); header( 'Cache-Control: public, max-age=3600' ); echo $content; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- plain-text file body, served as text/plain. exit; } /** @return string The path this site's llms.txt lives at, subdirectory installs included. */ public static function llms_path() { $home_path = wp_parse_url( home_url( '/' ), PHP_URL_PATH ); return trailingslashit( is_string( $home_path ) && '' !== $home_path ? $home_path : '/' ) . 'llms.txt'; } /** * The llms.txt body, cached for 12 hours. Empty string means "nothing to * serve". A failed fetch falls back to the last good copy so a network blip * never takes a working file offline. * * @return string */ public static function get_llms_content() { $cached = get_transient( self::LLMS_TRANSIENT ); if ( false !== $cached ) { return (string) $cached; } $settings = self::settings(); $api_key = ! empty( $settings['api_key'] ) ? $settings['api_key'] : ''; $site_id = (int) get_option( 'schema_ai_saas_site_id', 0 ); if ( '' === $api_key || $site_id <= 0 ) { set_transient( self::LLMS_TRANSIENT, '', HOUR_IN_SECONDS ); return ''; } $response = wp_remote_get( SCHEMA_AI_API_BASE . '/sites/' . $site_id . '/llms-txt', array( 'timeout' => 8, 'headers' => array( 'X-API-Key' => $api_key ), ) ); if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) { $last = (string) get_option( self::LLMS_LAST_GOOD, '' ); set_transient( self::LLMS_TRANSIENT, $last, 15 * MINUTE_IN_SECONDS ); return $last; } $body = json_decode( wp_remote_retrieve_body( $response ), true ); $content = isset( $body['data']['content'] ) && is_string( $body['data']['content'] ) ? $body['data']['content'] : ''; if ( '' !== $content ) { update_option( self::LLMS_LAST_GOOD, $content, false ); } set_transient( self::LLMS_TRANSIENT, $content, '' !== $content ? 12 * HOUR_IN_SECONDS : HOUR_IN_SECONDS ); return $content; } // ── 2. AI crawlers in robots.txt ──────────────────────────────────────── /** * Add explicit allow groups for the AI crawlers to WordPress's virtual * robots.txt. Left alone when the switch is off, or when the site is set * to discourage search engines (that choice is the owner's to undo). * * @param string $output Robots.txt body so far. * @param bool|string $is_public Whether the site allows indexing (WordPress passes a bool). * @return string */ public function filter_robots_txt( $output, $is_public ) { if ( ! self::crawlers_allowed() || ! $is_public ) { return $output; } $lines = array( '', '# Added by AI Schema Gen: let AI assistants read this site' ); foreach ( array_keys( self::crawlers() ) as $agent ) { $lines[] = 'User-agent: ' . $agent; $lines[] = 'Allow: /'; $lines[] = ''; } return rtrim( (string) $output ) . "\n" . implode( "\n", $lines ); } /** * Which AI crawlers the live robots.txt blocks, read the way a crawler * reads it: its own group if it has one, otherwise the "*" group. Cached * for 10 minutes. Only whole-site blocks ("Disallow: /") count. * * @return array{fetched:bool,physical:bool,crawlers:array} */ public static function crawler_status() { $cached = get_transient( self::ROBOTS_STATUS_CACHE ); if ( is_array( $cached ) ) { return $cached; } $result = array( 'fetched' => false, 'physical' => self::physical_file_exists( 'robots.txt' ), 'crawlers' => array(), ); $response = wp_remote_get( home_url( '/robots.txt' ), array( 'timeout' => 8 ) ); if ( ! is_wp_error( $response ) && 200 === wp_remote_retrieve_response_code( $response ) ) { $result['fetched'] = true; $groups = self::parse_robots( (string) wp_remote_retrieve_body( $response ) ); foreach ( array_keys( self::crawlers() ) as $agent ) { $result['crawlers'][ $agent ] = self::is_blocked( $groups, $agent ) ? 'blocked' : 'allowed'; } } set_transient( self::ROBOTS_STATUS_CACHE, $result, 10 * MINUTE_IN_SECONDS ); return $result; } /** * @param string $body robots.txt text. * @return array */ private static function parse_robots( $body ) { $groups = array(); $current = null; $in_ua = false; foreach ( preg_split( '/\r\n|\r|\n/', $body ) as $raw ) { $line = trim( preg_replace( '/#.*$/', '', $raw ) ); if ( '' === $line || false === strpos( $line, ':' ) ) { continue; } list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) ); $field = strtolower( $field ); if ( 'user-agent' === $field ) { if ( ! $in_ua || null === $current ) { $groups[] = array( 'agents' => array(), 'allow_root' => false, 'disallow_root' => false ); $current = count( $groups ) - 1; } $groups[ $current ]['agents'][] = strtolower( $value ); $in_ua = true; continue; } $in_ua = false; if ( null === $current ) { continue; } if ( 'disallow' === $field && '/' === $value ) { $groups[ $current ]['disallow_root'] = true; } elseif ( 'allow' === $field && '/' === $value ) { $groups[ $current ]['allow_root'] = true; } } return $groups; } private static function is_blocked( array $groups, $agent ) { $agent = strtolower( $agent ); $matching = array_filter( $groups, function ( $g ) use ( $agent ) { return in_array( $agent, $g['agents'], true ); } ); if ( empty( $matching ) ) { $matching = array_filter( $groups, function ( $g ) { return in_array( '*', $g['agents'], true ); } ); } $disallow = false; $allow = false; foreach ( $matching as $g ) { $disallow = $disallow || $g['disallow_root']; $allow = $allow || $g['allow_root']; } return $disallow && ! $allow; } // ── 3. "Discourage search engines" warning ────────────────────────────── public function hidden_site_notice() { if ( '0' !== (string) get_option( 'blog_public' ) || ! current_user_can( 'manage_options' ) || get_option( self::HIDDEN_DISMISSED ) ) { return; } $screen = function_exists( 'get_current_screen' ) ? get_current_screen() : null; $id = $screen ? (string) $screen->id : ''; if ( 'dashboard' !== $id && false === strpos( $id, 'ai-schema-gen' ) && 'options-reading' !== $id ) { return; } $fix_url = wp_nonce_url( admin_url( 'admin-post.php?action=schema_ai_make_visible' ), 'schema_ai_make_visible' ); $dismiss_url = wp_nonce_url( admin_url( 'admin-post.php?action=schema_ai_dismiss_hidden' ), 'schema_ai_dismiss_hidden' ); ?>

Reading, so Google, ChatGPT, Perplexity and others are told not to read it.', 'ai-schema-gen' ); ?>