link references. * * Advertises the active AI Discovery Files to automated tools that scan * robots.txt or the homepage . Without this, the files serve correctly * but are invisible to crawlers that don't already know the canonical URLs. * * Only files the user has actually activated in plugin settings are emitted. * * @package AIDF * @since 1.3.9 */ if ( ! defined( 'ABSPATH' ) ) { exit; } /** * Emits discoverability hints into robots.txt and the HTML head. * * Two hooks, both honour the active_files setting: * * - robots_txt filter (WP 3.0+) — appends a block listing every currently- * active discovery file URL. URLs are emitted as comments (lines starting * with "#") so they are safe with any robots.txt parser, per RFC 9309. * Note this filter only fires when WordPress is generating robots.txt * virtually; sites with a physical robots.txt at the webroot will need to * copy the block in by hand, which the admin UI exposes. * * - wp_head action — emits a for every active file * so tools that scan the head (the same way they scan for RSS feeds) can * discover them. */ class AIDF_Discovery { /** * Slug -> human-readable title used in attribute. * * @var array */ private static $head_titles = array( 'llms-txt' => 'AI Discovery (llms.txt)', 'llm-txt' => 'AI Discovery legacy alias (llm.txt)', 'llms-html' => 'AI Discovery, HTML (llms.html)', 'ai-txt' => 'AI Usage Policy (ai.txt)', 'ai-json' => 'AI Permissions (ai.json)', 'identity-json' => 'Business Identity (identity.json)', 'brand-txt' => 'Brand Terminology (brand.txt)', 'faq-ai-txt' => 'FAQs for AI (faq-ai.txt)', 'developer-ai-txt' => 'Developer Context (developer-ai.txt)', 'robots-ai-txt' => 'AI Crawler Access (robots-ai.txt)', ); /** * Register hooks. * * @return void */ public static function init() { add_filter( 'robots_txt', array( __CLASS__, 'inject_discovery_refs' ), 20, 2 ); add_action( 'wp_head', array( __CLASS__, 'inject_head_links' ), 5 ); } /** * Filter callback for robots_txt. * * Appends a labelled block listing every active AI Discovery File URL. * If the site is set to "discourage search engines" ($public === false), * we don't add anything — WordPress will already be returning * Disallow: / and the discovery files shouldn't be advertised. * * @param string $output Existing robots.txt body. * @param int $public Whether the blog is public. * @return string */ public static function inject_discovery_refs( $output, $public ) { if ( ! $public ) { return $output; } $block = self::get_robots_txt_block(); if ( '' === $block ) { return $output; } // Make sure we land on a new line cleanly. $separator = ( '' === $output || "\n" === substr( $output, -1 ) ) ? '' : "\n"; return $output . $separator . $block; } /** * Action callback for wp_head. * * Emits one per active discovery file so * crawlers that scan the head (the way they scan for feeds) can * pick the URLs up. Skipped in admin and feed contexts. * * @return void */ public static function inject_head_links() { if ( is_admin() || is_feed() ) { return; } $entries = self::get_active_entries(); if ( empty( $entries ) ) { return; } echo "\n\n"; foreach ( $entries as $entry ) { printf( '' . "\n", esc_attr( $entry['short_type'] ), esc_url( $entry['url'] ), esc_attr( $entry['title'] ) ); } echo "\n\n"; } /** * Build the labelled robots.txt block for the active discovery files. * * Returned without a trailing newline. Returns an empty string when no * files are active so callers can decide whether to emit anything. * * This method is public so the admin UI can show it to the user when * a physical robots.txt is detected (and the filter therefore can't * inject it automatically). * * @return string */ public static function get_robots_txt_block() { $entries = self::get_active_entries(); if ( empty( $entries ) ) { return ''; } $lines = array(); $lines[] = '# AI Discovery Files'; $lines[] = '# Specification: https://www.ai-visibility.org.uk/specifications/'; $lines[] = '# Generated by AI Discovery Files for WordPress v' . AIDF_VERSION; foreach ( $entries as $entry ) { $lines[] = '# ' . $entry['filename'] . ': ' . $entry['url']; } return implode( "\n", $lines ); } /** * Whether a physical robots.txt exists at the WordPress root. * * When true, WordPress will serve that file as-is and the robots_txt * filter never fires. The admin UI uses this to warn the user that * they need to copy the block into their robots.txt by hand. * * @return bool */ public static function has_physical_robots_txt() { $abspath = untrailingslashit( ABSPATH ); return file_exists( $abspath . '/robots.txt' ); } /** * Resolve the list of active discovery files to displayable entries. * * Each entry has: * - slug : the file slug (e.g. 'llms-txt') * - filename : the on-disk filename (e.g. 'llms.txt') * - url : the absolute URL where the plugin serves it * - short_type : the MIME type without parameters (e.g. 'text/plain') * - title : the human-readable title used in * * Skips: * - Slugs that are not present in AIDF_Plugin::get_file_types(). * - llm-txt (it is a 301 redirect to llms.txt — advertising both would * just send crawlers through an extra hop, so we list only llms-txt). * * @return array> */ private static function get_active_entries() { $settings = AIDF_Plugin::get_settings(); $active = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array(); $file_types = AIDF_Plugin::get_file_types(); $out = array(); foreach ( $active as $slug ) { if ( 'llm-txt' === $slug ) { // llm.txt is a redirect to llms.txt, no point advertising it separately. continue; } if ( ! isset( $file_types[ $slug ] ) ) { continue; } $meta = $file_types[ $slug ]; $out[] = array( 'slug' => $slug, 'filename' => $meta['filename'], 'url' => home_url( '/' . $meta['filename'] ), 'short_type' => self::short_type( $meta['content_type'] ), 'title' => isset( self::$head_titles[ $slug ] ) ? self::$head_titles[ $slug ] : $meta['filename'], ); } return $out; } /** * Strip the charset parameter from a Content-Type so it can be used in * a attribute (which expects bare MIME types). * * 'text/plain; charset=utf-8' -> 'text/plain' * * @param string $content_type Full Content-Type header value. * @return string */ private static function short_type( $content_type ) { $parts = explode( ';', $content_type, 2 ); return trim( $parts[0] ); } }