link references.
*
* Advertises the active AI Discovery Files to automated tools that scan
* robots.txt or the homepage
. Without this, the files serve correctly
* but are invisible to crawlers that don't already know the canonical URLs.
*
* Only files the user has actually activated in plugin settings are emitted.
*
* @package AIDF
* @since 1.3.9
*/
if ( ! defined( 'ABSPATH' ) ) {
exit;
}
/**
* Emits discoverability hints into robots.txt and the HTML head.
*
* Two hooks, both honour the active_files setting:
*
* - robots_txt filter (WP 3.0+) — appends a block listing every currently-
* active discovery file URL. URLs are emitted as comments (lines starting
* with "#") so they are safe with any robots.txt parser, per RFC 9309.
* Note this filter only fires when WordPress is generating robots.txt
* virtually; sites with a physical robots.txt at the webroot will need to
* copy the block in by hand, which the admin UI exposes.
*
* - wp_head action — emits a for every active file
* so tools that scan the head (the same way they scan for RSS feeds) can
* discover them.
*/
class AIDF_Discovery {
/**
* Slug -> human-readable title used in attribute.
*
* @var array
*/
private static $head_titles = array(
'llms-txt' => 'AI Discovery (llms.txt)',
'llm-txt' => 'AI Discovery legacy alias (llm.txt)',
'llms-html' => 'AI Discovery, HTML (llms.html)',
'ai-txt' => 'AI Usage Policy (ai.txt)',
'ai-json' => 'AI Permissions (ai.json)',
'identity-json' => 'Business Identity (identity.json)',
'brand-txt' => 'Brand Terminology (brand.txt)',
'faq-ai-txt' => 'FAQs for AI (faq-ai.txt)',
'developer-ai-txt' => 'Developer Context (developer-ai.txt)',
'robots-ai-txt' => 'AI Crawler Access (robots-ai.txt)',
);
/**
* Register hooks.
*
* @return void
*/
public static function init() {
add_filter( 'robots_txt', array( __CLASS__, 'inject_discovery_refs' ), 20, 2 );
add_action( 'wp_head', array( __CLASS__, 'inject_head_links' ), 5 );
}
/**
* Filter callback for robots_txt.
*
* Appends a labelled block listing every active AI Discovery File URL.
* If the site is set to "discourage search engines" ($public === false),
* we don't add anything — WordPress will already be returning
* Disallow: / and the discovery files shouldn't be advertised.
*
* @param string $output Existing robots.txt body.
* @param int $public Whether the blog is public.
* @return string
*/
public static function inject_discovery_refs( $output, $public ) {
if ( ! $public ) {
return $output;
}
$block = self::get_robots_txt_block();
if ( '' === $block ) {
return $output;
}
// Make sure we land on a new line cleanly.
$separator = ( '' === $output || "\n" === substr( $output, -1 ) ) ? '' : "\n";
return $output . $separator . $block;
}
/**
* Action callback for wp_head.
*
* Emits one per active discovery file so
* crawlers that scan the head (the way they scan for feeds) can
* pick the URLs up. Skipped in admin and feed contexts.
*
* @return void
*/
public static function inject_head_links() {
if ( is_admin() || is_feed() ) {
return;
}
$entries = self::get_active_entries();
if ( empty( $entries ) ) {
return;
}
echo "\n\n";
foreach ( $entries as $entry ) {
printf(
'' . "\n",
esc_attr( $entry['short_type'] ),
esc_url( $entry['url'] ),
esc_attr( $entry['title'] )
);
}
echo "\n\n";
}
/**
* Build the labelled robots.txt block for the active discovery files.
*
* Returned without a trailing newline. Returns an empty string when no
* files are active so callers can decide whether to emit anything.
*
* This method is public so the admin UI can show it to the user when
* a physical robots.txt is detected (and the filter therefore can't
* inject it automatically).
*
* @return string
*/
public static function get_robots_txt_block() {
$entries = self::get_active_entries();
if ( empty( $entries ) ) {
return '';
}
$lines = array();
$lines[] = '# AI Discovery Files';
$lines[] = '# Specification: https://www.ai-visibility.org.uk/specifications/';
$lines[] = '# Generated by AI Discovery Files for WordPress v' . AIDF_VERSION;
foreach ( $entries as $entry ) {
$lines[] = '# ' . $entry['filename'] . ': ' . $entry['url'];
}
return implode( "\n", $lines );
}
/**
* Whether a physical robots.txt exists at the WordPress root.
*
* When true, WordPress will serve that file as-is and the robots_txt
* filter never fires. The admin UI uses this to warn the user that
* they need to copy the block into their robots.txt by hand.
*
* @return bool
*/
public static function has_physical_robots_txt() {
$abspath = untrailingslashit( ABSPATH );
return file_exists( $abspath . '/robots.txt' );
}
/**
* Resolve the list of active discovery files to displayable entries.
*
* Each entry has:
* - slug : the file slug (e.g. 'llms-txt')
* - filename : the on-disk filename (e.g. 'llms.txt')
* - url : the absolute URL where the plugin serves it
* - short_type : the MIME type without parameters (e.g. 'text/plain')
* - title : the human-readable title used in
*
* Skips:
* - Slugs that are not present in AIDF_Plugin::get_file_types().
* - llm-txt (it is a 301 redirect to llms.txt — advertising both would
* just send crawlers through an extra hop, so we list only llms-txt).
*
* @return array>
*/
private static function get_active_entries() {
$settings = AIDF_Plugin::get_settings();
$active = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array();
$file_types = AIDF_Plugin::get_file_types();
$out = array();
foreach ( $active as $slug ) {
if ( 'llm-txt' === $slug ) {
// llm.txt is a redirect to llms.txt, no point advertising it separately.
continue;
}
if ( ! isset( $file_types[ $slug ] ) ) {
continue;
}
$meta = $file_types[ $slug ];
$out[] = array(
'slug' => $slug,
'filename' => $meta['filename'],
'url' => home_url( '/' . $meta['filename'] ),
'short_type' => self::short_type( $meta['content_type'] ),
'title' => isset( self::$head_titles[ $slug ] ) ? self::$head_titles[ $slug ] : $meta['filename'],
);
}
return $out;
}
/**
* Strip the charset parameter from a Content-Type so it can be used in
* a attribute (which expects bare MIME types).
*
* 'text/plain; charset=utf-8' -> 'text/plain'
*
* @param string $content_type Full Content-Type header value.
* @return string
*/
private static function short_type( $content_type ) {
$parts = explode( ';', $content_type, 2 );
return trim( $parts[0] );
}
}