*/
public static function get_conflicts() {
$alerts = array();
// Check WordPress "Discourage search engines from indexing this site" setting.
// When enabled, WordPress adds Disallow: / for all user agents in robots.txt,
// which blocks every AI crawler from accessing the site.
if ( '0' === get_option( 'blog_public' ) ) {
$alerts[] = array(
'type' => 'error',
'message' => sprintf(
/* translators: %s: URL to the Reading Settings page */
__( 'Your site is set to discourage search engines from indexing. This blocks all AI crawlers via robots.txt. Disable this in %s if you want AI bots to access your site.', 'ai-discovery-files' ),
'' . esc_html__( 'Settings → Reading', 'ai-discovery-files' ) . ''
),
'action_url' => admin_url( 'options-reading.php' ),
'action_label' => __( 'Reading Settings', 'ai-discovery-files' ),
);
}
// Attempt to load cached parsed rules.
$rules = get_transient( self::TRANSIENT_KEY );
if ( false === $rules ) {
$body = self::fetch_robots_txt();
// If we cannot fetch robots.txt, return what we have so far.
if ( false === $body ) {
return apply_filters( 'aidf_crawler_conflicts', $alerts, array() );
}
$rules = self::parse_robots_txt( $body );
set_transient( self::TRANSIENT_KEY, $rules, self::TRANSIENT_TTL );
}
$alerts = array_merge( $alerts, self::generate_alerts( $rules ) );
/**
* Filters the crawler conflict alerts array.
*
* Third-party code may add, remove, or modify alerts. Each entry must
* contain 'type', 'message', 'action_url', and 'action_label' keys.
*
* @since 1.3.0
*
* @param array $alerts Conflict alerts.
* @param array $rules Parsed robots.txt rules.
*/
return apply_filters( 'aidf_crawler_conflicts', $alerts, $rules );
}
// -------------------------------------------------------------------------
// Fetch
// -------------------------------------------------------------------------
/**
* Fetch the site's robots.txt content.
*
* Uses {@see wp_remote_get()} with a short timeout so the admin page is
* never blocked. Only returns the body when the response is HTTP 200.
*
* @since 1.3.0
*
* @return string|false Body string on success, false on failure or non-200.
*/
public static function fetch_robots_txt() {
$url = trailingslashit( get_site_url() ) . 'robots.txt';
$response = wp_remote_get(
$url,
array( 'timeout' => 5 )
);
if ( is_wp_error( $response ) ) {
return false;
}
$status_code = wp_remote_retrieve_response_code( $response );
if ( 200 !== (int) $status_code ) {
return false;
}
return wp_remote_retrieve_body( $response );
}
// -------------------------------------------------------------------------
// Parse
// -------------------------------------------------------------------------
/**
* Parse a robots.txt body string into a structured rule set.
*
* Processes the file line by line. Recognises `User-agent`, `Disallow`,
* and `Allow` directives (case-insensitive). Ignores comments, `Sitemap`,
* `Crawl-delay`, and any other directives.
*
* @since 1.3.0
*
* @param string $content Raw robots.txt file content.
* @return array{user_agent_rules: array>}
*/
public static function parse_robots_txt( $content ) {
$result = array( 'user_agent_rules' => array() );
$current_agents = array();
$lines = preg_split( '/\r\n|\r|\n/', $content );
if ( ! is_array( $lines ) ) {
return $result;
}
foreach ( $lines as $raw_line ) {
// Strip inline comments and surrounding whitespace.
$line = trim( (string) $raw_line );
// Skip blank lines and full-line comments.
if ( '' === $line || '#' === $line[0] ) {
// A blank line signals the end of the current group.
if ( '' === $line ) {
$current_agents = array();
}
continue;
}
// Split into directive and value at the first colon.
$colon_pos = strpos( $line, ':' );
if ( false === $colon_pos ) {
continue;
}
$directive = strtolower( trim( substr( $line, 0, $colon_pos ) ) );
$value = trim( substr( $line, $colon_pos + 1 ) );
// Strip inline comments from the value.
$hash_pos = strpos( $value, '#' );
if ( false !== $hash_pos ) {
$value = trim( substr( $value, 0, $hash_pos ) );
}
switch ( $directive ) {
case 'user-agent':
if ( '' !== $value ) {
$current_agents[] = $value;
// Initialise the agent's rule list if not yet set.
if ( ! isset( $result['user_agent_rules'][ $value ] ) ) {
$result['user_agent_rules'][ $value ] = array();
}
}
break;
case 'disallow':
case 'allow':
if ( ! empty( $current_agents ) ) {
$entry = array(
'directive' => ucfirst( $directive ), // Normalise to 'Disallow'/'Allow'.
'path' => $value,
);
foreach ( $current_agents as $agent ) {
$result['user_agent_rules'][ $agent ][] = $entry;
}
}
break;
// Silently ignore Sitemap, Crawl-delay, and anything else.
default:
break;
}
}
return $result;
}
// -------------------------------------------------------------------------
// Access Check
// -------------------------------------------------------------------------
/**
* Determine whether a specific bot user-agent is allowed or blocked.
*
* Checks the bot's own named rules first; falls back to wildcard `*` rules
* when no specific entry exists. The effective access is determined by the
* standard robots.txt precedence: `Allow` beats `Disallow` for the same path.
*
* @since 1.3.0
*
* @param array $rules Parsed rules from {@see parse_robots_txt()}.
* @param string $bot_ua Bot user-agent token (e.g. 'GPTBot').
* @return string 'blocked' or 'allowed'.
*/
public static function check_bot_access( $rules, $bot_ua ) {
$ua_rules = isset( $rules['user_agent_rules'] ) ? $rules['user_agent_rules'] : array();
// Build the effective rule list: specific rules take precedence over wildcard.
$effective_rules = array();
// Case-insensitive key look-up for the specific bot.
foreach ( $ua_rules as $agent => $agent_rules ) {
if ( 0 === strcasecmp( $agent, $bot_ua ) ) {
$effective_rules = $agent_rules;
break;
}
}
// Fallback to wildcard rules when no specific rules exist.
if ( empty( $effective_rules ) && isset( $ua_rules['*'] ) ) {
$effective_rules = $ua_rules['*'];
}
if ( empty( $effective_rules ) ) {
return 'allowed';
}
// Check for full-site block: Disallow: / with no compensating Allow: /
$has_full_disallow = false;
$has_full_allow = false;
foreach ( $effective_rules as $rule ) {
if ( 'Disallow' === $rule['directive'] && '/' === $rule['path'] ) {
$has_full_disallow = true;
}
if ( 'Allow' === $rule['directive'] && '/' === $rule['path'] ) {
$has_full_allow = true;
}
}
if ( $has_full_disallow && ! $has_full_allow ) {
return 'blocked';
}
return 'allowed';
}
// -------------------------------------------------------------------------
// Alert Generation
// -------------------------------------------------------------------------
/**
* Generate actionable alerts by cross-referencing parsed rules with settings.
*
* Checks for three categories of conflict:
* 1. An enabled bot is blocked while the crawler policy is permissive.
* 2. A wildcard `Disallow: /` blocks all crawlers with no specific AI allows.
* 3. One or more active AI discovery file paths are blocked by Disallow rules.
*
* @since 1.3.0
*
* @param array $rules Parsed rules from {@see parse_robots_txt()}.
* @return array
*/
public static function generate_alerts( $rules ) {
$alerts = array();
$settings = AIDF_Plugin::get_settings();
$crawler_policy = isset( $settings['crawler_policy'] ) ? $settings['crawler_policy'] : 'allow_known';
$enabled_bot_keys = isset( $settings['crawler_enabled_bots'] ) ? (array) $settings['crawler_enabled_bots'] : array();
$all_bots = AIDF_Crawler_Registry::get_bots();
$permissions_url = admin_url( 'admin.php?page=ai-discovery-files&tab=permissions' );
// ------------------------------------------------------------------
// 1. Bot-level block conflicts.
// ------------------------------------------------------------------
if ( in_array( $crawler_policy, array( 'allow_all', 'allow_known' ), true ) ) {
foreach ( $enabled_bot_keys as $bot_key ) {
if ( ! isset( $all_bots[ $bot_key ] ) ) {
continue;
}
$bot = $all_bots[ $bot_key ];
$bot_ua = $bot['ua'];
$access = self::check_bot_access( $rules, $bot_ua );
if ( 'blocked' === $access ) {
$alerts[] = array(
'type' => 'warning',
'message' => sprintf(
/* translators: 1: Bot name (e.g. GPTBot), 2: Policy label (e.g. Allow All) */
__( 'Your robots.txt blocks %1$s, but your Crawler Policy is set to “%2$s”.', 'ai-discovery-files' ),
esc_html( $bot['name'] ),
esc_html( self::get_policy_label( $crawler_policy ) )
),
'action_url' => $permissions_url,
'action_label' => __( 'Review Permissions', 'ai-discovery-files' ),
);
}
}
}
// ------------------------------------------------------------------
// 2. Wildcard Disallow: / with no AI-specific allow rules.
// ------------------------------------------------------------------
$ua_rules = isset( $rules['user_agent_rules'] ) ? $rules['user_agent_rules'] : array();
$wildcard_rules = isset( $ua_rules['*'] ) ? $ua_rules['*'] : array();
$wildcard_blocks_all = false;
foreach ( $wildcard_rules as $rule ) {
if ( 'Disallow' === $rule['directive'] && '/' === $rule['path'] ) {
$wildcard_blocks_all = true;
break;
}
}
if ( $wildcard_blocks_all ) {
// Determine whether any known AI bot has its own explicit rules.
$has_specific_ai_rules = false;
foreach ( $all_bots as $bot ) {
foreach ( $ua_rules as $agent => $agent_rules ) {
if ( 0 === strcasecmp( $agent, $bot['ua'] ) && ! empty( $agent_rules ) ) {
$has_specific_ai_rules = true;
break 2;
}
}
}
if ( ! $has_specific_ai_rules ) {
$alerts[] = array(
'type' => 'warning',
'message' => __( 'Your robots.txt has a wildcard Disallow: / with no specific allow rules for AI crawlers. All bots — including AI assistants — are blocked.', 'ai-discovery-files' ),
'action_url' => $permissions_url,
'action_label' => __( 'Review Permissions', 'ai-discovery-files' ),
);
}
}
// ------------------------------------------------------------------
// 3. Active AI discovery file paths blocked by Disallow rules.
// ------------------------------------------------------------------
$active_files = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array();
$all_types = AIDF_Plugin::get_file_types();
$blocked_files = array();
foreach ( $active_files as $file_key ) {
if ( ! isset( $all_types[ $file_key ] ) ) {
continue;
}
$filename = $all_types[ $file_key ]['filename'];
$file_path = '/' . ltrim( $filename, '/' );
if ( self::is_path_blocked( $ua_rules, $file_path ) ) {
$blocked_files[] = $filename;
}
}
if ( ! empty( $blocked_files ) ) {
$alerts[] = array(
'type' => 'error',
'message' => sprintf(
/* translators: %s: comma-separated list of blocked file names */
_n(
'Your robots.txt blocks the following AI discovery file: %s. AI crawlers may not be able to read it.',
'Your robots.txt blocks the following AI discovery files: %s. AI crawlers may not be able to read them.',
count( $blocked_files ),
'ai-discovery-files'
),
'' . implode( ', ', array_map( 'esc_html', $blocked_files ) ) . ''
),
'action_url' => '',
'action_label' => '',
);
}
return $alerts;
}
// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------
/**
* Determine whether any agent's Disallow rules cover a given path.
*
* Checks both the wildcard `*` group and any agent-specific groups that
* have `Disallow` for the exact path (or a parent path via prefix match).
* An `Allow` for the same or more-specific path takes precedence.
*
* @since 1.3.0
*
* @param array> $ua_rules Parsed user-agent rules.
* @param string $file_path Absolute path starting with '/'.
* @return bool True if the path is effectively blocked for at least one agent group.
*/
private static function is_path_blocked( $ua_rules, $file_path ) {
// Check wildcard rules.
if ( isset( $ua_rules['*'] ) ) {
if ( self::rules_block_path( $ua_rules['*'], $file_path ) ) {
return true;
}
}
// Check every other agent group for good measure.
foreach ( $ua_rules as $agent => $rules ) {
if ( '*' === $agent ) {
continue;
}
if ( self::rules_block_path( $rules, $file_path ) ) {
return true;
}
}
return false;
}
/**
* Evaluate a flat rule list against a single path.
*
* Applies standard robots.txt precedence: Allow beats Disallow when both
* match the same path. Uses prefix matching (a rule for `/` blocks everything,
* a rule for `/llms` would block `/llms.txt`).
*
* @since 1.3.0
*
* @param array $rules Flat list of directive/path pairs.
* @param string $file_path Absolute path to check.
* @return bool True if effectively blocked.
*/
private static function rules_block_path( $rules, $file_path ) {
$is_disallowed = false;
foreach ( $rules as $rule ) {
$rule_path = $rule['path'];
// An empty Disallow means "allow everything" — skip it.
if ( '' === $rule_path ) {
continue;
}
// Prefix match: rule '/llms' covers '/llms.txt'.
if ( 0 === strpos( $file_path, $rule_path ) ) {
if ( 'Disallow' === $rule['directive'] ) {
$is_disallowed = true;
} elseif ( 'Allow' === $rule['directive'] ) {
// An explicit Allow overrides any Disallow.
return false;
}
}
}
return $is_disallowed;
}
/**
* Return a human-readable label for a crawler_policy value.
*
* @since 1.3.0
*
* @param string $policy Policy key: 'allow_all', 'allow_known', or 'restrict'.
* @return string Translated label.
*/
private static function get_policy_label( $policy ) {
$labels = array(
'allow_all' => __( 'Allow All', 'ai-discovery-files' ),
'allow_known' => __( 'Allow Known', 'ai-discovery-files' ),
'restrict' => __( 'Restrict', 'ai-discovery-files' ),
);
return isset( $labels[ $policy ] ) ? $labels[ $policy ] : $policy;
}
}