name. */ public static function assistants() { return [ 'chatgpt' => 'ChatGPT', 'gemini' => 'Gemini', 'perplexity' => 'Perplexity', 'copilot' => 'Copilot', 'claude' => 'Claude', 'meta' => 'Meta AI', 'grok' => 'Grok', 'deepseek' => 'DeepSeek', 'mistral' => 'Le Chat', 'other' => __('Other AI assistant', 'ai-flash-tune'), ]; } /** * AI crawlers and fetchers seen on the storefront. * purpose: search (puts the store in AI answers), user (fetches a page a * person asked about), training (collects text to train models). * * @return array key => label, owner, purpose, ua (regex fragment) */ public static function crawlers() { return [ 'oai-searchbot' => ['OAI-SearchBot', 'OpenAI', 'search', 'OAI-SearchBot'], 'chatgpt-user' => ['ChatGPT-User', 'OpenAI', 'user', 'ChatGPT-User'], 'gptbot' => ['GPTBot', 'OpenAI', 'training', 'GPTBot'], 'perplexitybot' => ['PerplexityBot', 'Perplexity', 'search', 'PerplexityBot'], 'perplexity-user' => ['Perplexity-User', 'Perplexity', 'user', 'Perplexity-User'], 'claude-searchbot' => ['Claude-SearchBot', 'Anthropic', 'search', 'Claude-SearchBot'], 'claude-user' => ['Claude-User', 'Anthropic', 'user', 'Claude-User'], 'claudebot' => ['ClaudeBot', 'Anthropic', 'training', 'ClaudeBot'], 'googlebot' => ['Googlebot', 'Google', 'search', 'Googlebot'], 'google-agent' => ['Google-Agent', 'Google', 'user', 'Google-Agent'], 'bingbot' => ['Bingbot', 'Microsoft', 'search', 'bingbot'], 'applebot' => ['Applebot', 'Apple', 'search', 'Applebot'], 'meta-fetcher' => ['meta-externalfetcher', 'Meta', 'user', 'meta-externalfetcher'], 'meta-agent' => ['meta-externalagent', 'Meta', 'training', 'meta-externalagent'], 'amazonbot' => ['Amazonbot', 'Amazon', 'search', 'Amazonbot'], 'duckassist' => ['DuckAssistBot', 'DuckDuckGo', 'search', 'DuckAssistBot'], 'bytespider' => ['Bytespider', 'ByteDance', 'training', 'Bytespider'], 'ccbot' => ['CCBot', 'Common Crawl', 'training', 'CCBot'], ]; } /** Crawlers whose block keeps products out of AI shopping answers. */ public static function search_crawler_tokens() { return ['OAI-SearchBot', 'ChatGPT-User', 'PerplexityBot', 'Claude-SearchBot', 'Googlebot', 'Bingbot', 'Applebot']; } // ------------------------------------------------------------------ // Sources // ------------------------------------------------------------------ /** A source key from the tracking script, or '' when it is not one we know. */ public static function normalize_source($raw) { $raw = strtolower(trim((string) $raw)); if (preg_match('/^ai:([a-z]{2,12})$/', $raw, $m)) { return isset(self::assistants()[$m[1]]) ? $raw : 'ai:other'; } return in_array($raw, ['search', 'paid', 'social', 'email', 'referral', 'direct'], true) ? $raw : ''; } public static function is_ai_source($source) { return strpos((string) $source, 'ai:') === 0; } /** "ChatGPT", "Organic search"... for a source key. */ public static function source_label($source) { $source = (string) $source; if (self::is_ai_source($source)) { return self::assistants()[substr($source, 3)] ?? self::assistants()['other']; } $labels = [ 'search' => __('Organic search', 'ai-flash-tune'), 'paid' => __('Paid ads', 'ai-flash-tune'), 'social' => __('Social', 'ai-flash-tune'), 'email' => __('Email', 'ai-flash-tune'), 'referral' => __('Another website', 'ai-flash-tune'), 'direct' => __('Direct', 'ai-flash-tune'), ]; return $labels[$source] ?? __('Unknown', 'ai-flash-tune'); } /** The source of a visit, from its funnel rows. */ public static function source_for_session($session_id) { global $wpdb; $table = $wpdb->prefix . 'aiflt_funnel_data'; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned table; the value is a placeholder. $source = (string) $wpdb->get_var($wpdb->prepare("SELECT source FROM {$table} WHERE session_id = %s AND source IS NOT NULL AND source != '' ORDER BY id ASC LIMIT 1", (string) $session_id)); return self::normalize_source($source); } // ------------------------------------------------------------------ // Automated visitors // ------------------------------------------------------------------ /** Whether this request comes from a script, crawler or AI fetcher rather than a person's browser. */ public static function is_automated_request() { // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Only matched against patterns. $ua = isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''; $automated = self::is_automated_ua($ua); /** * @param bool $automated * @param string $ua */ return (bool) apply_filters('aiflt_is_automated_request', $automated, $ua); } public static function is_automated_ua($ua) { $ua = trim((string) $ua); if ($ua === '') { return true; } if (self::crawler_for_ua($ua) !== '') { return true; } return (bool) preg_match('/bot\b|bot\/|crawler|spider|slurp|headless|phantomjs|puppeteer|playwright|selenium|webdriver|lighthouse|pagespeed|gtmetrix|pingdom|uptime|curl\/|wget\/|python-|python\/|httpclient|go-http|okhttp|java\/|axios\/|node-fetch|postman|scrapy|facebookexternalhit/i', $ua); } /** The key of a known AI crawler in this user agent, or ''. */ public static function crawler_for_ua($ua) { foreach (self::crawlers() as $key => $c) { if (stripos((string) $ua, $c[3]) !== false) { return $key; } } return ''; } /** Count one visit kept out of the reports. */ public static function count_filtered() { $all = get_option(self::FILTERED_OPTION, []); $all = is_array($all) ? $all : []; $day = current_time('Y-m-d'); $all[$day] = (int) ($all[$day] ?? 0) + 1; ksort($all); update_option(self::FILTERED_OPTION, array_slice($all, -self::KEEP_DAYS, null, true), false); } /** Automated visits kept out over the last $days days. */ public static function filtered_total($days = 30) { $all = get_option(self::FILTERED_OPTION, []); $from = wp_date('Y-m-d', current_time('timestamp') - ($days - 1) * DAY_IN_SECONDS, new DateTimeZone('UTC')); $n = 0; foreach ((array) $all as $day => $count) { if ($day >= $from) { $n += (int) $count; } } return $n; } // ------------------------------------------------------------------ // AI crawlers on the storefront // ------------------------------------------------------------------ public static function init() { add_action('template_redirect', [__CLASS__, 'notice_crawler'], 1); } /** * Note a known AI crawler on a storefront page. At most one write per * crawler per 10 minutes, so busy crawlers cost next to nothing. * Pages served from a full-page cache never reach PHP, so on cached sites * this undercounts; it still shows whether a crawler comes at all. */ public static function notice_crawler() { if (is_admin() || wp_doing_ajax() || wp_doing_cron() || (defined('REST_REQUEST') && REST_REQUEST)) { return; } // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Only matched against patterns. $key = self::crawler_for_ua(isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''); if ($key === '') { return; } self::record_crawler($key, time()); } public static function record_crawler($key, $now) { $all = get_option(self::CRAWLERS_OPTION, []); $all = is_array($all) ? $all : []; $bucket = (int) floor($now / (10 * MINUTE_IN_SECONDS)); $c = is_array($all[$key] ?? null) ? $all[$key] : ['bucket' => 0, 'last' => 0, 'days' => []]; if ((int) $c['bucket'] === $bucket) { return; } $day = wp_date('Y-m-d', $now); $c['bucket'] = $bucket; $c['last'] = $now; $c['days'][$day] = (int) ($c['days'][$day] ?? 0) + 1; ksort($c['days']); $c['days'] = array_slice($c['days'], -self::KEEP_DAYS, null, true); $all[$key] = $c; update_option(self::CRAWLERS_OPTION, $all, false); } /** * Each AI crawler seen: label, owner, purpose, last visit (Unix), and in how * many 10-minute windows it came over the last 7 and the 28 days before. */ public static function crawler_stats() { $all = get_option(self::CRAWLERS_OPTION, []); $out = []; $today = current_time('timestamp'); $d7 = wp_date('Y-m-d', $today - 6 * DAY_IN_SECONDS, new DateTimeZone('UTC')); $d35 = wp_date('Y-m-d', $today - 34 * DAY_IN_SECONDS, new DateTimeZone('UTC')); foreach (self::crawlers() as $key => $c) { $seen = is_array($all[$key] ?? null) ? $all[$key] : null; if (!$seen) { continue; } $recent = 0; $before = 0; foreach ((array) $seen['days'] as $day => $n) { if ($day >= $d7) { $recent += (int) $n; } elseif ($day >= $d35) { $before += (int) $n; } } $out[$key] = ['label' => $c[0], 'owner' => $c[1], 'purpose' => $c[2], 'last' => (int) $seen['last'], 'last_7' => $recent, 'before_28' => $before]; } return $out; } // ------------------------------------------------------------------ // robots.txt // ------------------------------------------------------------------ /** The robots.txt this site serves: the file when there is one, else WordPress' own. */ public static function robots_txt() { $file = ABSPATH . 'robots.txt'; if (is_readable($file)) { return (string) file_get_contents($file); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Local file. } $public = (string) get_option('blog_public'); $output = "User-agent: *\n"; if ($public === '0') { $output .= "Disallow: /\n"; } else { $path = (string) wp_parse_url(site_url(), PHP_URL_PATH); $output .= "Disallow: {$path}/wp-admin/\nAllow: {$path}/wp-admin/admin-ajax.php\n"; } // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress core hook. return (string) apply_filters('robots_txt', $output, $public); } /** * The crawlers in $tokens that robots.txt keeps off the whole site. * * @return string[] tokens blocked */ public static function robots_blocked(array $tokens = [], $robots = null) { $tokens = $tokens ?: self::search_crawler_tokens(); $robots = $robots === null ? self::robots_txt() : (string) $robots; // Groups: user agents followed by their rules. $groups = []; $current = null; $last_was_agent = false; foreach (preg_split('/\r\n|\r|\n/', $robots) as $line) { $line = trim(preg_replace('/#.*$/', '', $line)); if ($line === '' || strpos($line, ':') === false) { continue; } [$field, $value] = array_map('trim', explode(':', $line, 2)); $field = strtolower($field); if ($field === 'user-agent') { if (!$last_was_agent || $current === null) { $groups[] = ['agents' => [], 'rules' => []]; $current = count($groups) - 1; } $groups[$current]['agents'][] = strtolower($value); $last_was_agent = true; } elseif ($current !== null && in_array($field, ['disallow', 'allow'], true)) { $groups[$current]['rules'][] = [$field, $value]; $last_was_agent = false; } } $site_root = rtrim((string) wp_parse_url(home_url('/'), PHP_URL_PATH), '/') . '/'; $blocked = []; foreach ($tokens as $token) { $rules = null; foreach ($groups as $g) { foreach ($g['agents'] as $agent) { if ($agent !== '*' && stripos($token, $agent) === 0) { $rules = array_merge((array) $rules, $g['rules']); } } } if ($rules === null) { foreach ($groups as $g) { if (in_array('*', $g['agents'], true)) { $rules = array_merge((array) $rules, $g['rules']); } } } $disallowed = false; $allowed = false; foreach ((array) $rules as [$type, $path]) { if ($type === 'disallow' && ($path === '/' || $path === $site_root || $path === '/*')) { $disallowed = true; } if ($type === 'allow' && ($path === '/' || $path === $site_root)) { $allowed = true; } } if ($disallowed && !$allowed) { $blocked[] = $token; } } return $blocked; } }