$bot_data ) { $bot_signals = array(); // Values stored before 1.3.2 were not checked against the vocabulary, so normalize on the way out too. $bot_search = ayudawp_acs_sanitize_signal_value( $bot_data['search'] ?? '' ); $bot_ai_input = ayudawp_acs_sanitize_signal_value( $bot_data['ai_input'] ?? '' ); $bot_ai_train = ayudawp_acs_sanitize_signal_value( $bot_data['ai_train'] ?? '' ); if ( '' !== $bot_search ) { $bot_signals[] = 'search=' . $bot_search; } if ( '' !== $bot_ai_input ) { $bot_signals[] = 'ai-input=' . $bot_ai_input; } if ( '' !== $bot_ai_train ) { $bot_signals[] = 'ai-train=' . $bot_ai_train; } if ( ! empty( $bot_signals ) ) { $output .= 'User-Agent: ' . $bot . "\n"; $output .= 'Content-Signal: ' . implode( ', ', $bot_signals ) . "\n"; $output .= "Allow: /\n\n"; } } // Custom bots. $custom_bots = $settings['custom_bots'] ?? array(); foreach ( $custom_bots as $bot ) { if ( empty( $bot['name'] ) ) { continue; } $custom_signals = array(); // Use specific setting if set, otherwise use global. Anything outside the // vocabulary normalizes to '', which falls back to the global signal. $search_value = ayudawp_acs_sanitize_signal_value( $bot['search'] ?? '' ); $ai_input_value = ayudawp_acs_sanitize_signal_value( $bot['ai_input'] ?? '' ); $ai_train_value = ayudawp_acs_sanitize_signal_value( $bot['ai_train'] ?? '' ); $search_value = '' !== $search_value ? $search_value : $effective['search']; $ai_input_value = '' !== $ai_input_value ? $ai_input_value : $effective['ai_input']; $ai_train_value = '' !== $ai_train_value ? $ai_train_value : $effective['ai_train']; if ( '' !== $search_value ) { $custom_signals[] = 'search=' . $search_value; } if ( '' !== $ai_input_value ) { $custom_signals[] = 'ai-input=' . $ai_input_value; } if ( '' !== $ai_train_value ) { $custom_signals[] = 'ai-train=' . $ai_train_value; } if ( ! empty( $custom_signals ) ) { $output .= 'User-Agent: ' . $bot['name'] . "\n"; $output .= 'Content-Signal: ' . implode( ', ', $custom_signals ) . "\n"; $output .= "Allow: /\n\n"; } } return $output . "# END AI Content Signals\n"; } /** * Get legal text for Content Signals Policy * * @return string Legal text block as robots.txt comments. */ function ayudawp_acs_get_legal_text() { $text = "# As a condition of accessing this website, you agree to abide by the following content signals:\n"; $text .= "# (a) If a content-signal = yes, you may collect content for the corresponding use.\n"; $text .= "# (b) If a content-signal = no, you may not collect content for the corresponding use.\n"; $text .= "# (c) If the website operator does not include a content signal for a corresponding use,\n"; $text .= "# the website operator neither grants nor restricts permission via content signal\n"; $text .= "# with respect to the corresponding use.\n"; $text .= "#\n"; $text .= "# The content signals and their meanings are:\n"; $text .= "# search: building a search index and providing search results\n"; $text .= "# (e.g., returning hyperlinks and short excerpts from your website's contents).\n"; $text .= "# Search does not include providing AI-generated search summaries.\n"; $text .= "# ai-input: inputting content into one or more AI models\n"; $text .= "# (e.g., retrieval augmented generation, grounding, or other real-time\n"; $text .= "# taking of content for generative AI search answers).\n"; $text .= "# ai-train: training or fine-tuning AI models.\n"; $text .= "#\n"; $text .= "# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF RIGHTS\n"; $text .= "# UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED\n"; $text .= "# RIGHTS IN THE DIGITAL SINGLE MARKET.\n\n"; return $text; } /** * Create physical robots.txt with basic WordPress rules * * @return bool True on success, false on failure. */ function ayudawp_acs_create_physical_robots() { // The file is shared by the whole network, so the bar is the network // capability. Checked here and not only at the caller: every writer of this // file carries its own gate, so a future caller cannot bypass it. if ( ! ayudawp_acs_can_write_shared_robots() ) { return false; } if ( ! ayudawp_acs_init_filesystem() ) { return false; } global $wp_filesystem; $robots_file = ABSPATH . 'robots.txt'; // Don't create if already exists. if ( $wp_filesystem->exists( $robots_file ) ) { return false; } // Check if root directory is writable. if ( ! $wp_filesystem->is_writable( ABSPATH ) ) { return false; } // Basic WordPress robots.txt content. $content = "# Basic WordPress robots.txt\n"; $content .= "# Created by AI Content Signals\n\n"; $content .= "User-agent: *\n"; $content .= "Disallow: /wp-admin/\n"; $content .= "Allow: /wp-admin/admin-ajax.php\n"; $content .= "Disallow: /wp-includes/\n"; $content .= "Disallow: /wp-content/plugins/\n"; $content .= "Disallow: /wp-content/themes/\n"; $content .= "Disallow: /wp-content/cache/\n"; $content .= "Disallow: /readme.html\n"; $content .= "Disallow: /license.txt\n\n"; // Add sitemap if available (native sitemaps ship with WordPress 5.5.1+, the plugin minimum). $sitemap_url = get_sitemap_url( 'index' ); if ( $sitemap_url ) { $content .= 'Sitemap: ' . esc_url( $sitemap_url ) . "\n"; } // Write to file. return $wp_filesystem->put_contents( $robots_file, $content, FS_CHMOD_FILE ); } /** * Update physical robots.txt file if it exists * * @param array|null $settings Optional settings array; fetched from DB if null. * @return bool True on success, false on failure. */ function ayudawp_acs_update_physical_robots( $settings = null ) { // Same shared file, same bar. Note this runs on every save once the file // exists, whatever the create_physical_robots setting says: that setting // only governs creating the file, not rewriting it. if ( ! ayudawp_acs_can_write_shared_robots() ) { return false; } if ( ! ayudawp_acs_init_filesystem() ) { return false; } global $wp_filesystem; $robots_file = ABSPATH . 'robots.txt'; // If settings not provided, get from database. if ( null === $settings ) { $settings = get_option( 'ayudawp_acs_settings', array() ); } // Create physical robots.txt if option is enabled and file doesn't exist. if ( ! $wp_filesystem->exists( $robots_file ) ) { if ( ( $settings['create_physical_robots'] ?? 'no' ) === 'yes' ) { ayudawp_acs_create_physical_robots(); } else { return false; } } if ( ! $wp_filesystem->exists( $robots_file ) || ! $wp_filesystem->is_writable( $robots_file ) ) { return false; } // Read current content. $current_content = $wp_filesystem->get_contents( $robots_file ); if ( false === $current_content ) { return false; } // Remove ALL Content Signals blocks using BEGIN/END markers. // // Replace with a newline, never with an empty string. The leading \n* in the // pattern swallows the line breaks in front of the block, so when the block // sits between two others (a very common case: other plugins append their own // blocks to robots.txt too) removing it would glue the preceding line to the // following one. A directive followed by a comment on the same line still // parses as that directive, so the join can silently turn a harmless removal // into an active rule the site owner never wrote. // The blank line keeps the spacing the block had, and cannot pile up: the // leading \n* eats every previous blank line before putting this one back. $pattern = '/\n*# BEGIN AI Content Signals\n.*?# END AI Content Signals\n*/s'; $current_content = preg_replace( $pattern, "\n\n", $current_content ); // Trim any extra newlines. The leading ones only appear when the removed // block was at the very top of the file. $current_content = rtrim( ltrim( $current_content, "\n" ) ); // Generate new Content Signals. $content_signals = ayudawp_acs_generate_content_signals(); // Combine content, always leaving exactly one newline at the end so whatever // writes to robots.txt after us starts on its own line. $new_content = $current_content . "\n" . rtrim( $content_signals ) . "\n"; // Write to file. return $wp_filesystem->put_contents( $robots_file, $new_content, FS_CHMOD_FILE ); } /** * Tell a subsite administrator why the physical robots.txt did not change. * * Saying it beats failing silently: the settings themselves were saved, and * the signals do reach the crawlers through the virtual robots.txt, which is * per site. Only the shared file on disk is out of reach. * * Registered through add_settings_error(), which options.php stores in the * settings_errors transient before redirecting (wp-admin/options.php:367-376), * so the notice survives the redirect and shows on the settings page. * * @since 1.3.3 */ function ayudawp_acs_notify_network_only() { if ( ! function_exists( 'add_settings_error' ) ) { return; } // What the site actually serves decides which half of the story is true, so // check the file rather than assume the reassuring case. if ( ayudawp_acs_physical_robots_exists() ) { $message = __( 'Your settings were saved, but the robots.txt file was not changed: it is a physical file at the root of the network, shared by every site on it, so only a network administrator can write it. While that file exists it is what every site serves, so your Content Signals do not reach robots.txt. The HTML meta tag, HTTP header and TDMRep surfaces below are per site and do work. Ask your network administrator to update the shared file.', 'ai-content-signals' ); } else { $message = __( 'Your settings were saved. The physical robots.txt file was not created: it would sit at the root of the network, shared by every site on it, so only a network administrator can create it. Your Content Signals are served through this site own virtual robots.txt, which works normally.', 'ai-content-signals' ); } add_settings_error( 'ayudawp_acs', 'ayudawp_acs_robots_network_only', esc_html( $message ), 'warning' ); } /** * Hook to update robots.txt after settings are saved * * @param array $old_value Previous settings. * @param array $new_value New settings. */ function ayudawp_acs_update_after_save( $old_value, $new_value ) { // Bail before touching the filesystem at all. The write functions carry // their own gate, so this is not what makes the save safe; it saves the // pointless work on every subsite save, and it is where there is enough // context to explain the outcome on screen. if ( ! ayudawp_acs_can_write_shared_robots() ) { if ( ayudawp_acs_physical_robots_is_network_only() ) { ayudawp_acs_notify_network_only(); } return; } ayudawp_acs_update_physical_robots( $new_value ); } add_action( 'update_option_ayudawp_acs_settings', 'ayudawp_acs_update_after_save', 10, 2 ); /** * Hook to update robots.txt after settings are added for the first time * * @param string $option Option name. * @param mixed $value Option value. */ function ayudawp_acs_update_after_add( $option, $value ) { // Same reasoning as ayudawp_acs_update_after_save(). No notice here: this // fires on activation, where the administrator did not ask for anything to // be written and a warning would only be noise. if ( ! ayudawp_acs_can_write_shared_robots() ) { return; } ayudawp_acs_update_physical_robots( $value ); } add_action( 'add_option_ayudawp_acs_settings', 'ayudawp_acs_update_after_add', 10, 2 ); /** * Remove Content Signals from physical robots.txt * * Called from uninstall.php during plugin removal. * * @return bool True on success, false on failure. */ function ayudawp_acs_remove_from_physical_robots() { // Same shared file, same bar, with one documented exception: during // uninstall there is no user in context (WP-CLI, cron), and core has // already authorised the actor, because delete_plugins maps to // do_not_allow for anyone who is not a super admin on a network // (wp-includes/capabilities.php:619-643, WordPress 7.1). WP_UNINSTALL_PLUGIN // is defined by wp_uninstall_plugin() alone, never by a request. if ( ! defined( 'WP_UNINSTALL_PLUGIN' ) && ! ayudawp_acs_can_write_shared_robots() ) { return false; } if ( ! ayudawp_acs_init_filesystem() ) { return false; } global $wp_filesystem; $robots_file = ABSPATH . 'robots.txt'; if ( ! $wp_filesystem->exists( $robots_file ) || ! $wp_filesystem->is_writable( $robots_file ) ) { return false; } // Read current content. $current_content = $wp_filesystem->get_contents( $robots_file ); if ( false === $current_content ) { return false; } // Remove ALL Content Signals blocks using BEGIN/END markers. Replaced with a // newline rather than an empty string for the reason documented in // ayudawp_acs_update_physical_robots(): an empty string glues the line before // the block to the line after it. $pattern = '/\n*# BEGIN AI Content Signals\n.*?# END AI Content Signals\n*/s'; $new_content = preg_replace( $pattern, "\n\n", $current_content ); // Trim any extra newlines. $new_content = rtrim( ltrim( $new_content, "\n" ) ); // Our block was all the file had: delete it rather than leave an empty // robots.txt behind. A web server serves a physical robots.txt even when it // is empty, and that shadows the virtual one WordPress generates, so the site // would silently lose its core directives and its Sitemap line. if ( '' === $new_content ) { return (bool) $wp_filesystem->delete( $robots_file ); } // Leave exactly one newline at the end, so the next plugin that appends to // robots.txt starts on its own line. return $wp_filesystem->put_contents( $robots_file, $new_content . "\n", FS_CHMOD_FILE ); }