this, switch to chunked mode. const CHUNK_SIZE = 8192; // 8 KB per chunk. const CHUNK_OVERLAP = 500; // 500 bytes of overlap between consecutive chunks. const MAX_CHUNKS = 5; // never process more than this many chunks. /** * Register the AJAX endpoints. */ public static function init() { add_action( 'wp_ajax_aidf_suggest_url', array( __CLASS__, 'ajax_suggest_url' ) ); add_action( 'wp_ajax_aidf_import_extract', array( __CLASS__, 'ajax_import_extract' ) ); } /** * AJAX: suggest a hint URL for the requested ability. */ public static function ajax_suggest_url() { check_ajax_referer( self::NONCE_ACTION, 'nonce' ); if ( ! current_user_can( 'manage_options' ) ) { wp_send_json_error( array( 'message' => __( 'Permission denied.', 'ai-discovery-files' ) ), 403 ); } $slug = isset( $_POST['ability'] ) ? sanitize_text_field( wp_unslash( $_POST['ability'] ) ) : ''; if ( '' === $slug ) { wp_send_json_error( array( 'message' => __( 'Missing ability.', 'ai-discovery-files' ) ), 400 ); } $url = self::suggest_url_for_ability( $slug ); wp_send_json_success( array( 'url' => $url ) ); } /** * AJAX: fetch URL, extract content, run ability (possibly chunked), return cleaned + aggregated result. */ public static function ajax_import_extract() { check_ajax_referer( self::NONCE_ACTION, 'nonce' ); if ( ! current_user_can( 'manage_options' ) ) { wp_send_json_error( array( 'message' => __( 'Permission denied.', 'ai-discovery-files' ) ), 403 ); } $slug = isset( $_POST['ability'] ) ? sanitize_text_field( wp_unslash( $_POST['ability'] ) ) : ''; $url = isset( $_POST['url'] ) ? esc_url_raw( wp_unslash( $_POST['url'] ) ) : ''; if ( '' === $slug || '' === $url ) { wp_send_json_error( array( 'message' => __( 'Missing ability or URL.', 'ai-discovery-files' ) ), 400 ); } if ( ! AIDF_AI_Extract::is_available() ) { wp_send_json_error( array( 'message' => __( 'AI Import is not available on this site.', 'ai-discovery-files' ) ), 400 ); } // Same-network URL gate. Rejects competitor.com etc. if ( ! self::is_same_network_url( $url ) ) { wp_send_json_error( array( 'message' => __( 'For your security, this plugin can only import from your own site (or another site in this WordPress network). The URL you entered points to a different domain.', 'ai-discovery-files' ), ), 400 ); } // Throttle. $throttle_key = 'aidf_import_' . get_current_user_id() . '_' . md5( $slug ); if ( get_transient( $throttle_key ) ) { wp_send_json_error( array( 'message' => sprintf( /* translators: %d: number of seconds the user must wait before re-running an import. */ __( 'Please wait %d seconds before running this import again.', 'ai-discovery-files' ), self::THROTTLE_WINDOW ), ), 429 ); } set_transient( $throttle_key, 1, self::THROTTLE_WINDOW ); // Fetch + reduce. Returns either a single string (small page) or an array of chunks (long page). $reduced = self::fetch_and_reduce( $url ); if ( is_wp_error( $reduced ) ) { wp_send_json_error( array( 'message' => $reduced->get_error_message() ), 400 ); } // Call the ability. if ( ! function_exists( 'wp_get_ability' ) ) { wp_send_json_error( array( 'message' => __( 'WordPress AI Abilities API is not available on this site. Update to WordPress 7.0 or later.', 'ai-discovery-files' ) ), 400 ); } $ability = wp_get_ability( $slug ); if ( ! $ability ) { wp_send_json_error( array( 'message' => sprintf( /* translators: 1: ability slug, 2: diagnostic summary of the registration attempt. */ __( 'Could not load AI ability "%1$s". Diagnostic: %2$s', 'ai-discovery-files' ), $slug, AIDF_AI_Extract::debug_summary() ), ), 400 ); } // Decide chunking strategy per ability. $chunks = is_array( $reduced ) ? $reduced : array( $reduced ); $use_chunks = self::ability_supports_chunks( $slug ) ? $chunks : array( $chunks[0] ); $chunk_count = count( $use_chunks ); $aggregated = null; $aggregated_flags = array(); $total_tokens = 0; foreach ( $use_chunks as $i => $chunk_text ) { // $ability->execute() returns the schema-compliant output (e.g. { items: [...] } // or { paragraph: "..." }). Fingerprint flags and token usage are stashed on // AIDF_AI_Extract as a side channel because the Abilities API validates // the return shape against the declared output_schema. $result = $ability->execute( array( 'content' => $chunk_text, 'source_url' => $url, ) ); if ( is_wp_error( $result ) ) { // First chunk failure aborts; later chunk failure is logged but does not destroy work so far. if ( 0 === $i ) { wp_send_json_error( array( 'message' => $result->get_error_message() ), 500 ); } continue; } $chunk_flags = AIDF_AI_Extract::consume_last_flags(); $chunk_usage = AIDF_AI_Extract::consume_last_token_usage(); $aggregated = self::merge_chunk_result( $slug, $aggregated, $result ); if ( ! empty( $chunk_flags ) ) { foreach ( $chunk_flags as $path => $flags_arr ) { $aggregated_flags[ 'chunk' . $i . '.' . $path ] = $flags_arr; } } if ( isset( $chunk_usage['total_tokens'] ) ) { $total_tokens += (int) $chunk_usage['total_tokens']; } } // Persist last-used URL if the setting is on. self::maybe_remember_url( $slug, $url ); // Bump telemetry. $count = (int) get_option( 'aidf_ai_import_count', 0 ); update_option( 'aidf_ai_import_count', $count + 1, false ); update_option( 'aidf_ai_import_last_run', time(), false ); wp_send_json_success( array( 'data' => $aggregated, 'flags' => $aggregated_flags, 'token_usage' => array( 'total_tokens' => $total_tokens ), 'chunks_used' => $chunk_count, ) ); } /** * Whether an ability merges results across multiple chunks. * * @param string $slug Ability slug. * @return bool */ private static function ability_supports_chunks( $slug ) { $chunked_abilities = array( 'ai-discovery-files/extract-faq', 'ai-discovery-files/extract-key-people', 'ai-discovery-files/extract-services', ); return in_array( $slug, $chunked_abilities, true ); } /** * Merge a chunk's ability result into the accumulator. * * @param string $slug Ability slug. * @param array|null $acc Accumulator (null on first chunk). * @param array $result Ability result for current chunk. * @return array Merged accumulator. */ private static function merge_chunk_result( $slug, $acc, $result ) { // $result is the schema-compliant ability output directly (no wrapper key). $data = is_array( $result ) ? $result : array(); // Single-shot abilities (boilerplate, taglines, socials, brand-voice): // accumulator = first chunk verbatim. Later chunks are ignored because // these read header/footer/first-page content only. $single_shot = array( 'ai-discovery-files/extract-boilerplate', 'ai-discovery-files/extract-taglines', 'ai-discovery-files/extract-socials', 'ai-discovery-files/extract-brand-voice', ); if ( in_array( $slug, $single_shot, true ) ) { return null === $acc ? $data : $acc; } // FAQ / Key People / Services: append items, dedupe. $existing_items = ( null !== $acc && isset( $acc['items'] ) && is_array( $acc['items'] ) ) ? $acc['items'] : array(); $new_items = isset( $data['items'] ) && is_array( $data['items'] ) ? $data['items'] : array(); $key_field = self::dedup_key_for( $slug ); foreach ( $new_items as $item ) { if ( self::item_key_seen( $existing_items, $item, $key_field ) ) { continue; } $existing_items[] = $item; } return array( 'items' => $existing_items ); } /** * Return the field name used to dedupe items between chunks. * * @param string $slug Ability slug. * @return string Field name (e.g. 'q', 'name'). */ private static function dedup_key_for( $slug ) { switch ( $slug ) { case 'ai-discovery-files/extract-faq': return 'q'; case 'ai-discovery-files/extract-key-people': case 'ai-discovery-files/extract-services': return 'name'; default: return ''; } } /** * Whether an item's key-field value is already present in a list. * Comparison is case-insensitive and trimmed. * * @param array $list Existing items. * @param array $item Candidate item. * @param string $key_field Field to compare. * @return bool */ private static function item_key_seen( $list, $item, $key_field ) { if ( '' === $key_field || ! isset( $item[ $key_field ] ) ) { return false; } $needle = strtolower( trim( (string) $item[ $key_field ] ) ); if ( '' === $needle ) { return false; } foreach ( $list as $existing ) { if ( isset( $existing[ $key_field ] ) && strtolower( trim( (string) $existing[ $key_field ] ) ) === $needle ) { return true; } } return false; } /** * Validate that a URL belongs to the current WP site or its network. * * Accepted hosts: * 1. The host of home_url() (current site). * 2. On multisite, any host returned by get_sites() (network). * 3. A subdomain of, or a parent domain of, the current host. * * @param string $url URL to validate. * @return bool */ public static function is_same_network_url( $url ) { $url_host = wp_parse_url( $url, PHP_URL_HOST ); if ( ! $url_host ) { return false; } $url_host = strtolower( $url_host ); $current_host = strtolower( (string) wp_parse_url( home_url( '/' ), PHP_URL_HOST ) ); if ( '' === $current_host ) { return false; } if ( self::hosts_match_or_subdomain( $url_host, $current_host ) ) { return true; } if ( function_exists( 'is_multisite' ) && is_multisite() && function_exists( 'get_sites' ) ) { $sites = get_sites( array( 'number' => 1000 ) ); foreach ( $sites as $site ) { $site_host = strtolower( (string) ( isset( $site->domain ) ? $site->domain : '' ) ); if ( '' === $site_host ) { continue; } if ( self::hosts_match_or_subdomain( $url_host, $site_host ) ) { return true; } } } return false; } /** * True if $a equals $b, or $a is a subdomain of $b, or $b is a subdomain of $a. * * @param string $a Lowercased host A. * @param string $b Lowercased host B. * @return bool */ private static function hosts_match_or_subdomain( $a, $b ) { if ( $a === $b ) { return true; } $suffix_a = '.' . $b; $suffix_b = '.' . $a; $len_sa = strlen( $suffix_a ); $len_sb = strlen( $suffix_b ); if ( strlen( $a ) > $len_sa && substr( $a, -$len_sa ) === $suffix_a ) { return true; } if ( strlen( $b ) > $len_sb && substr( $b, -$len_sb ) === $suffix_b ) { return true; } return false; } /** * Suggest a hint URL for an ability slug. Honours last-used URL * if the "Remember last-used hint URLs" setting is on and a URL * was previously saved for this ability; otherwise falls back to * a WP_Query best-guess based on keyword search. * * @param string $slug Ability slug. * @return string Suggested URL (may be empty). */ private static function suggest_url_for_ability( $slug ) { $remembered = self::remembered_url_for( $slug ); if ( '' !== $remembered ) { return $remembered; } $search_terms = array( 'ai-discovery-files/extract-faq' => array( 'faq', 'frequently asked questions' ), 'ai-discovery-files/extract-key-people' => array( 'team', 'about us', 'leadership', 'people', 'staff' ), 'ai-discovery-files/extract-services' => array( 'services', 'what we do', 'offerings', 'products' ), 'ai-discovery-files/extract-boilerplate' => array( 'about', 'about us' ), 'ai-discovery-files/extract-taglines' => array(), // always homepage 'ai-discovery-files/extract-socials' => array(), // always homepage (socials in footer) 'ai-discovery-files/extract-brand-voice' => array( 'about', 'about us' ), ); // Abilities that always read from the homepage (socials in footer, taglines in hero). if ( in_array( $slug, array( 'ai-discovery-files/extract-taglines', 'ai-discovery-files/extract-socials' ), true ) ) { return home_url( '/' ); } if ( ! isset( $search_terms[ $slug ] ) ) { return ''; } // Special case: extract-boilerplate and extract-brand-voice both prefer // the WP "About" page if one is configured under Settings > Reading. if ( in_array( $slug, array( 'ai-discovery-files/extract-boilerplate', 'ai-discovery-files/extract-brand-voice' ), true ) ) { $about_id = (int) get_option( 'page_about' ); if ( $about_id > 0 ) { $about_url = get_permalink( $about_id ); if ( $about_url ) { return $about_url; } } } foreach ( $search_terms[ $slug ] as $term ) { $query = new WP_Query( array( 'post_type' => array( 'page', 'post' ), 'post_status' => 'publish', 's' => $term, 'posts_per_page' => 1, 'no_found_rows' => true, 'fields' => 'ids', ) ); if ( ! empty( $query->posts ) ) { $permalink = get_permalink( $query->posts[0] ); if ( $permalink ) { return $permalink; } } } return ''; } /** * Read the remembered URL for an ability slug from plugin settings. * * @param string $slug Ability slug. * @return string Remembered URL, or empty string. */ private static function remembered_url_for( $slug ) { $settings = AIDF_Plugin::get_settings(); if ( empty( $settings['ai_import_remember_urls'] ) ) { return ''; } $last = isset( $settings['ai_import_last_urls'] ) && is_array( $settings['ai_import_last_urls'] ) ? $settings['ai_import_last_urls'] : array(); return isset( $last[ $slug ] ) ? (string) $last[ $slug ] : ''; } /** * If the "remember URLs" setting is on, persist this URL as the last-used * URL for the given ability slug. * * @param string $slug Ability slug. * @param string $url URL the user just used. * @return void */ private static function maybe_remember_url( $slug, $url ) { $settings = AIDF_Plugin::get_settings(); if ( empty( $settings['ai_import_remember_urls'] ) ) { return; } $last = isset( $settings['ai_import_last_urls'] ) && is_array( $settings['ai_import_last_urls'] ) ? $settings['ai_import_last_urls'] : array(); $last[ $slug ] = $url; $settings['ai_import_last_urls'] = $last; update_option( 'aidf_settings', $settings ); } /** * Fetch a URL via wp_remote_get(), strip to readable plain text. If the * cleaned text is small (<= CHUNK_THRESHOLD), returns a single string. * If it is large, returns an array of overlapping chunks (max MAX_CHUNKS). * * Returns WP_Error on fetch failure or if the page is too short. * * @param string $url URL to fetch. * @return string|array|WP_Error */ private static function fetch_and_reduce( $url ) { $response = wp_remote_get( $url, array( 'timeout' => 15, 'user-agent' => 'AIDF-Plugin/2.0 (+' . home_url( '/' ) . ')', ) ); if ( is_wp_error( $response ) ) { return new WP_Error( 'aidf_fetch_failed', __( 'We could not reach that URL. Check it is correct and accessible to your site.', 'ai-discovery-files' ) ); } $code = wp_remote_retrieve_response_code( $response ); if ( $code < 200 || $code >= 300 ) { return new WP_Error( 'aidf_bad_status', sprintf( /* translators: %d: HTTP status code returned by the URL the user entered (e.g. 404, 500). */ __( 'The URL returned HTTP %d. Check the address.', 'ai-discovery-files' ), $code ) ); } $body = wp_remote_retrieve_body( $response ); if ( '' === $body ) { return new WP_Error( 'aidf_empty_body', __( 'The URL returned an empty page.', 'ai-discovery-files' ) ); } // Strip scripts/styles before tag-stripping so their content is removed. $body = preg_replace( '#]*>.*?#is', '', $body ); $body = preg_replace( '#]*>.*?#is', '', $body ); $text = wp_strip_all_tags( $body, true ); $text = preg_replace( '/\s+/', ' ', $text ); $text = trim( $text ); if ( str_word_count( $text ) < self::MIN_CONTENT_WORDS ) { return new WP_Error( 'aidf_too_short', __( 'There is not enough content on that page to extract from. Try a different URL.', 'ai-discovery-files' ) ); } // Small page: return as a single string. if ( strlen( $text ) <= self::CHUNK_THRESHOLD ) { return $text; } // Long page: split into overlapping chunks. return self::split_into_chunks( $text ); } /** * Split long cleaned text into overlapping chunks. * * @param string $text Full cleaned text. * @return array Up to MAX_CHUNKS chunks. */ private static function split_into_chunks( $text ) { $chunks = array(); $total = strlen( $text ); $start = 0; $step = self::CHUNK_SIZE - self::CHUNK_OVERLAP; while ( $start < $total && count( $chunks ) < self::MAX_CHUNKS ) { $chunks[] = substr( $text, $start, self::CHUNK_SIZE ); $start += $step; } return $chunks; } }