this, switch to chunked mode.
const CHUNK_SIZE = 8192; // 8 KB per chunk.
const CHUNK_OVERLAP = 500; // 500 bytes of overlap between consecutive chunks.
const MAX_CHUNKS = 5; // never process more than this many chunks.
/**
* Register the AJAX endpoints.
*/
public static function init() {
add_action( 'wp_ajax_aidf_suggest_url', array( __CLASS__, 'ajax_suggest_url' ) );
add_action( 'wp_ajax_aidf_import_extract', array( __CLASS__, 'ajax_import_extract' ) );
}
/**
* AJAX: suggest a hint URL for the requested ability.
*/
public static function ajax_suggest_url() {
check_ajax_referer( self::NONCE_ACTION, 'nonce' );
if ( ! current_user_can( 'manage_options' ) ) {
wp_send_json_error( array( 'message' => __( 'Permission denied.', 'ai-discovery-files' ) ), 403 );
}
$slug = isset( $_POST['ability'] ) ? sanitize_text_field( wp_unslash( $_POST['ability'] ) ) : '';
if ( '' === $slug ) {
wp_send_json_error( array( 'message' => __( 'Missing ability.', 'ai-discovery-files' ) ), 400 );
}
$url = self::suggest_url_for_ability( $slug );
wp_send_json_success( array( 'url' => $url ) );
}
/**
* AJAX: fetch URL, extract content, run ability (possibly chunked), return cleaned + aggregated result.
*/
public static function ajax_import_extract() {
check_ajax_referer( self::NONCE_ACTION, 'nonce' );
if ( ! current_user_can( 'manage_options' ) ) {
wp_send_json_error( array( 'message' => __( 'Permission denied.', 'ai-discovery-files' ) ), 403 );
}
$slug = isset( $_POST['ability'] ) ? sanitize_text_field( wp_unslash( $_POST['ability'] ) ) : '';
$url = isset( $_POST['url'] ) ? esc_url_raw( wp_unslash( $_POST['url'] ) ) : '';
if ( '' === $slug || '' === $url ) {
wp_send_json_error( array( 'message' => __( 'Missing ability or URL.', 'ai-discovery-files' ) ), 400 );
}
if ( ! AIDF_AI_Extract::is_available() ) {
wp_send_json_error( array( 'message' => __( 'AI Import is not available on this site.', 'ai-discovery-files' ) ), 400 );
}
// Same-network URL gate. Rejects competitor.com etc.
if ( ! self::is_same_network_url( $url ) ) {
wp_send_json_error(
array(
'message' => __( 'For your security, this plugin can only import from your own site (or another site in this WordPress network). The URL you entered points to a different domain.', 'ai-discovery-files' ),
),
400
);
}
// Throttle.
$throttle_key = 'aidf_import_' . get_current_user_id() . '_' . md5( $slug );
if ( get_transient( $throttle_key ) ) {
wp_send_json_error(
array(
'message' => sprintf(
/* translators: %d: number of seconds the user must wait before re-running an import. */
__( 'Please wait %d seconds before running this import again.', 'ai-discovery-files' ),
self::THROTTLE_WINDOW
),
),
429
);
}
set_transient( $throttle_key, 1, self::THROTTLE_WINDOW );
// Fetch + reduce. Returns either a single string (small page) or an array of chunks (long page).
$reduced = self::fetch_and_reduce( $url );
if ( is_wp_error( $reduced ) ) {
wp_send_json_error( array( 'message' => $reduced->get_error_message() ), 400 );
}
// Call the ability.
if ( ! function_exists( 'wp_get_ability' ) ) {
wp_send_json_error(
array( 'message' => __( 'WordPress AI Abilities API is not available on this site. Update to WordPress 7.0 or later.', 'ai-discovery-files' ) ),
400
);
}
$ability = wp_get_ability( $slug );
if ( ! $ability ) {
wp_send_json_error(
array(
'message' => sprintf(
/* translators: 1: ability slug, 2: diagnostic summary of the registration attempt. */
__( 'Could not load AI ability "%1$s". Diagnostic: %2$s', 'ai-discovery-files' ),
$slug,
AIDF_AI_Extract::debug_summary()
),
),
400
);
}
// Decide chunking strategy per ability.
$chunks = is_array( $reduced ) ? $reduced : array( $reduced );
$use_chunks = self::ability_supports_chunks( $slug ) ? $chunks : array( $chunks[0] );
$chunk_count = count( $use_chunks );
$aggregated = null;
$aggregated_flags = array();
$total_tokens = 0;
foreach ( $use_chunks as $i => $chunk_text ) {
// $ability->execute() returns the schema-compliant output (e.g. { items: [...] }
// or { paragraph: "..." }). Fingerprint flags and token usage are stashed on
// AIDF_AI_Extract as a side channel because the Abilities API validates
// the return shape against the declared output_schema.
$result = $ability->execute(
array(
'content' => $chunk_text,
'source_url' => $url,
)
);
if ( is_wp_error( $result ) ) {
// First chunk failure aborts; later chunk failure is logged but does not destroy work so far.
if ( 0 === $i ) {
wp_send_json_error( array( 'message' => $result->get_error_message() ), 500 );
}
continue;
}
$chunk_flags = AIDF_AI_Extract::consume_last_flags();
$chunk_usage = AIDF_AI_Extract::consume_last_token_usage();
$aggregated = self::merge_chunk_result( $slug, $aggregated, $result );
if ( ! empty( $chunk_flags ) ) {
foreach ( $chunk_flags as $path => $flags_arr ) {
$aggregated_flags[ 'chunk' . $i . '.' . $path ] = $flags_arr;
}
}
if ( isset( $chunk_usage['total_tokens'] ) ) {
$total_tokens += (int) $chunk_usage['total_tokens'];
}
}
// Persist last-used URL if the setting is on.
self::maybe_remember_url( $slug, $url );
// Bump telemetry.
$count = (int) get_option( 'aidf_ai_import_count', 0 );
update_option( 'aidf_ai_import_count', $count + 1, false );
update_option( 'aidf_ai_import_last_run', time(), false );
wp_send_json_success(
array(
'data' => $aggregated,
'flags' => $aggregated_flags,
'token_usage' => array( 'total_tokens' => $total_tokens ),
'chunks_used' => $chunk_count,
)
);
}
/**
* Whether an ability merges results across multiple chunks.
*
* @param string $slug Ability slug.
* @return bool
*/
private static function ability_supports_chunks( $slug ) {
$chunked_abilities = array(
'ai-discovery-files/extract-faq',
'ai-discovery-files/extract-key-people',
'ai-discovery-files/extract-services',
);
return in_array( $slug, $chunked_abilities, true );
}
/**
* Merge a chunk's ability result into the accumulator.
*
* @param string $slug Ability slug.
* @param array|null $acc Accumulator (null on first chunk).
* @param array $result Ability result for current chunk.
* @return array Merged accumulator.
*/
private static function merge_chunk_result( $slug, $acc, $result ) {
// $result is the schema-compliant ability output directly (no wrapper key).
$data = is_array( $result ) ? $result : array();
// Single-shot abilities (boilerplate, taglines, socials, brand-voice):
// accumulator = first chunk verbatim. Later chunks are ignored because
// these read header/footer/first-page content only.
$single_shot = array(
'ai-discovery-files/extract-boilerplate',
'ai-discovery-files/extract-taglines',
'ai-discovery-files/extract-socials',
'ai-discovery-files/extract-brand-voice',
);
if ( in_array( $slug, $single_shot, true ) ) {
return null === $acc ? $data : $acc;
}
// FAQ / Key People / Services: append items, dedupe.
$existing_items = ( null !== $acc && isset( $acc['items'] ) && is_array( $acc['items'] ) ) ? $acc['items'] : array();
$new_items = isset( $data['items'] ) && is_array( $data['items'] ) ? $data['items'] : array();
$key_field = self::dedup_key_for( $slug );
foreach ( $new_items as $item ) {
if ( self::item_key_seen( $existing_items, $item, $key_field ) ) {
continue;
}
$existing_items[] = $item;
}
return array( 'items' => $existing_items );
}
/**
* Return the field name used to dedupe items between chunks.
*
* @param string $slug Ability slug.
* @return string Field name (e.g. 'q', 'name').
*/
private static function dedup_key_for( $slug ) {
switch ( $slug ) {
case 'ai-discovery-files/extract-faq':
return 'q';
case 'ai-discovery-files/extract-key-people':
case 'ai-discovery-files/extract-services':
return 'name';
default:
return '';
}
}
/**
* Whether an item's key-field value is already present in a list.
* Comparison is case-insensitive and trimmed.
*
* @param array $list Existing items.
* @param array $item Candidate item.
* @param string $key_field Field to compare.
* @return bool
*/
private static function item_key_seen( $list, $item, $key_field ) {
if ( '' === $key_field || ! isset( $item[ $key_field ] ) ) {
return false;
}
$needle = strtolower( trim( (string) $item[ $key_field ] ) );
if ( '' === $needle ) {
return false;
}
foreach ( $list as $existing ) {
if ( isset( $existing[ $key_field ] ) && strtolower( trim( (string) $existing[ $key_field ] ) ) === $needle ) {
return true;
}
}
return false;
}
/**
* Validate that a URL belongs to the current WP site or its network.
*
* Accepted hosts:
* 1. The host of home_url() (current site).
* 2. On multisite, any host returned by get_sites() (network).
* 3. A subdomain of, or a parent domain of, the current host.
*
* @param string $url URL to validate.
* @return bool
*/
public static function is_same_network_url( $url ) {
$url_host = wp_parse_url( $url, PHP_URL_HOST );
if ( ! $url_host ) {
return false;
}
$url_host = strtolower( $url_host );
$current_host = strtolower( (string) wp_parse_url( home_url( '/' ), PHP_URL_HOST ) );
if ( '' === $current_host ) {
return false;
}
if ( self::hosts_match_or_subdomain( $url_host, $current_host ) ) {
return true;
}
if ( function_exists( 'is_multisite' ) && is_multisite() && function_exists( 'get_sites' ) ) {
$sites = get_sites( array( 'number' => 1000 ) );
foreach ( $sites as $site ) {
$site_host = strtolower( (string) ( isset( $site->domain ) ? $site->domain : '' ) );
if ( '' === $site_host ) {
continue;
}
if ( self::hosts_match_or_subdomain( $url_host, $site_host ) ) {
return true;
}
}
}
return false;
}
/**
* True if $a equals $b, or $a is a subdomain of $b, or $b is a subdomain of $a.
*
* @param string $a Lowercased host A.
* @param string $b Lowercased host B.
* @return bool
*/
private static function hosts_match_or_subdomain( $a, $b ) {
if ( $a === $b ) {
return true;
}
$suffix_a = '.' . $b;
$suffix_b = '.' . $a;
$len_sa = strlen( $suffix_a );
$len_sb = strlen( $suffix_b );
if ( strlen( $a ) > $len_sa && substr( $a, -$len_sa ) === $suffix_a ) {
return true;
}
if ( strlen( $b ) > $len_sb && substr( $b, -$len_sb ) === $suffix_b ) {
return true;
}
return false;
}
/**
* Suggest a hint URL for an ability slug. Honours last-used URL
* if the "Remember last-used hint URLs" setting is on and a URL
* was previously saved for this ability; otherwise falls back to
* a WP_Query best-guess based on keyword search.
*
* @param string $slug Ability slug.
* @return string Suggested URL (may be empty).
*/
private static function suggest_url_for_ability( $slug ) {
$remembered = self::remembered_url_for( $slug );
if ( '' !== $remembered ) {
return $remembered;
}
$search_terms = array(
'ai-discovery-files/extract-faq' => array( 'faq', 'frequently asked questions' ),
'ai-discovery-files/extract-key-people' => array( 'team', 'about us', 'leadership', 'people', 'staff' ),
'ai-discovery-files/extract-services' => array( 'services', 'what we do', 'offerings', 'products' ),
'ai-discovery-files/extract-boilerplate' => array( 'about', 'about us' ),
'ai-discovery-files/extract-taglines' => array(), // always homepage
'ai-discovery-files/extract-socials' => array(), // always homepage (socials in footer)
'ai-discovery-files/extract-brand-voice' => array( 'about', 'about us' ),
);
// Abilities that always read from the homepage (socials in footer, taglines in hero).
if ( in_array( $slug, array( 'ai-discovery-files/extract-taglines', 'ai-discovery-files/extract-socials' ), true ) ) {
return home_url( '/' );
}
if ( ! isset( $search_terms[ $slug ] ) ) {
return '';
}
// Special case: extract-boilerplate and extract-brand-voice both prefer
// the WP "About" page if one is configured under Settings > Reading.
if ( in_array( $slug, array( 'ai-discovery-files/extract-boilerplate', 'ai-discovery-files/extract-brand-voice' ), true ) ) {
$about_id = (int) get_option( 'page_about' );
if ( $about_id > 0 ) {
$about_url = get_permalink( $about_id );
if ( $about_url ) {
return $about_url;
}
}
}
foreach ( $search_terms[ $slug ] as $term ) {
$query = new WP_Query(
array(
'post_type' => array( 'page', 'post' ),
'post_status' => 'publish',
's' => $term,
'posts_per_page' => 1,
'no_found_rows' => true,
'fields' => 'ids',
)
);
if ( ! empty( $query->posts ) ) {
$permalink = get_permalink( $query->posts[0] );
if ( $permalink ) {
return $permalink;
}
}
}
return '';
}
/**
* Read the remembered URL for an ability slug from plugin settings.
*
* @param string $slug Ability slug.
* @return string Remembered URL, or empty string.
*/
private static function remembered_url_for( $slug ) {
$settings = AIDF_Plugin::get_settings();
if ( empty( $settings['ai_import_remember_urls'] ) ) {
return '';
}
$last = isset( $settings['ai_import_last_urls'] ) && is_array( $settings['ai_import_last_urls'] )
? $settings['ai_import_last_urls']
: array();
return isset( $last[ $slug ] ) ? (string) $last[ $slug ] : '';
}
/**
* If the "remember URLs" setting is on, persist this URL as the last-used
* URL for the given ability slug.
*
* @param string $slug Ability slug.
* @param string $url URL the user just used.
* @return void
*/
private static function maybe_remember_url( $slug, $url ) {
$settings = AIDF_Plugin::get_settings();
if ( empty( $settings['ai_import_remember_urls'] ) ) {
return;
}
$last = isset( $settings['ai_import_last_urls'] ) && is_array( $settings['ai_import_last_urls'] )
? $settings['ai_import_last_urls']
: array();
$last[ $slug ] = $url;
$settings['ai_import_last_urls'] = $last;
update_option( 'aidf_settings', $settings );
}
/**
* Fetch a URL via wp_remote_get(), strip to readable plain text. If the
* cleaned text is small (<= CHUNK_THRESHOLD), returns a single string.
* If it is large, returns an array of overlapping chunks (max MAX_CHUNKS).
*
* Returns WP_Error on fetch failure or if the page is too short.
*
* @param string $url URL to fetch.
* @return string|array|WP_Error
*/
private static function fetch_and_reduce( $url ) {
$response = wp_remote_get(
$url,
array(
'timeout' => 15,
'user-agent' => 'AIDF-Plugin/2.0 (+' . home_url( '/' ) . ')',
)
);
if ( is_wp_error( $response ) ) {
return new WP_Error( 'aidf_fetch_failed', __( 'We could not reach that URL. Check it is correct and accessible to your site.', 'ai-discovery-files' ) );
}
$code = wp_remote_retrieve_response_code( $response );
if ( $code < 200 || $code >= 300 ) {
return new WP_Error(
'aidf_bad_status',
sprintf(
/* translators: %d: HTTP status code returned by the URL the user entered (e.g. 404, 500). */
__( 'The URL returned HTTP %d. Check the address.', 'ai-discovery-files' ),
$code
)
);
}
$body = wp_remote_retrieve_body( $response );
if ( '' === $body ) {
return new WP_Error( 'aidf_empty_body', __( 'The URL returned an empty page.', 'ai-discovery-files' ) );
}
// Strip scripts/styles before tag-stripping so their content is removed.
$body = preg_replace( '##is', '', $body );
$body = preg_replace( '##is', '', $body );
$text = wp_strip_all_tags( $body, true );
$text = preg_replace( '/\s+/', ' ', $text );
$text = trim( $text );
if ( str_word_count( $text ) < self::MIN_CONTENT_WORDS ) {
return new WP_Error( 'aidf_too_short', __( 'There is not enough content on that page to extract from. Try a different URL.', 'ai-discovery-files' ) );
}
// Small page: return as a single string.
if ( strlen( $text ) <= self::CHUNK_THRESHOLD ) {
return $text;
}
// Long page: split into overlapping chunks.
return self::split_into_chunks( $text );
}
/**
* Split long cleaned text into overlapping chunks.
*
* @param string $text Full cleaned text.
* @return array Up to MAX_CHUNKS chunks.
*/
private static function split_into_chunks( $text ) {
$chunks = array();
$total = strlen( $text );
$start = 0;
$step = self::CHUNK_SIZE - self::CHUNK_OVERLAP;
while ( $start < $total && count( $chunks ) < self::MAX_CHUNKS ) {
$chunks[] = substr( $text, $start, self::CHUNK_SIZE );
$start += $step;
}
return $chunks;
}
}