+ the main
* article HTML. Admin-only (enforced by the REST permission callback).
*
* @package AICitare
*/
if ( ! defined( 'ABSPATH' ) ) {
exit;
}
class AICitare_Import {
public static function import( $url, $opts = array() ) {
$defaults = array(
'images' => true, // Download images into the media library.
'alt' => true, // Preserve original alt text on media.
'featured' => true, // Set the first image as the featured image.
'links' => 'nofollow', // External/affiliate links: keep | nofollow | strip.
);
$opts = wp_parse_args( is_array( $opts ) ? $opts : array(), $defaults );
if ( ! in_array( $opts['links'], array( 'keep', 'nofollow', 'strip' ), true ) ) {
$opts['links'] = 'nofollow';
}
$url = esc_url_raw( trim( (string) $url ) );
if ( ! $url || ! preg_match( '#^https?://#i', $url ) ) {
return new WP_Error( 'bad_url', 'Please enter a valid http(s) URL.' );
}
$res = wp_remote_get( $url, array(
'timeout' => 20,
'redirection' => 3,
'headers' => array( 'Accept' => 'text/html' ),
'user-agent' => 'GEO-Dashboard-Importer/1.0',
) );
if ( is_wp_error( $res ) ) {
return $res;
}
$code = (int) wp_remote_retrieve_response_code( $res );
if ( $code >= 400 ) {
return new WP_Error( 'fetch', 'Could not fetch URL (HTTP ' . $code . ').' );
}
$html = wp_remote_retrieve_body( $res );
if ( '' === trim( (string) $html ) ) {
return new WP_Error( 'empty', 'Empty response from URL.' );
}
$title = '';
$content = '';
$source = 'html';
// 1) Prefer the WordPress REST "alternate" link for clean content.
$api = self::find_rest_link( $html );
if ( $api ) {
$r2 = wp_remote_get( $api, array( 'timeout' => 20, 'redirection' => 3 ) );
if ( ! is_wp_error( $r2 ) && (int) wp_remote_retrieve_response_code( $r2 ) < 400 ) {
$j = json_decode( wp_remote_retrieve_body( $r2 ), true );
if ( isset( $j['title']['rendered'] ) ) {
$title = $j['title']['rendered'];
$content = isset( $j['content']['rendered'] ) ? $j['content']['rendered'] : '';
$source = 'wp-rest';
}
}
}
// 2) Fallback: Open Graph / + main article HTML.
if ( '' === $title ) {
if ( preg_match( '#<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)#i', $html, $m ) ) {
$title = $m[1];
} elseif ( preg_match( '#<title[^>]*>(.*?)#is', $html, $m ) ) {
$title = $m[1];
}
$content = self::extract_main( $html );
}
$title = trim( wp_strip_all_tags( html_entity_decode( $title, ENT_QUOTES ) ) );
if ( '' === $title ) {
$title = 'Imported post';
}
if ( '' === trim( wp_strip_all_tags( $content ) ) ) {
if ( preg_match( '#]+name=["\']description["\'][^>]+content=["\']([^"\']+)#i', $html, $m ) ) {
$content = '' . esc_html( html_entity_decode( $m[1], ENT_QUOTES ) ) . '
';
}
}
// Handle external/affiliate links per the user's choice.
$content = self::process_links( $content, $url, $opts['links'] );
$pid = wp_insert_post( array(
'post_title' => $title,
'post_content' => wp_kses_post( $content ),
'post_status' => 'draft',
'post_type' => 'post',
), true );
if ( is_wp_error( $pid ) ) {
return $pid;
}
update_post_meta( $pid, '_aicitare_import_source', $url );
// Download images into the media library and rewrite to local URLs.
$imgcount = 0;
if ( $opts['images'] ) {
$content2 = self::sideload_images( $content, $url, $pid, $imgcount, $opts );
if ( $content2 !== $content ) {
wp_update_post( array( 'ID' => $pid, 'post_content' => wp_kses_post( $content2 ) ) );
}
}
AICitare_Log::add( 'import', 'Imported draft #' . $pid . ' (' . $imgcount . ' images, links=' . $opts['links'] . ') from ' . $url );
return array(
'ok' => true,
'postId' => (int) $pid,
'title' => $title,
'source' => $source,
'images' => $imgcount,
'opts' => $opts,
'editUrl' => admin_url( 'post.php?post=' . $pid . '&action=edit' ),
'viewUrl' => get_permalink( $pid ),
);
}
/**
* Import every post found on a category / archive URL as drafts.
* Discovers post links via the WordPress REST API (by category/tag slug)
* and falls back to scraping the archive HTML. Skips already-imported URLs.
*/
public static function import_category( $url, $opts = array(), $limit = 10, $urls = null ) {
$limit = max( 1, min( 50, (int) $limit ) );
// If the caller passed an explicit list of selected URLs, import exactly those.
if ( is_array( $urls ) && ! empty( $urls ) ) {
$links = array();
foreach ( $urls as $u ) {
$u = esc_url_raw( trim( (string) $u ) );
if ( $u && preg_match( '#^https?://#i', $u ) ) {
$links[] = $u;
}
}
$links = array_slice( array_values( array_unique( $links ) ), 0, $limit );
if ( empty( $links ) ) {
return new WP_Error( 'none', 'No valid post URLs were selected.' );
}
} else {
$url = esc_url_raw( trim( (string) $url ) );
if ( ! $url || ! preg_match( '#^https?://#i', $url ) ) {
return new WP_Error( 'bad_url', 'Please enter a valid http(s) URL.' );
}
$items = self::discover_items( $url, $limit );
if ( is_wp_error( $items ) ) {
return $items;
}
$links = wp_list_pluck( $items, 'url' );
if ( empty( $links ) ) {
return new WP_Error( 'none', 'No posts found at that URL. Make sure it is a category or archive page.' );
}
}
@set_time_limit( 0 ); // phpcs:ignore
$items = array();
$imported = 0;
$skipped = 0;
foreach ( $links as $link ) {
if ( count( $items ) >= $limit ) {
break;
}
$exist = self::find_by_source( $link );
if ( $exist ) {
$items[] = array( 'url' => $link, 'status' => 'skipped', 'reason' => 'already imported', 'postId' => $exist, 'editUrl' => admin_url( 'post.php?post=' . $exist . '&action=edit' ) );
$skipped++;
continue;
}
$r = self::import( $link, $opts );
if ( is_wp_error( $r ) ) {
$items[] = array( 'url' => $link, 'status' => 'error', 'reason' => $r->get_error_message() );
continue;
}
$items[] = array( 'url' => $link, 'status' => 'ok', 'title' => $r['title'], 'postId' => $r['postId'], 'images' => $r['images'], 'editUrl' => $r['editUrl'] );
$imported++;
}
AICitare_Log::add( 'import', 'Bulk import from ' . $url . ': ' . $imported . ' imported, ' . $skipped . ' skipped.' );
return array(
'ok' => true,
'bulk' => true,
'source' => $url,
'found' => count( $links ),
'imported' => $imported,
'skipped' => $skipped,
'items' => $items,
);
}
/**
* Preview the posts on a category/archive URL without importing.
* Returns each post's url, title, and whether it was already imported.
*/
public static function preview_category( $url, $limit = 20 ) {
$url = esc_url_raw( trim( (string) $url ) );
if ( ! $url || ! preg_match( '#^https?://#i', $url ) ) {
return new WP_Error( 'bad_url', 'Please enter a valid http(s) URL.' );
}
$limit = max( 1, min( 50, (int) $limit ) );
$items = self::discover_items( $url, $limit );
if ( is_wp_error( $items ) ) {
return $items;
}
if ( empty( $items ) ) {
return new WP_Error( 'none', 'No posts found at that URL. Make sure it is a category or archive page.' );
}
foreach ( $items as &$it ) {
$it['imported'] = self::find_by_source( $it['url'] ) ? true : false;
}
unset( $it );
return array( 'ok' => true, 'source' => $url, 'found' => count( $items ), 'items' => $items );
}
/** Discover posts on a category/archive page. Returns [ {url, title} ] (REST first, HTML scrape fallback). */
private static function discover_items( $url, $limit ) {
$p = wp_parse_url( $url );
if ( empty( $p['scheme'] ) || empty( $p['host'] ) ) {
return array();
}
$root = $p['scheme'] . '://' . $p['host'] . ( isset( $p['port'] ) ? ':' . $p['port'] : '' );
$path = isset( $p['path'] ) ? trim( $p['path'], '/' ) : '';
$segs = ( '' === $path ) ? array() : explode( '/', $path );
$slug = $segs ? end( $segs ) : '';
// 1) WordPress REST API by category/tag slug.
if ( $slug ) {
foreach ( array( 'categories' => 'categories', 'tags' => 'tags' ) as $tax => $param ) {
$tres = wp_remote_get( $root . '/wp-json/wp/v2/' . $tax . '?slug=' . rawurlencode( $slug ), array( 'timeout' => 15 ) );
if ( is_wp_error( $tres ) || (int) wp_remote_retrieve_response_code( $tres ) >= 400 ) {
continue;
}
$tj = json_decode( wp_remote_retrieve_body( $tres ), true );
if ( is_array( $tj ) && isset( $tj[0]['id'] ) ) {
$tid = (int) $tj[0]['id'];
$pres = wp_remote_get( $root . '/wp-json/wp/v2/posts?' . $param . '=' . $tid . '&per_page=' . $limit . '&_fields=link,title', array( 'timeout' => 20 ) );
if ( ! is_wp_error( $pres ) ) {
$pj = json_decode( wp_remote_retrieve_body( $pres ), true );
$items = array();
if ( is_array( $pj ) ) {
foreach ( $pj as $row ) {
if ( empty( $row['link'] ) ) {
continue;
}
$title = isset( $row['title']['rendered'] ) ? trim( wp_strip_all_tags( html_entity_decode( $row['title']['rendered'], ENT_QUOTES ) ) ) : '';
$items[] = array( 'url' => $row['link'], 'title' => $title );
}
}
if ( $items ) {
return array_slice( $items, 0, $limit );
}
}
}
}
}
// 2) Fallback: scrape the archive HTML for post links (title derived from slug).
$res = wp_remote_get( $url, array( 'timeout' => 20, 'redirection' => 3, 'user-agent' => 'GEO-Dashboard-Importer/1.0' ) );
if ( is_wp_error( $res ) ) {
return $res;
}
$html = wp_remote_retrieve_body( $res );
$links = array_slice( self::extract_post_links( $html, $root, $url ), 0, $limit );
$items = array();
foreach ( $links as $l ) {
$lp = wp_parse_url( $l, PHP_URL_PATH );
$lsegs = $lp ? array_filter( explode( '/', trim( $lp, '/' ) ) ) : array();
$ls = $lsegs ? end( $lsegs ) : '';
$title = $ls ? ucwords( str_replace( array( '-', '_' ), ' ', $ls ) ) : '';
$items[] = array( 'url' => $l, 'title' => $title );
}
return $items;
}
/** Heuristic extraction of post permalinks from an archive page's HTML. */
private static function extract_post_links( $html, $root, $base ) {
$out = array();
if ( preg_match_all( '#<(?:h1|h2|h3)[^>]*>\s*]*href=["\']([^"\']+)["\']#i', $html, $m ) ) {
$out = array_merge( $out, $m[1] );
}
if ( preg_match_all( '#]*class=["\'][^"\']*(?:entry-title|post-title|read-more|more-link)[^"\']*["\'][^>]*href=["\']([^"\']+)["\']#i', $html, $m2 ) ) {
$out = array_merge( $out, $m2[1] );
}
if ( preg_match_all( '#href=["\']([^"\']+)["\'][^>]*class=["\'][^"\']*(?:entry-title|post-title|read-more|more-link)[^"\']*["\']#i', $html, $m3 ) ) {
$out = array_merge( $out, $m3[1] );
}
$host = wp_parse_url( $root, PHP_URL_HOST );
$basenrm = rtrim( preg_replace( '#\#.*$#', '', $base ), '/' );
$clean = array();
foreach ( $out as $h ) {
$abs = self::absolutize( html_entity_decode( $h, ENT_QUOTES ), $base );
if ( ! $abs ) {
continue;
}
$abs = preg_replace( '#\#.*$#', '', $abs );
$hh = wp_parse_url( $abs, PHP_URL_HOST );
if ( ! $hh || 0 !== strcasecmp( $hh, $host ) ) {
continue;
}
if ( preg_match( '#/(category|tag|author|page|wp-login|wp-admin|feed|comment)/#i', $abs ) ) {
continue;
}
$abs = rtrim( $abs, '/' ) . '/';
if ( rtrim( $abs, '/' ) === $basenrm ) {
continue; // The archive page itself.
}
$clean[ $abs ] = true; // Dedup by key, preserve order.
}
return array_keys( $clean );
}
/** Return the post ID previously imported from a given source URL, or 0. */
private static function find_by_source( $url ) {
global $wpdb;
$id = $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM {$wpdb->postmeta} WHERE meta_key = '_aicitare_import_source' AND meta_value = %s LIMIT 1", $url ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
return $id ? (int) $id : 0;
}
/**
* Process external links in imported content.
* keep = leave as-is; nofollow = add rel="nofollow sponsored"; strip = unwrap to plain text.
* Internal links (same host as source) are always left untouched.
*/
private static function process_links( $content, $base, $mode ) {
if ( 'keep' === $mode || '' === trim( (string) $content ) ) {
return $content;
}
$host = wp_parse_url( $base, PHP_URL_HOST );
return preg_replace_callback( '#(]*>)(.*?)#is', function ( $m ) use ( $host, $mode ) {
$open = $m[1];
$inner = $m[2];
if ( ! preg_match( '#href=["\']([^"\']+)["\']#i', $open, $hm ) ) {
return $m[0];
}
$href = html_entity_decode( $hm[1], ENT_QUOTES );
$h = wp_parse_url( $href, PHP_URL_HOST );
// Only act on absolute http(s) links to a different host.
if ( ! $h || ! preg_match( '#^https?://#i', $href ) || ( $host && 0 === strcasecmp( $h, $host ) ) ) {
return $m[0];
}
if ( 'strip' === $mode ) {
return $inner; // Keep the anchor text, drop the link.
}
// nofollow: rewrite the opening tag's rel attribute.
$o = preg_replace( '#\srel=["\'][^"\']*["\']#i', '', $open );
$o = preg_replace( '#';
}, $content );
}
/** Download ![]()
sources into the media library; rewrite content to local URLs. */
private static function sideload_images( $content, $base, $pid, &$count, $opts = array() ) {
$want_alt = ! isset( $opts['alt'] ) || $opts['alt'];
$want_feat = ! isset( $opts['featured'] ) || $opts['featured'];
$count = 0;
if ( ! preg_match_all( '#]*>#i', $content, $tags ) ) {
return $content;
}
require_once ABSPATH . 'wp-admin/includes/file.php';
require_once ABSPATH . 'wp-admin/includes/media.php';
require_once ABSPATH . 'wp-admin/includes/image.php';
$map = array();
$first_att = 0;
$max = 20; // safety cap.
foreach ( $tags[0] as $tag ) {
if ( $count >= $max ) {
break;
}
$src = self::img_src( $tag );
if ( ! $src ) {
continue;
}
$abs = self::absolutize( $src, $base );
if ( ! $abs || 0 === strpos( $abs, 'data:' ) ) {
continue;
}
if ( isset( $map[ $abs ] ) ) {
$local = $map[ $abs ];
} else {
$alt = $want_alt ? self::img_alt( $tag ) : '';
$att = media_sideload_image( $abs, $pid, $alt ? $alt : null, 'id' );
if ( is_wp_error( $att ) ) {
continue;
}
$local = wp_get_attachment_url( $att );
if ( ! $local ) {
continue;
}
// Preserve the original alt text on the media item (SEO / GEO signal).
if ( $want_alt && '' !== $alt ) {
update_post_meta( $att, '_wp_attachment_image_alt', sanitize_text_field( $alt ) );
}
$map[ $abs ] = $local;
$count++;
if ( ! $first_att ) {
$first_att = $att;
}
}
$content = str_replace( $tag, self::rewrite_tag( $tag, $local ), $content );
}
if ( $want_feat && $first_att && ! has_post_thumbnail( $pid ) ) {
set_post_thumbnail( $pid, $first_att );
}
return $content;
}
/** Pick the real image URL from an ![]()
tag (handles lazy-load + srcset). */
private static function img_src( $tag ) {
foreach ( array( 'data-src', 'data-lazy-src', 'data-original', 'data-lazy' ) as $a ) {
if ( preg_match( '#\s' . $a . '=["\']([^"\']+)["\']#i', $tag, $m ) ) {
return trim( html_entity_decode( $m[1], ENT_QUOTES ) );
}
}
foreach ( array( 'data-srcset', 'srcset' ) as $a ) {
if ( preg_match( '#\s' . $a . '=["\']([^"\']+)["\']#i', $tag, $m ) ) {
$first = trim( explode( ',', $m[1] )[0] );
$first = preg_split( '#\s+#', trim( $first ) )[0];
if ( $first ) {
return html_entity_decode( $first, ENT_QUOTES );
}
}
}
if ( preg_match( '#\ssrc=["\']([^"\']+)["\']#i', $tag, $m ) ) {
$s = trim( $m[1] );
if ( 0 !== strpos( $s, 'data:' ) && ! preg_match( '#(blank|placeholder|spacer|lazy|1x1|pixel)\.(gif|png|svg)#i', $s ) ) {
return html_entity_decode( $s, ENT_QUOTES );
}
}
return '';
}
/** Extract the alt text from an ![]()
tag. */
private static function img_alt( $tag ) {
if ( preg_match( '#\salt=["\']([^"\']*)["\']#i', $tag, $m ) ) {
return trim( html_entity_decode( $m[1], ENT_QUOTES ) );
}
return '';
}
/** Resolve a possibly-relative URL against the page URL. */
private static function absolutize( $u, $base ) {
$u = trim( $u );
if ( preg_match( '#^https?://#i', $u ) ) {
return $u;
}
$p = wp_parse_url( $base );
if ( empty( $p['scheme'] ) || empty( $p['host'] ) ) {
return '';
}
$origin = $p['scheme'] . '://' . $p['host'] . ( isset( $p['port'] ) ? ':' . $p['port'] : '' );
if ( 0 === strpos( $u, '//' ) ) {
return $p['scheme'] . ':' . $u;
}
if ( 0 === strpos( $u, '/' ) ) {
return $origin . $u;
}
$dir = isset( $p['path'] ) ? preg_replace( '#/[^/]*$#', '/', $p['path'] ) : '/';
return $origin . $dir . $u;
}
/** Swap an ![]()
's src to the local URL and strip lazy attributes. */
private static function rewrite_tag( $tag, $local ) {
$t = preg_replace( '#\s(?:data-src|data-lazy-src|data-original|data-lazy|data-srcset|srcset|sizes)=["\'][^"\']*["\']#i', '', $tag );
if ( preg_match( '#\ssrc=["\'][^"\']*["\']#i', $t ) ) {
$t = preg_replace( '#\ssrc=["\'][^"\']*["\']#i', ' src="' . esc_url( $local ) . '"', $t, 1 );
} else {
$t = preg_replace( '#. */
private static function find_rest_link( $html ) {
if ( preg_match( '#]+rel=["\']alternate["\'][^>]+type=["\']application/json["\'][^>]+href=["\']([^"\']+)["\']#i', $html, $m ) ) {
return html_entity_decode( $m[1], ENT_QUOTES );
}
if ( preg_match( '#]+type=["\']application/json["\'][^>]+rel=["\']alternate["\'][^>]+href=["\']([^"\']+)["\']#i', $html, $m ) ) {
return html_entity_decode( $m[1], ENT_QUOTES );
}
return '';
}
/** Best-effort main content extraction from arbitrary HTML. */
private static function extract_main( $html ) {
$h = preg_replace( '#]*>.*?#is', ' ', $html );
$h = preg_replace( '#]*>.*?#is', ' ', $h );
foreach ( array( 'article', 'main' ) as $tag ) {
if ( preg_match( '#<' . $tag . '\b[^>]*>(.*?)#is', $h, $m ) ) {
return $m[1];
}
}
if ( preg_match( '#]*>(.*?) #is', $h, $m ) ) {
return $m[1];
}
return '';
}
}