create_log(
sprintf(
/* translators: %d: Post ID. */
esc_html__( 'Post ID %d has no source link, so its content cannot be fetched.', 'ai-content-writer' ),
absint( $post_id )
),
'error',
$campaign_id
);
return false;
}
$dom = new HtmlWeb();
$html = $dom->load( trailingslashit( $source_link ) );
if ( ! $html ) {
ai_content_writer()->create_log(
sprintf(
/* translators: 1: Source article URL. 2: Post ID. */
esc_html__( 'Could not fetch the source article %1$s for post ID %2$d. The page may be unreachable or blocking requests.', 'ai-content-writer' ),
esc_url( $source_link ),
absint( $post_id )
),
'error',
$campaign_id
);
return false;
}
$settings = aicw_get_campaign_image_settings( $campaign_id );
$with_images = 'source' === $settings['mode'];
$images = array();
$result = array(
'title' => '',
'content' => '',
);
$possible_title_selectors = array(
'h1.entry-title',
'h1.post-title',
'h1.article-title',
'article h1',
'.article-header h1',
'.post-header h1',
'header h1',
'h1',
'.headline',
'.title',
'#title',
);
foreach ( $possible_title_selectors as $selector ) {
$title_element = $html->find( $selector, 0 );
if ( $title_element ) {
$result['title'] = trim( $title_element->plaintext );
break;
}
}
if ( empty( $result['title'] ) ) {
$title_element = $html->find( 'title', 0 );
if ( $title_element ) {
$result['title'] = trim( $title_element->plaintext );
}
}
$content_elements = null;
$possible_content_selectors = array(
'article',
'.post-content',
'.entry-content',
'.article-content',
'.content',
'#content',
'.post-body',
'.article-body',
'.story-content',
'.main-content',
'main',
'.post',
'#main-content',
'.story',
'.entry',
'.blog-post',
'.post-entry',
);
foreach ( $possible_content_selectors as $selector ) {
$element = $html->find( $selector, 0 );
if ( $element ) {
$content_elements = $element;
break;
}
}
if ( empty( (array) $content_elements ) ) {
// Without a content container, keep the paragraphs that are not in headers, footers, sidebars and the like.
$content_paragraphs = array();
foreach ( $html->find( 'p' ) as $p ) {
$text = trim( $p->plaintext );
if ( strlen( $text ) < 10 ) {
continue;
}
$parent = $p->parent();
$parent_tag = $parent ? strtolower( $parent->tag ) : '';
$parent_class = $parent ? strtolower( $parent->class ) : '';
$parent_id = $parent ? strtolower( $parent->id ) : '';
$should_skip = false;
foreach ( array( 'footer', 'header', 'sidebar', 'comment', 'menu', 'nav', 'widget' ) as $skip_keyword ) {
if ( false !== strpos( $parent_class, $skip_keyword ) || false !== strpos( $parent_id, $skip_keyword ) || false !== strpos( $parent_tag, $skip_keyword ) ) {
$should_skip = true;
break;
}
}
if ( ! $should_skip ) {
$content_paragraphs[] = $text;
}
}
$result['content'] = implode( "\n\n", $content_paragraphs );
} else {
foreach ( $content_elements->find( 'aside, .sidebar, .widget, nav, .navigation, .menu, .comment, .comments, .post-meta, .related-posts, .social-share, .share-buttons, .sharing, .author-bio, script, style, iframe, form, .advertisement, .ads, .ad-container' ) as $remove ) {
$remove->outertext = '';
$remove->clear();
}
/**
* Allow cleanup HTML elements from the content.
* This will allow users to clean up specific elements using HTML tags, classes, or IDs.
*
* @param object $content_elements The content elements.
* @param int $post_id The post ID.
*
* @since 2.0.2
* @return object The modified content elements.
*/
$content_elements = apply_filters( 'aicw_cleanup_content_elements', $content_elements, $post_id );
$selector = 'p, h2, h3, h4, h5, h6, ol, ul, blockquote' . ( $with_images ? ', figure, img' : '' );
$paragraphs = array();
$last_heading = '';
foreach ( $content_elements->find( $selector ) as $node ) {
$tag_name = strtolower( $node->tag );
if ( 'figure' === $tag_name || 'img' === $tag_name ) {
$image = 'figure' === $tag_name ? self::get_figure_image( $node, $source_link ) : self::get_image( $node, $source_link, true );
if ( $image && count( $images ) < $settings['max'] && ! in_array( $image['url'], wp_list_pluck( $images, 'url' ), true ) ) {
if ( '' === $image['alt'] ) {
$image['alt'] = Plan::clean_text( '' !== $image['caption'] ? $image['caption'] : $last_heading, 125 );
}
$paragraphs[] = Plan::marker( count( $images ) );
$images[] = $image;
}
continue;
}
if ( preg_match( '/^h[2-6]$/', $tag_name ) ) {
$last_heading = trim( $node->plaintext );
}
if ( 'ol' === $tag_name || 'ul' === $tag_name ) {
$list_items = array();
foreach ( $node->find( 'li' ) as $li ) {
$li_text = trim( $li->plaintext );
if ( ! empty( $li_text ) ) {
$list_items[] = "
{$li_text}";
}
}
if ( ! empty( $list_items ) ) {
$paragraphs[] = "<{$tag_name}>" . implode( '', $list_items ) . "{$tag_name}>";
}
continue;
}
$text = trim( $node->plaintext );
if ( ! empty( $text ) ) {
$paragraphs[] = "<{$tag_name}>{$text}{$tag_name}>";
}
}
$result['content'] = implode( "\n", $paragraphs );
if ( empty( $result['content'] ) ) {
$result['content'] = trim( $content_elements->plaintext );
}
}
$featured = '';
if ( $with_images ) {
$og_image = $html->find( 'meta[property=og:image]', 0 );
$featured = (string) get_post_meta( $post_id, '_aicw_source_image', true );
if ( '' === $featured && $og_image && ! empty( $og_image->content ) ) {
$featured = self::absolute_url( trim( $og_image->content ), $source_link );
}
}
$result['content'] = preg_replace( '/\s+/', ' ', $result['content'] );
$result['content'] = trim( $result['content'] );
$result['title'] = trim( preg_replace( '/\s+/', ' ', $result['title'] ) );
$html->clear();
unset( $html );
if ( '' === trim( wp_strip_all_tags( Plan::strip( $result['content'] ) ) ) ) {
ai_content_writer()->create_log(
sprintf(
/* translators: 1: Source article URL. 2: Post ID. */
esc_html__( 'No readable article content could be extracted from %1$s for post ID %2$d. The page layout may not be supported.', 'ai-content-writer' ),
esc_url( $source_link ),
absint( $post_id )
),
'error',
$campaign_id
);
return false;
}
if ( $with_images ) {
$repeated = array_search( $featured, wp_list_pluck( $images, 'url' ), true );
if ( '' === $featured && ! empty( $images ) ) {
$featured = $images[0]['url'];
$repeated = 0;
}
if ( false !== $repeated ) {
$result['content'] = self::remove_image( $result['content'], $images, $repeated );
}
update_post_meta(
$post_id,
self::IMAGES_META,
array(
'featured' => $featured,
'inline' => $images,
)
);
}
$updated = wp_update_post(
array(
'ID' => $post_id,
'post_title' => wp_strip_all_tags( '' !== $result['title'] ? $result['title'] : $title ),
'post_content' => wp_kses_post( $result['content'] ),
)
);
if ( is_wp_error( $updated ) ) {
ai_content_writer()->create_log(
sprintf(
/* translators: 1: Post ID. 2: Error message. */
esc_html__( 'Could not save the generated content for post ID %1$d: %2$s', 'ai-content-writer' ),
absint( $post_id ),
esc_html( $updated->get_error_message() )
),
'error',
$campaign_id
);
return false;
}
return true;
}
/**
* Remove an image and its marker, renumbering the markers after it.
*
* @param string $content Content with markers.
* @param array $images Images, by slot; the image is removed from it.
* @param int $slot Slot to remove.
*
* @since 2.6.0
* @return string
*/
protected static function remove_image( $content, &$images, $slot ) {
$content = Plan::fill( $content, $slot );
for ( $next = $slot + 1, $count = count( $images ); $next < $count; $next++ ) {
$content = str_replace( Plan::marker( $next ), Plan::marker( $next - 1 ), $content );
}
array_splice( $images, $slot, 1 );
return $content;
}
/**
* The image of a figure, with its caption.
*
* @param object $figure Figure node.
* @param string $base Page URL.
*
* @since 2.6.0
* @return array|null `url`, `alt`, `caption`.
*/
protected static function get_figure_image( $figure, $base ) {
$img = $figure->find( 'img', 0 );
if ( ! $img ) {
return null;
}
$image = self::get_image( $img, $base, false );
if ( $image ) {
$caption = $figure->find( 'figcaption', 0 );
$image['caption'] = $caption ? Plan::clean_text( $caption->plaintext, 200 ) : '';
}
return $image;
}
/**
* A usable image from an img node.
*
* @param object $img Image node.
* @param string $base Page URL.
* @param bool $skip_in_figure Whether to skip images inside a figure, which are taken with the figure.
*
* @since 2.6.0
* @return array|null `url`, `alt`, `caption`.
*/
protected static function get_image( $img, $base, $skip_in_figure ) {
if ( $skip_in_figure ) {
for ( $parent = $img->parent(); $parent; $parent = $parent->parent() ) {
if ( 'figure' === strtolower( (string) $parent->tag ) ) {
return null;
}
}
}
$width = $img->getAttribute( 'width' );
if ( is_numeric( $width ) && (int) $width < self::MIN_IMAGE_WIDTH ) {
return null;
}
$url = self::get_largest_src( (string) $img->getAttribute( 'srcset' ) . ',' . (string) $img->getAttribute( 'data-srcset' ) );
foreach ( array( 'data-src', 'data-lazy-src', 'data-original', 'src' ) as $attribute ) {
if ( '' !== $url ) {
break;
}
$value = trim( (string) $img->getAttribute( $attribute ) );
$url = 0 === strpos( $value, 'data:' ) ? '' : $value;
}
$url = '' !== $url ? self::absolute_url( $url, $base ) : '';
if ( '' === $url ) {
return null;
}
$path = strtolower( (string) wp_parse_url( $url, PHP_URL_PATH ) );
$haystack = strtolower( basename( $path ) . ' ' . $img->getAttribute( 'class' ) . ' ' . $img->getAttribute( 'id' ) . ' ' . $img->getAttribute( 'alt' ) );
if ( '.svg' === substr( $path, -4 ) || preg_match( '/(^|[^a-z])(avatar|logo|icon|emoji|sprite|pixel|spinner|placeholder|gravatar|badge)/', $haystack ) ) {
return null;
}
// Image CDNs often carry the size in the query, e.g. ?width=32.
parse_str( (string) wp_parse_url( $url, PHP_URL_QUERY ), $query );
foreach ( array( 'width', 'w' ) as $key ) {
if ( isset( $query[ $key ] ) && is_numeric( $query[ $key ] ) && (int) $query[ $key ] < self::MIN_IMAGE_WIDTH ) {
return null;
}
}
return array(
'url' => $url,
'alt' => Plan::clean_text( (string) $img->getAttribute( 'alt' ), 125 ),
'caption' => '',
);
}
/**
* The widest candidate of a srcset.
*
* @param string $srcset Srcset value.
*
* @since 2.6.0
* @return string
*/
protected static function get_largest_src( $srcset ) {
$best = '';
$width = 0;
foreach ( explode( ',', $srcset ) as $candidate ) {
$parts = preg_split( '/\s+/', trim( $candidate ) );
if ( empty( $parts[0] ) || 0 === strpos( $parts[0], 'data:' ) ) {
continue;
}
$size = isset( $parts[1] ) ? (int) $parts[1] : 1;
if ( $size > $width ) {
$best = $parts[0];
$width = $size;
}
}
return $best;
}
/**
* Resolve a URL found on a page to an absolute http(s) URL.
*
* @param string $url The URL.
* @param string $base Page URL.
*
* @since 2.6.0
* @return string Empty when not http(s).
*/
public static function absolute_url( $url, $base ) {
$url = \WP_Http::make_absolute_url( html_entity_decode( $url, ENT_QUOTES, 'UTF-8' ), $base );
return in_array( wp_parse_url( $url, PHP_URL_SCHEME ), array( 'http', 'https' ), true ) ? esc_url_raw( $url ) : '';
}
}