create_log( sprintf( /* translators: %d: Post ID. */ esc_html__( 'Post ID %d has no source link, so its content cannot be fetched.', 'ai-content-writer' ), absint( $post_id ) ), 'error', $campaign_id ); return false; } $dom = new HtmlWeb(); $html = $dom->load( trailingslashit( $source_link ) ); if ( ! $html ) { ai_content_writer()->create_log( sprintf( /* translators: 1: Source article URL. 2: Post ID. */ esc_html__( 'Could not fetch the source article %1$s for post ID %2$d. The page may be unreachable or blocking requests.', 'ai-content-writer' ), esc_url( $source_link ), absint( $post_id ) ), 'error', $campaign_id ); return false; } $settings = aicw_get_campaign_image_settings( $campaign_id ); $with_images = 'source' === $settings['mode']; $images = array(); $result = array( 'title' => '', 'content' => '', ); $possible_title_selectors = array( 'h1.entry-title', 'h1.post-title', 'h1.article-title', 'article h1', '.article-header h1', '.post-header h1', 'header h1', 'h1', '.headline', '.title', '#title', ); foreach ( $possible_title_selectors as $selector ) { $title_element = $html->find( $selector, 0 ); if ( $title_element ) { $result['title'] = trim( $title_element->plaintext ); break; } } if ( empty( $result['title'] ) ) { $title_element = $html->find( 'title', 0 ); if ( $title_element ) { $result['title'] = trim( $title_element->plaintext ); } } $content_elements = null; $possible_content_selectors = array( 'article', '.post-content', '.entry-content', '.article-content', '.content', '#content', '.post-body', '.article-body', '.story-content', '.main-content', 'main', '.post', '#main-content', '.story', '.entry', '.blog-post', '.post-entry', ); foreach ( $possible_content_selectors as $selector ) { $element = $html->find( $selector, 0 ); if ( $element ) { $content_elements = $element; break; } } if ( empty( (array) $content_elements ) ) { // Without a content container, keep the paragraphs that are not in headers, footers, sidebars and the like. $content_paragraphs = array(); foreach ( $html->find( 'p' ) as $p ) { $text = trim( $p->plaintext ); if ( strlen( $text ) < 10 ) { continue; } $parent = $p->parent(); $parent_tag = $parent ? strtolower( $parent->tag ) : ''; $parent_class = $parent ? strtolower( $parent->class ) : ''; $parent_id = $parent ? strtolower( $parent->id ) : ''; $should_skip = false; foreach ( array( 'footer', 'header', 'sidebar', 'comment', 'menu', 'nav', 'widget' ) as $skip_keyword ) { if ( false !== strpos( $parent_class, $skip_keyword ) || false !== strpos( $parent_id, $skip_keyword ) || false !== strpos( $parent_tag, $skip_keyword ) ) { $should_skip = true; break; } } if ( ! $should_skip ) { $content_paragraphs[] = $text; } } $result['content'] = implode( "\n\n", $content_paragraphs ); } else { foreach ( $content_elements->find( 'aside, .sidebar, .widget, nav, .navigation, .menu, .comment, .comments, .post-meta, .related-posts, .social-share, .share-buttons, .sharing, .author-bio, script, style, iframe, form, .advertisement, .ads, .ad-container' ) as $remove ) { $remove->outertext = ''; $remove->clear(); } /** * Allow cleanup HTML elements from the content. * This will allow users to clean up specific elements using HTML tags, classes, or IDs. * * @param object $content_elements The content elements. * @param int $post_id The post ID. * * @since 2.0.2 * @return object The modified content elements. */ $content_elements = apply_filters( 'aicw_cleanup_content_elements', $content_elements, $post_id ); $selector = 'p, h2, h3, h4, h5, h6, ol, ul, blockquote' . ( $with_images ? ', figure, img' : '' ); $paragraphs = array(); $last_heading = ''; foreach ( $content_elements->find( $selector ) as $node ) { $tag_name = strtolower( $node->tag ); if ( 'figure' === $tag_name || 'img' === $tag_name ) { $image = 'figure' === $tag_name ? self::get_figure_image( $node, $source_link ) : self::get_image( $node, $source_link, true ); if ( $image && count( $images ) < $settings['max'] && ! in_array( $image['url'], wp_list_pluck( $images, 'url' ), true ) ) { if ( '' === $image['alt'] ) { $image['alt'] = Plan::clean_text( '' !== $image['caption'] ? $image['caption'] : $last_heading, 125 ); } $paragraphs[] = Plan::marker( count( $images ) ); $images[] = $image; } continue; } if ( preg_match( '/^h[2-6]$/', $tag_name ) ) { $last_heading = trim( $node->plaintext ); } if ( 'ol' === $tag_name || 'ul' === $tag_name ) { $list_items = array(); foreach ( $node->find( 'li' ) as $li ) { $li_text = trim( $li->plaintext ); if ( ! empty( $li_text ) ) { $list_items[] = "
  • {$li_text}
  • "; } } if ( ! empty( $list_items ) ) { $paragraphs[] = "<{$tag_name}>" . implode( '', $list_items ) . ""; } continue; } $text = trim( $node->plaintext ); if ( ! empty( $text ) ) { $paragraphs[] = "<{$tag_name}>{$text}"; } } $result['content'] = implode( "\n", $paragraphs ); if ( empty( $result['content'] ) ) { $result['content'] = trim( $content_elements->plaintext ); } } $featured = ''; if ( $with_images ) { $og_image = $html->find( 'meta[property=og:image]', 0 ); $featured = (string) get_post_meta( $post_id, '_aicw_source_image', true ); if ( '' === $featured && $og_image && ! empty( $og_image->content ) ) { $featured = self::absolute_url( trim( $og_image->content ), $source_link ); } } $result['content'] = preg_replace( '/\s+/', ' ', $result['content'] ); $result['content'] = trim( $result['content'] ); $result['title'] = trim( preg_replace( '/\s+/', ' ', $result['title'] ) ); $html->clear(); unset( $html ); if ( '' === trim( wp_strip_all_tags( Plan::strip( $result['content'] ) ) ) ) { ai_content_writer()->create_log( sprintf( /* translators: 1: Source article URL. 2: Post ID. */ esc_html__( 'No readable article content could be extracted from %1$s for post ID %2$d. The page layout may not be supported.', 'ai-content-writer' ), esc_url( $source_link ), absint( $post_id ) ), 'error', $campaign_id ); return false; } if ( $with_images ) { $repeated = array_search( $featured, wp_list_pluck( $images, 'url' ), true ); if ( '' === $featured && ! empty( $images ) ) { $featured = $images[0]['url']; $repeated = 0; } if ( false !== $repeated ) { $result['content'] = self::remove_image( $result['content'], $images, $repeated ); } update_post_meta( $post_id, self::IMAGES_META, array( 'featured' => $featured, 'inline' => $images, ) ); } $updated = wp_update_post( array( 'ID' => $post_id, 'post_title' => wp_strip_all_tags( '' !== $result['title'] ? $result['title'] : $title ), 'post_content' => wp_kses_post( $result['content'] ), ) ); if ( is_wp_error( $updated ) ) { ai_content_writer()->create_log( sprintf( /* translators: 1: Post ID. 2: Error message. */ esc_html__( 'Could not save the generated content for post ID %1$d: %2$s', 'ai-content-writer' ), absint( $post_id ), esc_html( $updated->get_error_message() ) ), 'error', $campaign_id ); return false; } return true; } /** * Remove an image and its marker, renumbering the markers after it. * * @param string $content Content with markers. * @param array $images Images, by slot; the image is removed from it. * @param int $slot Slot to remove. * * @since 2.6.0 * @return string */ protected static function remove_image( $content, &$images, $slot ) { $content = Plan::fill( $content, $slot ); for ( $next = $slot + 1, $count = count( $images ); $next < $count; $next++ ) { $content = str_replace( Plan::marker( $next ), Plan::marker( $next - 1 ), $content ); } array_splice( $images, $slot, 1 ); return $content; } /** * The image of a figure, with its caption. * * @param object $figure Figure node. * @param string $base Page URL. * * @since 2.6.0 * @return array|null `url`, `alt`, `caption`. */ protected static function get_figure_image( $figure, $base ) { $img = $figure->find( 'img', 0 ); if ( ! $img ) { return null; } $image = self::get_image( $img, $base, false ); if ( $image ) { $caption = $figure->find( 'figcaption', 0 ); $image['caption'] = $caption ? Plan::clean_text( $caption->plaintext, 200 ) : ''; } return $image; } /** * A usable image from an img node. * * @param object $img Image node. * @param string $base Page URL. * @param bool $skip_in_figure Whether to skip images inside a figure, which are taken with the figure. * * @since 2.6.0 * @return array|null `url`, `alt`, `caption`. */ protected static function get_image( $img, $base, $skip_in_figure ) { if ( $skip_in_figure ) { for ( $parent = $img->parent(); $parent; $parent = $parent->parent() ) { if ( 'figure' === strtolower( (string) $parent->tag ) ) { return null; } } } $width = $img->getAttribute( 'width' ); if ( is_numeric( $width ) && (int) $width < self::MIN_IMAGE_WIDTH ) { return null; } $url = self::get_largest_src( (string) $img->getAttribute( 'srcset' ) . ',' . (string) $img->getAttribute( 'data-srcset' ) ); foreach ( array( 'data-src', 'data-lazy-src', 'data-original', 'src' ) as $attribute ) { if ( '' !== $url ) { break; } $value = trim( (string) $img->getAttribute( $attribute ) ); $url = 0 === strpos( $value, 'data:' ) ? '' : $value; } $url = '' !== $url ? self::absolute_url( $url, $base ) : ''; if ( '' === $url ) { return null; } $path = strtolower( (string) wp_parse_url( $url, PHP_URL_PATH ) ); $haystack = strtolower( basename( $path ) . ' ' . $img->getAttribute( 'class' ) . ' ' . $img->getAttribute( 'id' ) . ' ' . $img->getAttribute( 'alt' ) ); if ( '.svg' === substr( $path, -4 ) || preg_match( '/(^|[^a-z])(avatar|logo|icon|emoji|sprite|pixel|spinner|placeholder|gravatar|badge)/', $haystack ) ) { return null; } // Image CDNs often carry the size in the query, e.g. ?width=32. parse_str( (string) wp_parse_url( $url, PHP_URL_QUERY ), $query ); foreach ( array( 'width', 'w' ) as $key ) { if ( isset( $query[ $key ] ) && is_numeric( $query[ $key ] ) && (int) $query[ $key ] < self::MIN_IMAGE_WIDTH ) { return null; } } return array( 'url' => $url, 'alt' => Plan::clean_text( (string) $img->getAttribute( 'alt' ), 125 ), 'caption' => '', ); } /** * The widest candidate of a srcset. * * @param string $srcset Srcset value. * * @since 2.6.0 * @return string */ protected static function get_largest_src( $srcset ) { $best = ''; $width = 0; foreach ( explode( ',', $srcset ) as $candidate ) { $parts = preg_split( '/\s+/', trim( $candidate ) ); if ( empty( $parts[0] ) || 0 === strpos( $parts[0], 'data:' ) ) { continue; } $size = isset( $parts[1] ) ? (int) $parts[1] : 1; if ( $size > $width ) { $best = $parts[0]; $width = $size; } } return $best; } /** * Resolve a URL found on a page to an absolute http(s) URL. * * @param string $url The URL. * @param string $base Page URL. * * @since 2.6.0 * @return string Empty when not http(s). */ public static function absolute_url( $url, $base ) { $url = \WP_Http::make_absolute_url( html_entity_decode( $url, ENT_QUOTES, 'UTF-8' ), $base ); return in_array( wp_parse_url( $url, PHP_URL_SCHEME ), array( 'http', 'https' ), true ) ? esc_url_raw( $url ) : ''; } }