(.*?)<\/title>/is', $html, $matches ) ) { $title = trim( html_entity_decode( $matches[1], ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ); } if ( empty( $title ) ) { $title = $url; } // Clean HTML content to obtain readable text. $text = self::clean_html( $html ); return array( 'title' => $title, 'text' => $text, ); } /** Reject loopback, private, link-local, and metadata hosts before the request. */ private static function is_public_host( string $host ): bool { $host = trim( strtolower( $host ), '[]' ); if ( 'localhost' === $host || substr( $host, -6 ) === '.local' || substr( $host, -6 ) === '.internal' ) { return false; } $ips = filter_var( $host, FILTER_VALIDATE_IP ) ? array( $host ) : ( gethostbynamel( $host ) ?: array() ); if ( empty( $ips ) ) { return false; } foreach ( $ips as $ip ) { if ( false === filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE ) ) { return false; } } return true; } /** * Strip out style tags, scripts, layout/headers, navigation menus and keep readable content. * * @param string $html Raw HTML content. * @return string Plain text representation. */ private static function clean_html( string $html ): string { // Strip comments. $html = preg_replace( '//ms', '', $html ); // Strip style, script and template blocks. $html = preg_replace( '/<(script|style|template|svg|noscript|iframe)[^>]*>.*?<\/\1>/is', '', $html ); // Strip headers, footers, and sidebars to target the main article/body text as much as possible. $html = preg_replace( '/<(header|footer|nav|aside)[^>]*>.*?<\/\1>/is', '', $html ); // Replace structural blocks with newlines to preserve spacing/paragraphs. $html = preg_replace( '/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/i', "\n", $html ); // Strip remaining HTML tags. $text = wp_strip_all_tags( $html ); // Decode entities. $text = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ); // Clean up extra spaces/newlines. $text = preg_replace( "/[ \t]+/u", ' ', $text ); $text = preg_replace( "/\n\s*\n+/u", "\n\n", $text ); return trim( $text ); } }

2048 ) { throw new \Exception( esc_html__( 'The URL is too long.', 'ai-support-assistant' ) ); } $response = wp_safe_remote_get( $url, array( 'timeout' => 30, 'redirection' => 0, 'limit_response_size' => 2 * 1024 * 1024, 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 WordPress-ai-support-assistant/' . ( defined( 'AI_SUPPORT_ASSISTANT_VERSION' ) ? AI_SUPPORT_ASSISTANT_VERSION : '1.0.2' ), ) ); if ( is_wp_error( $response ) ) { $err = $response->get_error_message(); Logger::error( "URL Crawl failed for $url: " . $err ); /* translators: %s: Crawl error details. */ throw new \Exception( esc_html( sprintf( __( 'Failed to fetch URL: %s', 'ai-support-assistant' ), $err ) ) ); } $code = wp_remote_retrieve_response_code( $response ); if ( 200 !== $code ) { Logger::error( "URL Crawl failed with HTTP $code for $url" ); /* translators: %d: HTTP status code. */ throw new \Exception( esc_html( sprintf( __( 'Failed to fetch URL, HTTP error code: %d', 'ai-support-assistant' ), $code ) ) ); } $html = wp_remote_retrieve_body( $response ); $content_type = strtolower( (string) wp_remote_retrieve_header( $response, 'content-type' ) ); if ( $content_type && false === strpos( $content_type, 'text/html' ) && false === strpos( $content_type, 'text/plain' ) ) { throw new \Exception( esc_html__( 'The URL did not return an HTML or text document.', 'ai-support-assistant' ) ); } if ( empty( $html ) ) { throw new \Exception( esc_html__( 'Webpage content is empty.', 'ai-support-assistant' ) ); } // Parse Title. $title = ''; if ( preg_match( '/(.*?)<\/title>/is', $html, $matches ) ) { $title = trim( html_entity_decode( $matches[1], ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ); } if ( empty( $title ) ) { $title = $url; } // Clean HTML content to obtain readable text. $text = self::clean_html( $html ); return array( 'title' => $title, 'text' => $text, ); } /** Reject loopback, private, link-local, and metadata hosts before the request. */ private static function is_public_host( string $host ): bool { $host = trim( strtolower( $host ), '[]' ); if ( 'localhost' === $host || substr( $host, -6 ) === '.local' || substr( $host, -6 ) === '.internal' ) { return false; } $ips = filter_var( $host, FILTER_VALIDATE_IP ) ? array( $host ) : ( gethostbynamel( $host ) ?: array() ); if ( empty( $ips ) ) { return false; } foreach ( $ips as $ip ) { if ( false === filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE ) ) { return false; } } return true; } /** * Strip out style tags, scripts, layout/headers, navigation menus and keep readable content. * * @param string $html Raw HTML content. * @return string Plain text representation. */ private static function clean_html( string $html ): string { // Strip comments. $html = preg_replace( '//ms', '', $html ); // Strip style, script and template blocks. $html = preg_replace( '/<(script|style|template|svg|noscript|iframe)[^>]*>.*?<\/\1>/is', '', $html ); // Strip headers, footers, and sidebars to target the main article/body text as much as possible. $html = preg_replace( '/<(header|footer|nav|aside)[^>]*>.*?<\/\1>/is', '', $html ); // Replace structural blocks with newlines to preserve spacing/paragraphs. $html = preg_replace( '/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/i', "\n", $html ); // Strip remaining HTML tags. $text = wp_strip_all_tags( $html ); // Decode entities. $text = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ); // Clean up extra spaces/newlines. $text = preg_replace( "/[ \t]+/u", ' ', $text ); $text = preg_replace( "/\n\s*\n+/u", "\n\n", $text ); return trim( $text ); } }