urls = $urls; $this->hash_key = $hash_key; } public function extract( string $html, string $document_url, string $x_robots_tag = '' ): HtmlExtraction { $bytes = strlen( $html ); $document = $this->urls->evaluate( $document_url ); if ( 'accepted' !== $document['state'] || ! is_string( $document['url'] ) ) { return $this->not_tested( $bytes, 'invalid_document_url' ); } if ( $bytes > self::MAX_HTML_BYTES ) { return $this->not_tested( $bytes, 'html_budget_exceeded' ); } if ( ! class_exists( DOMDocument::class ) || ! class_exists( DOMXPath::class ) ) { return $this->not_tested( $bytes, 'dom_extension_unavailable' ); } $previous = libxml_use_internal_errors( true ); try { $dom = new DOMDocument(); $loaded = $dom->loadHTML( '' . $html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_COMPACT ); $xpath = $loaded ? new DOMXPath( $dom ) : null; } catch ( Throwable $exception ) { $xpath = null; } finally { libxml_clear_errors(); libxml_use_internal_errors( $previous ); } if ( ! $xpath instanceof DOMXPath ) { return $this->not_tested( $bytes, 'html_parse_failed' ); } $document_url = $document['url']; try { $link_data = $this->links( $xpath, $document_url ); $signals = array( 'state' => 'observed', 'byte_count' => $bytes, 'document_reference' => $this->reference( $document_url ), 'language' => $this->language( $xpath ), 'title' => $this->single_text_signal( $this->nodes( $xpath, '//title' ) ), 'meta_description' => $this->single_attribute_signal( $this->nodes( $xpath, '//meta[translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="description"]' ), 'content' ), 'robots' => $this->robots( $xpath, $x_robots_tag ), 'canonical' => $this->canonical( $xpath, $document_url ), 'hreflang' => $this->hreflang( $xpath, $document_url ), 'headings' => $this->headings( $xpath ), 'links' => $link_data['signals'], 'media' => $this->media( $xpath ), 'structured_data' => $this->structured_data( $xpath ), 'answer_structure' => $this->answer_structure( $xpath ), 'provenance' => $this->provenance( $xpath, $link_data['signals'] ), 'publisher_controls' => $this->publisher_controls( $xpath ), 'limitations' => array( 'server_html_only', 'raw_content_not_retained', 'rendered_javascript_not_tested', 'search_inclusion_not_tested', ), ); } catch ( RuntimeException $exception ) { return $this->not_tested( $bytes, 'html_signal_extraction_failed' ); } return new HtmlExtraction( $signals, $link_data['urls'] ); } /** * @return array{state:string,value:string} */ private function language( DOMXPath $xpath ): array { $nodes = $this->nodes( $xpath, '/html' ); if ( 0 === $nodes->length || ! $nodes->item( 0 ) instanceof DOMElement ) { return array( 'state' => 'missing', 'value' => '', ); } $value = strtolower( trim( $nodes->item( 0 )->getAttribute( 'lang' ) ) ); if ( '' === $value ) { return array( 'state' => 'missing', 'value' => '', ); } $valid = 1 === preg_match( '/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/', $value ); return array( 'state' => $valid ? 'declared' : 'invalid', 'value' => $valid ? $value : '', ); } /** * @param DOMNodeList $nodes Matching nodes. * @return array{count:int,length:int,reference:string} */ private function single_text_signal( DOMNodeList $nodes ): array { $value = 0 < $nodes->length && $nodes->item( 0 ) instanceof DOMNode ? $this->clean_text( $this->node_text( $nodes->item( 0 ) ) ) : ''; return array( 'count' => $nodes->length, 'length' => $this->text_length( $value ), 'reference' => '' === $value ? '' : $this->reference( $value ), ); } /** * @param DOMNodeList $nodes Matching nodes. * @return array{count:int,length:int,reference:string} */ private function single_attribute_signal( DOMNodeList $nodes, string $attribute ): array { $value = 0 < $nodes->length && $nodes->item( 0 ) instanceof DOMElement ? $this->clean_text( $nodes->item( 0 )->getAttribute( $attribute ) ) : ''; return array( 'count' => $nodes->length, 'length' => $this->text_length( $value ), 'reference' => '' === $value ? '' : $this->reference( $value ), ); } /** * @return array{directives:list,meta_elements:int,http_directives_present:bool,http_directives:list} */ private function robots( DOMXPath $xpath, string $x_robots_tag ): array { $nodes = $this->nodes( $xpath, '//meta[translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="robots" or translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="googlebot"]' ); $directives = array(); foreach ( $nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $directives = array_merge( $directives, $this->parse_robots_directives( $node->getAttribute( 'content' ) ) ); } $http_directives = '' === $x_robots_tag ? array() : $this->parse_robots_directives( $x_robots_tag, true ); $directives = array_merge( $directives, $http_directives ); $directives = array_values( array_unique( $directives ) ); sort( $directives ); $http_directives = array_values( array_unique( $http_directives ) ); sort( $http_directives ); return array( 'directives' => $directives, 'meta_elements' => $nodes->length, 'http_directives_present' => '' !== $x_robots_tag, 'http_directives' => $http_directives, ); } /** * @return list */ private function parse_robots_directives( string $value, bool $strip_product_prefix = false ): array { $content = strtolower( trim( $value ) ); if ( '' === $content || strlen( $content ) > 1024 || 1 === preg_match( '/[\x00-\x1F\x7F]/', $content ) ) { return array(); } if ( $strip_product_prefix ) { $content = preg_replace( '/(?:^|,)\s*[a-z0-9_-]{1,64}\s*:\s*(?=(?:all|follow|index|no|none|max-|unavailable_after))/', ',', $content ); $content = is_string( $content ) ? $content : ''; } $directives = array(); if ( 1 === preg_match( '/(?:^|,)\s*unavailable_after\s*:/', $content ) ) { $directives[] = 'unavailable_after'; $content = preg_replace( '/(?:^|,)\s*unavailable_after\s*:[^,]*/', '', $content ); $content = is_string( $content ) ? $content : ''; } $content = preg_replace( '/(max-(?:image|video)-preview|max-snippet)\s*:\s*/', '$1:', $content ); $content = is_string( $content ) ? $content : ''; $parts = preg_split( '/[\s,]+/', $content ); foreach ( is_array( $parts ) ? $parts : array() as $directive ) { $directive = trim( $directive ); if ( 1 === preg_match( '/^(?:all|follow|index|indexifembedded|noarchive|nofollow|noimageindex|noindex|none|nositelinkssearchbox|nosnippet|notranslate|max-image-preview:(?:none|standard|large)|max-snippet:-?[0-9]+|max-video-preview:-?[0-9]+)$/', $directive ) ) { $directives[] = $directive; } } return array_values( array_unique( $directives ) ); } /** * @return array{count:int,self:int,same_origin:int,external:int,invalid:int,references:list,references_truncated:bool} */ private function canonical( DOMXPath $xpath, string $document_url ): array { $nodes = $this->nodes( $xpath, '//link[contains(concat(" ",normalize-space(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"))," ")," canonical ")]' ); $self = 0; $same = 0; $external = 0; $invalid = 0; $references = array(); foreach ( $nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $href = trim( $node->getAttribute( 'href' ) ); $decision = $this->urls->evaluate( $href, $document_url ); if ( 'accepted' === $decision['state'] && is_string( $decision['url'] ) ) { ++$same; if ( $decision['url'] === $document_url ) { ++$self; } $references[] = $this->reference( $decision['url'] ); } elseif ( 'external_origin' === $decision['reason'] ) { ++$external; $references[] = $this->reference( $href ); } else { ++$invalid; } } $references = array_values( array_unique( $references ) ); return array( 'count' => $nodes->length, 'self' => $self, 'same_origin' => $same, 'external' => $external, 'invalid' => $invalid, 'references' => array_slice( $references, 0, self::MAX_SIGNAL_VALUES ), 'references_truncated' => count( $references ) > self::MAX_SIGNAL_VALUES, ); } /** * @return array{count:int,declared_languages:list,same_origin:int,external:int,invalid:int,languages_truncated:bool} */ private function hreflang( DOMXPath $xpath, string $document_url ): array { $nodes = $this->nodes( $xpath, '//link[contains(concat(" ",normalize-space(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"))," ")," alternate ") and @hreflang]' ); $languages = array(); $same = 0; $external = 0; $invalid = 0; foreach ( $nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $language = strtolower( trim( $node->getAttribute( 'hreflang' ) ) ); if ( 'x-default' === $language || 1 === preg_match( '/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/', $language ) ) { $languages[] = $language; } else { ++$invalid; } $decision = $this->urls->evaluate( trim( $node->getAttribute( 'href' ) ), $document_url ); if ( 'accepted' === $decision['state'] ) { ++$same; } elseif ( 'external_origin' === $decision['reason'] ) { ++$external; } else { ++$invalid; } } $languages = array_values( array_unique( $languages ) ); sort( $languages ); return array( 'count' => $nodes->length, 'declared_languages' => array_slice( $languages, 0, self::MAX_SIGNAL_VALUES ), 'same_origin' => $same, 'external' => $external, 'invalid' => $invalid, 'languages_truncated' => count( $languages ) > self::MAX_SIGNAL_VALUES, ); } /** * @return array{counts:array,empty:int,level_skips:int,question_headings:int} */ private function headings( DOMXPath $xpath ): array { $nodes = $this->nodes( $xpath, '//h1|//h2|//h3|//h4|//h5|//h6' ); $counts = array_fill_keys( array( 'h1', 'h2', 'h3', 'h4', 'h5', 'h6' ), 0 ); $empty = 0; $skips = 0; $questions = 0; $previous = 0; foreach ( $nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $name = strtolower( $this->tag_name( $node ) ); if ( isset( $counts[ $name ] ) ) { ++$counts[ $name ]; } $text = $this->clean_text( $this->node_text( $node ) ); if ( '' === $text ) { ++$empty; } if ( 1 === preg_match( '/[??]$/u', $text ) ) { ++$questions; } $level = (int) substr( $name, 1 ); if ( $previous > 0 && $level > $previous + 1 ) { ++$skips; } $previous = $level; } return array( 'counts' => $counts, 'empty' => $empty, 'level_skips' => $skips, 'question_headings' => $questions, ); } /** * @return array{signals:array{same_origin:int,external:int,skipped:int,unique_same_origin:int,same_origin_references:list,truncated:bool},urls:list} */ private function links( DOMXPath $xpath, string $document_url ): array { $same = 0; $external = 0; $skipped = 0; $urls = array(); $truncated = false; foreach ( $this->nodes( $xpath, '//a[@href]' ) as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $decision = $this->urls->evaluate( $node->getAttribute( 'href' ), $document_url ); if ( 'accepted' === $decision['state'] && is_string( $decision['url'] ) ) { ++$same; if ( isset( $urls[ $decision['url'] ] ) ) { continue; } if ( count( $urls ) < self::MAX_DISCOVERED_LINKS ) { $urls[ $decision['url'] ] = true; } else { $truncated = true; } } elseif ( 'external_origin' === $decision['reason'] ) { ++$external; } else { ++$skipped; } } return array( 'signals' => array( 'same_origin' => $same, 'external' => $external, 'skipped' => $skipped, 'unique_same_origin' => count( $urls ), 'same_origin_references' => array_map( fn ( string $url ): string => $this->reference( $url ), array_keys( $urls ) ), 'truncated' => $truncated, ), 'urls' => array_keys( $urls ), ); } /** * @return array{images:int,missing_alt:int,empty_alt:int,missing_dimensions:int,lazy:int} */ private function media( DOMXPath $xpath ): array { $nodes = $this->nodes( $xpath, '//img' ); $missing_alt = 0; $empty_alt = 0; $missing_dimensions = 0; $lazy = 0; foreach ( $nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } if ( ! $node->hasAttribute( 'alt' ) ) { ++$missing_alt; } elseif ( '' === trim( $node->getAttribute( 'alt' ) ) ) { ++$empty_alt; } if ( ! $node->hasAttribute( 'width' ) || ! $node->hasAttribute( 'height' ) ) { ++$missing_dimensions; } if ( 'lazy' === strtolower( trim( $node->getAttribute( 'loading' ) ) ) ) { ++$lazy; } } return array( 'images' => $nodes->length, 'missing_alt' => $missing_alt, 'empty_alt' => $empty_alt, 'missing_dimensions' => $missing_dimensions, 'lazy' => $lazy, ); } /** * @return array{blocks:int,parsed:int,errors:int,truncated:bool,type_count:int,entity_count:int,types:list,entity_references:list} */ private function structured_data( DOMXPath $xpath ): array { $nodes = $this->nodes( $xpath, '//script[translate(@type,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="application/ld+json"]' ); $parsed = 0; $errors = 0; $truncated = $nodes->length > self::MAX_JSON_LD_BLOCKS; $types = array(); $entities = array(); $block_index = 0; foreach ( $nodes as $node ) { if ( ++$block_index > self::MAX_JSON_LD_BLOCKS || ! $node instanceof DOMNode ) { break; } $json = trim( $this->node_text( $node ) ); if ( '' === $json || strlen( $json ) > self::MAX_JSON_LD_BYTES ) { ++$errors; $truncated = $truncated || strlen( $json ) > self::MAX_JSON_LD_BYTES; continue; } $value = json_decode( $json, true, 32 ); if ( JSON_ERROR_NONE !== json_last_error() || ! is_array( $value ) ) { ++$errors; continue; } ++$parsed; $this->collect_schema_identifiers( $value, $types, $entities, 0 ); } $types = array_values( array_unique( $types ) ); $entities = array_values( array_unique( $entities ) ); sort( $types ); sort( $entities ); $type_count = count( $types ); $entity_count = count( $entities ); $truncated = $truncated || $type_count > self::MAX_SIGNAL_VALUES || $entity_count > self::MAX_SIGNAL_VALUES; return array( 'blocks' => $nodes->length, 'parsed' => $parsed, 'errors' => $errors, 'truncated' => $truncated, 'type_count' => $type_count, 'entity_count' => $entity_count, 'types' => array_slice( $types, 0, self::MAX_SIGNAL_VALUES ), 'entity_references' => array_slice( $entities, 0, self::MAX_SIGNAL_VALUES ), ); } /** * @param array $value JSON-LD value. * @param list $types Validated schema types. * @param list $entities Keyed entity references. */ private function collect_schema_identifiers( array $value, array &$types, array &$entities, int $depth ): void { if ( $depth > 16 ) { return; } foreach ( $value as $key => $child ) { if ( '@type' === $key ) { foreach ( is_array( $child ) ? $child : array( $child ) as $type ) { if ( is_string( $type ) && 1 === preg_match( '/^[A-Za-z][A-Za-z0-9:_-]{0,63}$/', $type ) ) { $types[] = $type; } } } elseif ( '@id' === $key && is_string( $child ) && '' !== trim( $child ) && strlen( $child ) <= 2048 ) { $entities[] = $this->reference( trim( $child ) ); } if ( is_array( $child ) ) { $this->collect_schema_identifiers( $child, $types, $entities, $depth + 1 ); } } } /** * @return array{eligible_sections:int,question_answer_pairs:int,concise_answer_pairs:int} */ private function answer_structure( DOMXPath $xpath ): array { $eligible = 0; $questions = 0; $concise = 0; foreach ( $this->nodes( $xpath, '//h2|//h3' ) as $heading ) { if ( ! $heading instanceof DOMElement ) { continue; } $next = $this->next_sibling( $heading ); while ( $next instanceof DOMNode && XML_TEXT_NODE === $this->node_type( $next ) && '' === trim( $this->node_text( $next ) ) ) { $next = $this->next_sibling( $next ); } if ( ! $next instanceof DOMElement || ! in_array( strtolower( $this->tag_name( $next ) ), array( 'p', 'ul', 'ol' ), true ) ) { continue; } $text_length = $this->text_length( $this->clean_text( $this->node_text( $next ) ) ); if ( 0 === $text_length ) { continue; } ++$eligible; if ( 1 === preg_match( '/[??]$/u', $this->clean_text( $this->node_text( $heading ) ) ) ) { ++$questions; if ( $text_length >= 40 && $text_length <= 600 ) { ++$concise; } } } return array( 'eligible_sections' => $eligible, 'question_answer_pairs' => $questions, 'concise_answer_pairs' => $concise, ); } /** * @param array{same_origin:int,external:int,skipped:int,unique_same_origin:int,same_origin_references:list,truncated:bool} $link_signals Link counts. * @return array{dated_elements:int,author_markers:int,external_reference_links:int} */ private function provenance( DOMXPath $xpath, array $link_signals ): array { return array( 'dated_elements' => $this->nodes( $xpath, '//time[@datetime]' )->length, 'author_markers' => $this->nodes( $xpath, '//*[@itemprop="author" or contains(concat(" ",normalize-space(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"))," ")," author ")]' )->length, 'external_reference_links' => $link_signals['external'], ); } /** * @return array{data_nosnippet:int,invalid_data_nosnippet_elements:int} */ private function publisher_controls( DOMXPath $xpath ): array { $all = $this->nodes( $xpath, '//*[@data-nosnippet]' )->length; $valid = $this->nodes( $xpath, '//div[@data-nosnippet]|//span[@data-nosnippet]|//section[@data-nosnippet]' )->length; return array( 'data_nosnippet' => $valid, 'invalid_data_nosnippet_elements' => max( 0, $all - $valid ), ); } private function clean_text( string $value ): string { $value = preg_replace( '/[\x00-\x1F\x7F]+/u', ' ', $value ); $value = is_string( $value ) ? preg_replace( '/\s+/u', ' ', $value ) : ''; return is_string( $value ) ? trim( $value ) : ''; } private function text_length( string $value ): int { return function_exists( 'mb_strlen' ) ? mb_strlen( $value, 'UTF-8' ) : strlen( $value ); } private function reference( string $value ): string { return hash_hmac( 'sha256', $value, $this->hash_key ); } private function node_text( DOMNode $node ): string { // phpcs:ignore WordPress.NamingConventions.ValidVariableName.UsedPropertyNotSnakeCase -- Native DOM API property. return $node->textContent; } private function tag_name( DOMElement $element ): string { // phpcs:ignore WordPress.NamingConventions.ValidVariableName.UsedPropertyNotSnakeCase -- Native DOM API property. return $element->tagName; } private function next_sibling( DOMNode $node ): ?DOMNode { // phpcs:ignore WordPress.NamingConventions.ValidVariableName.UsedPropertyNotSnakeCase -- Native DOM API property. return $node->nextSibling; } private function node_type( DOMNode $node ): int { // phpcs:ignore WordPress.NamingConventions.ValidVariableName.UsedPropertyNotSnakeCase -- Native DOM API property. return $node->nodeType; } /** * @return DOMNodeList */ private function nodes( DOMXPath $xpath, string $query ): DOMNodeList { $nodes = $xpath->query( $query ); if ( ! $nodes instanceof DOMNodeList ) { throw new RuntimeException( 'xpath_query_failed' ); } return $nodes; } private function not_tested( int $bytes, string $reason ): HtmlExtraction { return new HtmlExtraction( array( 'state' => 'not_tested', 'byte_count' => $bytes, 'reason' => $reason, 'limitations' => array( 'raw_content_not_retained', 'html_preconditions_not_met' ), ), array() ); } }