urls = $urls; $this->queue = $queue; $this->http = $http; $this->html = $html; $this->sitemaps = $sitemaps; } /** * Execute no more than one HTTP request and return a safe resumable cursor. * * @param array $cursor Previous task cursor. */ public function process_one( int $run_id, array $cursor, int $remaining_requests ): CrawlChunkResult { if ( $run_id < 1 || $remaining_requests < 0 || $remaining_requests > 500 ) { throw new InvalidArgumentException( 'invalid_crawl_chunk' ); } $cursor = $this->normalize_cursor( $cursor ); $item = $this->queue->next_queued( $run_id ); if ( null === $item && null === $cursor['robots_policy'] ) { $this->queue->enqueue_robots( $run_id ); $item = $this->queue->next_queued( $run_id ); } if ( null === $item ) { return new CrawlChunkResult( 'completed', $this->final_cursor( $cursor, $run_id ), $this->usage() ); } if ( 0 === $remaining_requests ) { return new CrawlChunkResult( 'budget_exhausted', $this->final_cursor( $cursor, $run_id ), $this->usage() ); } $robots = null === $cursor['robots_policy'] ? null : RobotsPolicy::from_checkpoint( $cursor['robots_policy'] ); if ( 'robots' !== $item['resource_class'] && null === $robots ) { $this->complete( $run_id, $item, 'skipped', $this->not_tested_signals( 'robots_policy_unavailable' ), null, 'none', 'robots_policy_unavailable' ); return new CrawlChunkResult( 'active', $cursor, $this->usage( 0, 1, 0 ) ); } if ( null !== $robots && ! $this->allowed( $robots, $item['url'], $item['resource_class'] ) ) { $this->complete( $run_id, $item, 'skipped', $this->not_tested_signals( 'robots_disallowed' ), null, 'none', 'robots_disallowed' ); return new CrawlChunkResult( 'active', $cursor, $this->usage( 0, 1, 0 ) ); } $fetch = $this->fetch( $item ); ++$cursor['requests']; $response = $fetch->response(); $bytes = null === $response ? 0 : strlen( $response->body() ); if ( 'redirect' === $fetch->state() ) { $this->process_redirect( $run_id, $item, $fetch ); return new CrawlChunkResult( 'active', $cursor, $this->usage( 1, 1, $bytes ) ); } if ( 'received' !== $fetch->state() || null === $response ) { $this->complete( $run_id, $item, 'skipped', $this->not_tested_signals( $fetch->reason() ), null, 'none', $fetch->reason() ); if ( 'robots' === $item['resource_class'] ) { $cursor['robots_policy'] = RobotsPolicy::from_http( 503, '' )->checkpoint(); } return new CrawlChunkResult( 'active', $cursor, $this->usage( 1, 1, $bytes ) ); } switch ( $item['resource_class'] ) { case 'robots': $this->process_robots( $run_id, $item, $response, $cursor ); break; case 'sitemap': $this->process_sitemap( $run_id, $item, $response, $robots ); break; case 'page': $this->process_page( $run_id, $item, $response, $robots ); break; case 'norm_query': case 'norm_slash': case 'norm_case': case 'norm_protocol': case 'error_probe': $this->process_diagnostic_page( $run_id, $item, $response ); break; case 'emerging_file': $this->process_emerging_file( $run_id, $item, $response ); break; default: throw new RuntimeException( 'unsupported_crawl_resource' ); } return new CrawlChunkResult( 'active', $cursor, $this->usage( 1, 1, $bytes ) ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. */ private function fetch( array $item ): HttpFetchResult { switch ( $item['resource_class'] ) { case 'robots': return $this->http->fetch_robots_hop( $item['url'] ); case 'sitemap': return $this->http->fetch_sitemap_hop( $item['url'] ); case 'page': return $this->http->fetch_page_hop( $item['url'] ); default: return $this->http->fetch_probe_hop( $item['url'], $item['resource_class'] ); } } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. */ private function process_redirect( int $run_id, array $item, HttpFetchResult $fetch ): void { $target = $fetch->redirect_url(); $enqueued = false; $reason = 'redirect_limit_exceeded'; if ( null !== $target && $item['redirect_hops'] < 5 ) { $enqueued = $this->queue->enqueue_redirect( $run_id, $target, $item['url_hash'], $item['depth'], $item['redirect_hops'] + 1, $item['resource_class'] ); $reason = $enqueued ? 'redirect_queued' : 'redirect_duplicate'; } $response = $fetch->response(); $target_is_parent = null !== $target && null !== $item['parent_hash'] && hash_equals( $item['parent_hash'], $this->queue->reference_url( $target ) ); $this->complete( $run_id, $item, 'observed', array( 'state' => 'redirect', 'outcome' => $reason, 'target_enqueued' => $enqueued, 'target_is_parent' => $target_is_parent, 'hops' => $item['redirect_hops'] + 1, 'limitations' => array( 'target_stored_as_masked_parent_edge' ), ), null === $response ? null : $response->status(), 'none', null ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. * @param array $cursor Updated by reference with normalized rules. */ private function process_robots( int $run_id, array $item, HttpTransportResponse $response, array &$cursor ): void { $policy = RobotsPolicy::from_http( $response->status(), $response->body() ); $cursor['robots_policy'] = $policy->checkpoint(); $sitemap_count = 0; $page_seeded = false; if ( ! $policy->truncated() && ! $response->truncated() && in_array( $policy->state(), array( 'parsed', 'unavailable' ), true ) ) { $declared = $policy->sitemaps(); $declared[] = $this->urls->default_sitemap_url(); foreach ( array_values( array_unique( $declared ) ) as $sitemap_url ) { if ( $this->allowed( $policy, $sitemap_url, 'sitemap' ) && $this->queue->enqueue_sitemap( $run_id, $sitemap_url, $item['url_hash'], 0 ) ) { ++$sitemap_count; } } if ( $this->allowed( $policy, $this->urls->start_url(), 'page' ) ) { $page_seeded = $this->queue->enqueue( $run_id, $this->urls->start_url(), $item['url_hash'], 0 ); } } $this->complete( $run_id, $item, 'observed', array( 'state' => $policy->state(), 'rule_count' => $policy->rule_count(), 'sitemap_candidates' => $sitemap_count, 'page_seeded' => $page_seeded, 'truncated' => $policy->truncated() || $response->truncated(), 'limitations' => array( 'selected_rules_runtime_only', 'raw_content_not_retained' ), ), $response->status(), 'text', null ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. */ private function process_sitemap( int $run_id, array $item, HttpTransportResponse $response, ?RobotsPolicy $robots ): void { if ( $response->status() < 200 || $response->status() >= 300 ) { $this->complete( $run_id, $item, 'observed', $this->http_signals( $response ), $response->status(), 'xml', null ); return; } if ( $response->truncated() ) { $this->complete( $run_id, $item, 'skipped', $this->not_tested_signals( 'sitemap_response_truncated', true ), $response->status(), 'xml', 'sitemap_not_tested' ); return; } $extraction = $this->sitemaps->extract( $response->body(), $item['url'] ); if ( null !== $robots && ! $robots->truncated() ) { foreach ( $extraction->sitemap_urls() as $url ) { if ( $item['depth'] < self::MAX_DEPTH && $this->allowed( $robots, $url, 'sitemap' ) ) { $this->queue->enqueue_sitemap( $run_id, $url, $item['url_hash'], $item['depth'] + 1 ); } } foreach ( $extraction->page_urls() as $url ) { if ( $item['depth'] < self::MAX_DEPTH && $this->allowed( $robots, $url, 'page' ) ) { $this->queue->enqueue( $run_id, $url, $item['url_hash'], $item['depth'] + 1 ); } } } $state = 'observed' === ( $extraction->signals()['state'] ?? null ) ? 'observed' : 'skipped'; $this->complete( $run_id, $item, $state, $extraction->signals(), $response->status(), 'xml', 'skipped' === $state ? 'sitemap_not_tested' : null ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. */ private function process_page( int $run_id, array $item, HttpTransportResponse $response, ?RobotsPolicy $robots ): void { $is_html = $this->is_html_response( $response ); if ( $response->status() < 200 || $response->status() >= 300 || ! $is_html ) { $this->complete( $run_id, $item, 'observed', $this->http_signals( $response, $is_html ? 'http_status_observed' : 'non_html_response' ), $response->status(), $is_html ? 'html' : 'other', null ); return; } if ( $response->truncated() ) { $this->complete( $run_id, $item, 'skipped', $this->not_tested_signals( 'html_response_truncated', true ), $response->status(), 'html', 'html_not_tested' ); return; } $extraction = $this->html->extract( $response->body(), $item['url'], $response->x_robots_tag() ); if ( null !== $robots && ! $robots->truncated() ) { foreach ( $extraction->discovered_urls() as $url ) { if ( $item['depth'] < self::MAX_DEPTH && $this->allowed( $robots, $url, 'page' ) ) { $this->queue->enqueue( $run_id, $url, $item['url_hash'], $item['depth'] + 1 ); } } } $this->queue->enqueue_diagnostics( $run_id, $item['url'], $item['url_hash'], $item['depth'] ); $state = 'observed' === ( $extraction->signals()['state'] ?? null ) ? 'observed' : 'skipped'; $this->complete( $run_id, $item, $state, $extraction->signals(), $response->status(), 'html', 'skipped' === $state ? 'html_not_tested' : null ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. */ private function process_diagnostic_page( int $run_id, array $item, HttpTransportResponse $response ): void { $is_html = $this->is_html_response( $response ); $signals = $this->http_signals( $response ); if ( $is_html && ! $response->truncated() ) { $signals = array_merge( $signals, $this->diagnostic_html_signals( $response->body(), $item ) ); } $this->complete( $run_id, $item, 'observed', $signals, $response->status(), $is_html ? 'html' : 'other', null ); } private function is_html_response( HttpTransportResponse $response ): bool { $content_type = strtolower( trim( $response->content_type() ) ); if ( false !== strpos( $content_type, 'text/html' ) || false !== strpos( $content_type, 'application/xhtml+xml' ) ) { return true; } if ( '' !== $content_type ) { return false; } $prefix = substr( $response->body(), 0, 2048 ); $prefix = ltrim( $prefix, "\xEF\xBB\xBF \t\r\n" ); return 1 === preg_match( '/\A(?:(?:<\?xml[^>]*>\s*)|(?:\s*))*?(?:body(); $line_count = '' === $body ? 0 : substr_count( $body, "\n" ) + 1; $signals = array( 'state' => 'observed', 'outcome' => $response->status() >= 200 && $response->status() < 300 ? 'file_present' : 'file_absent', 'status' => $response->status(), 'byte_count' => strlen( $body ), 'line_count' => min( 10000, $line_count ), 'h1_count' => (int) preg_match_all( '/^#\s+\S/m', $body ), 'h2_count' => (int) preg_match_all( '/^##\s+\S/m', $body ), 'markdown_links' => (int) preg_match_all( '/\[[^\]\r\n]{1,200}\]\([^\)\r\n]{1,200}\)/', $body ), 'truncated' => $response->truncated() || $line_count > 10000, 'limitations' => array( 'experimental_convention', 'raw_content_not_retained', 'provider_usage_not_proven' ), ); $this->complete( $run_id, $item, 'observed', $signals, $response->status(), 'text', null ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. * @return array */ private function diagnostic_html_signals( string $html, array $item ): array { $empty = array( 'parse_state' => 'unavailable', 'canonical_count' => 0, 'canonical_parent' => 0, 'title_present' => false, 'h1_count' => 0, 'internal_links' => 0, 'limitations' => array( 'raw_content_not_retained', 'visual_quality_not_assessed' ), ); if ( ! class_exists( DOMDocument::class ) || ! class_exists( DOMXPath::class ) ) { return $empty; } $previous = libxml_use_internal_errors( true ); try { $document = new DOMDocument(); $loaded = $document->loadHTML( '' . $html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_COMPACT ); $xpath = $loaded ? new DOMXPath( $document ) : null; } catch ( Throwable $exception ) { $xpath = null; } finally { libxml_clear_errors(); libxml_use_internal_errors( $previous ); } if ( ! $xpath instanceof DOMXPath ) { return $empty; } $canonical_nodes = $xpath->query( '//link[contains(concat(" ",normalize-space(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"))," ")," canonical ")]' ); $link_nodes = $xpath->query( '//a[@href]' ); $title_nodes = $xpath->query( '//title[normalize-space()]' ); $h1_nodes = $xpath->query( '//h1' ); $canonical_count = false === $canonical_nodes ? 0 : $canonical_nodes->length; $canonical_parent = 0; if ( false !== $canonical_nodes && null !== $item['parent_hash'] ) { foreach ( $canonical_nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $decision = $this->urls->evaluate_probe( trim( $node->getAttribute( 'href' ) ), $item['resource_class'], $item['url'] ); if ( 'accepted' === $decision['state'] && is_string( $decision['url'] ) && hash_equals( $item['parent_hash'], $this->queue->reference_url( $decision['url'] ) ) ) { ++$canonical_parent; } } } $internal_links = 0; if ( false !== $link_nodes ) { foreach ( $link_nodes as $node ) { if ( ! $node instanceof DOMElement ) { continue; } $decision = $this->urls->evaluate( trim( $node->getAttribute( 'href' ) ), preg_replace( '/\?.*$/', '', $item['url'] ) ?? $item['url'] ); if ( 'accepted' === $decision['state'] ) { ++$internal_links; } } } return array( 'parse_state' => 'observed', 'canonical_count' => $canonical_count, 'canonical_parent' => $canonical_parent, 'title_present' => false !== $title_nodes && $title_nodes->length > 0, 'h1_count' => false === $h1_nodes ? 0 : $h1_nodes->length, 'internal_links' => min( 10000, $internal_links ), 'limitations' => array( 'raw_content_not_retained', 'visual_quality_not_assessed' ), ); } private function allowed( RobotsPolicy $policy, string $url, string $resource_class ): bool { if ( 'sitemap' === $resource_class ) { $decision = $this->urls->evaluate_sitemap( $url ); } elseif ( in_array( $resource_class, array( 'norm_query', 'norm_slash', 'norm_case', 'norm_protocol', 'error_probe', 'emerging_file' ), true ) ) { $decision = $this->urls->evaluate_probe( $url, $resource_class ); } else { $decision = $this->urls->evaluate( $url ); } return 'accepted' === $decision['state'] && is_string( $decision['path'] ) && $policy->allows( $decision['path'] ); } /** * @param array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string} $item Queue row. * @param array $signals Masked signals. */ private function complete( int $run_id, array $item, string $state, array $signals, ?int $status, string $content_class, ?string $error_code ): void { if ( ! $this->queue->complete( $run_id, $item['id'], $item['url_hash'], $state, $signals, $status, $content_class, $error_code ) ) { throw new RuntimeException( 'crawl_queue_completion_conflict' ); } } /** * @param array $cursor Untrusted task cursor. * @return array */ private function normalize_cursor( array $cursor ): array { if ( array() === $cursor ) { return array( 'version' => self::CURSOR_VERSION, 'requests' => 0, 'robots_policy' => null, ); } if ( array( 'requests', 'robots_policy', 'version' ) !== $this->sorted_keys( $cursor ) || self::CURSOR_VERSION !== $cursor['version'] || ! is_int( $cursor['requests'] ) || $cursor['requests'] < 0 || $cursor['requests'] > 500 || ( null !== $cursor['robots_policy'] && ! is_array( $cursor['robots_policy'] ) ) ) { throw new InvalidArgumentException( 'invalid_crawl_cursor' ); } if ( is_array( $cursor['robots_policy'] ) ) { RobotsPolicy::from_checkpoint( $cursor['robots_policy'] ); } return $cursor; } /** * @param array $cursor Current cursor. * @return array */ private function final_cursor( array $cursor, int $run_id ): array { $counts = $this->queue->counts( $run_id ); return array( 'version' => self::CURSOR_VERSION, 'requests' => $cursor['requests'], 'counts' => $counts, ); } /** * @return array */ private function not_tested_signals( string $reason, bool $truncated = false ): array { return array( 'state' => 'not_tested', 'reason' => $reason, 'truncated' => $truncated, 'limitations' => array( 'raw_content_not_retained' ), ); } /** * @return array */ private function http_signals( HttpTransportResponse $response, string $outcome = 'http_status_observed' ): array { return array( 'state' => 'observed', 'outcome' => $outcome, 'status' => $response->status(), 'truncated' => $response->truncated(), 'limitations' => array( 'response_body_not_analyzed', 'raw_content_not_retained' ), ); } /** * @return array{milliseconds:int,items:int,requests:int,bytes:int} */ private function usage( int $requests = 0, int $items = 0, int $bytes = 0 ): array { return array( 'milliseconds' => 0, 'items' => $items, 'requests' => $requests, 'bytes' => $bytes, ); } /** * @param array $value Array to inspect. * @return list */ private function sorted_keys( array $value ): array { $keys = array_keys( $value ); sort( $keys ); return $keys; } }