urls = $urls; $this->transport = $transport; } public function fetch_page( string $url, int $remaining_requests ): HttpFetchResult { $decision = $this->urls->evaluate( $url ); return $this->fetch( $decision, $remaining_requests, self::MAX_HTML_BYTES, 'html', fn ( string $candidate, string $base ): array => $this->urls->evaluate( $candidate, $base ) ); } public function fetch_robots( int $remaining_requests ): HttpFetchResult { return $this->fetch( array( 'state' => 'accepted', 'reason' => 'fixed_robots_endpoint', 'url' => $this->urls->robots_url(), 'path' => '/robots.txt', ), $remaining_requests, self::MAX_ROBOTS_BYTES, 'robots', fn ( string $candidate, string $base ): array => $this->urls->evaluate_robots_redirect( $candidate, $base ) ); } public function fetch_sitemap( string $url, int $remaining_requests ): HttpFetchResult { $decision = $this->urls->evaluate_sitemap( $url ); return $this->fetch( $decision, $remaining_requests, self::MAX_SITEMAP_BYTES, 'sitemap', fn ( string $candidate, string $base ): array => $this->urls->evaluate_sitemap( $candidate, $base ) ); } /** * Execute exactly one page request. A validated redirect is returned to the * durable queue and is never followed by this worker invocation. */ public function fetch_page_hop( string $url ): HttpFetchResult { return $this->fetch_hop( $this->urls->evaluate( $url ), self::MAX_HTML_BYTES, 'html', fn ( string $candidate, string $base ): array => $this->urls->evaluate( $candidate, $base ) ); } public function fetch_robots_hop( string $url ): HttpFetchResult { return $this->fetch_hop( $this->urls->evaluate_robots_redirect( $url, $this->urls->robots_url() ), self::MAX_ROBOTS_BYTES, 'robots', fn ( string $candidate, string $base ): array => $this->urls->evaluate_robots_redirect( $candidate, $base ) ); } public function fetch_sitemap_hop( string $url ): HttpFetchResult { return $this->fetch_hop( $this->urls->evaluate_sitemap( $url ), self::MAX_SITEMAP_BYTES, 'sitemap', fn ( string $candidate, string $base ): array => $this->urls->evaluate_sitemap( $candidate, $base ) ); } public function fetch_probe_hop( string $url, string $kind ): HttpFetchResult { $publisher = 'emerging_file' === $kind; return $this->fetch_hop( $this->urls->evaluate_probe( $url, $kind ), $publisher ? self::MAX_PUBLISHER_BYTES : self::MAX_HTML_BYTES, $publisher ? 'publisher' : 'html', fn ( string $candidate, string $base ): array => $this->urls->evaluate_probe( $candidate, $kind, $base ) ); } /** * @param array{state:string,reason:string,url:string|null,path:string|null} $decision Initial URL decision. * @param callable(string,string):array{state:string,reason:string,url:string|null,path:string|null} $redirect_policy Redirect policy. */ private function fetch_hop( array $decision, int $max_body_bytes, string $resource_class, callable $redirect_policy ): HttpFetchResult { if ( 'accepted' !== $decision['state'] || ! is_string( $decision['url'] ) ) { $rejected_class = 'html' === $resource_class ? 'page' : $resource_class; return HttpFetchResult::not_tested( $rejected_class . '_url_rejected', 0, 0 ); } $current = $decision['url']; try { $response = $this->transport->get( $current, $max_body_bytes, $resource_class ); } catch ( Throwable $exception ) { return HttpFetchResult::error( 'transport_exception', 1, 0 ); } if ( 'network_error' === $response->state() ) { return HttpFetchResult::not_tested( 'network_error', 1, 0 ); } if ( $response->status() < 300 || $response->status() > 399 ) { return HttpFetchResult::received( $current, 1, 0, $response ); } $location = $response->location(); if ( null === $location || '' === $location ) { return HttpFetchResult::not_tested( 'redirect_location_missing', 1, 0 ); } $redirect = $redirect_policy( $location, $current ); if ( 'accepted' !== $redirect['state'] || ! is_string( $redirect['url'] ) ) { return HttpFetchResult::not_tested( 'redirect_target_rejected', 1, 0 ); } return HttpFetchResult::redirect( $current, $redirect['url'], $response ); } /** * @param array{state:string,reason:string,url:string|null,path:string|null} $decision Initial URL decision. * @param callable(string,string):array{state:string,reason:string,url:string|null,path:string|null} $redirect_policy Redirect policy. */ private function fetch( array $decision, int $remaining_requests, int $max_body_bytes, string $resource_class, callable $redirect_policy ): HttpFetchResult { if ( $remaining_requests < 1 || $remaining_requests > 500 ) { throw new InvalidArgumentException( 'invalid_crawl_request_budget' ); } if ( 'accepted' !== $decision['state'] || ! is_string( $decision['url'] ) ) { $rejected_class = 'html' === $resource_class ? 'page' : $resource_class; return HttpFetchResult::not_tested( $rejected_class . '_url_rejected', 0, 0 ); } $current = $decision['url']; $requests = 0; $redirects = 0; while ( true ) { if ( $requests >= $remaining_requests ) { return HttpFetchResult::not_tested( 'request_budget_exhausted', $requests, $redirects ); } ++$requests; try { $response = $this->transport->get( $current, $max_body_bytes, $resource_class ); } catch ( Throwable $exception ) { return HttpFetchResult::error( 'transport_exception', $requests, $redirects ); } if ( 'network_error' === $response->state() ) { return HttpFetchResult::not_tested( 'network_error', $requests, $redirects ); } if ( $response->status() < 300 || $response->status() > 399 ) { return HttpFetchResult::received( $current, $requests, $redirects, $response ); } if ( $redirects >= self::MAX_REDIRECTS ) { return HttpFetchResult::not_tested( 'redirect_limit_exceeded', $requests, $redirects ); } $location = $response->location(); if ( null === $location || '' === $location ) { return HttpFetchResult::not_tested( 'redirect_location_missing', $requests, $redirects ); } $redirect = $redirect_policy( $location, $current ); if ( 'accepted' !== $redirect['state'] || ! is_string( $redirect['url'] ) ) { return HttpFetchResult::not_tested( 'redirect_target_rejected', $requests, $redirects ); } $current = $redirect['url']; ++$redirects; } } }