database = $database; $this->table = Schema::tables( $database )['crawl_pages']; $this->urls = $urls; $this->signals = $signals; $this->hash_key = $hash_key; } /** * Add one normalized request candidate. Duplicate or over-budget candidates * are not inserted and never expand the queue. */ public function enqueue( int $run_id, string $url, ?string $parent_hash, int $depth ): bool { return $this->enqueue_resource( $run_id, $url, $parent_hash, $depth, 0, self::RESOURCE_PAGE ); } public function enqueue_robots( int $run_id ): bool { return $this->enqueue_resource( $run_id, $this->urls->robots_url(), null, 0, 0, self::RESOURCE_ROBOTS ); } public function enqueue_sitemap( int $run_id, string $url, ?string $parent_hash, int $depth ): bool { return $this->enqueue_resource( $run_id, $url, $parent_hash, $depth, 0, self::RESOURCE_SITEMAP ); } public function enqueue_diagnostics( int $run_id, string $page_url, string $parent_hash, int $depth ): int { if ( $run_id < 1 || $depth < 0 || $depth > self::MAX_CRAWL_DEPTH || 1 !== preg_match( '/^[a-f0-9]{64}$/', $parent_hash ) ) { throw new InvalidArgumentException( 'invalid_crawl_diagnostics' ); } $candidates = $this->urls->diagnostic_variants( $page_url ); $candidates['error_probe'] = $this->urls->error_probe_url( substr( hash_hmac( 'sha256', 'error-probe:' . $run_id, $this->hash_key ), 0, 24 ) ); $candidates['emerging_file'] = $this->urls->emerging_file_url(); $inserted = 0; foreach ( $candidates as $resource_class => $url ) { if ( ! in_array( $resource_class, self::RESOURCE_DIAGNOSTICS, true ) || $this->has_resource_class( $run_id, $resource_class ) ) { continue; } if ( $this->enqueue_resource( $run_id, $url, $parent_hash, $depth, 0, $resource_class ) ) { ++$inserted; } } return $inserted; } /** * Add a manually validated redirect target without changing its resource class. */ public function enqueue_redirect( int $run_id, string $url, string $parent_hash, int $depth, int $redirect_hops, string $resource_class ): bool { return $this->enqueue_resource( $run_id, $url, $parent_hash, $depth, $redirect_hops, $resource_class ); } public function reference_url( string $url ): string { return $this->reference( $url ); } private function enqueue_resource( int $run_id, string $url, ?string $parent_hash, int $depth, int $redirect_hops, string $resource_class ): bool { if ( $run_id < 1 || $depth < 0 || $depth > self::MAX_CRAWL_DEPTH || $redirect_hops < 0 || $redirect_hops > 5 ) { throw new InvalidArgumentException( 'invalid_crawl_candidate' ); } $this->assert_resource_class( $resource_class ); if ( null !== $parent_hash && 1 !== preg_match( '/^[a-f0-9]{64}$/', $parent_hash ) ) { throw new InvalidArgumentException( 'invalid_crawl_parent' ); } $decision = $this->evaluate_resource( $url, $resource_class ); if ( 'accepted' !== $decision['state'] || ! is_string( $decision['url'] ) || ! is_string( $decision['path'] ) ) { return false; } $runtime_path = in_array( $resource_class, self::RESOURCE_DIAGNOSTICS, true ) ? $decision['url'] : $decision['path']; $display_path = in_array( $resource_class, array( self::RESOURCE_PAGE, self::RESOURCE_ROBOTS, self::RESOURCE_SITEMAP, 'emerging_file' ), true ) ? $this->urls->safe_display_resource_path( $decision['path'] ) : null; if ( strlen( $runtime_path ) > self::MAX_RUNTIME_PATH ) { return false; } $url_hash = $this->reference( $decision['url'] ); $now = current_time( 'mysql', true ); // Table name is selected from the compiled Schema allowlist. INSERT IGNORE // makes duplicate discovery idempotent; the subquery enforces the run cap. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $result = $this->database->query( $this->database->prepare( "INSERT IGNORE INTO {$this->table} (run_id,url_hash,parent_hash,depth,redirect_hops,resource_class,state,runtime_path,display_path,signals_json,http_status,content_class,error_code,created_at,updated_at) SELECT %d,%s,NULLIF(%s,''),%d,%d,%s,%s,%s,NULLIF(%s,''),%s,NULL,%s,NULL,%s,%s FROM DUAL WHERE (SELECT COUNT(*) FROM {$this->table} WHERE run_id = %d) < %d", $run_id, $url_hash, $parent_hash, $depth, $redirect_hops, $resource_class, self::STATE_QUEUED, $runtime_path, $display_path, '{}', 'none', $now, $now, $run_id, self::MAX_PAGES ) ); if ( false === $result ) { throw new StorageException( 'crawl_enqueue_failed' ); } return 1 === $result; } /** * The URL is runtime-only. Callers must not put it in logs, task cursors, * Findings or exports. * * @return array{id:int,url:string,url_hash:string,parent_hash:string|null,depth:int,redirect_hops:int,resource_class:string}|null */ public function next_queued( int $run_id ): ?array { if ( $run_id < 1 ) { throw new InvalidArgumentException( 'invalid_run_id' ); } // Table name is selected from the compiled Schema allowlist. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $row = $this->database->get_row( $this->database->prepare( "SELECT id,url_hash,parent_hash,depth,redirect_hops,resource_class,runtime_path FROM {$this->table} WHERE run_id = %d AND state = %s AND runtime_path IS NOT NULL ORDER BY id ASC LIMIT 1", $run_id, self::STATE_QUEUED ), ARRAY_A ); if ( ! is_array( $row ) ) { return null; } $resource_class = (string) $row['resource_class']; $this->assert_resource_class( $resource_class ); $decision = $this->evaluate_resource( (string) $row['runtime_path'], $resource_class ); if ( 'accepted' !== $decision['state'] || ! is_string( $decision['url'] ) ) { throw new StorageException( 'invalid_stored_crawl_path' ); } return array( 'id' => (int) $row['id'], 'url' => $decision['url'], 'url_hash' => (string) $row['url_hash'], 'parent_hash' => null === $row['parent_hash'] ? null : (string) $row['parent_hash'], 'depth' => (int) $row['depth'], 'redirect_hops' => (int) $row['redirect_hops'], 'resource_class' => $resource_class, ); } /** * @param array $signals Masked page observations. */ public function complete( int $run_id, int $page_id, string $url_hash, string $state, array $signals, ?int $http_status, string $content_class, ?string $error_code = null ): bool { if ( $run_id < 1 || $page_id < 1 || 1 !== preg_match( '/^[a-f0-9]{64}$/', $url_hash ) ) { throw new InvalidArgumentException( 'invalid_crawl_completion' ); } if ( ! in_array( $state, array( self::STATE_OBSERVED, self::STATE_SKIPPED, self::STATE_ERROR ), true ) ) { throw new InvalidArgumentException( 'invalid_crawl_state' ); } if ( null !== $http_status && ( $http_status < 100 || $http_status > 599 ) ) { throw new InvalidArgumentException( 'invalid_crawl_http_status' ); } if ( ! in_array( $content_class, array( 'html', 'xml', 'text', 'other', 'none' ), true ) ) { throw new InvalidArgumentException( 'invalid_crawl_content_class' ); } if ( null !== $error_code && 1 !== preg_match( '/^[a-z][a-z0-9_]{1,63}$/', $error_code ) ) { throw new InvalidArgumentException( 'invalid_crawl_error_code' ); } $this->signals->validate( $signals ); $json = wp_json_encode( $signals, JSON_UNESCAPED_SLASHES ); if ( false === $json || strlen( $json ) > self::MAX_SIGNAL_BYTES ) { throw new StorageException( 'crawl_signals_too_large' ); } // URL hash is a compare-and-swap guard; raw runtime path is cleared in the // same statement that commits the terminal masked observation. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $updated = $this->database->query( $this->database->prepare( "UPDATE {$this->table} SET state = %s, runtime_path = NULL, signals_json = %s, http_status = NULLIF(%d,0), content_class = %s, error_code = NULLIF(%s,''), updated_at = %s WHERE id = %d AND run_id = %d AND url_hash = %s AND state = %s", $state, $json, $http_status, $content_class, $error_code, current_time( 'mysql', true ), $page_id, $run_id, $url_hash, self::STATE_QUEUED ) ); if ( false === $updated ) { throw new StorageException( 'crawl_completion_failed' ); } return 1 === $updated; } public function clear_pending( int $run_id, string $error_code ): int { if ( $run_id < 1 || 1 !== preg_match( '/^[a-z][a-z0-9_]{1,63}$/', $error_code ) ) { throw new InvalidArgumentException( 'invalid_crawl_clear' ); } // Table name is selected from the compiled Schema allowlist. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $result = $this->database->query( $this->database->prepare( "UPDATE {$this->table} SET state = %s, runtime_path = NULL, signals_json = %s, content_class = %s, error_code = %s, updated_at = %s WHERE run_id = %d AND state = %s", self::STATE_SKIPPED, '{}', 'none', $error_code, current_time( 'mysql', true ), $run_id, self::STATE_QUEUED ) ); if ( false === $result ) { throw new StorageException( 'crawl_clear_failed' ); } return $result; } /** * Read only terminal masked signals; runtime paths are never selected. * * @return list,http_status:int|null,content_class:string,error_code:string|null}> */ public function snapshot( int $run_id, int $offset = 0, int $limit = 100 ): array { if ( $run_id < 1 || $offset < 0 || $limit < 1 || $limit > 100 ) { throw new InvalidArgumentException( 'invalid_crawl_snapshot_query' ); } // Table name is selected from the compiled Schema allowlist; bounded LIMIT // and OFFSET are validated integers. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $rows = $this->database->get_results( $this->database->prepare( "SELECT url_hash,parent_hash,display_path,depth,redirect_hops,resource_class,state,signals_json,http_status,content_class,error_code FROM {$this->table} WHERE run_id = %d AND state <> %s ORDER BY id ASC LIMIT %d OFFSET %d", $run_id, self::STATE_QUEUED, $limit, $offset ), ARRAY_A ); $snapshot = array(); foreach ( $rows as $row ) { $signals = $this->decode_signals( (string) $row['signals_json'] ); $snapshot[] = array( 'url_hash' => (string) $row['url_hash'], 'parent_hash' => null === $row['parent_hash'] ? null : (string) $row['parent_hash'], 'display_path' => null === $row['display_path'] ? null : (string) $row['display_path'], 'depth' => (int) $row['depth'], 'redirect_hops' => (int) $row['redirect_hops'], 'resource_class' => (string) $row['resource_class'], 'state' => (string) $row['state'], 'signals' => $signals, 'http_status' => null === $row['http_status'] ? null : (int) $row['http_status'], 'content_class' => (string) $row['content_class'], 'error_code' => null === $row['error_code'] ? null : (string) $row['error_code'], ); } return $snapshot; } /** * @return array{queued:int,observed:int,skipped:int,error:int,total:int} */ public function counts( int $run_id ): array { if ( $run_id < 1 ) { throw new InvalidArgumentException( 'invalid_run_id' ); } $counts = array_fill_keys( array( self::STATE_QUEUED, self::STATE_OBSERVED, self::STATE_SKIPPED, self::STATE_ERROR ), 0 ); // Table name is selected from the compiled Schema allowlist. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $rows = $this->database->get_results( $this->database->prepare( "SELECT state,COUNT(*) AS page_count FROM {$this->table} WHERE run_id = %d GROUP BY state", $run_id ), ARRAY_A ); foreach ( $rows as $row ) { $state = (string) $row['state']; if ( array_key_exists( $state, $counts ) ) { $counts[ $state ] = (int) $row['page_count']; } } return array( 'queued' => $counts[ self::STATE_QUEUED ], 'observed' => $counts[ self::STATE_OBSERVED ], 'skipped' => $counts[ self::STATE_SKIPPED ], 'error' => $counts[ self::STATE_ERROR ], 'total' => array_sum( $counts ), ); } /** * Safe live progress for the administrator screen. No origin, query, * fragment, credential or rejected path segment is returned. * * @return array{processed:int,queued:int,total:int,current_path:string|null} */ public function progress( int $run_id ): array { $counts = $this->counts( $run_id ); // Table name is selected from the compiled Schema allowlist. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $current = $this->database->get_var( $this->database->prepare( "SELECT display_path FROM {$this->table} WHERE run_id = %d AND state = %s ORDER BY id ASC LIMIT 1", $run_id, self::STATE_QUEUED ) ); return array( 'processed' => $counts['observed'] + $counts['skipped'] + $counts['error'], 'queued' => $counts['queued'], 'total' => $counts['total'], 'current_path' => is_string( $current ) && '' !== $current ? $current : null, ); } private function reference( string $url ): string { return hash_hmac( 'sha256', $url, $this->hash_key ); } /** * @return array{state:string,reason:string,url:string|null,path:string|null} */ private function evaluate_resource( string $url, string $resource_class ): array { switch ( $resource_class ) { case self::RESOURCE_ROBOTS: return $this->urls->evaluate_robots_redirect( $url, $this->urls->robots_url() ); case self::RESOURCE_SITEMAP: return $this->urls->evaluate_sitemap( $url ); default: return in_array( $resource_class, self::RESOURCE_DIAGNOSTICS, true ) ? $this->urls->evaluate_probe( $url, $resource_class ) : $this->urls->evaluate( $url ); } } private function assert_resource_class( string $resource_class ): void { if ( ! in_array( $resource_class, array_merge( array( self::RESOURCE_PAGE, self::RESOURCE_ROBOTS, self::RESOURCE_SITEMAP ), self::RESOURCE_DIAGNOSTICS ), true ) ) { throw new InvalidArgumentException( 'invalid_crawl_resource_class' ); } } private function has_resource_class( int $run_id, string $resource_class ): bool { // Table name is selected from the compiled Schema allowlist. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared $count = $this->database->get_var( $this->database->prepare( "SELECT COUNT(*) FROM {$this->table} WHERE run_id = %d AND resource_class = %s", $run_id, $resource_class ) ); return (int) $count > 0; } /** * @return array */ private function decode_signals( string $json ): array { if ( strlen( $json ) > self::MAX_SIGNAL_BYTES ) { throw new StorageException( 'invalid_crawl_signals' ); } try { $signals = json_decode( $json, true, self::MAX_CRAWL_DEPTH, JSON_THROW_ON_ERROR ); } catch ( JsonException $exception ) { // Previous exception remains local and is never rendered. // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped throw new StorageException( 'invalid_crawl_signals', 0, $exception ); } if ( ! is_array( $signals ) ) { throw new StorageException( 'invalid_crawl_signals' ); } $this->signals->validate( $signals ); return $signals; } }