scope = $scope; $this->files = $files; $this->extractor = $extractor; $this->snippets = $snippets; $this->seed_on_initialize = $seed_on_initialize; $this->include_snippets = $include_snippets; $this->include_vendor_dependencies = $include_vendor_dependencies; } /** * @param array $cursor Safe persisted cursor. */ public function process_one( int $run_id, array $cursor ): CodeChunkResult { if ( $run_id < 1 ) { throw new InvalidArgumentException( 'invalid_code_chunk' ); } $cursor = $this->normalize_cursor( $cursor ); $started = microtime( true ); $items = 0; $bytes = 0; if ( ! $cursor['initialized'] && $this->seed_on_initialize ) { foreach ( $this->scope->seeds() as $seed ) { $this->files->enqueue( $run_id, $seed['scope'], $seed['path'], $seed['type'] ); } } $cursor['initialized'] = true; while ( $items < self::MAX_ITEMS_PER_CALL && $bytes < self::MAX_BYTES_PER_CALL && microtime( true ) - $started < self::MAX_CALL_SECONDS ) { $item = $this->files->next_queued( $run_id ); if ( null === $item ) { $snippet_state = $this->include_snippets ? $this->process_snippet( $run_id, $cursor, $bytes ) : 'completed'; if ( 'progress' === $snippet_state ) { ++$items; continue; } return new CodeChunkResult( 'completed', $this->final_cursor( $cursor, $run_id ), $this->usage( $started, $items, $bytes ) ); } if ( 'directory' === $item['item_type'] ) { if ( $this->expand_directory( $run_id, $item, $started + self::MAX_CALL_SECONDS ) ) { ++$cursor['directories']; } } else { $file_result = $this->process_file( $run_id, $item, self::MAX_BYTES_PER_CALL - $bytes ); $bytes += $file_result['bytes']; if ( ! $file_result['completed'] ) { return new CodeChunkResult( 'active', $cursor, $this->usage( $started, $items, $bytes ) ); } ++$cursor['files']; } ++$items; } return new CodeChunkResult( 'active', $cursor, $this->usage( $started, $items, $bytes ) ); } /** * @param array{id:int,file_hash:string,scope:string,item_type:string,extension:string,path:string,size_bytes:int,directory_offset:int} $item Queue row. */ private function expand_directory( int $run_id, array $item, float $deadline ): bool { try { $entries = new FilesystemIterator( $item['path'], FilesystemIterator::SKIP_DOTS ); $entries->seek( $item['directory_offset'] ); } catch ( Throwable $exception ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'directory_unreadable', true ); return true; } $limit_hit = false; $processed = 0; $next_offset = $item['directory_offset']; while ( $entries->valid() && $processed < self::MAX_DIRECTORY_ENTRIES_PER_CHUNK && $next_offset < self::MAX_DIRECTORY_ENTRIES && microtime( true ) < $deadline ) { $entry = $entries->current(); $path = $entry->getPathname(); ++$processed; ++$next_offset; $entries->next(); if ( $entry->isLink() ) { if ( ! $this->files->record_skipped_reference( $run_id, $item['scope'], $path, 'symlink_skipped' ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } continue; } if ( $entry->isDir() ) { if ( '.git' === $entry->getFilename() ) { if ( ! $this->files->record_metadata( $run_id, $item['scope'], $path, 'metadata.git' ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } continue; } $directory_name = strtolower( $entry->getFilename() ); $excluded = in_array( $directory_name, array( 'node_modules', '.svn', '.hg' ), true ) || ( 'vendor' === $directory_name && ! $this->include_vendor_dependencies ); if ( $excluded ) { $error_code = 'vendor' === $directory_name ? 'vendor_dependency_scope_excluded' : ( 'node_modules' === $directory_name ? 'build_dependency_scope_excluded' : 'repository_metadata_scope_excluded' ); if ( ! $this->files->record_skipped_reference( $run_id, $item['scope'], $path, $error_code ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } continue; } if ( ! $this->files->enqueue( $run_id, $item['scope'], $path, 'directory' ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } continue; } if ( ! $entry->isFile() ) { continue; } $metadata = $this->metadata_signal( strtolower( $entry->getFilename() ) ); if ( null !== $metadata ) { if ( ! $this->files->record_metadata( $run_id, $item['scope'], $path, $metadata ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } continue; } $extension = strtolower( (string) pathinfo( $entry->getFilename(), PATHINFO_EXTENSION ) ); if ( in_array( $extension, array( 'php', 'inc', 'phtml', 'js', 'mjs', 'cjs', 'css' ), true ) ) { if ( ! $this->files->enqueue( $run_id, $item['scope'], $path, 'file' ) && $this->files->counts( $run_id )['total'] >= 10000 ) { $limit_hit = true; break; } } } if ( $entries->valid() && $next_offset >= self::MAX_DIRECTORY_ENTRIES ) { $limit_hit = true; } if ( ! $limit_hit && $entries->valid() ) { if ( 0 === $processed ) { return false; } if ( ! $this->files->checkpoint_directory( $run_id, $item['id'], $item['file_hash'], $next_offset ) ) { throw new RuntimeException( 'code_directory_checkpoint_conflict' ); } return false; } if ( ! $this->files->finish_directory( $run_id, $item['id'], $item['file_hash'], $limit_hit ? 'scan_item_limit' : null ) ) { throw new RuntimeException( 'code_directory_completion_conflict' ); } return true; } /** * @param array{id:int,file_hash:string,scope:string,item_type:string,extension:string,path:string,size_bytes:int,directory_offset:int} $item Queue row. * @return array{completed:bool,bytes:int} */ private function process_file( int $run_id, array $item, int $remaining_bytes ): array { $current_size = filesize( $item['path'] ); if ( false === $current_size ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'file_stat_failed', true ); return array( 'completed' => true, 'bytes' => 0, ); } if ( $current_size > self::MAX_FILE_BYTES ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'file_size_limit' ); return array( 'completed' => true, 'bytes' => 0, ); } if ( $current_size > $remaining_bytes ) { return array( 'completed' => false, 'bytes' => 0, ); } if ( ! is_readable( $item['path'] ) ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'file_unreadable' ); return array( 'completed' => true, 'bytes' => 0, ); } // Canonical path was checked against CodeScopePolicy immediately before read. // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents $source = file_get_contents( $item['path'], false, null, 0, min( self::MAX_FILE_BYTES, $remaining_bytes ) + 1 ); if ( false === $source ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'file_read_failed', true ); return array( 'completed' => true, 'bytes' => 0, ); } if ( strlen( $source ) > self::MAX_FILE_BYTES ) { $this->files->skip( $run_id, $item['id'], $item['file_hash'], 'file_size_limit' ); return array( 'completed' => true, 'bytes' => strlen( $source ), ); } if ( strlen( $source ) > $remaining_bytes ) { return array( 'completed' => false, 'bytes' => strlen( $source ), ); } $signals = $this->extractor->extract( $source, $item['extension'] ); if ( ! $this->files->complete( $run_id, $item['id'], $item['file_hash'], $signals ) ) { throw new RuntimeException( 'code_file_completion_conflict' ); } return array( 'completed' => true, 'bytes' => strlen( $source ), ); } /** * @param array $cursor Safe cursor updated by reference. */ private function process_snippet( int $run_id, array &$cursor, int &$bytes ): string { if ( 'unknown' === $cursor['snippets']['state'] ) { $cursor['snippets']['state'] = $this->snippets->available() ? 'active' : 'not_available'; } if ( 'active' !== $cursor['snippets']['state'] ) { return 'completed'; } $snippet = $this->snippets->next_at( $cursor['snippets']['offset'] ); if ( null === $snippet ) { $cursor['snippets']['state'] = 'completed'; return 'completed'; } ++$cursor['snippets']['offset']; ++$cursor['snippets']['observed']; if ( 1000 === $cursor['snippets']['observed'] ) { $this->files->mark_discovery_limitation( $run_id, 'snippet_item_limit' ); } if ( $snippet['oversized'] ) { if ( ! $this->files->record_virtual_skip( $run_id, 'code_snippets', (string) $snippet['id'], 'file_size_limit', $snippet['size_bytes'] ) ) { $this->files->mark_discovery_limitation( $run_id, 'scan_item_limit' ); } ++$cursor['snippets']['skipped']; return 'progress'; } $source = 0 === strpos( ltrim( $snippet['code'] ), 'files->record_virtual_code( $run_id, 'code_snippets', (string) $snippet['id'], $this->extractor->extract( $source, 'php' ), strlen( $source ) ) ) { $this->files->mark_discovery_limitation( $run_id, 'scan_item_limit' ); ++$cursor['snippets']['skipped']; } $bytes += strlen( $source ); return 'progress'; } private function metadata_signal( string $filename ): ?string { $map = array( 'composer.json' => 'metadata.composer_manifest', 'composer.lock' => 'metadata.composer_lock', 'package.json' => 'metadata.package_manifest', 'package-lock.json' => 'metadata.package_lock', 'pnpm-lock.yaml' => 'metadata.package_lock', 'yarn.lock' => 'metadata.package_lock', ); return $map[ $filename ] ?? null; } /** * @param array $cursor Untrusted task cursor. * @return array */ private function normalize_cursor( array $cursor ): array { if ( array() === $cursor ) { return array( 'version' => self::CURSOR_VERSION, 'initialized' => false, 'directories' => 0, 'files' => 0, 'snippets' => array( 'state' => 'unknown', 'offset' => 0, 'observed' => 0, 'skipped' => 0, ), ); } $valid = isset( $cursor['version'], $cursor['initialized'], $cursor['directories'], $cursor['files'], $cursor['snippets'] ) && self::CURSOR_VERSION === $cursor['version'] && is_bool( $cursor['initialized'] ) && is_int( $cursor['directories'] ) && $cursor['directories'] >= 0 && $cursor['directories'] <= 10000 && is_int( $cursor['files'] ) && $cursor['files'] >= 0 && $cursor['files'] <= 10000 && is_array( $cursor['snippets'] ) && isset( $cursor['snippets']['state'], $cursor['snippets']['offset'], $cursor['snippets']['observed'], $cursor['snippets']['skipped'] ) && in_array( $cursor['snippets']['state'], array( 'unknown', 'active', 'not_available', 'completed' ), true ) && is_int( $cursor['snippets']['offset'] ) && $cursor['snippets']['offset'] >= 0 && $cursor['snippets']['offset'] <= 1000 && is_int( $cursor['snippets']['observed'] ) && $cursor['snippets']['observed'] >= 0 && $cursor['snippets']['observed'] <= 1000 && is_int( $cursor['snippets']['skipped'] ) && $cursor['snippets']['skipped'] >= 0 && $cursor['snippets']['skipped'] <= 1000; if ( ! $valid ) { throw new InvalidArgumentException( 'invalid_code_cursor' ); } return $cursor; } /** * @param array $cursor Current cursor. * @return array */ private function final_cursor( array $cursor, int $run_id ): array { $cursor['counts'] = $this->files->counts( $run_id ); return $cursor; } /** @return array{milliseconds:int,items:int,requests:int,bytes:int} */ private function usage( float $started, int $items, int $bytes ): array { return array( 'milliseconds' => max( 1, (int) ceil( ( microtime( true ) - $started ) * 1000 ) ), 'items' => $items, 'requests' => 0, 'bytes' => $bytes, ); } }