*/ private array $crawlers; /** @var array> */ private array $robotsTxtTokens; private int $schemaVersion; private string $updatedAt; /** * @param array $crawlers * @param array> $robotsTxtTokens */ private function __construct(array $crawlers, array $robotsTxtTokens, int $schemaVersion, string $updatedAt) { $this->crawlers = $crawlers; $this->robotsTxtTokens = $robotsTxtTokens; $this->schemaVersion = $schemaVersion; $this->updatedAt = $updatedAt; } public static function defaultFilePath(): string { return dirname(__DIR__) . '/data/crawlers.json'; } public static function fromDefaultFile(): self { return self::fromFile(self::defaultFilePath()); } public static function fromFile(string $path): self { if (!is_readable($path)) { throw new RuntimeException('Cannot read crawlers.json: ' . json_encode($path, self::MESSAGE_VALUE_FLAGS)); } $raw = file_get_contents($path); if ($raw === false) { throw new RuntimeException('Failed to read crawlers.json: ' . json_encode($path, self::MESSAGE_VALUE_FLAGS)); } $decoded = json_decode($raw, true); if (!is_array($decoded)) { throw new InvalidArgumentException('crawlers.json contains invalid JSON: ' . json_encode(json_last_error_msg(), self::MESSAGE_VALUE_FLAGS)); } return self::fromArray($decoded); } /** * @param array $data */ public static function fromArray(array $data): self { if (!isset($data['schema_version']) || !is_int($data['schema_version'])) { throw new InvalidArgumentException('Missing or non-integer schema_version'); } if ($data['schema_version'] !== 1) { throw new InvalidArgumentException('Unsupported schema_version: ' . json_encode($data['schema_version'], self::MESSAGE_VALUE_FLAGS)); } if (!isset($data['crawlers']) || !is_array($data['crawlers'])) { throw new InvalidArgumentException('Missing crawlers array'); } $crawlers = []; $seenIds = []; foreach ($data['crawlers'] as $index => $entry) { if (!is_array($entry)) { throw new InvalidArgumentException('crawlers[' . json_encode($index, self::MESSAGE_VALUE_FLAGS) . '] is not an object'); } $crawler = self::buildCrawler($entry, (string) $index); if (isset($seenIds[$crawler->id])) { throw new InvalidArgumentException('Duplicate crawler id: ' . json_encode($crawler->id, self::MESSAGE_VALUE_FLAGS)); } $seenIds[$crawler->id] = true; $crawlers[] = $crawler; } $tokens = []; foreach ($data['robots_txt_tokens'] ?? [] as $index => $entry) { if (!is_array($entry) || !isset($entry['id'], $entry['vendor'], $entry['token'])) { throw new InvalidArgumentException('robots_txt_tokens[' . json_encode($index, self::MESSAGE_VALUE_FLAGS) . '] is missing required fields (id, vendor, token)'); } $tokens[] = array_map('strval', $entry); } return new self( $crawlers, $tokens, $data['schema_version'], isset($data['updated_at']) && is_string($data['updated_at']) ? $data['updated_at'] : '' ); } /** * @param array $entry */ private static function buildCrawler(array $entry, string $where): CrawlerDefinition { foreach (['id', 'vendor', 'ua_pattern', 'category', 'verify'] as $key) { if (!isset($entry[$key])) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] is missing ' . json_encode($key, self::MESSAGE_VALUE_FLAGS)); } } if (!array_key_exists('docs_url', $entry)) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] is missing docs_url (use an explicit empty string if unpublished)'); } $category = (string) $entry['category']; if (!in_array($category, CrawlerDefinition::CATEGORIES, true)) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] has an invalid category: ' . json_encode($category, self::MESSAGE_VALUE_FLAGS)); } $verify = $entry['verify']; if (!is_array($verify) || !isset($verify['method'])) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] is missing verify.method'); } if (!in_array($verify['method'], CrawlerDefinition::VERIFY_METHODS, true)) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] has an invalid verify.method: ' . json_encode($verify['method'], self::MESSAGE_VALUE_FLAGS)); } if ($verify['method'] === CrawlerDefinition::VERIFY_RDNS && empty($verify['domains'])) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] uses rdns verification but has no domains'); } if ($verify['method'] === CrawlerDefinition::VERIFY_IP_LIST && empty($verify['ip_list_url'])) { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] uses ip_list verification but has no ip_list_url'); } $uaPattern = (string) $entry['ua_pattern']; if ($uaPattern === '') { throw new InvalidArgumentException('crawlers[' . json_encode($where, self::MESSAGE_VALUE_FLAGS) . '] has an empty ua_pattern'); } /** @var array{method: string, ip_list_url?: string, domains?: array} $verify */ return new CrawlerDefinition( (string) $entry['id'], (string) $entry['vendor'], $uaPattern, $category, $verify, (string) $entry['docs_url'], isset($entry['note']) ? (string) $entry['note'] : null ); } /** * @return array */ public function crawlers(): array { return $this->crawlers; } public function findById(string $id): ?CrawlerDefinition { foreach ($this->crawlers as $crawler) { if ($crawler->id === $id) { return $crawler; } } return null; } /** * @return array> */ public function robotsTxtTokens(): array { return $this->robotsTxtTokens; } public function schemaVersion(): int { return $this->schemaVersion; } public function updatedAt(): string { return $this->updatedAt; } }