*/ private array $rules; /** @var list */ private array $sitemaps; private bool $truncated; /** * @param list $rules Selected rules. * @param list $sitemaps Declared sitemap URLs. */ private function __construct( string $state, array $rules, array $sitemaps, bool $truncated ) { $this->state = $state; $this->rules = $rules; $this->sitemaps = $sitemaps; $this->truncated = $truncated; } public static function from_http( int $status, string $body, string $crawler_product = 'px320siteaudit' ): self { if ( 429 === $status || $status >= 500 ) { return new self( 'temporarily_unavailable', array(), array(), false ); } if ( $status >= 400 ) { return new self( 'unavailable', array(), array(), false ); } if ( $status < 200 || $status >= 300 ) { return new self( 'not_tested', array(), array(), false ); } $truncated = strlen( $body ) > self::MAX_BODY_BYTES; $body = substr( $body, 0, self::MAX_BODY_BYTES ); $body = 0 === strpos( $body, "\xEF\xBB\xBF" ) ? substr( $body, 3 ) : $body; $groups = array(); $sitemaps = array(); $current = array( 'agents' => array(), 'rules' => array(), ); $has_rules = false; $lines = preg_split( '/\r\n|\r|\n/', $body ); foreach ( is_array( $lines ) ? $lines : array() as $line ) { $line = preg_replace( '/\s*#.*$/', '', $line ); $line = is_string( $line ) ? trim( $line ) : ''; if ( '' === $line || false === strpos( $line, ':' ) ) { continue; } list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) ); $field = strtolower( $field ); if ( 'sitemap' === $field ) { if ( self::valid_value( $value, 2048 ) && count( $sitemaps ) < self::MAX_SITEMAPS ) { $sitemaps[] = $value; } elseif ( '' !== $value ) { $truncated = true; } continue; } if ( 'user-agent' === $field ) { if ( $has_rules ) { $groups[] = $current; $current = array( 'agents' => array(), 'rules' => array(), ); $has_rules = false; } if ( self::valid_value( $value, 128 ) ) { $current['agents'][] = strtolower( $value ); } continue; } if ( ! in_array( $field, array( 'allow', 'disallow' ), true ) || array() === $current['agents'] ) { continue; } $has_rules = true; if ( '' === $value ) { continue; } if ( ! self::valid_value( $value, self::MAX_RULE_BYTES ) ) { $truncated = true; continue; } if ( count( $current['rules'] ) >= self::MAX_RULES ) { $truncated = true; continue; } $current['rules'][] = array( 'allow' => 'allow' === $field, 'pattern' => $value, 'specificity' => strlen( str_replace( array( '*', '$' ), '', $value ) ), ); } if ( array() !== $current['agents'] ) { $groups[] = $current; } $crawler_product = strtolower( trim( $crawler_product ) ); $exact = array(); $fallback = array(); foreach ( $groups as $group ) { if ( in_array( $crawler_product, $group['agents'], true ) ) { $exact = array_merge( $exact, $group['rules'] ); } elseif ( in_array( '*', $group['agents'], true ) ) { $fallback = array_merge( $fallback, $group['rules'] ); } } $selected = array() !== $exact ? $exact : $fallback; if ( count( $selected ) > self::MAX_RULES ) { $selected = array_slice( $selected, 0, self::MAX_RULES ); $truncated = true; } return new self( 'parsed', $selected, array_values( array_unique( $sitemaps ) ), $truncated ); } /** * Runtime-only normalized policy for the resumable crawler task cursor. * It is replaced with aggregate counts as soon as crawl collection ends. * * @return array{state:string,rules:list,truncated:bool} */ public function checkpoint(): array { return array( 'state' => $this->state, 'rules' => $this->rules, 'truncated' => $this->truncated, ); } /** * @param array $checkpoint Runtime task cursor projection. */ public static function from_checkpoint( array $checkpoint ): self { $state = $checkpoint['state'] ?? null; $rules = $checkpoint['rules'] ?? null; $truncated = $checkpoint['truncated'] ?? null; if ( ! is_string( $state ) || ! in_array( $state, array( 'parsed', 'unavailable', 'temporarily_unavailable', 'not_tested' ), true ) || ! is_array( $rules ) || ! is_bool( $truncated ) || count( $rules ) > self::MAX_RULES ) { throw new \InvalidArgumentException( 'invalid_robots_checkpoint' ); } $normalized = array(); foreach ( $rules as $rule ) { if ( ! is_array( $rule ) || ! isset( $rule['allow'], $rule['pattern'], $rule['specificity'] ) || ! is_bool( $rule['allow'] ) || ! is_string( $rule['pattern'] ) || ! is_int( $rule['specificity'] ) || $rule['specificity'] < 0 || ! self::valid_value( $rule['pattern'], self::MAX_RULE_BYTES ) ) { throw new \InvalidArgumentException( 'invalid_robots_checkpoint' ); } $expected = strlen( str_replace( array( '*', '$' ), '', $rule['pattern'] ) ); if ( $expected !== $rule['specificity'] ) { throw new \InvalidArgumentException( 'invalid_robots_checkpoint' ); } $normalized[] = $rule; } return new self( $state, $normalized, array(), $truncated ); } public function state(): string { return $this->state; } public function truncated(): bool { return $this->truncated; } public function rule_count(): int { return count( $this->rules ); } /** * Sitemap values remain runtime crawl candidates and must pass the same-site * URL policy before any request or persisted evidence projection. * * @return list */ public function sitemaps(): array { return $this->sitemaps; } public function allows( string $path ): bool { if ( 'temporarily_unavailable' === $this->state || 'not_tested' === $this->state ) { return false; } if ( 'unavailable' === $this->state ) { return true; } $path = '' === $path ? '/' : $path; $best_length = -1; $best_allow = true; foreach ( $this->rules as $rule ) { if ( ! $this->matches( $rule['pattern'], $path ) ) { continue; } if ( $rule['specificity'] > $best_length || ( $rule['specificity'] === $best_length && $rule['allow'] ) ) { $best_length = $rule['specificity']; $best_allow = $rule['allow']; } } return $best_allow; } private function matches( string $pattern, string $path ): bool { $end_anchor = '$' === substr( $pattern, -1 ); if ( $end_anchor ) { $pattern = substr( $pattern, 0, -1 ); } $quoted = preg_quote( $pattern, '~' ); $quoted = str_replace( '\\*', '.*', $quoted ); $regex = '~^' . $quoted . ( $end_anchor ? '$' : '' ) . '~'; return 1 === preg_match( $regex, $path ); } private static function valid_value( string $value, int $max_bytes ): bool { return '' !== $value && strlen( $value ) <= $max_bytes && 1 === preg_match( '//u', $value ) && 0 === preg_match( '/[\x00-\x1F\x7F]/', $value ); } }