parse( trim( $origin_url ) ); if ( null === $parts || isset( $parts['user'] ) || isset( $parts['pass'] ) || isset( $parts['query'] ) || isset( $parts['fragment'] ) ) { throw new InvalidArgumentException( 'invalid_crawl_origin' ); } $this->scheme = strtolower( $parts['scheme'] ); $this->host = $this->normalize_host( $parts['host'] ); $this->port = isset( $parts['port'] ) ? $parts['port'] : $this->default_port( $this->scheme ); $this->origin = $this->scheme . '://' . $this->host; if ( $this->port !== $this->default_port( $this->scheme ) ) { $this->origin .= ':' . $this->port; } $path = isset( $parts['path'] ) && '' !== $parts['path'] ? $this->normalize_path( $parts['path'] ) : '/'; if ( strlen( $path ) > self::MAX_PATH_BYTES || $this->is_action_path( $path ) || $this->is_asset_path( $path ) ) { throw new InvalidArgumentException( 'invalid_crawl_origin_path' ); } $this->scope_path = '/' === $path ? '/' : rtrim( $path, '/' ) . '/'; } public function origin(): string { return $this->origin; } public function start_url(): string { return $this->origin . $this->scope_path; } /** * Return a queryless same-origin path that is safe to show to an administrator. * * The crawler still keys pages by HMAC. This projection exists only to make a * local report actionable. It rejects paths that look like credentials, * private tokens, email addresses, phone numbers or opaque personal IDs. */ public function safe_display_path( string $url ): ?string { $decision = $this->evaluate( $url ); if ( 'accepted' !== $decision['state'] || ! is_string( $decision['path'] ) ) { return null; } return $this->safe_display_resource_path( $decision['path'] ); } /** * Project an already policy-accepted queryless resource path for local UI. */ public function safe_display_resource_path( string $path ): ?string { if ( '' === $path || '/' !== $path[0] || false !== strpos( $path, '?' ) || false !== strpos( $path, '#' ) ) { return null; } $normalized = $this->normalize_path( $path ); if ( $normalized !== $path || strlen( $path ) > self::MAX_PATH_BYTES ) { return null; } return $this->path_has_sensitive_segment( $path ) ? null : $path; } public function default_sitemap_url(): string { return rtrim( $this->start_url(), '/' ) . '/wp-sitemap.xml'; } /** * Build a bounded set of read-only diagnostics for one already accepted page. * * Every returned URL is runtime-only. The ordinary crawler remains queryless; * only these internally generated candidates may contain the fixed probe query * or the alternate HTTP scheme. * * @return array */ public function diagnostic_variants( string $page_url ): array { $accepted = $this->evaluate( $page_url ); if ( 'accepted' !== $accepted['state'] || ! is_string( $accepted['url'] ) || ! is_string( $accepted['path'] ) ) { return array(); } $path = $accepted['path']; $variants = array( 'norm_query' => $accepted['url'] . '?' . self::PROBE_QUERY, 'norm_protocol' => $this->alternate_origin() . $path, ); if ( '/' !== $path && rtrim( $this->scope_path, '/' ) !== rtrim( $path, '/' ) ) { $variants['norm_slash'] = '/' === substr( $path, -1 ) ? rtrim( $accepted['url'], '/' ) : $accepted['url'] . '/'; } $case_path = preg_replace_callback( '/[A-Za-z]/', static function ( array $case_match ): string { return strtolower( $case_match[0] ) === $case_match[0] ? strtoupper( $case_match[0] ) : strtolower( $case_match[0] ); }, $path, 1 ); if ( is_string( $case_path ) && $case_path !== $path && ! $this->is_action_path( $case_path ) ) { $variants['norm_case'] = $this->origin . $case_path; } return $variants; } public function error_probe_url( string $reference ): string { if ( 1 !== preg_match( '/^[a-f0-9]{24}$/', $reference ) ) { throw new InvalidArgumentException( 'invalid_error_probe_reference' ); } return rtrim( $this->start_url(), '/' ) . '/px320-site-audit-not-found-' . $reference . '/'; } public function emerging_file_url(): string { return $this->origin . '/llms.txt'; } /** * Validate only an internally generated diagnostic URL or its redirect. * * @return array{state:string,reason:string,url:string|null,path:string|null} */ public function evaluate_probe( string $candidate, string $kind, string $base_url = '' ): array { if ( ! in_array( $kind, self::PROBE_KINDS, true ) || 1 === preg_match( '/[\x00-\x20\x7F]/', $candidate ) ) { return $this->rejected( 'invalid_probe' ); } $candidate = trim( $candidate ); if ( '' === $candidate || strlen( $candidate ) > self::MAX_URL_BYTES || false !== strpos( $candidate, '#' ) || false !== strpos( $candidate, ';' ) ) { return $this->rejected( 'invalid_probe' ); } $absolute = $this->resolve_probe( $candidate, $base_url ); $parts = null === $absolute ? null : $this->parse( $absolute ); if ( null === $parts || isset( $parts['user'] ) || isset( $parts['pass'] ) || isset( $parts['fragment'] ) ) { return $this->rejected( 'invalid_probe_url' ); } $scheme = strtolower( $parts['scheme'] ); $host = $this->normalize_host( $parts['host'] ); $port = isset( $parts['port'] ) ? $parts['port'] : $this->default_port( $scheme ); if ( $host !== $this->host || ! $this->probe_origin_allowed( $kind, $scheme, $port ) ) { return $this->rejected( 'external_probe_origin' ); } $query = isset( $parts['query'] ) ? (string) $parts['query'] : ''; if ( ( 'norm_query' === $kind && ! in_array( $query, array( '', self::PROBE_QUERY ), true ) ) || ( 'norm_query' !== $kind && '' !== $query ) ) { return $this->rejected( 'invalid_probe_query' ); } $path = isset( $parts['path'] ) && '' !== $parts['path'] ? $this->normalize_path( $parts['path'] ) : '/'; if ( strlen( $path ) > self::MAX_PATH_BYTES || $this->is_action_path( $path ) ) { return $this->rejected( 'invalid_probe_path' ); } if ( 'emerging_file' === $kind ) { if ( '/llms.txt' !== strtolower( $path ) ) { return $this->rejected( 'invalid_emerging_file_path' ); } } elseif ( ! $this->is_in_scope( $path ) || $this->is_asset_path( $path ) ) { return $this->rejected( 'invalid_probe_path' ); } $origin = $scheme . '://' . $host; if ( $port !== $this->default_port( $scheme ) ) { $origin .= ':' . $port; } return array( 'state' => 'accepted', 'reason' => 'same_host_' . $kind, 'url' => $origin . $path . ( '' === $query ? '' : '?' . $query ), 'path' => $path, ); } /** * Resolve and classify a crawl candidate without performing DNS or HTTP. * * Accepted URLs are same-origin, queryless public-page candidates. The URL is * runtime-only input for the crawler and must be removed before Finding storage. * * @return array{state:string,reason:string,url:string|null,path:string|null} */ public function evaluate( string $candidate, string $base_url = '' ): array { return $this->evaluate_candidate( $candidate, $base_url, 'page', true ); } /** * Evaluate only same-scope, queryless XML sitemap resources. * * @return array{state:string,reason:string,url:string|null,path:string|null} */ public function evaluate_sitemap( string $candidate, string $base_url = '' ): array { return $this->evaluate_candidate( $candidate, $base_url, 'sitemap', true ); } public function robots_url(): string { return $this->origin . '/robots.txt'; } /** * A robots redirect starts at the fixed origin-root endpoint and may leave a * subdirectory installation path, but never the exact HTTP origin. * * @return array{state:string,reason:string,url:string|null,path:string|null} */ public function evaluate_robots_redirect( string $candidate, string $base_url ): array { return $this->evaluate_candidate( $candidate, $base_url, 'robots', false ); } /** * @return array{state:string,reason:string,url:string|null,path:string|null} */ private function evaluate_candidate( string $candidate, string $base_url, string $resource_class, bool $enforce_scope ): array { if ( 1 === preg_match( '/[\x00-\x20\x7F]/', $candidate ) ) { return $this->rejected( 'control_character' ); } $candidate = trim( $candidate ); if ( '' === $candidate || strlen( $candidate ) > self::MAX_URL_BYTES ) { return $this->rejected( '' === $candidate ? 'empty' : 'url_too_long' ); } $lower = strtolower( $candidate ); foreach ( array( '#', 'mailto:', 'tel:', 'sms:', 'javascript:', 'data:', 'blob:' ) as $prefix ) { if ( 0 === strpos( $lower, $prefix ) ) { return $this->rejected( 'non_http_reference' ); } } $fragment = strpos( $candidate, '#' ); if ( false !== $fragment ) { $candidate = substr( $candidate, 0, $fragment ); if ( '' === $candidate ) { return $this->rejected( 'fragment_only' ); } } if ( false !== strpos( $candidate, '?' ) ) { return $this->rejected( 'query_string' ); } if ( false !== strpos( $candidate, ';' ) ) { return $this->rejected( 'path_parameter' ); } if ( 1 === preg_match( '/%(?![0-9a-f]{2})/i', $candidate ) ) { return $this->rejected( 'invalid_percent_encoding' ); } if ( 1 === preg_match( '/%(?:25)*(?:[01][0-9a-f]|23|25|2f|3b|3f|5c|7f)/i', $candidate ) ) { return $this->rejected( 'ambiguous_encoding' ); } $absolute = $this->resolve( $candidate, $base_url, $enforce_scope ); $parts = null === $absolute ? null : $this->parse( $absolute ); if ( null === $parts || isset( $parts['user'] ) || isset( $parts['pass'] ) || isset( $parts['query'] ) ) { return $this->rejected( 'invalid_url' ); } $scheme = strtolower( $parts['scheme'] ); $host = $this->normalize_host( $parts['host'] ); $port = isset( $parts['port'] ) ? $parts['port'] : $this->default_port( $scheme ); if ( $this->scheme !== $scheme || $this->host !== $host || $this->port !== $port ) { return $this->rejected( 'external_origin' ); } $path = isset( $parts['path'] ) && '' !== $parts['path'] ? $this->normalize_path( $parts['path'] ) : '/'; if ( strlen( $path ) > self::MAX_PATH_BYTES ) { return $this->rejected( 'path_too_long' ); } if ( $enforce_scope && ! $this->is_in_scope( $path ) ) { return $this->rejected( 'outside_site_path' ); } if ( $this->is_action_path( $path ) ) { return $this->rejected( 'action_path' ); } if ( 'sitemap' === $resource_class && 1 !== preg_match( '/\.xml$/i', $path ) ) { return $this->rejected( 'invalid_sitemap_path' ); } if ( 'robots' === $resource_class && $this->is_asset_path( $path ) && 1 !== preg_match( '/\.txt$/i', $path ) ) { return $this->rejected( 'invalid_robots_path' ); } if ( 'page' === $resource_class && $this->is_asset_path( $path ) ) { return $this->rejected( 'non_html_asset' ); } return array( 'state' => 'accepted', 'reason' => 'same_origin_' . $resource_class, 'url' => $this->origin . $path, 'path' => $path, ); } /** * @return array{scheme:string,host:string,port?:int,path?:string,user?:string,pass?:string,query?:string,fragment?:string}|null */ private function parse( string $url ): ?array { if ( '' === $url || strlen( $url ) > self::MAX_URL_BYTES ) { return null; } $parts = wp_parse_url( $url ); if ( ! is_array( $parts ) || ! isset( $parts['scheme'], $parts['host'] ) ) { return null; } if ( ! in_array( strtolower( $parts['scheme'] ), array( 'http', 'https' ), true ) || '' === $parts['host'] ) { return null; } if ( ! $this->valid_host( $parts['host'] ) ) { return null; } /** @var array{scheme:string,host:string,port?:int,path?:string,user?:string,pass?:string,query?:string,fragment?:string} $parts */ return $parts; } private function resolve( string $candidate, string $base_url, bool $enforce_scope ): ?string { if ( 1 === preg_match( '~^https?://~i', $candidate ) ) { return $candidate; } if ( 0 === strpos( $candidate, '//' ) ) { return $this->scheme . ':' . $candidate; } $base = '' !== $base_url ? $this->evaluate_base( $base_url, $enforce_scope ) : $this->start_url(); if ( null === $base ) { return null; } $base_parts = $this->parse( $base ); if ( null === $base_parts ) { return null; } if ( 0 === strpos( $candidate, '/' ) ) { $path = $candidate; } else { $base_path = isset( $base_parts['path'] ) ? $base_parts['path'] : '/'; $directory = substr( $base_path, -1 ) === '/' ? $base_path : dirname( $base_path ) . '/'; $path = $directory . $candidate; } return $this->origin . $this->normalize_path( $path ); } private function resolve_probe( string $candidate, string $base_url ): ?string { if ( 1 === preg_match( '~^https?://~i', $candidate ) ) { return $candidate; } $base_parts = '' === $base_url ? null : $this->parse( $base_url ); $base = is_array( $base_parts ) ? $base_parts : $this->parse( $this->start_url() ); if ( ! is_array( $base ) ) { return null; } $scheme = strtolower( $base['scheme'] ); $host = $this->normalize_host( $base['host'] ); $port = isset( $base['port'] ) ? $base['port'] : $this->default_port( $scheme ); $origin = $scheme . '://' . $host . ( $port === $this->default_port( $scheme ) ? '' : ':' . $port ); if ( 0 === strpos( $candidate, '//' ) ) { return $scheme . ':' . $candidate; } if ( 0 === strpos( $candidate, '/' ) ) { return $origin . $candidate; } $base_path = isset( $base['path'] ) ? (string) $base['path'] : '/'; $directory = '/' === substr( $base_path, -1 ) ? $base_path : dirname( $base_path ) . '/'; return $origin . $this->normalize_path( $directory . $candidate ); } private function probe_origin_allowed( string $kind, string $scheme, int $port ): bool { if ( $scheme === $this->scheme && $port === $this->port ) { return true; } return 'norm_protocol' === $kind && $scheme !== $this->scheme && $port === $this->default_port( $scheme ); } private function alternate_origin(): string { $scheme = 'https' === $this->scheme ? 'http' : 'https'; return $scheme . '://' . $this->host; } private function evaluate_base( string $base_url, bool $enforce_scope ): ?string { $parts = $this->parse( trim( $base_url ) ); if ( null === $parts || isset( $parts['user'] ) || isset( $parts['pass'] ) || isset( $parts['query'] ) || isset( $parts['fragment'] ) ) { return null; } $scheme = strtolower( $parts['scheme'] ); $host = $this->normalize_host( $parts['host'] ); $port = isset( $parts['port'] ) ? $parts['port'] : $this->default_port( $scheme ); if ( $scheme !== $this->scheme || $host !== $this->host || $port !== $this->port ) { return null; } $path = isset( $parts['path'] ) && '' !== $parts['path'] ? $this->normalize_path( $parts['path'] ) : '/'; return ! $enforce_scope || $this->is_in_scope( $path ) ? $this->origin . $path : null; } private function normalize_path( string $path ): string { $trailing = strlen( $path ) > 1 && '/' === substr( $path, -1 ); $segments = array(); foreach ( explode( '/', str_replace( '\\', '/', $path ) ) as $segment ) { $segment = preg_replace_callback( '/%[0-9a-f]{2}/i', static function ( array $encoded_octet ): string { $character = chr( (int) hexdec( substr( $encoded_octet[0], 1 ) ) ); return 1 === preg_match( '/^[A-Za-z0-9._~-]$/', $character ) ? $character : strtoupper( $encoded_octet[0] ); }, $segment ); $segment = is_string( $segment ) ? $segment : ''; $decoded = strtolower( rawurldecode( $segment ) ); if ( '' === $segment || '.' === $decoded ) { continue; } if ( '..' === $decoded ) { array_pop( $segments ); continue; } $segments[] = $segment; } $normalized = '/' . implode( '/', $segments ); if ( $trailing && '/' !== $normalized ) { $normalized .= '/'; } return $normalized; } private function is_action_path( string $path ): bool { $decoded = strtolower( rawurldecode( $path ) ); if ( 1 === preg_match( '~^/(?:wp-admin(?:/|$)|wp-comments-post\.php(?:/|$)|wp-login\.php(?:/|$)|wp-cron\.php(?:/|$)|xmlrpc\.php(?:/|$))~', $decoded ) ) { return true; } return 1 === preg_match( '~/(?:add-to-cart|cart|checkout|compare|delete-account|logout|lost-password|my-account|order-pay|order-received|remove-account|reset-password|unsubscribe|wishlist)(?:/|$)~', $decoded ); } private function is_asset_path( string $path ): bool { return 1 === preg_match( '~\.(?:avif|bmp|css|csv|docx?|eot|gif|gz|ico|jpe?g|js|json|map|mp3|mp4|mpeg|odt|pdf|png|pptx?|rar|rss|svg|tar|tgz|tiff?|txt|webm|webp|woff2?|xlsx?|xml|zip)$~i', $path ); } private function is_in_scope( string $path ): bool { if ( '/' === $this->scope_path ) { return true; } return rtrim( $this->scope_path, '/' ) === $path || 0 === strpos( $path, $this->scope_path ); } private function path_has_sensitive_segment( string $path ): bool { foreach ( array_filter( explode( '/', $path ), static fn ( string $segment ): bool => '' !== $segment ) as $encoded_segment ) { $segment = rawurldecode( $encoded_segment ); if ( '' === $segment || 1 !== preg_match( '//u', $segment ) ) { return true; } $lower = strtolower( $segment ); if ( in_array( $lower, array( 'auth', 'key', 'nonce', 'password', 'private', 'secret', 'session', 'token' ), true ) ) { return true; } if ( false !== filter_var( $segment, FILTER_VALIDATE_EMAIL ) || false !== filter_var( $segment, FILTER_VALIDATE_IP ) || 1 === preg_match( '/^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i', $segment ) || 1 === preg_match( '/^[a-f0-9]{16,}$/i', $segment ) || 1 === preg_match( '/^(?:[A-Za-z0-9_-]{24,}|[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,})$/', $segment ) || 1 === preg_match( '/^(?:\+?\d[\d(). _-]{6,}\d)$/', $segment ) ) { return true; } } return false; } private function normalize_host( string $host ): string { if ( '[' === substr( $host, 0, 1 ) && ']' === substr( $host, -1 ) ) { return '[' . strtolower( substr( $host, 1, -1 ) ) . ']'; } return rtrim( strtolower( $host ), '.' ); } private function valid_host( string $host ): bool { if ( '[' === substr( $host, 0, 1 ) && ']' === substr( $host, -1 ) ) { return false !== filter_var( substr( $host, 1, -1 ), FILTER_VALIDATE_IP, FILTER_FLAG_IPV6 ); } return '' !== $host && false === strpbrk( $host, ':#?[]' ); } private function default_port( string $scheme ): int { return 'https' === strtolower( $scheme ) ? 443 : 80; } /** * @return array{state:string,reason:string,url:null,path:null} */ private function rejected( string $reason ): array { return array( 'state' => 'skipped', 'reason' => $reason, 'url' => null, 'path' => null, ); } }