"\'`\\\\]/u', $decoded) === 1) { return $url; } return $decoded; } /** * Normalize a URL for duplicate detection. */ public static function normalize(string $url): string { $u = trim($url); if ($u === '') { return ''; } if (!self::is_crawlable_href_candidate($u)) { return ''; } // Drop fragment (anchor) for duplicate detection. $u = preg_replace('/#.*$/', '', $u); // Scheme-relative. if (strpos($u, '//') === 0) { $u = 'https:' . $u; } // Parse with the WP parser (tests shim wp_parse_url when standalone). $parts = wp_parse_url($u); if (!is_array($parts)) { return $u; } $scheme = isset($parts['scheme']) ? strtolower((string) $parts['scheme']) : ''; $host = isset($parts['host']) ? strtolower((string) $parts['host']) : ''; $port = isset($parts['port']) ? ':' . (int) $parts['port'] : ''; $path = isset($parts['path']) ? (string) $parts['path'] : ''; $query = isset($parts['query']) ? (string) $parts['query'] : ''; if ($path === '') { $path = '/'; } $path = preg_replace('#/{2,}#', '/', $path); $path = self::normalize_path_encoding($path); $path = self::normalize_trailing_slash($path); $query = self::normalize_query($query); // Relative URLs: return path + query. if ($scheme === '' && $host === '') { return $path . ($query !== '' ? ('?' . $query) : ''); } $scheme = $scheme !== '' ? $scheme : 'https'; $out = $scheme . '://' . $host . $port . $path; if ($query !== '') { $out .= '?' . $query; } return $out; } /** * Resolve relative URL against base. */ public static function resolve(string $base, string $rel): string { $rel = trim($rel); if ($rel === '') { return $base; } if (!self::is_crawlable_href_candidate($rel)) { return ''; } // Drop fragment in resolved URL (anchors are not separate pages). $rel_no_frag = preg_replace('/#.*$/', '', $rel); if ($rel_no_frag === null) { $rel_no_frag = $rel; } // Absolute. if (stripos($rel_no_frag, 'http://') === 0 || stripos($rel_no_frag, 'https://') === 0) { return $rel_no_frag; } // Scheme-relative. if (strpos($rel_no_frag, '//') === 0) { $scheme = wp_parse_url($base, PHP_URL_SCHEME) ?: 'https'; return $scheme . ':' . $rel_no_frag; } // Separate query from path. $query = ''; $path_rel = $rel_no_frag; if (strpos($rel_no_frag, '?') !== false) { [$path_rel, $query] = explode('?', $rel_no_frag, 2); } $parsed = wp_parse_url($base); if (!$parsed || empty($parsed['scheme']) || empty($parsed['host'])) { // Best effort for relative base. return $rel_no_frag; } $scheme = (string) $parsed['scheme']; $host = (string) $parsed['host']; $port = isset($parsed['port']) ? ':' . (int) $parsed['port'] : ''; $base_path = isset($parsed['path']) ? (string) $parsed['path'] : '/'; $base_dir = preg_replace('#/[^/]*$#', '/', $base_path); // Query-only links like "?x=1". if ($path_rel === '') { $new_path = $base_path; } elseif (strpos($path_rel, '/') === 0) { $new_path = $path_rel; } else { $new_path = $base_dir . $path_rel; } // Clean ./ and ../ $segments = []; foreach (explode('/', $new_path) as $seg) { if ($seg === '' || $seg === '.') { continue; } if ($seg === '..') { array_pop($segments); continue; } $segments[] = $seg; } $clean = '/' . implode('/', $segments); $out = $scheme . '://' . $host . $port . $clean; if ($query !== '') { $out .= '?' . $query; } return $out; } /** * Validate raw href/loc strings before they enter crawl queues. * * WordPress builders occasionally output malformed anchors or scripts that * look like text rather than URLs. Treating "وداعًا لحفظ الأرقام..." as a * relative URL creates false 404/title/viewport findings, so reject raw * whitespace/control characters and non-page schemes up front. */ public static function is_crawlable_href_candidate(string $href): bool { $href = trim(html_entity_decode($href, ENT_QUOTES | ENT_HTML5, 'UTF-8')); if ($href === '' || $href[0] === '#') { return false; } if (preg_match('/[\x00-\x1F\x7F\s<>"\']+/u', $href) === 1) { return false; } $lower = strtolower($href); foreach (['mailto:', 'tel:', 'javascript:', 'data:', 'blob:', 'sms:', 'whatsapp:', 'skype:', 'viber:'] as $scheme) { if (strpos($lower, $scheme) === 0) { return false; } } if (preg_match('/^[a-z][a-z0-9+.-]*:/i', $href) === 1 && strpos($lower, 'http://') !== 0 && strpos($lower, 'https://') !== 0) { return false; } return true; } public static function is_probably_non_html_url(string $url): bool { $url = trim(rawurldecode(str_replace('\\', '/', $url))); if ($url === '') { return false; } $parts = wp_parse_url($url); $path = ''; $query = ''; if (is_array($parts)) { $path = isset($parts['path']) ? (string) $parts['path'] : ''; $query = isset($parts['query']) ? strtolower((string) $parts['query']) : ''; } else { $path = $url; } $path = strtolower(trim((string) $path)); $path = preg_replace('#/{2,}#', '/', $path); $target = $path !== '' ? $path : strtolower($url); if ($target === '') { return false; } if (preg_match('~\.(?:jpg|jpeg|png|gif|webp|avif|svg|bmp|ico|pdf|doc|docx|xls|xlsx|ppt|pptx|csv|txt|zip|rar|7z|gz|mp3|wav|ogg|m4a|mp4|m4v|mov|avi|wmv|webm|css|js|mjs|map|json|xml|rss|atom|kml|kmz|gpx|woff|woff2|ttf|otf|eot)$~i', $target) === 1) { return true; } if (strpos($target, '/wp-content/uploads/') !== false) { return true; } if (strpos($target, '/wp-json/') !== false || strpos($query, 'rest_route=') !== false) { return true; } if (preg_match('~/(?:feed|comments/feed|embed)/?$~i', $target) === 1) { return true; } return false; } private static function normalize_path_encoding(string $path): string { if ($path === '') { return '/'; } $decoded = rawurldecode($path); $segments = explode('/', $decoded); $encoded_segments = []; foreach ($segments as $segment) { if ($segment === '') { $encoded_segments[] = ''; continue; } $encoded_segments[] = rawurlencode($segment); } $normalized = implode('/', $encoded_segments); if ($path[0] === '/' && ($normalized === '' || $normalized[0] !== '/')) { $normalized = '/' . $normalized; } return $normalized === '' ? '/' : $normalized; } private static function normalize_query(string $query): string { $query = ltrim(trim($query), '?'); if ($query === '') { return ''; } $arr = []; parse_str($query, $arr); if (!is_array($arr) || empty($arr)) { return ''; } $deny_keys = [ 'gclid', 'fbclid', 'msclkid', 'yclid', '_ga', '_gl', 'mc_cid', 'mc_eid', 'igshid', 'replytocom', // 4.0.17: the crawler's own cache-bypass token. Pages that echo the // current request URL (core's cancel-comment-reply-link, language // switchers, share buttons) used to feed it back into the queue. 'aetos_audit_t', ]; foreach (array_keys($arr) as $k) { $lk = strtolower((string) $k); if (strpos($lk, 'utm_') === 0 || in_array($lk, $deny_keys, true)) { unset($arr[$k]); } } if (empty($arr)) { return ''; } ksort($arr); return (string) http_build_query($arr, '', '&', PHP_QUERY_RFC3986); } private static function normalize_trailing_slash(string $path): string { if ($path === '/') { return '/'; } $is_file = (bool) preg_match('/\.[a-z0-9]{1,6}$/i', $path); if ($is_file) { return rtrim($path, '/'); } $use_trailing = self::site_uses_trailing_slash(); return $use_trailing ? trailingslashit($path) : untrailingslashit($path); } private static function site_uses_trailing_slash(): bool { $struct = (string) get_option('permalink_structure', ''); if ($struct === '') { // Pretty permalinks disabled => usually no trailing slash. return false; } return substr($struct, -1) === '/'; } /** * Default URL exclusions (to keep crawl reports focused on crawlable HTML pages). * * You can override/extend via filter: * - `aetos_seo_default_excluded_paths` * - `aetos_seo_is_url_excluded` */ public static function is_default_excluded_url(string $url): bool { $url = trim($url); if ($url === '') { return false; } $filtered = apply_filters('aetos_seo_is_url_excluded', null, $url); if (is_bool($filtered)) { return $filtered; } if (self::is_probably_non_html_url($url)) { return true; } $parts = wp_parse_url($url); $path = ''; $query = ''; if (is_array($parts)) { $path = isset($parts['path']) ? (string) $parts['path'] : ''; $query = isset($parts['query']) ? strtolower((string) $parts['query']) : ''; } elseif (strpos($url, '/') === 0 || strpos($url, '?') === 0) { $path = $url; } $path = strtolower((string) $path); $path = preg_replace('#/{2,}#', '/', $path); if ($path === '') { $path = '/'; } if (strpos($query, 'rest_route=') !== false || strpos($query, 'preview=true') !== false) { return true; } // v1.0.73: skip search results, attachment pages, replytocom. if (strpos($query, 's=') !== false && preg_match('/(?:^|&)s=/', $query)) { return true; } if (strpos($query, 'attachment_id=') !== false) { return true; } if (strpos($query, 'replytocom=') !== false) { return true; } // v1.0.73: skip archive/taxonomy/pagination/author URLs — these are // not real content pages and inflate the crawl queue from ~200 to // 7000+ on sites with many tags/categories/date archives. if (preg_match('~/page/\d+/?~', $path)) { return true; // pagination: /page/2/, /page/3/... } if (preg_match('~/(?:category|tag|tags|التصنيف|الوسم)/~', $path)) { return true; // taxonomy archives (EN + AR slugs) } if (preg_match('~/author/~', $path)) { return true; // author archives } if (preg_match('~/(?:attachment|الملف)/~', $path)) { return true; // attachment pages } // Date archives: /2024/ or /2024/01/ (but NOT /2024/slug-name/) if (preg_match('~^/(?:19|20)\d{2}/?$~', $path) || preg_match('~^/(?:19|20)\d{2}/\d{1,2}/?$~', $path)) { return true; } $defaults = [ '/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-json', '/feed', '/comments/feed', '/embed', ]; $prefixes = apply_filters('aetos_seo_default_excluded_paths', $defaults); if (!is_array($prefixes)) { $prefixes = $defaults; } $path_n = rtrim($path, '/'); if ($path_n === '') { $path_n = '/'; } foreach ($prefixes as $pfx) { if (!is_string($pfx) || $pfx === '') { continue; } $pfx = strtolower(trim($pfx)); if ($pfx[0] !== '/') { $pfx = '/' . $pfx; } $pfx = rtrim($pfx, '/'); if ($pfx === '') { continue; } if ($path_n === $pfx || strpos($path_n, $pfx . '/') === 0) { return true; } } return false; } }