<?php

declare(strict_types=1);

namespace AetosSeo\Collectors;

if (!defined('ABSPATH')) {
    exit;
}

use AetosSeo\Core\Settings;
use AetosSeo\Utils\Url;

/**
 * Lightweight HTML analyzer used by CrawlCollector.
 */
class HtmlCollector
{
    /**
     * Schema fields preserved by compact_jsonld_block(). Must stay aligned with
     * the field-presence assertions in LocalSeoCheck, AdvancedSchemaCheck,
     * EeatCheck, VoiceSearchCheck, AiFriendlyCheck — anything dropped here
     * becomes a silent false negative on field-presence checks.
     *
     * Promoted to a class constant in v1.0.92 so the 38-element array isn't
     * reallocated ~25k times per audit run (recursive call from compact_jsonld_block
     * fans out per @graph entry × 200 sample pages).
     *
     * @var string[]
     */
    private const KEEP_FIELDS = [
        // Core entity identity.
        '@type', '@id', '@context', 'name', 'url', 'sameAs', 'author', 'publisher', 'logo',
        'datePublished', 'dateModified', 'description', 'mainEntity', 'mainEntityOfPage',
        // LocalBusiness / NAP signals.
        'address', 'telephone', 'email', 'contactPoint',
        'openingHours', 'openingHoursSpecification',
        'geo', 'hasMap', 'priceRange', 'image', 'slogan', 'foundingDate',
        // Reviews / ratings.
        'aggregateRating', 'ratingValue', 'reviewCount', 'bestRating', 'worstRating', 'review',
        // Ecommerce / Product rich result signals.
        'offers', 'price', 'priceCurrency', 'availability', 'sku', 'mpn',
        'gtin', 'gtin8', 'gtin12', 'gtin13', 'gtin14', 'brand',
        'itemCondition', 'seller', 'shippingDetails', 'hasMerchantReturnPolicy',
        'reviewRating',
        // PostalAddress nested fields.
        'streetAddress', 'addressLocality', 'addressRegion', 'postalCode', 'addressCountry',
        // GeoCoordinates nested fields.
        'latitude', 'longitude',
        // OpeningHoursSpecification nested fields.
        'dayOfWeek', 'opens', 'closes', 'validFrom', 'validThrough',
        // VideoObject / Speakable / BreadcrumbList.
        'speakable', 'thumbnailUrl', 'contentUrl', 'embedUrl', 'duration', 'uploadDate',
        'itemListElement', 'position', 'item',
        // E-E-A-T signals.
        'jobTitle', 'worksFor', 'knowsAbout', 'affiliation', 'reviewedBy',
        // v3.3.47 — FAQPage Question/Answer fields. Round-2 false-positive
        // fix: AdvancedSchemaCheck + EeatCheck both reported "questions
        // without acceptedAnswer" on 100% of FAQPages because the field was
        // silently stripped here before the check ran. A production site
        // /سيو-المستشفيات-والعيادات/ verified — 6 questions, 6 answers,
        // flagged as "9 blocks have questions without acceptedAnswer".
        'acceptedAnswer', 'text',
    ];

    /**
     * Analyze a HTML document and extract SEO + crawl signals.
     *
     * @return array<string,mixed>
     */
    public function analyze(string $html, string $base_url = ''): array
    {
        $out = [
            'title' => '',
            'meta_desc' => '',
            'canonical' => '',
            'robots' => '',
            'viewport' => '',
            'robots_noindex' => false,
            'password_protected' => false,
            'hreflang_count' => 0,
            'list_count' => 0,
            'table_count' => 0,
            'dl_count' => 0,
            'h1_count' => 0,
            'h2_count' => 0,
            'h3_count' => 0,
            // v2.1.0 — capture heading text for question/intent analysis.
            'h1_texts' => [],
            'h2_texts' => [],
            'h3_texts' => [],
            // v2.1.5 — also capture H4 because Arabic FAQ pages often nest
            // questions as H4 under an H2 "الأسئلة الشائعة" section. Without
            // h4_texts the Question Coverage check undercounts.
            'h4_texts' => [],
            'first_paragraph' => '',
            'word_count' => 0,
            'images_missing_alt' => 0,
            'images_without_dimensions' => 0,
            'jsonld_types' => [],
            'jsonld_raw' => [],
            'jsonld_raw_truncated' => false, // v3.3.52 P2.3 — raw blocks / @graph were capped.
            'internal_links' => [],
            'nav_links' => [],
            'outbound_links_count' => 0,
            'css_count' => 0,
            'js_count' => 0,
            'css_origins' => [],
            'js_origins' => [],
            'js_blocking_head' => 0,
            'lazyload_detected' => false,
            'webp_images' => 0,
            'images_total' => 0,
            // Open Graph signals
            'og_title' => '',
            'og_description' => '',
            'og_image' => '',
            'og_type' => '',
            // Twitter Card signals
            'twitter_card' => '',
            'twitter_title' => '',
            // Page-level signals
            'lang' => '',
            // v3.3.46 — explicit parse-success flag. Stays false when the HTML
            // body is empty or DOMDocument::loadHTML() fails, so downstream
            // checks can tell "feature genuinely absent" apart from "page
            // never parsed" instead of treating every default value as a gap.
            '_html_parsed' => false,
            'has_favicon' => false,
            // Author / E-E-A-T signals
            'author_name' => '',
            'has_author_link' => false,
            'article_published_time' => '',
            'article_modified_time' => '',
            // New: gap-closing fields
            'text_to_html_ratio' => 0.0,
            'heading_hierarchy_valid' => true,
            'heading_sequence' => [],
            'has_srcset' => false,
            'srcset_count' => 0,
            'avif_images' => 0,
            'font_display_swap_detected' => false,
            'has_breadcrumb_nav' => false,
            'has_breadcrumb_schema' => false,
            'nofollow_internal_count' => 0,
            'has_publisher_schema' => false,
            'schema_types_detailed' => [],
            // v2.1.1 — actual visible video presence (excludes GTM noscript iframes
            // and other non-video iframes). Used by SerpFeatureCheck::evaluate_video()
            // to avoid emitting "video schema missing" on pages whose only iframe
            // is the GTM container.
            'has_video_embed' => false,
            'video_provider' => '',
            // v3.3.48 — surfaces UTF-8 BOM before <!DOCTYPE>. Strong signal
            // that a PHP source file (theme functions.php / wp-config.php /
            // plugin) is saved with BOM and emitting it on every response.
            // Some browsers tolerate it, but it breaks XML/sitemap parsers
            // and disrupts wp_head ordering on a few hosts.
            'has_bom_before_doctype' => false,
            // v3.3.50 — inline analytics signals captured by regex against the
            // raw body BEFORE DOMDocument parsing. GTM/GA4/Clarity are routinely
            // loaded via inline script snippets (especially behind WP Rocket
            // Delay JS / Perfmatters Script Manager), so they never appear as
            // external script-src references (googletagmanager.com/...) in the parsed
            // JS origin list. AnalyticsStackCheck reads these flags so it stops
            // emitting "measurement stack missing" on instrumented sites.
            'analytics_signals' => [
                'gtm_inline'        => false,
                'ga4_inline'        => false,
                'gtag_inline'       => false,
                'clarity_inline'    => false,
                'datalayer_inline'  => false,
                'meta_pixel_inline' => false,
                'gtm_ids'           => [],
                'ga4_ids'           => [],
            ],
        ];

        // v3.3.48 — detect BOM BEFORE trim so leading whitespace doesn't
        // hide it. We strip for our DOMDocument parse path AND surface the
        // flag for the audit check.
        $had_bom_before_doctype = (substr($html, 0, 3) === "\xEF\xBB\xBF");
        $html = trim($html);
        // v3.3.47 — strip UTF-8 BOM before parsing. Some WP themes / Rank Math
        // sites emit a BOM before <!DOCTYPE>, which derails DOMDocument's
        // XML-prolog encoding hack below and leaves the root <html> element's
        // attributes (lang, dir, prefix) empty even though they are present
        // in the raw HTML. Round-2 false-positive bug confirmed on
        // production validation on 2026-05-24.
        if (substr($html, 0, 3) === "\xEF\xBB\xBF") {
            $html = substr($html, 3);
            $had_bom_before_doctype = true;
        }
        $out['has_bom_before_doctype'] = $had_bom_before_doctype;
        if ($html === '') {
            return $out;
        }

        // v3.3.50 — run analytics-signal regex BEFORE DOM parsing. Inline
        // GTM/GA/Clarity snippets are always in the raw body. Running before
        // the DOM load means even pages whose HTML fails DOMDocument validation
        // (malformed markup) still surface their measurement stack correctly.
        $out['analytics_signals'] = $this->detect_analytics_signals($html);

        $prev = libxml_use_internal_errors(true);
        // v2.8.0 — explicit XXE/billion-laughs defense. PHP 8 default is safe
        // but we lock it down regardless: no external entity loading even if
        // a future runtime regresses to libxml2 < 2.9 defaults, and the
        // setter call is silently ignored on incompatible versions.
        if (function_exists('libxml_set_external_entity_loader')) {
            libxml_set_external_entity_loader(static function () { return null; });
        }
        $dom = new \DOMDocument();
        // Best effort: enforce UTF-8.
        $loaded = @$dom->loadHTML(
            '<?xml encoding="utf-8" ?>' . $html,
            LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_NONET | LIBXML_COMPACT
        );
        libxml_clear_errors();
        libxml_use_internal_errors($prev);

        if (!$loaded) {
            return $out;
        }

        // v3.3.46 — DOM loaded successfully: every signal extracted below is
        // real. A '' / false / 0 default from here on means the feature is
        // genuinely absent, not that the crawl failed.
        $out['_html_parsed'] = true;

        $xpath = new \DOMXPath($dom);

        // HTML lang attribute
        $htmlNodes = $dom->getElementsByTagName('html');
        if ($htmlNodes && $htmlNodes->length > 0) {
            $htmlEl = $htmlNodes->item(0);
            if ($htmlEl instanceof \DOMElement) {
                $lang = trim((string) $htmlEl->getAttribute('lang'));
                $out['lang'] = $lang !== '' ? $lang : (string) $htmlEl->getAttribute('xml:lang');
                $out['lang'] = trim($out['lang']);
            }
        }
        // v3.3.47 — defense in depth for the round-2 false-positive bug.
        // Even after the BOM strip, DOMDocument occasionally fails to read
        // attributes off the root <html> element when the opening tag contains
        // attribute values with embedded colons + spaces (e.g.
        // prefix="og: https://ogp.me/ns#"), since the XML-prolog hack pushes
        // libxml2 toward strict XML namespace parsing. Fall back to a
        // bracket-tolerant regex against the raw HTML when DOM gave us nothing.
        if ($out['lang'] === '' && preg_match('/<html\b([^>]*)>/i', $html, $hm)) {
            $htmlAttrs = $hm[1];
            if (preg_match('/\blang\s*=\s*["\']([A-Za-z][A-Za-z0-9_\-]*)/i', $htmlAttrs, $lm)
                || preg_match('/\bxml:lang\s*=\s*["\']([A-Za-z][A-Za-z0-9_\-]*)/i', $htmlAttrs, $lm)) {
                $out['lang'] = trim($lm[1]);
            }
        }

        // Favicon detection
        $faviconNodes = $xpath->query('//link[contains(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"icon")]');
        $out['has_favicon'] = ($faviconNodes && $faviconNodes->length > 0);
        if (!$out['has_favicon']) {
            // Check shortcut icon too
            $shortcutNodes = $xpath->query('//link[contains(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"shortcut")]');
            $out['has_favicon'] = ($shortcutNodes && $shortcutNodes->length > 0);
        }

        // Title
        $titleNode = $dom->getElementsByTagName('title');
        if ($titleNode && $titleNode->length > 0) {
            $out['title'] = trim((string) $titleNode->item(0)->textContent);
        }
        if ($out['title'] === '') {
            $out['title'] = $this->meta_content($xpath, 'property', 'og:title');
        }

        // Meta description
        $out['meta_desc'] = $this->meta_content($xpath, 'name', 'description');
        if ($out['meta_desc'] === '') {
            $out['meta_desc'] = $this->meta_content($xpath, 'property', 'og:description');
        }

        // Open Graph signals
        $out['og_title'] = $this->meta_content($xpath, 'property', 'og:title');
        $out['og_description'] = $this->meta_content($xpath, 'property', 'og:description');
        $out['og_image'] = $this->meta_content($xpath, 'property', 'og:image');
        $out['og_type'] = $this->meta_content($xpath, 'property', 'og:type');

        // Twitter Card signals
        $out['twitter_card'] = $this->meta_content($xpath, 'name', 'twitter:card');
        if ($out['twitter_card'] === '') {
            $out['twitter_card'] = $this->meta_content($xpath, 'property', 'twitter:card');
        }
        $out['twitter_title'] = $this->meta_content($xpath, 'name', 'twitter:title');
        if ($out['twitter_title'] === '') {
            $out['twitter_title'] = $this->meta_content($xpath, 'property', 'twitter:title');
        }

        // Author signals
        $out['author_name'] = $this->meta_content($xpath, 'name', 'author');
        $articleAuthorNodes = $xpath->query('//a[@rel and contains(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"author")]');
        $out['has_author_link'] = ($articleAuthorNodes && $articleAuthorNodes->length > 0);

        // Article publish/modified time (for content freshness)
        $out['article_published_time'] = $this->meta_content($xpath, 'property', 'article:published_time');
        $out['article_modified_time'] = $this->meta_content($xpath, 'property', 'article:modified_time');
        if ($out['article_modified_time'] === '') {
            $out['article_modified_time'] = $this->meta_content($xpath, 'name', 'dcterms.modified');
        }

        // Canonical
        $canonical = '';
        $canonNodes = $xpath->query('//link[translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="canonical"]');
        if ($canonNodes && $canonNodes->length > 0) {
            $canonical = trim((string) $canonNodes->item(0)->getAttribute('href'));
        }
        if ($canonical !== '') {
            $canonical = $base_url !== '' ? Url::resolve($base_url, $canonical) : $canonical;
            $out['canonical'] = $canonical;
        }

        // Robots meta
        $out['robots'] = $this->meta_content($xpath, 'name', 'robots');
        $robots_l = strtolower((string) $out['robots']);
        $out['robots_noindex'] = (strpos($robots_l, 'noindex') !== false) || (strpos($robots_l, 'none') !== false);

        // 4.0.18: a password-protected post shows WordPress's password form
        // instead of its body. Content-quality checks must not judge it.
        $pw_form = $xpath->query('//form[contains(concat(" ", normalize-space(@class), " "), " post-password-form ")] | //input[@name="post_password"]');
        $out['password_protected'] = ($pw_form && $pw_form->length > 0);

        // Viewport
        $out['viewport'] = $this->meta_content($xpath, 'name', 'viewport');

        // Hreflang
        $hreflangNodes = $xpath->query('//link[contains(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"alternate") and @hreflang]');
        $out['hreflang_count'] = $hreflangNodes ? (int) $hreflangNodes->length : 0;

        // Structural counts
        $out['list_count'] = (int) ($dom->getElementsByTagName('ul')->length + $dom->getElementsByTagName('ol')->length);
        $out['table_count'] = (int) $dom->getElementsByTagName('table')->length;
        $out['dl_count'] = (int) $dom->getElementsByTagName('dl')->length;
        $out['h1_count'] = (int) $dom->getElementsByTagName('h1')->length;
        $out['h2_count'] = (int) $dom->getElementsByTagName('h2')->length;
        $out['h3_count'] = (int) $dom->getElementsByTagName('h3')->length;

        // v2.1.0: collect heading texts (capped) for question/intent analysis.
        $out['h1_texts'] = $this->collect_heading_texts($dom, 'h1', 5);
        $out['h2_texts'] = $this->collect_heading_texts($dom, 'h2', 30);
        $out['h3_texts'] = $this->collect_heading_texts($dom, 'h3', 40);
        // v2.1.5 — capture H4 too (FAQ accordions often use H4)
        $out['h4_texts'] = $this->collect_heading_texts($dom, 'h4', 40);

        // Word count (main content only)
        $text = $this->extract_main_content_text($dom, $xpath);
        $out['word_count'] = $this->count_words($text);

        // v2.1.0: capture first paragraph (~first 600 chars of main text) for direct-answer analysis.
        if ($text !== '') {
            $first = function_exists('mb_substr') ? mb_substr($text, 0, 600, 'UTF-8') : substr($text, 0, 600);
            $out['first_paragraph'] = trim((string) $first);
        }

        // Images
        $imgs = $dom->getElementsByTagName('img');
        $missing_alt = 0;
        $missing_dims = 0;
        $lazy = false;
        $webp = 0;
        if ($imgs) {
            foreach ($imgs as $img) {
                if (!$img instanceof \DOMElement) {
                    continue;
                }
                $alt = trim((string) $img->getAttribute('alt'));
                if ($alt === '') {
                    $missing_alt++;
                }

                // Images without width/height (CLS issue)
                $has_width = $img->hasAttribute('width') || $img->hasAttribute('data-width');
                $has_height = $img->hasAttribute('height') || $img->hasAttribute('data-height');
                if (!$has_width || !$has_height) {
                    $missing_dims++;
                }

                $loading = strtolower(trim((string) $img->getAttribute('loading')));
                $cls = strtolower(trim((string) $img->getAttribute('class')));
                $has_data_src = $img->hasAttribute('data-src') || $img->hasAttribute('data-lazy-src') || $img->hasAttribute('data-original');
                if ($loading === 'lazy' || $has_data_src || strpos($cls, 'lazy') !== false) {
                    $lazy = true;
                }

                $src = strtolower(trim((string) $img->getAttribute('src')));
                if ($src !== '' && strpos($src, '.webp') !== false) {
                    $webp++;
                }
            }
        }
        $out['images_without_dimensions'] = $missing_dims;
        // picture <source type="image/webp">
        $webpSources = $xpath->query('//picture//source[contains(translate(@type,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"image/webp")]');
        if ($webpSources) {
            $webp += (int) $webpSources->length;
        }

        $out['images_total'] = $imgs ? (int) $imgs->length : 0;
        $out['images_missing_alt'] = $missing_alt;
        $out['lazyload_detected'] = $lazy;
        $out['webp_images'] = $webp;

        // Assets: CSS + JS counts + origin summaries
        // v1.0.64: track external CSS links and inline style tags SEPARATELY.
        // The previous combined `css_count` produced misleading findings on
        // Elementor/Astra sites that have only 2 external CSS files but 30+
        // inline style blocks (which is normal Elementor output, not a
        // round-trip problem). External requests and inline weight are two
        // completely different optimization concerns.
        $cssLinks = $xpath->query('//link[contains(translate(@rel,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"stylesheet")]');
        $styleTags = $dom->getElementsByTagName('style');
        $cssOrigins = [];
        $external_css_count = 0;
        // v1.0.65: track stylesheet URLs (stripped of query strings) to detect
        // the same physical file being loaded twice — a real render-blocker
        // commonly caused by misbehaving font/asset plugins.
        $css_url_seen = [];
        if ($cssLinks) {
            foreach ($cssLinks as $link) {
                if (!$link instanceof \DOMElement) {
                    continue;
                }
                $href = trim((string) $link->getAttribute('href'));
                if ($href === '') {
                    continue;
                }
                $external_css_count++;
                $this->collect_asset_origin($cssOrigins, $href, $base_url, 'css');
                $key = strtok($href, '?');
                if (!is_string($key) || $key === '') {
                    $key = $href;
                }
                $css_url_seen[$key] = ($css_url_seen[$key] ?? 0) + 1;
            }
        }
        $inline_css_count = $styleTags ? (int) $styleTags->length : 0;

        // Measure inline CSS total bytes — the real cost of inline styles
        // is HTML weight, not request count.
        $inline_css_bytes = 0;
        $largest_inline_css = 0;
        if ($styleTags) {
            foreach ($styleTags as $st) {
                if (!$st instanceof \DOMElement) {
                    continue;
                }
                $css_text = (string) $st->textContent;
                $size = strlen($css_text);
                $inline_css_bytes += $size;
                if ($size > $largest_inline_css) {
                    $largest_inline_css = $size;
                }
            }
        }

        $out['external_css_count']  = $external_css_count;
        $out['inline_css_count']    = $inline_css_count;
        $out['inline_css_bytes']    = $inline_css_bytes;
        $out['largest_inline_css_bytes'] = $largest_inline_css;
        // Backward-compat: keep `css_count` as the COMBINED figure for any
        // legacy code that still reads it. New checks should use the split.
        $out['css_count'] = $external_css_count + $inline_css_count;
        $this->collect_inline_asset_origin($cssOrigins, 'style', $inline_css_count);

        $scripts = $xpath->query('//script[@src]');
        $jsOrigins = [];
        $js_url_seen = [];
        if ($scripts) {
            foreach ($scripts as $script) {
                if (!$script instanceof \DOMElement) {
                    continue;
                }
                $src = trim((string) $script->getAttribute('src'));
                if ($src === '') {
                    continue;
                }
                $this->collect_asset_origin($jsOrigins, $src, $base_url, 'js');
                $key = strtok($src, '?');
                if (!is_string($key) || $key === '') {
                    $key = $src;
                }
                $js_url_seen[$key] = ($js_url_seen[$key] ?? 0) + 1;
            }
        }
        $out['js_count'] = $scripts ? (int) $scripts->length : 0;

        // v1.0.65: emit duplicate-asset list (same base URL loaded 2+ times,
        // ignoring ?ver= cache busters). Render-blocking, easy to overlook.
        $duplicate_assets = [];
        foreach ($css_url_seen as $u => $n) {
            if ($n >= 2) {
                $duplicate_assets[] = ['type' => 'css', 'url' => (string) $u, 'count' => (int) $n];
            }
        }
        foreach ($js_url_seen as $u => $n) {
            if ($n >= 2) {
                $duplicate_assets[] = ['type' => 'js', 'url' => (string) $u, 'count' => (int) $n];
            }
        }
        $out['duplicate_assets'] = $duplicate_assets;
        $out['css_origins'] = $this->finalize_asset_origins($cssOrigins);
        $out['js_origins'] = $this->finalize_asset_origins($jsOrigins);

        // Blocking JS in head: script files without defer/async/module.
        $blocking = 0;
        $headScripts = $xpath->query('//head//script[@src]');
        if ($headScripts) {
            foreach ($headScripts as $s) {
                if (!$s instanceof \DOMElement) {
                    continue;
                }
                $async = $s->hasAttribute('async');
                $defer = $s->hasAttribute('defer');
                $type = strtolower(trim((string) $s->getAttribute('type')));
                if (!$async && !$defer && $type !== 'module') {
                    $blocking++;
                }
            }
        }
        $out['js_blocking_head'] = $blocking;

        // JSON-LD types + raw + author for entity analysis
        $jsonld_data = $this->extract_jsonld_data($xpath);
        $out['jsonld_types'] = $jsonld_data['types'];
        $out['jsonld_raw'] = $jsonld_data['raw'];
        $out['jsonld_raw_truncated'] = !empty($jsonld_data['raw_truncated']); // v3.3.52 P2.3

        // Bug fix v1.0.55 + v1.0.56: many themes (Astra, Yoast/Rank Math driven
        // layouts) declare the author only via JSON-LD Person/Article.author, not
        // via <meta name="author"> or rel="author". v1.0.55 added a JSON-LD fallback
        // but it ran against the COMPACTED blocks, where compact_jsonld_block() had
        // already sliced @graph down to the first 5 entries — Yoast's Person entity
        // is usually at position 6+ so it was lost. v1.0.56 moves the extraction
        // *into* extract_jsonld_data() so it sees the full decoded payload.
        $jsonld_author = isset($jsonld_data['author']) && is_array($jsonld_data['author'])
            ? $jsonld_data['author']
            : ['name' => '', 'url' => '', 'has_link' => false];
        if (($out['author_name'] === '') && $jsonld_author['name'] !== '') {
            $out['author_name'] = $jsonld_author['name'];
        }
        if (!$out['has_author_link'] && !empty($jsonld_author['has_link'])) {
            $out['has_author_link'] = true;
        }

        // Internal links
        $out['internal_links'] = $this->extract_internal_links($xpath, $base_url);
        // 4.0.18: links inside header, footer, menus and sidebars, kept apart
        // so the audit can tell "linked only from the menu" from "linked from
        // nowhere" (a true orphan reachable only through the sitemap).
        $out['nav_links'] = $this->extract_nav_links($xpath, $base_url);

        // Outbound links count (external links)
        $out['outbound_links_count'] = $this->count_outbound_links($xpath, $base_url);

        // -----------------------------------------------
        // NEW: Gap-closing data collection
        // -----------------------------------------------

        // Text-to-HTML ratio
        $html_len = strlen($html);
        $text_len = strlen($text);
        $out['text_to_html_ratio'] = $html_len > 0 ? round($text_len / $html_len, 4) : 0.0;

        // Heading hierarchy validation (H1→H3 skip = invalid)
        $heading_sequence = [];
        $body_nodes = $xpath->query('//body//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]');
        if ($body_nodes) {
            foreach ($body_nodes as $hn) {
                if ($hn instanceof \DOMElement) {
                    $heading_sequence[] = (int) substr($hn->nodeName, 1);
                }
            }
        }
        $out['heading_sequence'] = array_slice($heading_sequence, 0, 30);
        $hierarchy_valid = true;
        $skip_at_index = -1;
        for ($i = 1, $c = count($heading_sequence); $i < $c; $i++) {
            if ($heading_sequence[$i] > $heading_sequence[$i - 1] + 1) {
                $hierarchy_valid = false;
                $skip_at_index = $i;
                break;
            }
        }
        $out['heading_hierarchy_valid'] = $hierarchy_valid;
        // v1.0.80: store where the skip happens so the finding can show
        // the exact break point (e.g. "H1→H3 at position 5") instead of
        // showing a truncated sequence that looks valid.
        $out['heading_skip_at'] = $skip_at_index;
        // v2.1.4 — capture the offending heading TEXT at the skip position
        // and the heading right before it. Without this, evidence only shows
        // numeric positions in a truncated sequence. The user has to inspect
        // the page manually to find which heading broke the hierarchy.
        $out['heading_skip_from_text'] = '';
        $out['heading_skip_to_text'] = '';
        if (!$hierarchy_valid && $skip_at_index >= 1 && $body_nodes && $body_nodes->length > $skip_at_index) {
            $from_node = $body_nodes->item($skip_at_index - 1);
            $to_node   = $body_nodes->item($skip_at_index);
            if ($from_node instanceof \DOMElement) {
                $txt = trim((string) $from_node->textContent);
                $out['heading_skip_from_text'] = $this->truncate_for_evidence($txt, 200);
            }
            if ($to_node instanceof \DOMElement) {
                $txt = trim((string) $to_node->textContent);
                $out['heading_skip_to_text'] = $this->truncate_for_evidence($txt, 200);
            }
        }

        // srcset detection (responsive images)
        $srcset_count = 0;
        $avif_count = 0;
        if ($imgs) {
            foreach ($imgs as $img) {
                if (!$img instanceof \DOMElement) {
                    continue;
                }
                if ($img->hasAttribute('srcset') || $img->hasAttribute('data-srcset')) {
                    $srcset_count++;
                }
                $src_l = strtolower(trim((string) $img->getAttribute('src')));
                if (strpos($src_l, '.avif') !== false) {
                    $avif_count++;
                }
            }
        }
        $avif_sources = $xpath->query('//picture//source[contains(translate(@type,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"image/avif")]');
        if ($avif_sources) {
            $avif_count += (int) $avif_sources->length;
        }
        $out['has_srcset'] = $srcset_count > 0;
        $out['srcset_count'] = $srcset_count;
        $out['avif_images'] = $avif_count;

        // font-display: swap detection (in inline style blocks)
        $font_swap = false;
        if ($styleTags) {
            foreach ($styleTags as $st) {
                $css_text = $st instanceof \DOMElement ? $st->textContent : '';
                if (stripos($css_text, 'font-display') !== false && stripos($css_text, 'swap') !== false) {
                    $font_swap = true;
                    break;
                }
            }
        }
        $out['font_display_swap_detected'] = $font_swap;

        // v2.1.1 — actual video detection. Only flag the page as having a
        // video if there's a real video player. We explicitly EXCLUDE GTM
        // noscript iframes and other utility iframes (Tag Manager, Recaptcha,
        // etc.) to stop SerpFeatureCheck from claiming "video schema missing"
        // on pages whose only iframe is the GTM container.
        $video_provider = '';
        $has_video_embed = false;

        $video_tags = $dom->getElementsByTagName('video');
        if ($video_tags && $video_tags->length > 0) {
            $has_video_embed = true;
            $video_provider = 'html5';
        }

        if (!$has_video_embed) {
            $iframes = $dom->getElementsByTagName('iframe');
            if ($iframes) {
                foreach ($iframes as $if) {
                    if (!$if instanceof \DOMElement) {
                        continue;
                    }
                    $src = strtolower(trim((string) $if->getAttribute('src')));
                    if ($src === '') {
                        continue;
                    }
                    // Skip Google Tag Manager noscript iframes — they're a tracking
                    // container, not a video player.
                    if (strpos($src, 'googletagmanager.com/ns.html') !== false) {
                        continue;
                    }
                    if (strpos($src, 'recaptcha') !== false) {
                        continue;
                    }
                    if (preg_match('~(youtube\.com|youtu\.be|youtube-nocookie\.com)~', $src)) {
                        $has_video_embed = true;
                        $video_provider = 'youtube';
                        break;
                    }
                    if (strpos($src, 'vimeo.com') !== false || strpos($src, 'player.vimeo.com') !== false) {
                        $has_video_embed = true;
                        $video_provider = 'vimeo';
                        break;
                    }
                    if (strpos($src, 'wistia.com') !== false || strpos($src, 'wistia.net') !== false) {
                        $has_video_embed = true;
                        $video_provider = 'wistia';
                        break;
                    }
                    if (strpos($src, 'dailymotion.com') !== false) {
                        $has_video_embed = true;
                        $video_provider = 'dailymotion';
                        break;
                    }
                    if (strpos($src, 'facebook.com/plugins/video') !== false) {
                        $has_video_embed = true;
                        $video_provider = 'facebook';
                        break;
                    }
                    if (strpos($src, 'tiktok.com') !== false) {
                        $has_video_embed = true;
                        $video_provider = 'tiktok';
                        break;
                    }
                }
            }
        }

        if (!$has_video_embed) {
            // <oembed> block (used by some block editors)
            $oembed = $dom->getElementsByTagName('oembed');
            if ($oembed && $oembed->length > 0) {
                $has_video_embed = true;
                $video_provider = 'oembed';
            }
        }

        $out['has_video_embed'] = $has_video_embed;
        $out['video_provider'] = $video_provider;

        // Breadcrumb nav detection
        $breadcrumb_nav = $xpath->query('//*[contains(translate(@class,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"breadcrumb") or contains(translate(@id,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"breadcrumb") or @aria-label="Breadcrumb" or @aria-label="breadcrumb"]');
        $out['has_breadcrumb_nav'] = ($breadcrumb_nav && $breadcrumb_nav->length > 0);

        // Breadcrumb + Publisher schema in JSON-LD
        $has_breadcrumb_schema = false;
        $has_publisher_schema = false;
        $schema_types_detailed = [];
        foreach ($out['jsonld_raw'] as $block) {
            $block_str = is_string($block) ? $block : (string) json_encode($block);
            if (stripos($block_str, 'BreadcrumbList') !== false) {
                $has_breadcrumb_schema = true;
            }
            if (stripos($block_str, '"publisher"') !== false) {
                $has_publisher_schema = true;
            }
        }
        foreach ($out['jsonld_types'] as $jtype) {
            $schema_types_detailed[] = $jtype;
        }
        $out['has_breadcrumb_schema'] = $has_breadcrumb_schema;
        $out['has_publisher_schema'] = $has_publisher_schema;
        $out['schema_types_detailed'] = array_unique($schema_types_detailed);

        // nofollow on internal links
        $nofollow_internal = 0;
        $all_links = $xpath->query('//a[@href]');
        if ($all_links && $base_url !== '') {
            $site_host = (string) wp_parse_url($base_url, PHP_URL_HOST);
            foreach ($all_links as $a) {
                if (!$a instanceof \DOMElement) {
                    continue;
                }
                $href = trim((string) $a->getAttribute('href'));
                if (!Url::is_crawlable_href_candidate($href)) {
                    continue;
                }
                // 4.0.17: WordPress core marks its own comment controls
                // (cancel reply, reply-to-comment) nofollow on purpose. They are
                // not content links, so they must not count as internal nofollow.
                $a_class = ' ' . strtolower((string) $a->getAttribute('class')) . ' ';
                if (
                    strtolower((string) $a->getAttribute('id')) === 'cancel-comment-reply-link'
                    || strpos($a_class, ' comment-reply-link ') !== false
                    || stripos($href, 'replytocom=') !== false
                    || preg_match('~#(respond|comment-\d+)$~i', $href)
                ) {
                    continue;
                }
                $resolved_href = Url::resolve($base_url, $href);
                if ($resolved_href === '') {
                    continue;
                }
                $link_host = (string) wp_parse_url($resolved_href, PHP_URL_HOST);
                if ($link_host === $site_host || $link_host === '') {
                    $rel = strtolower(trim((string) $a->getAttribute('rel')));
                    if (strpos($rel, 'nofollow') !== false) {
                        $nofollow_internal++;
                    }
                }
            }
        }
        $out['nofollow_internal_count'] = $nofollow_internal;

        // ── Readability Score (Flesch-Kincaid approximation, language-aware) ──
        $out['readability_score'] = $this->compute_readability($text, (string) $out['lang']);

        // ── JS Framework / SPA detection signals ─────────────────────────────
        $out['js_framework_signals'] = $this->detect_js_frameworks($html, $out['js_origins']);

        // ── Content freshness signal ──────────────────────────────────────────
        // Age in days since article_modified_time or article_published_time
        $date_str = $out['article_modified_time'] !== '' ? $out['article_modified_time'] : $out['article_published_time'];
        if ($date_str !== '') {
            $ts = strtotime($date_str);
            $out['content_age_days'] = is_int($ts) && $ts > 0 ? (int) floor((time() - $ts) / 86400) : -1;
        } else {
            $out['content_age_days'] = -1;
        }

        return $out;
    }

    /**
     * Compute a simplified readability score (0-100, higher = easier to read).
     * Uses word/sentence/syllable approximation. For Arabic: word-based sentence length.
     */
    private function compute_readability(string $text, string $lang): float
    {
        $text = trim($text);
        if ($text === '') {
            return 0.0;
        }

        $is_arabic = (strpos($lang, 'ar') === 0);

        // Count sentences (split on . ! ? ؟ ۔)
        $sentences = preg_split('/[.!?؟۔]+/', $text, -1, PREG_SPLIT_NO_EMPTY);
        $sentence_count = $sentences ? max(1, count(array_filter($sentences, static function ($s) {
            return trim($s) !== '';
        }))) : 1;

        // Count words
        $words = preg_split('/\s+/', $text, -1, PREG_SPLIT_NO_EMPTY);
        $word_count = $words ? count($words) : 0;
        if ($word_count === 0) {
            return 50.0;
        }

        $avg_words_per_sentence = $word_count / $sentence_count;

        if ($is_arabic) {
            // For Arabic: simpler heuristic based on avg sentence length
            // Ideal: 10-15 words/sentence = score ~80
            if ($avg_words_per_sentence <= 10) {
                return 85.0;
            } elseif ($avg_words_per_sentence <= 15) {
                return 75.0;
            } elseif ($avg_words_per_sentence <= 20) {
                return 60.0;
            } elseif ($avg_words_per_sentence <= 30) {
                return 45.0;
            }
            return 30.0;
        }

        // English: simplified Flesch Reading Ease approximation
        // Syllable count: vowel-based approximation
        $syllable_count = 0;
        if (is_array($words)) {
            foreach ($words as $word) {
                $syllable_count += max(1, (int) preg_match_all('/[aeiouAEIOU]/', (string) $word));
            }
        }
        $avg_syllables = $syllable_count / $word_count;

        // Flesch: 206.835 - 1.015*(words/sentences) - 84.6*(syllables/words)
        $score = 206.835 - (1.015 * $avg_words_per_sentence) - (84.6 * $avg_syllables);
        return round(max(0.0, min(100.0, $score)), 1);
    }

    /**
     * Detect JavaScript framework signals from HTML source and JS origins.
     *
     * @param array<string,mixed> $js_origins
     * @return array<string,mixed>
     */
    private function detect_js_frameworks(string $html, array $js_origins): array
    {
        $signals = [];

        // React
        if (strpos($html, 'data-reactroot') !== false || strpos($html, '__NEXT_DATA__') !== false || strpos($html, 'react-dom') !== false) {
            $signals[] = ['framework' => 'React/Next.js', 'risk' => 'CSR possible — verify SSR/SSG is used for main content'];
        }
        // Vue
        if (strpos($html, 'data-v-') !== false || strpos($html, '__vue__') !== false || strpos($html, 'nuxt') !== false) {
            $signals[] = ['framework' => 'Vue/Nuxt.js', 'risk' => 'CSR possible — verify SSR is used for main content'];
        }
        // Angular
        if (strpos($html, 'ng-version') !== false || strpos($html, '_nghost') !== false) {
            $signals[] = ['framework' => 'Angular', 'risk' => 'Angular apps often require server-side rendering for SEO'];
        }
        // Generic SPA signals
        if (strpos($html, '<div id="app"></div>') !== false || strpos($html, '<div id="root"></div>') !== false) {
            $signals[] = ['framework' => 'SPA mount point detected', 'risk' => 'Empty div suggests client-side rendering — check rendered HTML'];
        }

        // Check JS origins for known frameworks
        $js_origins_flat = json_encode($js_origins);
        if ($js_origins_flat !== false && (strpos($js_origins_flat, 'react') !== false || strpos($js_origins_flat, 'next') !== false)) {
            if (empty($signals)) {
                $signals[] = ['framework' => 'React/Next.js (from JS files)', 'risk' => 'Verify SSR/SSG for main content'];
            }
        }

        return $signals;
    }

    /**
     * v3.3.50 — Detect inline analytics/measurement signals in raw HTML.
     *
     * Modern GTM/GA4/Microsoft Clarity are loaded via inline script snippets,
     * often behind WP Rocket Delay JS or Perfmatters Script Manager. They never
     * appear as external script-src references in the parsed JS origin
     * list, so the AnalyticsStackCheck used to false-flag instrumented sites as
     * "measurement stack missing" (wsend.pro run-5, 2026-05-25).
     *
     * Uses regex against the raw body BEFORE DOM parsing so we catch signals
     * even when DOMDocument fails on malformed HTML.
     *
     * @return array<string,mixed>
     */
    private function detect_analytics_signals(string $html): array
    {
        $sig = [
            'gtm_inline'        => false,
            'ga4_inline'        => false,
            'gtag_inline'       => false,
            'clarity_inline'    => false,
            'datalayer_inline'  => false,
            'meta_pixel_inline' => false,
            // v3.3.52 P2.2 — privacy-first / non-Google measurement stacks.
            'matomo_inline'     => false,
            'plausible_inline'  => false,
            'segment_inline'    => false,
            'posthog_inline'    => false,
            'gtm_ids'           => [],
            'ga4_ids'           => [],
        ];

        // GTM container IDs (GTM-XXXXXXX). Captures both the inline gtm.js
        // bootstrap and the noscript iframe form.
        if (preg_match_all('/GTM-[A-Z0-9]{4,10}/', $html, $m)) {
            $sig['gtm_inline'] = true;
            $sig['gtm_ids'] = array_values(array_unique($m[0]));
        }
        if (stripos($html, 'googletagmanager.com/gtm.js') !== false
            || stripos($html, 'googletagmanager.com/ns.html') !== false) {
            $sig['gtm_inline'] = true;
        }

        // GA4 measurement IDs (G-XXXXXXXXXX). Universal Analytics UA-* is
        // EOL — intentionally not matched.
        if (preg_match_all('/\bG-[A-Z0-9]{8,12}\b/', $html, $m)) {
            $sig['ga4_inline'] = true;
            $sig['ga4_ids'] = array_values(array_unique($m[0]));
        }
        // gtag('config', ...) or window.gtag = ...
        if (stripos($html, "gtag('config'") !== false
            || stripos($html, 'gtag("config"') !== false
            || stripos($html, 'window.gtag') !== false) {
            $sig['gtag_inline'] = true;
        }

        // dataLayer initialization. The canonical GTM bootstrap snippet passes
        // the layer name as a quoted literal ('dataLayer') and references it via
        // w[l] — so `window.dataLayer` / `dataLayer.push(` literals may be
        // absent. Match those forms PLUS the quoted literal and the gtm.start
        // bootstrap signature so the standard snippet is never missed.
        if (stripos($html, 'window.dataLayer') !== false
            || stripos($html, 'dataLayer.push(') !== false
            || stripos($html, "'dataLayer'") !== false
            || stripos($html, '"dataLayer"') !== false
            || stripos($html, 'gtm.start') !== false
            || preg_match('/\bdataLayer\s*=\s*(window\.dataLayer\s*\|\||\[)/i', $html)) {
            $sig['datalayer_inline'] = true;
        }

        // Microsoft Clarity (clarity.ms script + clarity('event'…) calls).
        if (stripos($html, 'clarity.ms/tag') !== false
            || stripos($html, '.clarity.ms/') !== false
            || stripos($html, "clarity('event'") !== false
            || stripos($html, 'clarity("event"') !== false
            || preg_match('/window\.clarity\s*=/', $html)) {
            $sig['clarity_inline'] = true;
        }

        // Meta (Facebook) Pixel.
        if (stripos($html, 'connect.facebook.net/') !== false
            || stripos($html, 'fbq(') !== false
            || preg_match('/\bfbq\s*=\s*function/', $html)) {
            $sig['meta_pixel_inline'] = true;
        }

        // v3.3.52 P2.2 — non-Google measurement stacks. A site on one of these
        // IS instrumented; the old check flagged it "measurement stack missing".
        // Matomo (formerly Piwik): _paq queue + matomo.js/piwik.js loader.
        if (stripos($html, 'matomo.js') !== false
            || stripos($html, 'matomo.php') !== false
            || stripos($html, 'piwik.js') !== false
            || strpos($html, '_paq.push') !== false
            || preg_match('/\bvar\s+_paq\b/', $html)) {
            $sig['matomo_inline'] = true;
        }
        // Plausible Analytics (cloud or self-hosted).
        if (stripos($html, 'plausible.io/js/') !== false
            || stripos($html, 'cdn.plausible.io') !== false
            || preg_match('/\bsrc=["\'][^"\']*plausible[^"\']*\.js/i', $html)
            || preg_match('/\bplausible\s*=\s*window\.plausible/i', $html)) {
            $sig['plausible_inline'] = true;
        }
        // Segment.
        if (stripos($html, 'cdn.segment.com') !== false
            || preg_match('/\banalytics\.load\(/', $html)) {
            $sig['segment_inline'] = true;
        }
        // PostHog.
        if (stripos($html, 'posthog.com') !== false
            || preg_match('/\bposthog\.init\(/', $html)
            || preg_match('/\bwindow\.posthog\b/', $html)) {
            $sig['posthog_inline'] = true;
        }

        return $sig;
    }

    /**
     * @param array<string,array<string,mixed>> $bucket
     */
    private function collect_asset_origin(array &$bucket, string $asset_url, string $base_url, string $asset_type): void
    {
        $resolved = $base_url !== '' ? Url::resolve($base_url, $asset_url) : $asset_url;
        $resolved = trim($resolved);
        if ($resolved === '') {
            return;
        }

        $origin = $this->classify_asset_origin($resolved, $asset_type);
        $label = isset($origin['label']) ? (string) $origin['label'] : '';
        if ($label === '') {
            return;
        }

        if (!isset($bucket[$label])) {
            $bucket[$label] = [
                'type' => (string) ($origin['type'] ?? 'site'),
                'slug' => (string) ($origin['slug'] ?? 'custom'),
                'label' => $label,
                'count' => 0,
                'examples' => [],
            ];
        }

        $bucket[$label]['count'] = (int) $bucket[$label]['count'] + 1;
        $example = isset($origin['example']) ? trim((string) $origin['example']) : '';
        if ($example !== '' && !in_array($example, $bucket[$label]['examples'], true) && count($bucket[$label]['examples']) < 3) {
            $bucket[$label]['examples'][] = $example;
        }
    }

    /**
     * @param array<string,array<string,mixed>> $bucket
     */
    private function collect_inline_asset_origin(array &$bucket, string $asset_type, int $count): void
    {
        if ($count <= 0) {
            return;
        }

        $label = 'inline:' . strtolower($asset_type);
        $bucket[$label] = [
            'type' => 'inline',
            'slug' => strtolower($asset_type),
            'label' => $label,
            'count' => $count,
            'examples' => ['inline ' . strtolower($asset_type)],
        ];
    }

    /**
     * @return array<string,string>
     */
    private function classify_asset_origin(string $asset_url, string $asset_type): array
    {
        $normalized = Url::normalize($asset_url);
        if (stripos($normalized, 'data:') === 0) {
            return [
                'type' => 'inline',
                'slug' => strtolower($asset_type),
                'label' => 'inline:' . strtolower($asset_type),
                'example' => 'data-uri',
            ];
        }

        $parts = function_exists('wp_parse_url') ? wp_parse_url($normalized) : @wp_parse_url($normalized);
        $host = is_array($parts) && isset($parts['host']) ? strtolower((string) $parts['host']) : '';
        $path = is_array($parts) && isset($parts['path']) ? rawurldecode((string) $parts['path']) : '';
        $path = is_string($path) ? preg_replace('#/{2,}#', '/', $path) : '';
        $path = is_string($path) ? $path : '';

        $siteParts = function_exists('wp_parse_url') ? wp_parse_url(home_url('/')) : @wp_parse_url(home_url('/'));
        $siteHost = is_array($siteParts) && isset($siteParts['host']) ? strtolower((string) $siteParts['host']) : '';

        $type = 'site';
        $slug = 'custom';
        if ($path !== '' && preg_match('~/wp-content/plugins/([^/]+)/~i', $path, $m) === 1) {
            $type = 'plugin';
            $slug = $this->normalize_asset_slug((string) $m[1], 'plugin');
        } elseif ($path !== '' && preg_match('~/wp-content/themes/([^/]+)/~i', $path, $m) === 1) {
            $type = 'theme';
            $slug = $this->normalize_asset_slug((string) $m[1], 'theme');
        } elseif ($path !== '' && (strpos($path, '/wp-includes/') !== false || strpos($path, '/wp-admin/') !== false)) {
            $type = 'core';
            $slug = 'wordpress';
        } elseif ($host !== '' && $siteHost !== '' && $host !== $siteHost) {
            $type = 'external';
            $slug = $this->normalize_asset_slug($host, 'external');
        }

        return [
            'type' => $type,
            'slug' => $slug,
            'label' => $type . ':' . $slug,
            'example' => $this->normalize_asset_example($normalized, $host, $siteHost, $path),
        ];
    }

    private function normalize_asset_slug(string $value, string $fallback): string
    {
        $slug = strtolower(trim($value));
        $slug = preg_replace('/[^a-z0-9._-]+/i', '-', $slug);
        $slug = is_string($slug) ? trim($slug, '-.') : '';
        return $slug !== '' ? $slug : $fallback;
    }

    private function normalize_asset_example(string $normalized, string $host, string $siteHost, string $path): string
    {
        $cleanPath = $path !== '' ? preg_replace('/\?.*$/', '', $path) : '';
        $cleanPath = is_string($cleanPath) ? $cleanPath : '';
        if ($host !== '' && $siteHost !== '' && $host !== $siteHost) {
            return $host . ($cleanPath !== '' ? $cleanPath : '/');
        }
        if ($cleanPath !== '') {
            return $cleanPath;
        }
        return preg_replace('/\?.*$/', '', $normalized) ?: $normalized;
    }

    /**
     * @param array<string,array<string,mixed>> $bucket
     * @return array<int,array<string,mixed>>
     */
    private function finalize_asset_origins(array $bucket): array
    {
        if (empty($bucket)) {
            return [];
        }

        $rows = array_values($bucket);
        usort($rows, static function (array $a, array $b): int {
            $countA = (int) ($a['count'] ?? 0);
            $countB = (int) ($b['count'] ?? 0);
            if ($countA === $countB) {
                return strcmp((string) ($a['label'] ?? ''), (string) ($b['label'] ?? ''));
            }
            return $countB <=> $countA;
        });

        $rows = array_slice($rows, 0, 12);
        foreach ($rows as &$row) {
            $row['examples'] = array_values(array_slice(array_unique(array_map('strval', (array) ($row['examples'] ?? []))), 0, 3));
        }
        unset($row);

        return $rows;
    }

    private function meta_content(\DOMXPath $xpath, string $attr, string $value): string
    {
        $attr_l = strtolower($attr);
        $value_l = strtolower($value);
        $q = sprintf(
            '//meta[translate(@%s,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")="%s"]',
            $attr_l,
            $value_l
        );
        $nodes = $xpath->query($q);
        if ($nodes && $nodes->length > 0) {
            $content = trim((string) $nodes->item(0)->getAttribute('content'));
            return $content;
        }
        return '';
    }

    /**
     * @return array<int,string>
     */
    private function extract_internal_links(\DOMXPath $xpath, string $base_url): array
    {
        $limit = 200;
        try {
            $s = Settings::get();
            $limit = (int) ($s['scan_limits']['internal_links_extract_limit'] ?? 200);
        } catch (\Throwable $e) {
            // ignore
        }
        $limit = max(20, min(5000, $limit));

        $exclude = 1;
        try {
            $s = Settings::get();
            $exclude = (int) ($s['scan_limits']['exclude_global_nav_links'] ?? 1);
        } catch (\Throwable $e) {
            // ignore
        }
        $exclude = $exclude ? 1 : 0;

        $query = '//a[@href]';
        if ($exclude) {
            // Best-effort: exclude common global navigation areas.
            $query = '//a[@href and not(ancestor::header) and not(ancestor::footer) and not(ancestor::nav) and not(ancestor::aside) and not(ancestor::*[@role="navigation"])]';
        }

        $nodes = $xpath->query($query);
        if (!$nodes) {
            return [];
        }

        $seen = [];
        $out = [];
        foreach ($nodes as $a) {
            if (!$a instanceof \DOMElement) {
                continue;
            }
            $href = trim((string) $a->getAttribute('href'));
            if (!Url::is_crawlable_href_candidate($href)) {
                continue;
            }

            $abs = $base_url !== '' ? Url::resolve($base_url, $href) : $href;
            if ($abs === '') {
                continue;
            }
            if (!Url::is_internal_url($abs)) {
                continue;
            }
            if (Url::is_probably_non_html_url($abs) || Url::is_default_excluded_url($abs)) {
                continue;
            }

            $norm = Url::normalize($abs);
            if ($norm === '') {
                continue;
            }
            if (isset($seen[$norm])) {
                continue;
            }
            $seen[$norm] = true;
            $out[] = $norm;
            if (count($out) >= $limit) {
                break;
            }
        }

        return $out;
    }

    /**
     * Internal links inside global navigation areas (header, footer, nav,
     * aside, role=navigation). Normalized and de-duplicated, capped at 100.
     *
     * @return array<int,string>
     */
    private function extract_nav_links(\DOMXPath $xpath, string $base_url): array
    {
        $nodes = $xpath->query('//a[@href and (ancestor::header or ancestor::footer or ancestor::nav or ancestor::aside or ancestor::*[@role="navigation"])]');
        if (!$nodes) {
            return [];
        }
        $seen = [];
        foreach ($nodes as $a) {
            if (!$a instanceof \DOMElement) {
                continue;
            }
            $href = trim((string) $a->getAttribute('href'));
            if (!Url::is_crawlable_href_candidate($href)) {
                continue;
            }
            $abs = $base_url !== '' ? Url::resolve($base_url, $href) : $href;
            if ($abs === '' || !Url::is_internal_url($abs) || Url::is_probably_non_html_url($abs)) {
                continue;
            }
            $norm = Url::normalize($abs);
            if ($norm !== '') {
                $seen[$norm] = true;
            }
            if (count($seen) >= 100) {
                break;
            }
        }
        return array_keys($seen);
    }

    /**
     * @return array{types:array<int,string>,raw:array<int,array<string,mixed>>,author:array{name:string,url:string,has_link:bool}}
     */
    private function extract_jsonld_data(\DOMXPath $xpath): array
    {
        $types = [];
        $raw_blocks = [];
        $seen = [];
        // Bug fix v1.0.56: extract author from the FULL decoded JSON-LD here, BEFORE
        // compact_jsonld_block() truncates @graph to 5 entries. In Yoast / Rank Math
        // schemas the Person entity often lives at @graph[6..15] which used to be
        // sliced away, leaving extract_author_from_jsonld() (which read the compacted
        // blocks) to find nothing on every page.
        $author_state = ['name' => '', 'url' => '', 'has_link' => false];
        // v3.3.52 P2.3 — surface when raw blocks / @graph entries were capped so
        // field-completeness checks can avoid a definitive "missing" verdict on
        // schema that simply lives beyond the storage cap.
        $raw_truncated = false;

        $nodes = $xpath->query('//script[contains(translate(@type,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"),"ld+json")]');
        if (!$nodes) {
            return ['types' => [], 'raw' => [], 'author' => $author_state, 'raw_truncated' => false];
        }

        foreach ($nodes as $n) {
            if (!$n instanceof \DOMElement) {
                continue;
            }
            $raw = trim((string) $n->textContent);
            if ($raw === '') {
                continue;
            }

            $decoded = json_decode($raw, true);
            if (!is_array($decoded)) {
                continue;
            }

            // Run author extraction against the FULL decoded payload (not the
            // compacted/sliced version below). Stop as soon as we find a name.
            if ($author_state['name'] === '') {
                $found_author = $this->extract_author_from_jsonld([$decoded]);
                if ($found_author['name'] !== '') {
                    $author_state = $found_author;
                }
            }

            // Keep a compact raw block for entity analysis (limit size).
            // v3.3.52 P2.3 — cap raised 5→8 blocks; flag when exceeded or when a
            // single @graph is larger than the 40-entry compaction cap.
            if (count($raw_blocks) < 8) {
                if (isset($decoded['@graph']) && is_array($decoded['@graph']) && count($decoded['@graph']) > 40) {
                    $raw_truncated = true;
                }
                $raw_blocks[] = $this->compact_jsonld_block($decoded);
            } else {
                $raw_truncated = true;
            }

            $found = $this->walk_jsonld_for_types($decoded);
            foreach ($found as $t) {
                $t = trim((string) $t);
                if ($t === '') {
                    continue;
                }
                if (!isset($seen[$t])) {
                    $seen[$t] = true;
                    $types[] = $t;
                }
            }
        }

        return ['types' => $types, 'raw' => $raw_blocks, 'author' => $author_state, 'raw_truncated' => $raw_truncated];
    }

    /**
     * Keep only entity-relevant fields for compact storage. The keep list lives
     * on self::KEEP_FIELDS so the 38-element array isn't reallocated on every
     * recursive call. The @graph cap exists because Yoast/Rank Math wrap
     * everything in a single @graph that can run into the dozens of entries —
     * truncating heavy schema graphs keeps analysis_json bounded.
     *
     * @param mixed $node
     * @return array<string,mixed>
     */
    private function compact_jsonld_block($node): array
    {
        if (!is_array($node)) {
            return [];
        }
        $out = [];
        foreach (self::KEEP_FIELDS as $k) {
            if (isset($node[$k])) {
                $out[$k] = $this->compact_jsonld_value($node[$k]);
            }
        }
        if (isset($node['@graph']) && is_array($node['@graph'])) {
            $graph = [];
            // v3.3.52 P2.3 — cap raised 30→40 to cover schema-heavy Yoast/Rank
            // Math graphs without hiding entities; extract_jsonld_data() flags
            // any graph that still exceeds it.
            foreach (array_slice($node['@graph'], 0, 40) as $item) {
                $graph[] = $this->compact_jsonld_block($item);
            }
            $out['@graph'] = $graph;
        }
        return $out;
    }

    /**
     * Recursively compact a single JSON-LD value while preserving the right
     * shape for sequential arrays. The previous implementation called
     * compact_jsonld_block() on every array, which silently nuked sequential
     * arrays of strings (e.g. sameAs, openingHours, dayOfWeek) to `[]` because
     * none of their numeric keys matched KEEP_FIELDS — defeating
     * LocalSeoCheck/VoiceSearchCheck field-presence assertions on schema-rich
     * sites.
     *
     * Sequential arrays now keep their elements (strings get the 200-char cap,
     * nested arrays/objects recurse). Associative arrays/objects go through
     * compact_jsonld_block as before. Capped at 12 elements per list to bound
     * cost — 6 sameAs profiles + 7-day openingHours both fit comfortably.
     *
     * @param mixed $v
     * @return mixed
     */
    private function compact_jsonld_value($v)
    {
        if (is_string($v)) {
            return substr($v, 0, 200);
        }
        if (!is_array($v)) {
            return $v;
        }
        // Distinguish a sequential list (numeric 0..n-1 keys) from an
        // associative object, without the WP 6.5 / PHP 8.1 array_is_list()
        // dependency: a list equals its own re-indexed values (empty-safe).
        $is_list = ($v === array_values($v));
        if ($is_list) {
            $compacted = [];
            $count = 0;
            foreach ($v as $item) {
                if ($count >= 12) {
                    break;
                }
                $compacted[] = $this->compact_jsonld_value($item);
                $count++;
            }
            return $compacted;
        }
        return $this->compact_jsonld_block($v);
    }

    /**
     * Extract author signal (name + has_link flag) from compact JSON-LD blocks.
     *
     * Used as a fallback for HTML <meta name="author"> + rel="author" detection so
     * that EeatCheck / AiFriendlyCheck / AdvancedSchemaCheck stop reporting false
     * "missing author" findings on sites that only declare authors via Article.author
     * or @graph Person entities (very common with Yoast / Rank Math).
     *
     * @param array<int,array<string,mixed>> $raw_blocks
     * @return array{name:string, url:string, has_link:bool}
     */
    private function extract_author_from_jsonld(array $raw_blocks): array
    {
        $best = ['name' => '', 'url' => '', 'has_link' => false];

        $extract_from_node = static function ($node) use (&$best, &$extract_from_node): void {
            if (!is_array($node)) {
                return;
            }

            // Normalize @type (could be string, array, or missing).
            $type_raw = $node['@type'] ?? '';
            if (is_array($type_raw)) {
                $type_raw = (string) reset($type_raw);
            }
            $type = strtolower(trim((string) $type_raw));

            // Article-like nodes: dive into their author field.
            if (in_array($type, ['article', 'blogposting', 'newsarticle', 'webpage'], true)
                && !empty($node['author'])
            ) {
                $author = $node['author'];
                $candidates = [];
                if (is_string($author) && trim($author) !== '') {
                    $candidates[] = ['name' => trim($author)];
                } elseif (is_array($author)) {
                    // Could be a single author object or a list of authors.
                    $is_list = isset($author[0]) && is_array($author[0]);
                    if ($is_list) {
                        foreach ($author as $a) {
                            if (is_array($a)) {
                                $candidates[] = $a;
                            } elseif (is_string($a) && trim($a) !== '') {
                                $candidates[] = ['name' => trim($a)];
                            }
                        }
                    } else {
                        $candidates[] = $author;
                    }
                }

                foreach ($candidates as $c) {
                    if (!is_array($c)) {
                        continue;
                    }
                    $name = isset($c['name']) ? trim((string) $c['name']) : '';
                    $url  = isset($c['url'])  ? trim((string) $c['url'])  : '';
                    $has_same_as = !empty($c['sameAs']);
                    if ($name !== '' && $best['name'] === '') {
                        $best['name']     = $name;
                        $best['url']      = $url;
                        $best['has_link'] = ($url !== '' || $has_same_as);
                    }
                }
            }

            // Standalone Person entity (very common in @graph).
            if ($type === 'person' && !empty($node['name'])) {
                $name = trim((string) $node['name']);
                $url  = isset($node['url']) ? trim((string) $node['url']) : '';
                $has_same_as = !empty($node['sameAs']);
                if ($name !== '' && $best['name'] === '') {
                    $best['name']     = $name;
                    $best['url']      = $url;
                    $best['has_link'] = ($url !== '' || $has_same_as);
                }
            }

            // Recurse into @graph and any other nested arrays.
            if (isset($node['@graph']) && is_array($node['@graph'])) {
                foreach ($node['@graph'] as $item) {
                    $extract_from_node($item);
                }
            }
            foreach ($node as $k => $v) {
                if ($k === '@graph' || $k === 'author') {
                    continue;
                }
                if (is_array($v)) {
                    $extract_from_node($v);
                }
            }
        };

        foreach ($raw_blocks as $block) {
            $extract_from_node($block);
            if ($best['name'] !== '') {
                break;
            }
        }

        return $best;
    }

    /**
     * Count external (outbound) links on the page.
     */
    private function count_outbound_links(\DOMXPath $xpath, string $base_url): int
    {
        $nodes = $xpath->query('//a[@href]');
        if (!$nodes) {
            return 0;
        }

        $siteParts = function_exists('wp_parse_url') ? wp_parse_url(home_url('/')) : @wp_parse_url(home_url('/'));
        $siteHost = is_array($siteParts) && isset($siteParts['host']) ? strtolower((string) $siteParts['host']) : '';

        $count = 0;
        foreach ($nodes as $a) {
            if (!$a instanceof \DOMElement) {
                continue;
            }
            $href = trim((string) $a->getAttribute('href'));
            if (!Url::is_crawlable_href_candidate($href)) {
                continue;
            }
            $abs = $base_url !== '' ? Url::resolve($base_url, $href) : $href;
            if ($abs === '') {
                continue;
            }
            $parts = function_exists('wp_parse_url') ? wp_parse_url($abs) : @wp_parse_url($abs);
            $host = is_array($parts) && isset($parts['host']) ? strtolower((string) $parts['host']) : '';
            if ($host !== '' && $siteHost !== '' && $host !== $siteHost) {
                $count++;
            }
        }
        return $count;
    }

    /**
     * @param mixed $node
     * @return array<int,string>
     */
    private function walk_jsonld_for_types($node): array
    {
        $out = [];
        if (is_array($node)) {
            // If this is a JSON-LD object.
            if (isset($node['@type'])) {
                $t = $node['@type'];
                if (is_string($t)) {
                    $out[] = $t;
                } elseif (is_array($t)) {
                    foreach ($t as $tt) {
                        if (is_string($tt)) {
                            $out[] = $tt;
                        }
                    }
                }
            }
            foreach ($node as $v) {
                foreach ($this->walk_jsonld_for_types($v) as $x) {
                    $out[] = $x;
                }
            }
        }
        return $out;
    }

    /**
     * Extract main content text, excluding header/footer/navigation/sidebar.
     *
     * v2.0.1: don't return the FIRST matching selector — Elementor pages often
     * have a `<main>` wrapper that contains only the header/breadcrumb (yielding
     * ~10 words) while the real content lives further down in `[data-elementor-id]`.
     * Instead, evaluate every candidate selector, pick the one that produces the
     * most visible text, and fall back to body-text when nothing yields ≥ 100
     * words. This fixes a systemic "thin content" false positive on Elementor
     * sites where pages with 1000+ rendered words were being scored as 11 words.
     */
    private function extract_main_content_text(\DOMDocument $dom, \DOMXPath $xpath): string
    {
        $main_selectors = [
            '//main',
            '//article',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-widget-theme-post-content ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-location-single ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " entry-content ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " post-content ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " content-area ")]',
            '//*[@role="main"]',
            '//*[@id="content"]//article',
            '//*[@id="main-content"]',
            '//*[@data-elementor-id and contains(concat(" ", normalize-space(@class), " "), " elementor ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-section-wrap ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " e-con ")]',
        ];

        $best_text = '';
        $best_words = 0;
        foreach ($main_selectors as $selector) {
            $nodes = $xpath->query($selector);
            if (!$nodes || $nodes->length === 0) {
                continue;
            }
            $main_node = $nodes->item(0);
            if (!($main_node instanceof \DOMElement)) {
                continue;
            }
            $candidate = $this->get_cleaned_text_from_element($main_node);
            $words = $candidate === '' ? 0 : count(preg_split('/\s+/u', trim($candidate)) ?: []);
            if ($words > $best_words) {
                $best_words = $words;
                $best_text = $candidate;
            }
        }

        // If none of the narrow selectors produced meaningful content, fall
        // back to the full <body> minus chrome (header/nav/footer/sidebars).
        if ($best_words < 100) {
            $body = $dom->getElementsByTagName('body');
            if ($body && $body->length > 0) {
                $body_node = $body->item(0);
                if ($body_node instanceof \DOMElement) {
                    $body_text = $this->get_cleaned_text_from_element($body_node);
                    $body_words = $body_text === '' ? 0 : count(preg_split('/\s+/u', trim($body_text)) ?: []);
                    if ($body_words > $best_words) {
                        return $body_text;
                    }
                }
            }
        }

        if ($best_text !== '') {
            return $best_text;
        }
        return wp_strip_all_tags($dom->saveHTML() ?: '');
    }

    /**
     * Get text from element, excluding navigation/chrome.
     */
    private function get_cleaned_text_from_element(\DOMElement $element): string
    {
        // Clone to avoid modifying original DOM
        $clone = $element->cloneNode(true);
        $temp_doc = new \DOMDocument();
        $imported = $temp_doc->importNode($clone, true);
        if (!$imported) {
            return '';
        }
        $temp_doc->appendChild($imported);
        $temp_xpath = new \DOMXPath($temp_doc);

        // Remove navigation and chrome elements.
        // v1.0.80: refined exclusions for Elementor compatibility.
        // Do NOT exclude bare <header>/<footer> tags — Elementor uses
        // <header> inside sections for semantic headings. Only exclude
        // site-level header/footer identified by role or Elementor location.
        $exclude_selectors = [
            '//nav',
            '//aside',
            '//script',
            '//style',
            '//noscript',
            '//svg',
            '//canvas',
            '//template',
            '//iframe',
            '//form',
            '//input',
            '//textarea',
            '//button',
            '//select',
            '//option',
            '//*[contains(concat(" ", normalize-space(@class), " "), " sidebar ")]',
            // v1.0.76: DON'T exclude "widget" generically — Elementor uses
            // "elementor-widget-*" on EVERY content block. Excluding "widget"
            // strips ALL page content and produces word_count=12. Only exclude
            // WordPress sidebar widgets (class="widget widget_*").
            '//*[contains(concat(" ", normalize-space(@class), " "), " widget_")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " widget-area ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " navigation ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " menu ")]',
            // Elementor header/footer locations (site-level, not section-level)
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-location-header ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-location-footer ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-location-popup ")]',
            '//*[contains(concat(" ", normalize-space(@class), " "), " elementor-nav-menu ")]',
            '//*[@role="navigation"]',
            '//*[@role="complementary"]',
            '//*[@role="banner"]',
            '//*[@role="contentinfo"]',
            '//*[contains(translate(@id,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"), "sidebar")]',
            // v1.0.80: only exclude site-level header/footer by ID, not bare tags.
            '//*[self::header and @id and contains(translate(@id,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"), "masthead")]',
            '//*[self::footer and @id and contains(translate(@id,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz"), "colophon")]',
            '//*[self::header and contains(concat(" ", normalize-space(@class), " "), " site-header ")]',
            '//*[self::footer and contains(concat(" ", normalize-space(@class), " "), " site-footer ")]',
        ];

        foreach ($exclude_selectors as $selector) {
            $nodes = $temp_xpath->query($selector);
            if ($nodes) {
                // Convert to array to avoid live NodeList issues
                $to_remove = [];
                foreach ($nodes as $node) {
                    $to_remove[] = $node;
                }
                foreach ($to_remove as $node) {
                    if ($node->parentNode) {
                        $node->parentNode->removeChild($node);
                    }
                }
            }
        }

        $txt = (string) ($temp_doc->textContent ?? '');
        return trim($txt);
    }

    private function count_words(string $text): int
    {
        $text = trim($text);
        if ($text === '') {
            return 0;
        }
        // Unicode-safe tokenization.
        $parts = preg_split('/\s+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
        return is_array($parts) ? count($parts) : 0;
    }

    /**
     * v2.1.4 — normalize + truncate a heading text for evidence display.
     */
    private function truncate_for_evidence(string $text, int $max): string
    {
        $t = preg_replace('/\s+/u', ' ', $text) ?? $text;
        $t = trim($t);
        if ($t === '') {
            return '';
        }
        if (function_exists('mb_strlen') && mb_strlen($t, 'UTF-8') > $max) {
            return mb_substr($t, 0, $max, 'UTF-8') . '…';
        }
        if (strlen($t) > $max) {
            return substr($t, 0, $max) . '…';
        }
        return $t;
    }

    /**
     * v2.1.0: Capture heading texts for question/intent analysis. Each heading
     * is normalized (whitespace collapsed, trimmed) and truncated to 240 chars.
     *
     * @return array<int,string>
     */
    private function collect_heading_texts(\DOMDocument $dom, string $tag, int $cap): array
    {
        $out = [];
        $nodes = $dom->getElementsByTagName($tag);
        if (!$nodes || $nodes->length === 0) {
            return $out;
        }
        $count = 0;
        foreach ($nodes as $n) {
            if ($count >= $cap) {
                break;
            }
            $text = trim((string) $n->textContent);
            if ($text === '') {
                continue;
            }
            $text = preg_replace('/\s+/u', ' ', $text) ?? $text;
            if (function_exists('mb_strlen') && mb_strlen($text, 'UTF-8') > 240) {
                $text = mb_substr($text, 0, 240, 'UTF-8');
            } elseif (strlen($text) > 240) {
                $text = substr($text, 0, 240);
            }
            $out[] = $text;
            $count++;
        }
        return $out;
    }
}