2, // Fail fast to avoid hanging the scan
'redirection' => 0, // Do not follow redirects, observe them
'user-agent' => 'ActiveTwist-SEO-Inspector/1.0; ' . home_url(),
'sslverify' => false, // Local dev/loopback often has self-signed certs
];
$response = wp_remote_get($url, $args);
// Normalized result structure
$result = [
'code' => 0,
'canonicals' => [],
'robots_meta' => [],
'x_robots_tag' => [],
'error' => null
];
if (is_wp_error($response)) {
$result['error'] = $response->get_error_message();
return $result;
}
$result['code'] = wp_remote_retrieve_response_code($response);
$headers = wp_remote_retrieve_headers($response);
// Extract X-Robots-Tag (can be multiple headers or comma-separated)
$x_robots = isset($headers['x-robots-tag']) ? $headers['x-robots-tag'] : [];
if (!is_array($x_robots)) {
$x_robots = [$x_robots];
}
$result['x_robots_tag'] = $x_robots;
$body = wp_remote_retrieve_body($response);
if (empty($body)) {
return $result;
}
// Parse Body using DOMDocument
$dom = new \DOMDocument();
$internal_errors = libxml_use_internal_errors(true);
// loadHTML expects a full document. mb_convert_encoding helps with charset issues.
$dom->loadHTML(mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8'));
libxml_clear_errors();
libxml_use_internal_errors($internal_errors);
$xpath = new \DOMXPath($dom);
// Canonical:
foreach ($xpath->query('//link[@rel="canonical"]/@href') as $node) {
$result['canonicals'][] = $node->nodeValue;
}
// Robots Meta:
foreach ($xpath->query('//meta[@name="robots"]/@content') as $node) {
$result['robots_meta'][] = $node->nodeValue;
}
return $result;
}
}