<?php

if (!defined('ABSPATH')) {
    exit;
}

/**
 * Find Statistics Step
 * 
 * Searches the web specifically for statistics, data, and research
 * related to the topic. Results are stored in context for use by
 * subsequent steps (like generate_draft).
 * 
 * This differs from refine_add_statistics which asks AI to add stats
 * to existing content - this step finds REAL statistics before generation.
 */
class AIGH_Step_Find_Statistics extends AIGH_Pipeline_Step {
    
    protected $step_type = 'find_statistics';
    protected $step_name = 'Find Statistics';
    protected $description = 'Search the web for statistics, data, and research studies on the topic';
    protected $category = 'research';
    protected $estimated_duration = 45;
    
    protected $required_inputs = array('topic');
    protected $output_keys = array('statistics_results', 'statistics_brief');
    
    /**
     * @var string Serper API key
     */
    private $serper_api_key;
    
    public function __construct($config = array()) {
        parent::__construct($config);
        $this->serper_api_key = get_option('aigh_serper_api_key', '');
    }
    
    protected function get_default_config() {
        return array(
            'num_results' => 5,
            'num_sources_to_fetch' => 3,
            'include_year' => true,
            'query_patterns' => array(
                '{topic} statistics',
                '{topic} data research',
                '{topic} study findings'
            )
        );
    }
    
    public function execute($context) {
        $this->log_start($context);
        
        // Check for Serper API key
        if (empty($this->serper_api_key)) {
            $this->log_end($context, false);
            return $this->create_error('no_api_key', 'Serper API key not configured. Please add it in Settings.');
        }
        
        // Get topic from context
        $topic = $this->get_from_context($context, 'topic', '');
        if (empty($topic)) {
            $this->log_end($context, false);
            return $this->create_error('no_topic', 'No topic found in context');
        }
        
        // Build search queries
        $queries = $this->build_statistics_queries($topic);
        
        // Execute searches
        $all_results = array();
        $num_results = $this->get_config_value('num_results', 5);
        
        foreach ($queries as $query) {
            $results = $this->search_web($query, $num_results);
            
            if (is_wp_error($results)) {
                continue;
            }
            
            // Merge results, avoiding duplicates
            foreach ($results as $key => $result) {
                if (!is_numeric($key)) {
                    // Keep answer_box and knowledge_graph
                    if (!isset($all_results[$key])) {
                        $all_results[$key] = $result;
                    }
                    continue;
                }
                
                // Check for duplicate URLs
                $url = $result['url'] ?? '';
                $is_duplicate = false;
                foreach ($all_results as $existing) {
                    if (is_array($existing) && isset($existing['url']) && $existing['url'] === $url) {
                        $is_duplicate = true;
                        break;
                    }
                }
                
                if (!$is_duplicate) {
                    $all_results[] = $result;
                }
            }
        }
        
        if (empty($all_results)) {
            $this->set_in_context($context, 'statistics_results', array());
            $this->set_in_context($context, 'statistics_brief', '');
            $this->set_in_context($context, 'statistics_skipped', true);
            $this->log_end($context, true);
            return $context;
        }
        
        // Store raw search results
        $this->set_in_context($context, 'statistics_results', $all_results);
        
        // Fetch content from top results
        $num_sources = $this->get_config_value('num_sources_to_fetch', 3);
        $fetched_content = $this->fetch_top_results($all_results, $num_sources);
        
        // Create statistics brief
        $brief = $this->create_statistics_brief($fetched_content, $all_results, $topic);
        $this->set_in_context($context, 'statistics_brief', $brief);
        
        // Append to existing research_brief if present
        $existing_brief = $this->get_from_context($context, 'research_brief', '');
        if (!empty($existing_brief)) {
            $combined = $existing_brief . "\n\n" . $brief;
            $this->set_in_context($context, 'research_brief', $combined);
        } else {
            $this->set_in_context($context, 'research_brief', $brief);
        }
        
        $this->set_in_context($context, 'statistics_skipped', false);
        $this->log_end($context, true);
        return $context;
    }
    
    /**
     * Build statistics-focused search queries
     */
    private function build_statistics_queries($topic) {
        $queries = array();
        $patterns = $this->get_config_value('query_patterns', array(
            '{topic} statistics',
            '{topic} data research',
            '{topic} study findings'
        ));
        
        // Extract core topic (shorter version for better search)
        $core_topic = $this->extract_core_topic($topic);
        
        // Add year if configured
        $include_year = $this->get_config_value('include_year', true);
        $year = gmdate('Y');
        
        foreach ($patterns as $pattern) {
            $query = str_replace('{topic}', $core_topic, $pattern);
            if ($include_year && strpos($pattern, '{year}') === false) {
                $query .= ' ' . $year;
            }
            $queries[] = $query;
        }
        
        return $queries;
    }
    
    /**
     * Extract core topic for search queries
     */
    private function extract_core_topic($topic) {
        // If topic is short enough, use as-is
        if (strlen($topic) <= 60) {
            return $topic;
        }
        
        // Extract key phrases for longer topics
        $words = str_word_count($topic, 1);
        $keywords = array_slice($words, 0, 6);
        return implode(' ', $keywords);
    }
    
    /**
     * Search the web using Serper API
     */
    private function search_web($query, $num_results = 5) {
        $response = wp_remote_post('https://google.serper.dev/search', array(
            'headers' => array(
                'X-API-KEY' => $this->serper_api_key,
                'Content-Type' => 'application/json'
            ),
            'body' => json_encode(array(
                'q' => $query,
                'num' => min($num_results * 2, 10)
            )),
            'timeout' => 30
        ));
        
        if (is_wp_error($response)) {
            return $response;
        }
        
        $status_code = wp_remote_retrieve_response_code($response);
        if ($status_code !== 200) {
            return new WP_Error('serper_error', 'Serper API error: HTTP ' . $status_code);
        }
        
        $body = wp_remote_retrieve_body($response);
        $data = json_decode($body, true);
        
        if (json_last_error() !== JSON_ERROR_NONE) {
            return new WP_Error('json_error', 'Failed to parse Serper response');
        }
        
        // Extract and filter results - prioritize statistical sources
        $results = array();
        
        // Domains to skip
        $skip_domains = array(
            'youtube.com', 'youtu.be', 'vimeo.com', 'dailymotion.com',
            'nytimes.com', 'wsj.com', 'ft.com', 'bloomberg.com', 'economist.com',
            'reddit.com', 'twitter.com', 'x.com', 'facebook.com', 'instagram.com',
            'pinterest.com', 'tiktok.com', 'linkedin.com'
        );
        
        // Domains to prioritize (statistical sources)
        $priority_domains = array(
            'statista.com', 'pewresearch.org', 'gallup.com', 'bls.gov',
            'census.gov', 'who.int', 'worldbank.org', 'oecd.org',
            'ncbi.nlm.nih.gov', 'sciencedirect.com', 'nature.com',
            'springer.com', 'journals.sagepub.com', 'tandfonline.com'
        );
        
        $priority_results = array();
        $other_results = array();
        
        if (!empty($data['organic'])) {
            foreach ($data['organic'] as $item) {
                $url = $item['link'] ?? '';
                $domain = strtolower(wp_parse_url($url, PHP_URL_HOST));
                
                // Skip blocked domains
                $should_skip = false;
                foreach ($skip_domains as $skip) {
                    if (strpos($domain, $skip) !== false) {
                        $should_skip = true;
                        break;
                    }
                }
                
                if ($should_skip) continue;
                
                $result = array(
                    'title' => $item['title'] ?? '',
                    'url' => $url,
                    'snippet' => $item['snippet'] ?? '',
                    'position' => $item['position'] ?? 0
                );
                
                // Check if priority domain
                $is_priority = false;
                foreach ($priority_domains as $priority) {
                    if (strpos($domain, $priority) !== false) {
                        $is_priority = true;
                        break;
                    }
                }
                
                if ($is_priority) {
                    $priority_results[] = $result;
                } else {
                    $other_results[] = $result;
                }
            }
        }
        
        // Combine priority first, then others
        $results = array_merge($priority_results, $other_results);
        $results = array_slice($results, 0, $num_results);
        
        // Re-index
        $indexed_results = array_values($results);
        
        // Include answer box if present
        if (!empty($data['answerBox'])) {
            $indexed_results['answer_box'] = array(
                'title' => $data['answerBox']['title'] ?? '',
                'answer' => $data['answerBox']['answer'] ?? $data['answerBox']['snippet'] ?? ''
            );
        }
        
        // Include knowledge graph if present
        if (!empty($data['knowledgeGraph'])) {
            $indexed_results['knowledge_graph'] = array(
                'title' => $data['knowledgeGraph']['title'] ?? '',
                'description' => $data['knowledgeGraph']['description'] ?? ''
            );
        }
        
        return $indexed_results;
    }
    
    /**
     * Fetch content from top search results
     */
    private function fetch_top_results($search_results, $num_sources) {
        $fetched = array(
            'sources' => array(),
            'content' => array(),
            'errors' => array()
        );
        
        $fetched_count = 0;
        
        foreach ($search_results as $key => $result) {
            if (!is_numeric($key)) continue;
            if ($fetched_count >= $num_sources) break;
            if (empty($result['url'])) continue;
            
            $content = $this->fetch_url_content($result['url']);
            
            if (is_wp_error($content)) {
                $fetched['errors'][] = $result['url'] . ': ' . $content->get_error_message();
                continue;
            }
            
            $fetched['sources'][] = array(
                'title' => $result['title'],
                'url' => $result['url']
            );
            
            $fetched['content'][] = array(
                'source' => $result['title'],
                'url' => $result['url'],
                'content' => $content
            );
            
            $fetched_count++;
        }
        
        return $fetched;
    }
    
    /**
     * Fetch and extract content from a URL
     */
    private function fetch_url_content($url) {
        $response = wp_remote_get($url, array(
            'timeout' => 15,
            'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        ));
        
        if (is_wp_error($response)) {
            return $response;
        }
        
        $status_code = wp_remote_retrieve_response_code($response);
        if ($status_code !== 200) {
            return new WP_Error('fetch_error', 'HTTP ' . $status_code);
        }
        
        $html = wp_remote_retrieve_body($response);
        if (empty($html)) {
            return new WP_Error('empty_response', 'Empty response');
        }
        
        // Extract text content
        $text = $this->extract_text_from_html($html);
        
        if (strlen($text) < 100) {
            return new WP_Error('insufficient_content', 'Content too short');
        }
        
        // Truncate if too long
        $max_length = 8000;
        if (strlen($text) > $max_length) {
            $text = substr($text, 0, $max_length);
            $last_period = strrpos($text, '. ');
            if ($last_period !== false && $last_period > ($max_length * 0.8)) {
                $text = substr($text, 0, $last_period + 1);
            }
            $text .= "\n\n[Content truncated...]";
        }
        
        return $text;
    }
    
    /**
     * Extract readable text from HTML
     */
    private function extract_text_from_html($html) {
        // Remove unwanted elements
        $html = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $html);
        $html = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $html);
        $html = preg_replace('/<nav\b[^>]*>(.*?)<\/nav>/is', '', $html);
        $html = preg_replace('/<footer\b[^>]*>(.*?)<\/footer>/is', '', $html);
        $html = preg_replace('/<header\b[^>]*>(.*?)<\/header>/is', '', $html);
        $html = preg_replace('/<aside\b[^>]*>(.*?)<\/aside>/is', '', $html);
        $html = preg_replace('/<!--.*?-->/s', '', $html);
        
        // Try to find main content
        $extracted = '';
        
        if (preg_match('/<article[^>]*>(.*?)<\/article>/is', $html, $matches)) {
            $extracted = $matches[1];
        } elseif (preg_match('/<main[^>]*>(.*?)<\/main>/is', $html, $matches)) {
            $extracted = $matches[1];
        } elseif (preg_match_all('/<p[^>]*>(.{50,}?)<\/p>/is', $html, $matches)) {
            $extracted = implode("\n\n", $matches[0]);
        } else {
            $extracted = $html;
        }
        
        // Convert to text
        $extracted = preg_replace('/<h[1-6][^>]*>(.*?)<\/h[1-6]>/is', "\n\n### $1 ###\n\n", $extracted);
        $extracted = preg_replace('/<li[^>]*>(.*?)<\/li>/is', "• $1\n", $extracted);
        $extracted = preg_replace('/<br\s*\/?>/i', "\n", $extracted);
        $extracted = preg_replace('/<\/p>/i', "\n\n", $extracted);
        
        $text = wp_strip_all_tags($extracted);
        $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
        $text = preg_replace('/[ \t]+/', ' ', $text);
        $text = preg_replace('/\n\s*\n\s*\n+/', "\n\n", $text);
        
        return trim($text);
    }
    
    /**
     * Create a statistics brief from search results
     */
    private function create_statistics_brief($fetched_content, $search_results, $topic) {
        $brief = "=== STATISTICS & DATA RESEARCH ===\n";
        $brief .= "Topic: {$topic}\n\n";
        
        // Add answer box if present
        if (!empty($search_results['answer_box'])) {
            $brief .= "### Key Data Point ###\n";
            $brief .= $search_results['answer_box']['answer'] . "\n\n";
        }
        
        // Add search snippets with statistics focus
        $brief .= "### Statistics Found ###\n";
        foreach ($search_results as $key => $result) {
            if (!is_numeric($key)) continue;
            
            // Look for numbers in snippet
            $snippet = $result['snippet'] ?? '';
            $has_numbers = preg_match('/\d+/', $snippet);
            
            if ($has_numbers) {
                $brief .= "• {$result['title']}: {$snippet}\n";
            }
        }
        $brief .= "\n";
        
        // Add fetched content
        if (!empty($fetched_content['content'])) {
            $brief .= "### Detailed Sources ###\n\n";
            foreach ($fetched_content['content'] as $idx => $item) {
                $num = $idx + 1;
                $brief .= "--- Source {$num}: {$item['source']} ---\n";
                $brief .= "URL: {$item['url']}\n\n";
                $brief .= $item['content'] . "\n\n";
            }
        }
        
        // Add sources list
        if (!empty($fetched_content['sources'])) {
            $brief .= "### Source URLs ###\n";
            foreach ($fetched_content['sources'] as $source) {
                $brief .= "• {$source['title']}: {$source['url']}\n";
            }
            $brief .= "\n";
        }
        
        $brief .= "=== END STATISTICS RESEARCH ===";
        
        return $brief;
    }
    
    public function get_config_schema() {
        return array(
            'num_results' => array(
                'type' => 'number',
                'label' => 'Results per Query',
                'description' => 'How many search results to retrieve per query',
                'min' => 1,
                'max' => 10,
                'default' => 5
            ),
            'num_sources_to_fetch' => array(
                'type' => 'number',
                'label' => 'Sources to Fetch',
                'description' => 'How many pages to fetch full content from',
                'min' => 1,
                'max' => 5,
                'default' => 3
            ),
            'include_year' => array(
                'type' => 'checkbox',
                'label' => 'Include Current Year',
                'description' => 'Add current year to search queries for recent statistics',
                'default' => true
            )
        );
    }
}