<?php

if (!defined('ABSPATH')) {
    exit;
}

/**
 * Research URLs Step
 * 
 * Fetches content from user-provided URLs and creates a research brief
 * that can be used by subsequent generation steps.
 */
class AIGH_Step_Research_Urls extends AIGH_Pipeline_Step {
    
    protected $step_type = 'research_urls';
    protected $step_name = 'Research URLs';
    protected $description = 'Fetch and analyze content from provided URLs for research';
    protected $category = 'research';
    protected $estimated_duration = 30;
    
    protected $required_inputs = array();
    protected $output_keys = array('research_brief', 'research_sources_count', 'research_errors');
    
    /**
     * @var AIGH_OpenAI_Client
     */
    private $openai_client;
    
    public function log($message) {
        if (defined('WP_DEBUG') && WP_DEBUG) {
            // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log -- Debug logging wrapped in WP_DEBUG check
            error_log('[AIGH_RESEARCH] ' . $message);
        }
    }

    public function __construct($config = array(), $openai_client = null) {
        parent::__construct($config);
        $this->openai_client = $openai_client ?: new AIGH_OpenAI_Client();
        // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_print_r -- Debug logging
        $this->log('Constructor called. Config: ' . print_r($config, true));
    }

    protected function get_default_config() {
        $this->log('get_default_config() called');

        $config = array(
            'urls' => '',
            'max_urls' => 5,
            'max_content_length' => 12000,
            'skip_on_empty' => true,
            'create_brief' => true,
        );

        // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_print_r -- Debug logging
        $this->log('get_default_config() returning: ' . print_r($config, true));
        return $config;
    }
    
    public function can_execute($context) {
        $this->log('can_execute() called');
        $urls = $this->get_urls_from_config_or_context($context);
        $this->log('URLs found: ' . count($urls));
        return true;
    }
    
    public function execute($context) {
        $this->log('========== EXECUTE START ==========');
        $this->log_start($context);
        
        // Step 1: Get URLs
        $this->log('Step 1: Getting URLs from config or context');
        $urls = $this->get_urls_from_config_or_context($context);
        $this->log('URLs parsed: ' . count($urls));
        $this->log('URL list: ' . implode(', ', $urls));
        
        // Handle empty URLs
        if (empty($urls)) {
            $this->log('No URLs found - skipping');
            $this->set_in_context($context, 'research_skipped', true);
            $this->set_in_context($context, 'research_sources_count', 0);
            $this->log_end($context, true);
            return $context;
        }
        
        // Step 2: Fetch content from all URLs
        $this->log('Step 2: Fetching content from URLs');
        $research_data = $this->fetch_all_urls($urls);
        $this->log('Fetch complete. Sources: ' . count($research_data['sources']) . ', Errors: ' . count($research_data['errors']));
        
        if (!empty($research_data['errors'])) {
            $this->log('Fetch errors: ' . implode(' | ', $research_data['errors']));
        }
        
        if (empty($research_data['content'])) {
            $this->log('No content fetched from any URL');
            $this->set_in_context($context, 'research_errors', $research_data['errors']);
            $this->set_in_context($context, 'research_sources_count', 0);
            $this->set_in_context($context, 'research_skipped', true);
            $this->log_end($context, true);
            return $context;
        }
        
        // Log content lengths
        foreach ($research_data['content'] as $idx => $item) {
            $this->log('Content ' . ($idx + 1) . ' from ' . $item['url'] . ': ' . strlen($item['content']) . ' chars');
        }
        
        // Step 3: Get topic for context
        $this->log('Step 3: Getting topic from context');
        $topic = $this->get_from_context($context, 'topic', '');
        $this->log('Topic: ' . ($topic ?: '(none)'));
        
        // Step 4: Create brief or format raw content
        $this->log('Step 4: Creating research brief');
        $create_brief = $this->get_config_value('create_brief', true);
        $this->log('create_brief config: ' . ($create_brief ? 'true' : 'false'));
        
        if ($create_brief) {
            $this->log('Calling create_research_brief()');
            $brief = $this->create_research_brief($research_data, $topic);
        } else {
            $this->log('Formatting raw content (no AI brief)');
            $brief = $this->format_raw_content($research_data);
        }
        
        if (is_wp_error($brief)) {
            $this->log('Brief creation FAILED: ' . $brief->get_error_message());
            $this->log_end($context, false);
            return $brief;
        }
        
        $this->log('Brief created. Length: ' . strlen($brief) . ' chars');
        $this->log('Brief preview (first 500 chars): ' . substr($brief, 0, 500));
        
        // Step 5: Store results in context
        $this->log('Step 5: Storing results in context');
        $this->set_in_context($context, 'research_brief', $brief);
        $this->set_in_context($context, 'research_sources_count', count($research_data['sources']));
        $this->set_in_context($context, 'research_sources', $research_data['sources']);
        $this->set_in_context($context, 'research_errors', $research_data['errors']);
        $this->set_in_context($context, 'research_skipped', false);
        
        $this->log('Context updated. research_sources_count: ' . count($research_data['sources']));
        $this->log('========== EXECUTE END ==========');
        
        $this->log_end($context, true);
        return $context;
    }
    
    private function get_urls_from_config_or_context($context) {
        $this->log('get_urls_from_config_or_context() called');
        
        // First check config
        $urls_string = $this->get_config_value('urls', '');
        $this->log('URLs from config: "' . $urls_string . '"');
        
        // If empty, check context
        if (empty($urls_string)) {
            $urls_string = $this->get_from_context($context, 'research_urls', '');
            $this->log('URLs from context: "' . $urls_string . '"');
        }
        
        if (empty($urls_string)) {
            $this->log('No URL string found');
            return array();
        }
        
        // Parse URLs
        $urls = preg_split('/[\r\n,]+/', $urls_string);
        $urls = array_map('trim', $urls);
        $this->log('After split: ' . count($urls) . ' items');
        
        $self = $this;
        $urls = array_filter($urls, function($url) use ($self) {
            $valid = !empty($url) && filter_var($url, FILTER_VALIDATE_URL);
            if (!$valid && !empty($url)) {
                $self->log('Invalid URL filtered out: ' . $url);
            } else if ($valid) {
                $self->log('Valid URL: ' . $url);
            }
            return $valid;
        });
        $this->log('After filter: ' . count($urls) . ' valid URLs');
        
        $max_urls = $this->get_config_value('max_urls', 5);
        $urls = array_slice(array_values($urls), 0, $max_urls);
        $this->log('After slice (max ' . $max_urls . '): ' . count($urls) . ' URLs');
        
        return $urls;
    }
    
    private function fetch_all_urls($urls) {
        $this->log('fetch_all_urls() called with ' . count($urls) . ' URLs');
        
        $result = array(
            'sources' => array(),
            'content' => array(),
            'errors' => array()
        );
        
        foreach ($urls as $idx => $url) {
            $this->log('Fetching URL ' . ($idx + 1) . ': ' . $url);
            
            $content = $this->fetch_url_content($url);
            
            if (is_wp_error($content)) {
                $error_msg = $url . ': ' . $content->get_error_message();
                $this->log('FETCH ERROR: ' . $error_msg);
                $result['errors'][] = $error_msg;
                continue;
            }
            
            $this->log('FETCH SUCCESS: ' . strlen($content) . ' chars');
            
            $title = $this->extract_title_from_url($url);
            
            $result['sources'][] = array(
                'title' => $title,
                'url' => $url
            );
            
            $result['content'][] = array(
                'source' => $title,
                'url' => $url,
                'content' => $content
            );
        }
        
        $this->log('fetch_all_urls() complete. Success: ' . count($result['sources']) . ', Errors: ' . count($result['errors']));
        return $result;
    }
    
    private function fetch_url_content($url) {
        $this->log('fetch_url_content() called for: ' . $url);
        
        $this->log('Making HTTP request...');
        $response = wp_remote_get($url, array(
            'timeout' => 60,
            'headers' => array(
                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.5',
            ),
            'redirection' => 5,
            'sslverify' => false
        ));
        
        if (is_wp_error($response)) {
            $this->log('HTTP request failed: ' . $response->get_error_message());
            return new WP_Error('fetch_failed', 'Could not fetch URL: ' . $response->get_error_message());
        }
        
        $status_code = wp_remote_retrieve_response_code($response);
        $this->log('HTTP status: ' . $status_code);
        
        if ($status_code !== 200) {
            $this->log('Bad HTTP status');
            return new WP_Error('bad_status', "HTTP Error {$status_code}");
        }
        
        $html = wp_remote_retrieve_body($response);
        $this->log('Raw HTML length: ' . strlen($html) . ' chars');
        
        if (empty($html)) {
            $this->log('Empty HTML body');
            return new WP_Error('empty_content', 'URL returned empty content');
        }
        
        // Extract content
        $this->log('Extracting main content...');
        $content = $this->extract_main_content($html);
        $this->log('Extracted content length: ' . strlen($content) . ' chars');
        
        if (empty($content) || strlen($content) < 100) {
            $this->log('Content too short after extraction');
            return new WP_Error('insufficient_content', 'Could not extract meaningful content from page');
        }
        
        // Truncate if needed
        $max_length = $this->get_config_value('max_content_length', 12000);
        if (strlen($content) > $max_length) {
            $this->log('Truncating from ' . strlen($content) . ' to ' . $max_length . ' chars');
            $content = $this->smart_truncate($content, $max_length);
        }
        
        $this->log('Final content length: ' . strlen($content) . ' chars');
        $this->log('Content preview (first 300 chars): ' . substr($content, 0, 300));
        
        return $content;
    }
    
    private function extract_main_content($html) {
        $this->log('extract_main_content() called. HTML length: ' . strlen($html));
        
        // Remove unwanted elements
        $html = preg_replace('/<script\b[^>]*>.*?<\/script>/is', '', $html);
        $html = preg_replace('/<style\b[^>]*>.*?<\/style>/is', '', $html);
        $html = preg_replace('/<noscript\b[^>]*>.*?<\/noscript>/is', '', $html);
        $html = preg_replace('/<iframe\b[^>]*>.*?<\/iframe>/is', '', $html);
        $html = preg_replace('/<!--.*?-->/s', '', $html);
        $html = preg_replace('/<nav\b[^>]*>.*?<\/nav>/is', '', $html);
        $html = preg_replace('/<footer\b[^>]*>.*?<\/footer>/is', '', $html);
        $html = preg_replace('/<aside\b[^>]*>.*?<\/aside>/is', '', $html);
        $html = preg_replace('/<form\b[^>]*>.*?<\/form>/is', '', $html);
        
        $this->log('After removing unwanted elements: ' . strlen($html) . ' chars');
        
        $content = '';
        
        // Strategy 1: Article tag
        if (preg_match('/<article[^>]*>(.*?)<\/article>/is', $html, $matches)) {
            $content = $matches[1];
            $this->log('Strategy 1 (article tag): Found ' . strlen($content) . ' chars');
        }
        
        // Strategy 2: Main tag
        if (empty($content) && preg_match('/<main[^>]*>(.*?)<\/main>/is', $html, $matches)) {
            $content = $matches[1];
            $this->log('Strategy 2 (main tag): Found ' . strlen($content) . ' chars');
        }
        
        // Strategy 3: Content class names
        if (empty($content)) {
            $content_patterns = array(
                '/<div[^>]+class=["\'][^"\']*\b(post-content|entry-content|article-content|article-body|post-body|content-body|main-content|page-content|story-body|story-content)[^"\']*["\'][^>]*>(.*?)<\/div>/is',
                '/<div[^>]+id=["\'][^"\']*\b(content|main-content|article|post|entry)[^"\']*["\'][^>]*>(.*?)<\/div>/is',
            );
            
            foreach ($content_patterns as $pattern) {
                if (preg_match($pattern, $html, $matches)) {
                    $content = $matches[2] ?? $matches[1];
                    $this->log('Strategy 3 (content class): Found ' . strlen($content) . ' chars');
                    break;
                }
            }
        }
        
        // Strategy 4: Paragraph clusters
        if (empty($content)) {
            if (preg_match_all('/<p[^>]*>(.{100,}?)<\/p>/is', $html, $matches)) {
                $content = implode("\n\n", $matches[0]);
                $this->log('Strategy 4 (paragraphs): Found ' . strlen($content) . ' chars from ' . count($matches[0]) . ' paragraphs');
            }
        }
        
        // Strategy 5: Body fallback
        if (empty($content)) {
            if (preg_match('/<body[^>]*>(.*?)<\/body>/is', $html, $matches)) {
                $content = $matches[1];
                $this->log('Strategy 5 (body fallback): Found ' . strlen($content) . ' chars');
            } else {
                $content = $html;
                $this->log('Strategy 5 (full HTML fallback): Using ' . strlen($content) . ' chars');
            }
        }
        
        // Clean up
        $content = $this->clean_extracted_content($content);
        $this->log('After cleanup: ' . strlen($content) . ' chars');
        
        return $content;
    }
    
    private function clean_extracted_content($html) {
        $this->log('clean_extracted_content() called. Input length: ' . strlen($html) . ' chars');
        
        $this->log('Replacing headings with markdown...');
        $html = preg_replace('/<h[1-6][^>]*>(.*?)<\/h[1-6]>/is', "\n\n### $1 ###\n\n", $html);
        
        $this->log('Replacing list items...');
        $html = preg_replace('/<li[^>]*>(.*?)<\/li>/is', "â€¢ $1\n", $html);
        
        $this->log('Replacing br tags...');
        $html = preg_replace('/<br\s*\/?>/i', "\n", $html);
        
        $this->log('Replacing p and div closing tags...');
        $html = preg_replace('/<\/p>/i', "\n\n", $html);
        $html = preg_replace('/<\/div>/i', "\n", $html);
        
        $this->log('Stripping remaining HTML tags...');
        $text = wp_strip_all_tags($html);
        
        $this->log('Decoding HTML entities...');
        $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
        
        $this->log('Cleaning up whitespace...');
        $text = preg_replace('/[ \t]+/', ' ', $text);
        $text = preg_replace('/\n\s*\n\s*\n+/', "\n\n", $text);
        $text = trim($text);
        
        $this->log('clean_extracted_content() done. Output length: ' . strlen($text) . ' chars');
        return $text;
    }
    
    private function smart_truncate($text, $max_length) {
        $this->log('smart_truncate() called. Input length: ' . strlen($text) . ', Max: ' . $max_length);
        
        if (strlen($text) <= $max_length) {
            $this->log('No truncation needed');
            return $text;
        }
        
        $truncated = substr($text, 0, $max_length);
        $this->log('Initial truncation to ' . strlen($truncated) . ' chars');
        
        $last_period = strrpos($truncated, '. ');
        $last_newline = strrpos($truncated, "\n");
        $this->log('Last period at: ' . ($last_period ?: 'none') . ', Last newline at: ' . ($last_newline ?: 'none'));
        
        $cut_point = max($last_period, $last_newline);
        
        if ($cut_point !== false && $cut_point > ($max_length * 0.8)) {
            $truncated = substr($truncated, 0, $cut_point + 1);
            $this->log('Cut at sentence/line boundary. Final length: ' . strlen($truncated));
        } else {
            $this->log('No good cut point found, using hard cut');
        }
        
        $result = trim($truncated) . "\n\n[Content truncated...]";
        $this->log('smart_truncate() done. Output length: ' . strlen($result));
        return $result;
    }
    
    private function extract_title_from_url($url) {
        $this->log('extract_title_from_url() called for: ' . $url);
        
        $parsed = wp_parse_url($url);
        // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_print_r -- Debug logging
        $this->log('Parsed URL: ' . print_r($parsed, true));
        
        $host = $parsed['host'] ?? $url;
        $this->log('Host before cleanup: ' . $host);
        
        $host = preg_replace('/^www\./', '', $host);
        $this->log('Host after removing www: ' . $host);
        
        return $host;
    }
    
    private function create_research_brief($research_data, $topic = '') {
        $this->log('create_research_brief() called');
        $this->log('Sources: ' . count($research_data['content']) . ', Topic: ' . ($topic ?: '(none)'));
        
        if (empty($research_data['content'])) {
            $this->log('No content to create brief from');
            return '';
        }
        
        // Build prompt
        $system_prompt = 'You are a research analyst. Your job is to extract EVERY specific fact, statistic, name, date, and example from the sources. Be exhaustive - do not summarize, EXTRACT.';
        
        $user_prompt = '';
        if (!empty($topic)) {
            $user_prompt .= "Topic context: {$topic}\n\n";
        }
        
        $user_prompt .= "I need you to extract ALL specific, citable information from these " . count($research_data['content']) . " source(s).\n\n";
        
        foreach ($research_data['content'] as $index => $item) {
            $num = $index + 1;
            $user_prompt .= "========== SOURCE {$num}: {$item['source']} ==========\n";
            $user_prompt .= "URL: {$item['url']}\n\n";
            $user_prompt .= $item['content'];
            $user_prompt .= "\n\n";
        }
        
        $user_prompt .= "========== EXTRACTION TASK ==========\n\n";
        $user_prompt .= "Extract and list EVERYTHING specific and citable:\n\n";
        $user_prompt .= "1. STATISTICS & NUMBERS (every percentage, dollar amount, count)\n";
        $user_prompt .= "2. SPECIFIC NAMES (people, companies, products, locations)\n";
        $user_prompt .= "3. QUOTES & STATEMENTS (who said what)\n";
        $user_prompt .= "4. EVENTS & DATES (what happened when)\n";
        $user_prompt .= "5. EXAMPLES & CASE STUDIES\n\n";
        $user_prompt .= "Be EXHAUSTIVE. Missing details = generic article.\n\n";
        $user_prompt .= "Begin extraction:";
        
        $this->log('Prompt length: ' . strlen($user_prompt) . ' chars');
        $this->log('Calling OpenAI API...');
        
        $brief = $this->openai_client->generate_text($system_prompt, $user_prompt, 0.2, 4000);
        
        if (is_wp_error($brief)) {
            $this->log('OpenAI API ERROR: ' . $brief->get_error_message());
            return $brief;
        }
        
        $this->log('OpenAI API success. Response length: ' . strlen($brief) . ' chars');
        $this->log('Brief preview: ' . substr($brief, 0, 500));
        
        // Format
        $formatted = "=== RESEARCH BRIEF ===\n";
        $formatted .= "Sources analyzed: " . count($research_data['sources']) . "\n\n";
        
        foreach ($research_data['sources'] as $index => $source) {
            $num = $index + 1;
            $formatted .= "[{$num}] {$source['title']} - {$source['url']}\n";
        }
        
        $formatted .= "\n=== EXTRACTED INFORMATION ===\n\n";
        $formatted .= $brief;
        $formatted .= "\n\n=== END RESEARCH BRIEF ===";
        
        $this->log('Final formatted brief length: ' . strlen($formatted) . ' chars');
        
        return $formatted;
    }
    
    private function format_raw_content($research_data) {
        $this->log('format_raw_content() called');
        
        $output = "=== RAW RESEARCH CONTENT ===\n";
        $output .= "Sources: " . count($research_data['sources']) . "\n\n";
        
        foreach ($research_data['content'] as $index => $item) {
            $num = $index + 1;
            $output .= "========== SOURCE {$num}: {$item['source']} ==========\n";
            $output .= "URL: {$item['url']}\n\n";
            $output .= $item['content'];
            $output .= "\n\n";
        }
        
        $output .= "=== END RAW RESEARCH CONTENT ===";
        
        $this->log('Raw content formatted. Length: ' . strlen($output) . ' chars');
        return $output;
    }
    
    public function set_openai_client($client) {
        $this->log('set_openai_client() called');
        $this->openai_client = $client;
        $this->log('OpenAI client set');
    }
    
    public function get_config_schema() {
        $this->log('get_config_schema() called');
        
        $schema = array(
            'urls' => array(
                'type' => 'textarea',
                'label' => 'Research URLs',
                'description' => 'URLs to research (one per line)',
                'rows' => 4
            ),
            'max_urls' => array(
                'type' => 'number',
                'label' => 'Maximum URLs',
                'min' => 1,
                'max' => 10,
                'default' => 5
            ),
            'max_content_length' => array(
                'type' => 'number',
                'label' => 'Max Content Length',
                'min' => 2000,
                'max' => 20000,
                'default' => 12000
            ),
            'skip_on_empty' => array(
                'type' => 'checkbox',
                'label' => 'Skip if no URLs',
                'default' => true
            ),
            'create_brief' => array(
                'type' => 'checkbox',
                'label' => 'Create AI Brief',
                'default' => true
            )
        );
        
        $this->log('get_config_schema() returning schema with ' . count($schema) . ' fields');
        return $schema;
    }
}