get_var( $wpdb->prepare( 'SELECT status FROM %i WHERE id = %d', $doc_table, $document_id ) ); if ( null === $status ) { return false; } $previous_chunk_count = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE document_id = %d', DatabaseManager::get_chunks_table(), $document_id ) ); $args = array( $document_id ); if ( false !== wp_next_scheduled( 'ai_support_assistant_index_document', $args ) ) { return true; } $wpdb->update( $doc_table, array( 'status' => 'pending', 'error_message' => null, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); $scheduled = wp_schedule_single_event( time() + 5, 'ai_support_assistant_index_document', $args ); if ( false === $scheduled || is_wp_error( $scheduled ) ) { if ( false !== wp_next_scheduled( 'ai_support_assistant_index_document', $args ) ) { return true; } $error_message = is_wp_error( $scheduled ) ? $scheduled->get_error_message() : __( 'Unable to schedule document indexing.', 'ai-support-assistant' ); if ( $previous_chunk_count > 0 ) { $error_message = 'Indexing failed; existing index remains available: ' . $error_message; } $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $error_message, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); Logger::error( 'Unable to schedule document indexing.', array( 'document_id' => $document_id, 'error' => $error_message, ) ); return false; } return true; } /** * Run a queued indexing job. * * @param int $document_id Document database ID. * @return void */ public static function process_queued_indexing( int $document_id ): void { try { self::index_document( $document_id ); } catch ( \Throwable $e ) { global $wpdb; Logger::error( 'Background indexing failed.', array( 'document_id' => $document_id, 'error' => $e->getMessage(), ) ); $doc_table = DatabaseManager::get_documents_table(); $status = $wpdb->get_var( $wpdb->prepare( 'SELECT status FROM %i WHERE id = %d', $doc_table, $document_id ) ); if ( 'failed' !== $status ) { $chunk_count = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE document_id = %d', DatabaseManager::get_chunks_table(), $document_id ) ); $error_message = $chunk_count > 0 ? 'Indexing failed; existing index remains available: ' . $e->getMessage() : $e->getMessage(); $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $error_message, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); } } } /** * Register a WordPress Media Library attachment for background indexing. * * @param int $attachment_id Media attachment ID. * @return int Document ID on success, 0 on failure. */ public static function upload_attachment( int $attachment_id ): int { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); $file_path = get_attached_file( $attachment_id ); if ( ! $file_path || ! file_exists( $file_path ) ) { Logger::error( "Attachment file not found for ID $attachment_id" ); return 0; } if ( filesize( $file_path ) > 20 * 1024 * 1024 ) { Logger::error( "Attachment exceeds the 20 MB indexing limit for ID $attachment_id" ); return 0; } $filename = basename( $file_path ); $mime_type = get_post_mime_type( $attachment_id ); $title = get_the_title( $attachment_id ); // Check if already registered. $existing_id = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT id FROM %i WHERE file_path = %s', $doc_table, $file_path ) ); $data = array( 'title' => $title, 'filename' => $filename, 'mime_type' => $mime_type, 'file_path' => $file_path, 'status' => 'pending', 'created_at' => current_time( 'mysql' ), 'updated_at' => current_time( 'mysql' ), ); if ( $existing_id > 0 ) { $document_id = $existing_id; } else { $wpdb->insert( $doc_table, $data ); $document_id = (int) $wpdb->insert_id; } if ( $document_id > 0 ) { Logger::info( 'Document registered. Queueing background indexing.', array( 'document_id' => $document_id ) ); return self::schedule_indexing( $document_id ) ? $document_id : 0; } return 0; } /** * Process text extraction, chunking, and vector embedding uploads. * * @param int $document_id Document database ID. * @return bool True on success, false on failure. */ public static function index_document( int $document_id ): bool { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); $chunks_table = DatabaseManager::get_chunks_table(); // Get document record. $doc = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $doc_table, $document_id ) ); if ( ! $doc ) { Logger::error( "Document ID $document_id not found during indexing." ); return false; } $previous_chunk_count = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE document_id = %d', $chunks_table, $document_id ) ); // Update status to indexing. $wpdb->update( $doc_table, array( 'status' => 'indexing', 'error_message' => null, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); $transaction_started = false; try { // Get settings. $settings = Settings::get_instance(); $chunk_size = $settings->get( 'chunk_size', 700 ); $chunk_overlap = $settings->get( 'chunk_overlap', 100 ); $vector_store = EmbeddingService::get_vector_store(); $table_status = $wpdb->get_row( $wpdb->prepare( 'SHOW TABLE STATUS WHERE Name = %s', $chunks_table ) ); if ( ! $table_status || 'innodb' !== strtolower( (string) $table_status->Engine ) ) { throw new \RuntimeException( 'Document indexing requires an InnoDB chunks table for safe re-indexing.' ); } // 1. Extract text from the source. $pages = null; if ( 'text/html' === $doc->mime_type ) { $crawled = UrlCrawler::crawl( $doc->file_path ); $text = $crawled['text']; $wpdb->update( $doc_table, array( 'title' => $crawled['title'] ), array( 'id' => $document_id ) ); } else { $extractor = ExtractorFactory::get_extractor( $doc->mime_type, $doc->filename ); if ( ! $extractor ) { throw new \Exception( 'Unsupported file type format: ' . $doc->mime_type ); } if ( $extractor instanceof PdfExtractor ) { $pages = $extractor->extract_pages( $doc->file_path ); $text = implode( "\n\n", $pages ); } else { $text = $extractor->extract( $doc->file_path ); } } if ( empty( trim( $text ) ) ) { if ( 'application/pdf' === $doc->mime_type || 'pdf' === strtolower( pathinfo( $doc->filename, PATHINFO_EXTENSION ) ) ) { throw new \Exception( 'Extracted PDF text content is empty. This document may be scanned or image-only; please ensure it contains selectable text.' ); } throw new \Exception( 'Extracted document text content is empty.' ); } // 2. Chunk text. $chunks = null !== $pages ? ChunkingService::chunk_pages( $pages, $chunk_size, $chunk_overlap ) : ChunkingService::chunk( $text, $chunk_size, $chunk_overlap ); if ( empty( $chunks ) ) { throw new \Exception( 'Failed to partition document text into chunks.' ); } if ( count( $chunks ) > 2000 ) { throw new \RuntimeException( 'This document is too large to index in one job. Split it into smaller documents.' ); } Logger::info( sprintf( 'Document chunked into %d elements.', count( $chunks ) ), array( 'document_id' => $document_id ) ); // Generate the full replacement index before removing the current chunks. $prepared_chunks = array(); foreach ( $chunks as $chunk ) { $vector = EmbeddingService::generate( $chunk['chunk_text'] ); if ( empty( $vector ) ) { throw new \Exception( "Failed to generate vector embedding for chunk #{$chunk['chunk_number']}." ); } $prepared_chunks[] = array( 'chunk' => $chunk, 'vector' => $vector, 'model' => EmbeddingService::$last_model, ); } // Swap chunks atomically so a failed re-index leaves the previous index intact. if ( false === $wpdb->query( 'START TRANSACTION' ) ) { throw new \RuntimeException( 'Unable to start the document index transaction.' ); } $transaction_started = true; if ( ! $vector_store->delete_document( $document_id ) ) { throw new \RuntimeException( 'Unable to remove the previous document chunks.' ); } foreach ( $prepared_chunks as $prepared_chunk ) { $chunk = $prepared_chunk['chunk']; $meta = array( 'page_number' => $chunk['page_number'], 'token_count' => $chunk['token_count'], 'filename' => $doc->filename, 'embedding_model' => $prepared_chunk['model'], ); if ( ! $vector_store->upsert( $document_id, $chunk['chunk_number'], $prepared_chunk['vector'], $chunk['chunk_text'], $meta ) ) { throw new \RuntimeException( "Failed to save vector embedding in store for chunk #{$chunk['chunk_number']}." ); } } if ( false === $wpdb->query( 'COMMIT' ) ) { throw new \RuntimeException( 'Unable to commit the document index transaction.' ); } $transaction_started = false; // Complete index status. if ( false === $wpdb->update( $doc_table, array( 'status' => 'completed', 'error_message' => null, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ) ) { throw new \RuntimeException( 'Unable to update document indexing status.' ); } delete_transient( 'ai_support_assistant_statistics' ); // Invalidate statistics cache. Logger::info( "Document ID $document_id indexed successfully." ); return true; } catch ( \Throwable $e ) { if ( $transaction_started ) { $wpdb->query( 'ROLLBACK' ); } Logger::error( "Indexing failed for Document ID $document_id: " . $e->getMessage() ); $error_message = $previous_chunk_count > 0 ? 'Indexing failed; existing index remains available: ' . $e->getMessage() : $e->getMessage(); $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $error_message, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); throw $e; } } /** * Register a webpage URL for background crawling and indexing. * * @param string $url Webpage URL. * @return int Document ID on success, 0 on failure. */ public static function crawl_and_index_url( string $url ): int { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); // Check if already registered. $existing_id = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT id FROM %i WHERE file_path = %s', $doc_table, $url ) ); $filename = wp_parse_url( $url, PHP_URL_HOST ) . wp_parse_url( $url, PHP_URL_PATH ); $filename = trim( $filename, '/' ); $data = array( 'title' => $filename, 'filename' => $filename, 'mime_type' => 'text/html', 'file_path' => $url, 'status' => 'pending', 'created_at' => current_time( 'mysql' ), 'updated_at' => current_time( 'mysql' ), ); if ( $existing_id > 0 ) { $document_id = $existing_id; } else { $wpdb->insert( $doc_table, $data ); $document_id = (int) $wpdb->insert_id; } if ( $document_id <= 0 ) { return 0; } Logger::info( 'URL registered. Queueing background crawl and indexing.', array( 'document_id' => $document_id ) ); return self::schedule_indexing( $document_id ) ? $document_id : 0; } /** * Delete a document and all associated text chunks and vector embeddings. * * @param int $document_id Document database ID. * @return bool True on success. */ public static function delete_document( int $document_id ): bool { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); $exists = $wpdb->get_var( $wpdb->prepare( 'SELECT id FROM %i WHERE id = %d', $doc_table, $document_id ) ); if ( ! $exists ) { return false; } wp_clear_scheduled_hook( 'ai_support_assistant_index_document', array( $document_id ) ); // Delete vectors from active store. $vector_store = EmbeddingService::get_vector_store(); if ( ! $vector_store->delete_document( $document_id ) ) { Logger::error( "Unable to remove vector chunks for document ID $document_id" ); return false; } // Delete from local cache tables. $wpdb->delete( DatabaseManager::get_chunks_table(), array( 'document_id' => $document_id ) ); $result = $wpdb->delete( $doc_table, array( 'id' => $document_id ) ); delete_transient( 'ai_support_assistant_statistics' ); Logger::info( "Document ID $document_id and all its chunks deleted." ); return false !== $result; } }