#!/usr/bin/env python3
"""
SACAP RAG Service - Consolidated Python Service
Provides RAG (Retrieval-Augmented Generation) functionality for the FAQ application.
Designed to run under a URL path for staging/production deployment.
"""

import os
import json
import uuid
import time
import re
import logging
from datetime import datetime
from typing import List, Dict, Any, Optional

import numpy as np
import requests
from flask import Flask, request, jsonify, Blueprint
from flask_cors import CORS
from dotenv import load_dotenv
import mysql.connector
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

# PDF processing imports
from langchain_community.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
import tempfile

# Load environment variables
load_dotenv(dotenv_path='../../.env')
load_dotenv(dotenv_path='.env.local', override=True)

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Initialize Flask app
app = Flask(__name__)
CORS(app)

# Create Blueprint for RAG service with URL prefix
rag_bp = Blueprint('rag', __name__, url_prefix='/rag-service')

# Global variables for models
gemini_model = None
embedding_model = None

def initialize_models():
    """Initialize AI models"""
    global gemini_model, embedding_model
    
    # Initialize Google Gemini AI
    try:
        import google.generativeai as genai
        
        api_key = os.getenv('GOOGLE_API_KEY')
        if not api_key:
            raise Exception("GOOGLE_API_KEY not found in environment variables")
        
        genai.configure(api_key=api_key)
        gemini_model = genai.GenerativeModel('gemini-1.5-flash')
        logger.info("✅ Google Gemini AI model loaded successfully")
    except Exception as e:
        logger.error(f"❌ Error loading Google Gemini AI: {e}")
        gemini_model = None

    # Initialize the sentence transformer model
    try:
        embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
        logger.info("✅ Sentence transformer model loaded successfully")
    except Exception as e:
        logger.error(f"❌ Error loading sentence transformer model: {e}")
        embedding_model = None

# Database configuration
DB_CONFIG = {
    'host': os.getenv('RAG_DB_HOST', 'db'),
    'port': int(os.getenv('RAG_DB_PORT', '3306')),
    'user': os.getenv('RAG_DB_USER', 'db'),
    'password': os.getenv('RAG_DB_PASSWORD', 'db'),
    'database': os.getenv('RAG_DB_DATABASE', 'db'),
    'charset': 'utf8mb4',
    'collation': 'utf8mb4_unicode_ci'
}

def get_db_connection():
    """Get database connection"""
    try:
        connection = mysql.connector.connect(**DB_CONFIG)
        return connection
    except Exception as e:
        logger.error(f"❌ Database connection error: {e}")
        return None

def store_conversation_in_laravel(conversation_data: Dict[str, Any]) -> bool:
    """Store conversation data in Laravel via API endpoint"""
    try:
        # Get Laravel app URL from environment or use default
        laravel_url = os.getenv('LARAVEL_APP_URL', 'http://localhost:8000')
        api_endpoint = f"{laravel_url}/api/chatbot/conversations"
        
        # Send POST request to Laravel API
        response = requests.post(
            api_endpoint,
            json=conversation_data,
            headers={'Content-Type': 'application/json'},
            timeout=5  # 5 second timeout
        )
        
        if response.status_code == 201:
            logger.info(f"✅ Conversation stored successfully in Laravel")
            return True
        else:
            logger.error(f"❌ Failed to store conversation in Laravel: {response.status_code} - {response.text}")
            return False
            
    except requests.exceptions.RequestException as e:
        logger.error(f"❌ Error sending conversation to Laravel: {e}")
        return False
    except Exception as e:
        logger.error(f"❌ Unexpected error storing conversation: {e}")
        return False

def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> List[str]:
    """Split text into overlapping chunks with intelligent boundary detection."""
    if len(text) <= chunk_size:
        return [text]
    
    chunks = []
    paragraphs = text.split('\n\n')
    current_chunk = ""
    
    for paragraph in paragraphs:
        paragraph = paragraph.strip()
        if not paragraph:
            continue
            
        if current_chunk and len(current_chunk) + len(paragraph) + 2 > chunk_size:
            chunk_parts = split_chunk_intelligently(current_chunk, chunk_size)
            chunks.extend(chunk_parts)
            current_chunk = ""
        
        if current_chunk:
            current_chunk += "\n\n" + paragraph
        else:
            current_chunk = paragraph
    
    if current_chunk:
        chunk_parts = split_chunk_intelligently(current_chunk, chunk_size)
        chunks.extend(chunk_parts)
    
    return apply_intelligent_overlap(chunks, overlap)

def split_chunk_intelligently(text: str, max_size: int) -> List[str]:
    """Split text intelligently by sentences and paragraphs."""
    if len(text) <= max_size:
        return [text]
    
    sentences = split_into_sentences(text)
    chunks = []
    current_chunk = ""
    
    for sentence in sentences:
        if len(sentence) > max_size:
            if current_chunk:
                chunks.append(current_chunk.strip())
                current_chunk = ""
            word_chunks = split_by_words(sentence, max_size)
            chunks.extend(word_chunks)
        elif current_chunk and len(current_chunk) + len(sentence) + 1 > max_size:
            chunks.append(current_chunk.strip())
            current_chunk = sentence
        else:
            if current_chunk:
                current_chunk += " " + sentence
            else:
                current_chunk = sentence
    
    if current_chunk:
        chunks.append(current_chunk.strip())
    
    return chunks

def split_into_sentences(text: str) -> List[str]:
    """Split text into sentences with improved handling."""
    sentence_endings = r'[.!?]'
    sentences = re.split(f'({sentence_endings})', text)
    
    result = []
    current = ""
    
    for i, part in enumerate(sentences):
        current += part
        if re.match(sentence_endings, part):
            if i + 1 < len(sentences) and sentences[i + 1] and not sentences[i + 1].isspace():
                current += sentences[i + 1]
                i += 1
            result.append(current.strip())
            current = ""
    
    if current.strip():
        result.append(current.strip())
    
    return [s for s in result if s.strip()]

def split_by_words(text: str, max_size: int) -> List[str]:
    """Split text by word boundaries."""
    words = text.split()
    chunks = []
    current_chunk = []
    current_length = 0
    
    for word in words:
        word_length = len(word)
        
        if current_length + word_length + len(current_chunk) > max_size:
            if current_chunk:
                chunks.append(" ".join(current_chunk))
                current_chunk = [word]
                current_length = word_length
            else:
                chunks.append(word)
                current_chunk = []
                current_length = 0
        else:
            current_chunk.append(word)
            current_length += word_length
    
    if current_chunk:
        chunks.append(" ".join(current_chunk))
    
    return chunks

def apply_intelligent_overlap(chunks: List[str], overlap_size: int) -> List[str]:
    """Apply overlap between chunks while respecting word boundaries."""
    if len(chunks) <= 1 or overlap_size <= 0:
        return chunks
    
    overlapped_chunks = [chunks[0]]
    
    for i in range(1, len(chunks)):
        current_chunk = chunks[i]
        previous_chunk = chunks[i-1]
        
        prev_words = previous_chunk.split()
        overlap_text = ""
        
        if len(prev_words) > 10:
            prev_sentences = split_into_sentences(previous_chunk)
            if len(prev_sentences) > 1:
                last_sentence = prev_sentences[-1]
                if len(last_sentence) <= overlap_size:
                    overlap_text = last_sentence
                else:
                    overlap_words = prev_words[-max(1, overlap_size // 10):]
                    overlap_text = " ".join(overlap_words)
            else:
                overlap_words = prev_words[-max(1, overlap_size // 10):]
                overlap_text = " ".join(overlap_words)
        
        if overlap_text and not current_chunk.startswith(overlap_text):
            overlapped_chunk = overlap_text + " " + current_chunk
            overlapped_chunks.append(overlapped_chunk)
        else:
            overlapped_chunks.append(current_chunk)
    
    return overlapped_chunks

def generate_embedding(text: str) -> List[float]:
    """Generate embedding for text using sentence transformer"""
    if embedding_model is None:
        raise Exception("Sentence transformer model not loaded")
    
    try:
        embedding = embedding_model.encode([text])[0]
        return embedding.tolist()
    except Exception as e:
        logger.error(f"❌ Error generating embedding: {e}")
        raise

def search_similar_chunks(query: str, top_k: int = 5, min_similarity: float = 0.3) -> List[Dict[str, Any]]:
    """Search for similar chunks using vector similarity"""
    connection = get_db_connection()
    if not connection:
        raise Exception("Failed to connect to database")
    
    try:
        query_embedding = generate_embedding(query)
        cursor = connection.cursor(dictionary=True)
        
        cursor.execute("""
            SELECT id, content, embedding, metadata, document_id
            FROM rag_chunks
        """)
        
        chunks = cursor.fetchall()
        
        if not chunks:
            return []
        
        similarities = []
        for chunk in chunks:
            try:
                chunk_embedding = json.loads(chunk['embedding'])
                similarity = cosine_similarity(
                    [query_embedding], 
                    [chunk_embedding]
                )[0][0]
                
                if similarity >= min_similarity:
                    similarities.append({
                        'chunk_id': chunk['id'],
                        'content': chunk['content'],
                        'similarity': float(similarity),
                        'metadata': json.loads(chunk['metadata']) if chunk['metadata'] else {},
                        'document_id': chunk['document_id']
                    })
            except Exception as e:
                logger.error(f"❌ Error processing chunk {chunk['id']}: {e}")
                continue
        
        similarities.sort(key=lambda x: x['similarity'], reverse=True)
        return similarities[:top_k]
        
    finally:
        cursor.close()
        connection.close()

def generate_answer(query: str, context_chunks: List[Dict[str, Any]]) -> str:
    """Generate answer using Google Gemini AI"""
    if gemini_model is None:
        return "I'm having trouble accessing my knowledge right now. Please contact SACAP support for assistance, and they'll help you get the information you need."
    
    try:
        # Check for identity/greeting questions first - be very specific with exact matches only
        query_lower = query.lower().strip()
        
        # Only trigger identity response for very specific, short identity questions
        exact_identity_questions = [
            "who are you",
            "who are you?",
            "what are you", 
            "what are you?",
            "introduce yourself",
            "introduce yourself.",
            "what is your name",
            "what is your name?",
            "what's your name",
            "what's your name?",
            "hello",
            "hello.",
            "hi",
            "hi.",
            "hey",
            "hey."
        ]
        
        # Only trigger if the query exactly matches one of these patterns (and is short)
        is_identity_question = (
            query_lower in exact_identity_questions and 
            len(query.split()) <= 5  # Ensure it's a short question
        )
        
        if is_identity_question:
            return """I'm your SACAP academic assistant! 🌟

I'm designed to help you navigate your SACAP journey by answering questions about:

• Policies and procedures
• Assignment requirements  
• Campus processes
• Academic guidance
• General student support

Think of me as your calm, supportive companion who's here to make your studies clearer and less stressful. 

How can I support you today?"""
        
        if not context_chunks:
            return "I don't have information about that topic in my current knowledge base. Try rephrasing your question, or feel free to contact SACAP support for personalized assistance."
        
        # Build context
        context = ""
        for i, chunk in enumerate(context_chunks):
            context += f"Information {i+1}:\n{chunk['content']}\n\n"
        
        # Read the prompt from main.md file
        try:
            with open('prompts/main.md', 'r', encoding='utf-8') as f:
                prompt_template = f.read()
            
            # Replace placeholders in the template
            prompt = prompt_template.replace('{context}', context).replace('{query}', query)
        except FileNotFoundError:
            logger.error("❌ prompts/main.md file not found, using fallback prompt")
            # Fallback prompt if main.md is not found
            prompt = f"""You are a SACAP academic assistant.

    You are a calm, emotionally intelligent AI assistant designed to support SACAP students.

    Your purpose is to help students understand and navigate SACAP policies, assignment requirements, campus processes, and general academic guidance. You should always be emotionally aware, supportive, and professional in tone.

    {context}

    Question: {query}

    Answer (as SACAP's academic assistant):"""
        except Exception as e:
            logger.error(f"❌ Error reading prompts/main.md: {e}, using fallback prompt")
            # Fallback prompt if there's any other error
            prompt = f"""You are a SACAP academic assistant.

    {context}

    Question: {query}

    Answer:"""
        
        response = gemini_model.generate_content(prompt)
        answer = response.text.strip()
        
        # Debug: Log the raw response to see what Gemini actually returns
        logger.info(f"🔍 Raw Gemini response: {repr(answer)}")
        
        # Check for generic responses
        generic_phrases = [
            "based on the provided text",
            "according to the information provided",
            "the provided information doesn't",
            "i don't have enough information"
        ]
        
        if any(phrase in answer.lower() for phrase in generic_phrases):
            return "I'm not finding specific information about that in the handbook. Could you try rephrasing your question or contact SACAP support for more detailed guidance?"
        
        return answer
    except Exception as e:
        error_str = str(e)
        logger.error(f"❌ Error generating answer: {e}")
        
        # Check for specific API quota/rate limit errors
        if "429" in error_str or "quota" in error_str.lower() or "exceeded" in error_str.lower():
            logger.warning("🔄 API quota exceeded, providing fallback response")
            return generate_fallback_response(query, context_chunks)
        elif "401" in error_str or "unauthorized" in error_str.lower() or "authentication" in error_str.lower():
            logger.warning("🔑 API authentication failed, providing fallback response")
            return generate_fallback_response(query, context_chunks)
        elif "503" in error_str or "service unavailable" in error_str.lower() or "temporarily unavailable" in error_str.lower():
            logger.warning("⏳ API service temporarily unavailable, providing fallback response")
            return generate_fallback_response(query, context_chunks)
        else:
            # Generic error fallback
            return "I'm experiencing some technical difficulties right now. Please try again in a moment, or contact SACAP support if you need immediate assistance."

def generate_fallback_response(query: str, context_chunks: List[Dict[str, Any]]) -> str:
    """
    Generate a fallback response when AI service is unavailable due to quota limits or other API issues.
    This provides basic context-based responses without requiring AI generation.
    """
    try:
        if not context_chunks:
            return """I'm currently experiencing high demand and my AI processing is temporarily limited. 

However, I don't have specific information about your query in my knowledge base. Please try:

• Rephrasing your question with different keywords
• Contacting SACAP support directly for immediate assistance
• Checking the SACAP student portal for relevant information

I'll be back to full capacity soon! 🌟"""
        
        # Extract the most relevant content from context chunks
        query_lower = query.lower()
        best_chunks = []
        
        # Score chunks based on keyword overlap with query
        for chunk in context_chunks[:3]:  # Limit to top 3 chunks
            content = chunk.get('content', '')
            score = sum(1 for word in query_lower.split() if word in content.lower() and len(word) > 2)
            if score > 0:
                best_chunks.append((score, content))
        
        if not best_chunks:
            # If no good matches, just use first chunk
            best_chunks = [(1, context_chunks[0].get('content', ''))]
        
        # Sort by score and get best content
        best_chunks.sort(reverse=True, key=lambda x: x[0])
        relevant_content = best_chunks[0][1]
        
        # Create a basic response with the relevant content
        response = f"""I'm currently experiencing high demand and my AI processing is temporarily limited. However, I found some relevant information in the SACAP handbook:

{relevant_content[:500]}{'...' if len(relevant_content) > 500 else ''}

For more detailed information or if this doesn't fully answer your question, please:
• Contact SACAP support directly 
• Check the SACAP student portal
• Try rephrasing your question

I'll be back to full capacity soon! 🌟"""
        
        return response
        
    except Exception as e:
        logger.error(f"❌ Error in fallback response generation: {e}")
        return """I'm currently experiencing high demand and my systems are temporarily limited. 

Please contact SACAP support directly for immediate assistance, or try again in a few minutes.

Thank you for your patience! 🌟"""

def extract_title_from_content(content: str, filename: str = None) -> str:
    """
    Extract title from document content using various strategies
    """
    try:
        if not content or not content.strip():
            return filename or "Untitled Document"
        
        # Clean the content for processing
        lines = [line.strip() for line in content.split('\n') if line.strip()]
        
        if not lines:
            return filename or "Untitled Document"
        
        # Strategy 1: Look for obvious title patterns in first few lines
        title_patterns = [
            r'^Title:\s*(.+)$',
            r'^TITLE:\s*(.+)$',
            r'^Subject:\s*(.+)$',
            r'^SUBJECT:\s*(.+)$',
            r'^Document:\s*(.+)$',
            r'^DOCUMENT:\s*(.+)$',
        ]
        
        for line in lines[:5]:  # Check first 5 lines
            for pattern in title_patterns:
                match = re.search(pattern, line, re.IGNORECASE)
                if match:
                    title = match.group(1).strip()
                    if len(title) > 3 and len(title) <= 100:
                        return title
        
        # Strategy 2: Use first line if it looks like a title
        first_line = lines[0]
        
        # Title-like characteristics
        is_title_like = (
            len(first_line) <= 100 and  # Not too long
            len(first_line) >= 3 and    # Not too short
            not first_line.endswith('.') and  # Doesn't end with period
            not first_line.endswith(',') and  # Doesn't end with comma
            not first_line.startswith('http') and  # Not a URL
            not re.search(r'\d{4}-\d{2}-\d{2}', first_line) and  # Not a date
            len(first_line.split()) <= 15  # Not too many words
        )
        
        if is_title_like:
            # Clean up the title
            title = re.sub(r'[^\w\s\-_.,;:()\[\]{}]', '', first_line)
            title = ' '.join(title.split())  # Normalize whitespace
            if title:
                return title
        
        # Strategy 3: Look for the first substantial sentence
        for line in lines[:10]:  # Check first 10 lines
            if (len(line) >= 10 and len(line) <= 120 and 
                not line.startswith('http') and
                not re.search(r'\d{4}-\d{2}-\d{2}', line)):
                
                # Clean up and use as title
                title = re.sub(r'[^\w\s\-_.,;:()\[\]{}]', '', line)
                title = ' '.join(title.split())
                if title and len(title) >= 5:
                    return title[:100]  # Limit to 100 characters
        
        # Strategy 4: Extract from filename if available
        if filename:
            # Remove file extension and clean up
            title = os.path.splitext(filename)[0]
            title = re.sub(r'[_\-]', ' ', title)  # Replace underscores and dashes with spaces
            title = ' '.join(word.capitalize() for word in title.split())
            if len(title) >= 3:
                return title
        
        # Fallback: Use truncated content
        fallback = ' '.join(lines[0].split()[:10])  # First 10 words
        if len(fallback) >= 3:
            return fallback + "..."
        
        return "Untitled Document"
        
    except Exception as e:
        logger.warning(f"Error extracting title: {e}")
        return filename or "Untitled Document"

def calculate_document_stats(content: str) -> Dict[str, int]:
    """Calculate document statistics"""
    try:
        word_count = len(content.split())
        character_count = len(content)
        return {
            'word_count': word_count,
            'character_count': character_count
        }
    except Exception as e:
        logger.warning(f"Error calculating document stats: {e}")
        return {'word_count': 0, 'character_count': 0}

def create_content_preview(content: str, max_length: int = 200) -> str:
    """Create a preview of the document content"""
    try:
        if not content:
            return ""
        
        # Clean the content
        preview = ' '.join(content.split())
        
        if len(preview) <= max_length:
            return preview
        
        # Truncate at word boundary
        truncated = preview[:max_length]
        last_space = truncated.rfind(' ')
        if last_space > max_length * 0.8:  # If we can find a good break point
            truncated = truncated[:last_space]
        
        return truncated + "..."
        
    except Exception as e:
        logger.warning(f"Error creating content preview: {e}")
        return content[:200] + "..." if len(content) > 200 else content

def store_document(content: str, session_id: str, clear_existing: bool = True, source: str = 'ingested_content', filename: str = None) -> Dict[str, Any]:
    """Store document and its chunks in the database"""
    connection = get_db_connection()
    if not connection:
        raise Exception("Failed to connect to database")
    
    try:
        cursor = connection.cursor()
        
        if clear_existing:
            cursor.execute("DELETE FROM rag_chunks")
            cursor.execute("DELETE FROM rag_documents")
            logger.info("🗑️ Cleared existing documents and chunks")
        
        # Extract title and calculate document stats
        title = extract_title_from_content(content, filename)
        stats = calculate_document_stats(content)
        content_preview = create_content_preview(content)
        
        logger.info(f"📝 Extracted title: '{title}'")
        
        # Insert document with title and additional metadata
        document_metadata = {
            'timestamp': str(int(time.time())),
            'content_type': 'user_upload',
            'session_id': session_id,
            'filename': filename,
            'auto_extracted_title': True
        }
        
        cursor.execute("""
            INSERT INTO rag_documents (content, title, content_preview, word_count, character_count, session_id, content_type, source, metadata, created_at, updated_at)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
        """, (content, title, content_preview, stats['word_count'], stats['character_count'], session_id, 'user_upload', source, json.dumps(document_metadata)))
        
        document_id = cursor.lastrowid
        
        # Chunk the content
        chunks = chunk_text(content)
        logger.info(f"📄 Split content into {len(chunks)} chunks")
        
        # Generate embeddings and store chunks
        for i, chunk in enumerate(chunks):
            try:
                embedding = generate_embedding(chunk)
                
                chunk_metadata = {
                    'chunk_index': i,
                    'total_chunks': len(chunks),
                    'timestamp': str(int(time.time())),
                    'session_id': session_id
                }
                
                cursor.execute("""
                    INSERT INTO rag_chunks (document_id, content, embedding, chunk_index, session_id, metadata, created_at, updated_at)
                    VALUES (%s, %s, %s, %s, %s, %s, NOW(), NOW())
                """, (document_id, chunk, json.dumps(embedding), i, session_id, json.dumps(chunk_metadata)))
                
            except Exception as e:
                logger.error(f"❌ Error processing chunk {i}: {e}")
                continue
        
        connection.commit()
        
        logger.info(f"✅ Successfully stored document with {len(chunks)} chunks")
        return {
            'document_id': document_id,
            'title': title,
            'chunks_count': len(chunks),
            'cleared_existing': clear_existing,
            'session_id': session_id
        }
        
    except Exception as e:
        connection.rollback()
        logger.error(f"❌ Error storing document: {e}")
        raise
    finally:
        cursor.close()
        connection.close()

def update_existing_document(document_id: int, content: str, session_id: str, source: str = 'content_update', filename: str = None, preserve_title: str = None) -> Dict[str, Any]:
    """Update an existing document with new content and regenerate its chunks"""
    connection = get_db_connection()
    if not connection:
        raise Exception("Failed to connect to database")
    
    try:
        cursor = connection.cursor()
        
        # Check if document exists
        cursor.execute("SELECT id FROM rag_documents WHERE id = %s", (document_id,))
        if not cursor.fetchone():
            raise Exception(f"Document with ID {document_id} not found")
        
        # Delete existing chunks for this document
        cursor.execute("DELETE FROM rag_chunks WHERE document_id = %s", (document_id,))
        logger.info(f"🗑️ Deleted existing chunks for document {document_id}")
        
        # Extract title and calculate document stats
        if preserve_title:
            title = preserve_title
            logger.info(f"📝 Preserving user-provided title for document {document_id}: '{title}'")
        else:
            title = extract_title_from_content(content, filename)
            logger.info(f"📝 Auto-extracted title for document {document_id}: '{title}'")
        
        stats = calculate_document_stats(content)
        content_preview = create_content_preview(content)
        
        # Update document with new content and metadata
        document_metadata = {
            'timestamp': str(int(time.time())),
            'content_type': 'user_upload',
            'session_id': session_id,
            'filename': filename,
            'auto_extracted_title': True,
            'last_updated': str(int(time.time()))
        }
        
        cursor.execute("""
            UPDATE rag_documents 
            SET content = %s, title = %s, content_preview = %s, word_count = %s, character_count = %s, 
                session_id = %s, source = %s, metadata = %s, updated_at = NOW()
            WHERE id = %s
        """, (content, title, content_preview, stats['word_count'], stats['character_count'], 
              session_id, source, json.dumps(document_metadata), document_id))
        
        # Chunk the new content
        chunks = chunk_text(content)
        logger.info(f"📄 Split updated content into {len(chunks)} chunks")
        
        # Generate embeddings and store new chunks
        for i, chunk in enumerate(chunks):
            try:
                embedding = generate_embedding(chunk)
                
                chunk_metadata = {
                    'chunk_index': i,
                    'total_chunks': len(chunks),
                    'timestamp': str(int(time.time())),
                    'session_id': session_id,
                    'updated_document': True
                }
                
                cursor.execute("""
                    INSERT INTO rag_chunks (document_id, content, embedding, chunk_index, session_id, metadata, created_at, updated_at)
                    VALUES (%s, %s, %s, %s, %s, %s, NOW(), NOW())
                """, (document_id, chunk, json.dumps(embedding), i, session_id, json.dumps(chunk_metadata)))
                
            except Exception as e:
                logger.error(f"❌ Error processing chunk {i} for document {document_id}: {e}")
                continue
        
        connection.commit()
        
        logger.info(f"✅ Successfully updated document {document_id} with {len(chunks)} chunks")
        return {
            'document_id': document_id,
            'title': title,
            'chunks_count': len(chunks),
            'updated_existing': True,
            'session_id': session_id
        }
        
    except Exception as e:
        connection.rollback()
        logger.error(f"❌ Error updating document {document_id}: {e}")
        raise
    finally:
        cursor.close()
        connection.close()

# RAG Service Routes
@rag_bp.route('/health', methods=['GET'])
def health_check():
    """Health check endpoint"""
    try:
        connection = get_db_connection()
        if connection:
            connection.close()
            db_status = "connected"
        else:
            db_status = "disconnected"
            
        return jsonify({
            'status': 'healthy',
            'service': 'rag-service',
            'database': db_status,
            'embedding_model_loaded': embedding_model is not None,
            'gemini_available': gemini_model is not None,
            'timestamp': datetime.now().isoformat()
        })
    except Exception as e:
        return jsonify({
            'status': 'error',
            'service': 'rag-service',
            'error': str(e),
            'timestamp': datetime.now().isoformat()
        }), 500

@rag_bp.route('/ingest', methods=['POST'])
def ingest_content():
    """Ingest content into the knowledge base"""
    try:
        data = request.get_json()
        
        if not data or 'content' not in data:
            return jsonify({'error': 'Content is required'}), 400
        
        content = data['content'].strip()
        if not content:
            return jsonify({'error': 'Content cannot be empty'}), 400
        
        clear_existing = data.get('clear_existing', True)
        session_id = data.get('session_id', str(uuid.uuid4()))
        source = data.get('source', 'ingested_content')
        filename = data.get('filename', None)  # Optional filename for title extraction
        existing_document_id = data.get('document_id', None)  # For document updates
        preserve_title = data.get('preserve_title', None)  # User-provided title to preserve
        
        logger.info(f"📥 Ingesting content (clear_existing: {clear_existing}, document_id: {existing_document_id}, preserve_title: {preserve_title})")
        
        # Store or update document with optional filename for title extraction
        if existing_document_id:
            result = update_existing_document(existing_document_id, content, session_id, source, filename, preserve_title)
        else:
            result = store_document(content, session_id, clear_existing, source, filename)
        
        return jsonify({
            'message': f'Content indexed successfully. Chunks: {result["chunks_count"]}.',
            'document_id': result['document_id'],
            'title': result['title'],
            'session_id': session_id,
            'cleared_existing': clear_existing,
            'chunks_count': result['chunks_count'],
            'chunks_created': result['chunks_count']  # Laravel expects this field
        })
        
    except Exception as e:
        error_msg = f"Failed to ingest data: {str(e)}"
        logger.error(f"❌ {error_msg}")
        return jsonify({'error': error_msg}), 500

@rag_bp.route('/query', methods=['POST'])
def query_knowledge_base():
    """Query the knowledge base"""
    start_time = time.time()
    
    try:
        data = request.get_json()
        
        if not data or 'query' not in data:
            return jsonify({'error': 'Query is required'}), 400
        
        query = data['query'].strip()
        if not query:
            return jsonify({'error': 'Query cannot be empty'}), 400
        
        # Extract optional parameters
        session_id = data.get('session_id', 'anonymous')
        user_ip = request.headers.get('X-Real-IP', request.remote_addr)
        user_agent = request.headers.get('User-Agent', '')
        
        logger.info(f"🔍 Querying: {query}")
        
        # Check if we have any data
        connection = get_db_connection()
        if not connection:
            raise Exception("Failed to connect to database")
        
        cursor = connection.cursor()
        cursor.execute("SELECT COUNT(*) FROM rag_chunks")
        chunk_count = cursor.fetchone()[0]
        cursor.close()
        connection.close()
        
        if chunk_count == 0:
            return jsonify({
                'error': 'Knowledge base not initialized. Please ingest data first.'
            }), 400
        
        # Search for similar chunks
        similar_chunks = search_similar_chunks(query, top_k=5, min_similarity=0.3)
        
        # Generate answer
        answer = generate_answer(query, similar_chunks)
        
        # Calculate processing time
        processing_time = time.time() - start_time
        
        # Calculate average similarity score if chunks are available
        similarity_score = None
        if similar_chunks:
            similarity_score = int(sum(chunk['similarity'] for chunk in similar_chunks) / len(similar_chunks) * 100)
        
        # Prepare conversation data for storage
        conversation_data = {
            'session_id': session_id,
            'question': query,
            'answer': answer,
            'context_chunks': [
                {
                    'content': chunk['content'],
                    'similarity': chunk['similarity'],
                    'metadata': chunk['metadata']
                } for chunk in similar_chunks[:3]
            ] if similar_chunks else None,
            'processing_time': round(processing_time, 3),
            'user_ip': user_ip,
            'user_agent': user_agent,
            'metadata': {
                'chunks_found': len(similar_chunks),
                'min_similarity_threshold': 0.3,
                'model_used': 'gemini-1.5-flash',
                'embedding_model': 'all-MiniLM-L6-v2'
            },
            'similarity_score': similarity_score,
            'has_context': len(similar_chunks) > 0
        }
        
        # Store conversation in Laravel (non-blocking) - DISABLED: Laravel handles this directly
        # try:
        #     store_conversation_in_laravel(conversation_data)
        # except Exception as e:
        #     # Don't fail the request if conversation storage fails
        #     logger.warning(f"⚠️ Failed to store conversation, but continuing: {e}")
        
        # Format sources for response
        sources = []
        if similar_chunks:
            for chunk in similar_chunks[:3]:
                sources.append({
                    'content': chunk['content'],
                    'metadata': chunk['metadata']
                })
        
        return jsonify({
            'answer': answer,
            'sources': sources,
            'processing_time': round(processing_time, 3),
            'session_id': session_id
        })
        
    except Exception as e:
        error_msg = f"Failed to query knowledge base: {str(e)}"
        logger.error(f"❌ {error_msg}")
        return jsonify({'error': error_msg}), 500

@rag_bp.route('/upload-pdf', methods=['POST'])
def upload_pdf():
    """Handle PDF file upload"""
    try:
        if 'pdf_file' not in request.files:
            return jsonify({'error': 'No PDF file provided'}), 400
        
        file = request.files['pdf_file']
        if file.filename == '':
            return jsonify({'error': 'No file selected'}), 400
        
        clear_existing = request.form.get('clear_existing', 'false').lower() == 'true'
        filename = request.form.get('filename', file.filename)
        
        logger.info(f"📄 Processing PDF upload: {filename}")
        
        # Save uploaded file temporarily
        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
            file.save(temp_file.name)
            temp_path = temp_file.name
        
        try:
            # Load and process PDF
            loader = PyPDFLoader(temp_path)
            documents = loader.load()
            
            full_content = ""
            for doc in documents:
                page_content = doc.page_content.strip()
                if page_content:
                    full_content += page_content + "\n\n"
            
            if not full_content.strip():
                return jsonify({'error': 'No readable content found in PDF'}), 400
            
            # Store the PDF content with filename for title extraction
            session_id = str(uuid.uuid4())
            result = store_document(full_content, session_id, clear_existing, f'pdf_upload:{filename}', filename)
            
            logger.info(f"✅ Successfully processed PDF with {result['chunks_count']} chunks")
            
            return jsonify({
                'message': f'PDF "{filename}" processed successfully',
                'document_id': result['document_id'],
                'title': result['title'],
                'chunks_created': result['chunks_count'],
                'pages_processed': len(documents),
                'session_id': session_id
            })
            
        finally:
            if os.path.exists(temp_path):
                os.unlink(temp_path)
                
    except Exception as e:
        error_msg = f"Failed to process PDF: {str(e)}"
        logger.error(f"❌ {error_msg}")
        return jsonify({'error': error_msg}), 500

@rag_bp.route('/clear', methods=['POST'])
def clear_knowledge_base():
    """Clear the knowledge base"""
    try:
        logger.info("🗑️ Clearing knowledge base")
        
        connection = get_db_connection()
        if not connection:
            raise Exception("Failed to connect to database")
        
        cursor = connection.cursor()
        cursor.execute("DELETE FROM rag_chunks")
        cursor.execute("DELETE FROM rag_documents")
        connection.commit()
        cursor.close()
        connection.close()
        
        logger.info("✅ Knowledge base cleared successfully")
        
        return jsonify({
            'message': 'Knowledge base cleared successfully.'
        })
        
    except Exception as e:
        error_msg = f"Failed to clear knowledge base: {str(e)}"
        logger.error(f"❌ {error_msg}")
        return jsonify({'error': error_msg}), 500

@rag_bp.route('/status', methods=['GET'])
def get_status():
    """Get system status"""
    try:
        connection = get_db_connection()
        if not connection:
            raise Exception("Database connection failed")
        
        cursor = connection.cursor()
        cursor.execute("SELECT COUNT(*) FROM rag_documents")
        doc_count = cursor.fetchone()[0]
        cursor.execute("SELECT COUNT(*) FROM rag_chunks")
        chunk_count = cursor.fetchone()[0]
        cursor.close()
        connection.close()
        
        return jsonify({
            'status': 'ready',
            'service': 'rag-service',
            'documents': doc_count,
            'chunks': chunk_count,
            'embedding_model_loaded': embedding_model is not None,
            'gemini_available': gemini_model is not None,
            'database': 'connected'
        })
        
    except Exception as e:
        return jsonify({
            'status': 'error',
            'service': 'rag-service',
            'error': str(e),
            'embedding_model_loaded': embedding_model is not None,
            'gemini_available': gemini_model is not None,
            'database': 'disconnected'
        }), 500

# Register the blueprint
app.register_blueprint(rag_bp)

# Root health check (for load balancer/reverse proxy)
@app.route('/health', methods=['GET'])
def root_health():
    """Root health check endpoint"""
    return jsonify({
        'status': 'healthy',
        'service': 'rag-service-root',
        'timestamp': datetime.now().isoformat()
    })

# Initialize models on startup
initialize_models()

if __name__ == '__main__':
    port = int(os.getenv('RAG_SERVICE_PORT', 5002))
    host = os.getenv('RAG_SERVICE_HOST', '0.0.0.0')
    debug = os.getenv('RAG_SERVICE_DEBUG', 'false').lower() == 'true'
    
    logger.info(f"🚀 Starting RAG Service on {host}:{port}")
    logger.info(f"🔗 Service available at: http://{host}:{port}/rag-service/")
    
    app.run(host=host, port=port, debug=debug)