"""NLP preprocessing and tokenization module."""

import re


def normalize_text(text: str) -> str:
    """Normalize text for matching."""
    text = text.lower().strip()
    text = re.sub(r'\s+', ' ', text)
    return text


def tokenize(text: str) -> list[str]:
    """Tokenize text into words."""
    text = normalize_text(text)
    tokens = re.findall(r'\b\w+\b', text)
    return tokens


def extract_keywords(text: str) -> set[str]:
    """Extract important keywords from text."""
    stopwords = {
        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
        'should', 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in',
        'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through',
        'during', 'before', 'after', 'above', 'below', 'between', 'under',
        'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',
        'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some',
        'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than',
        'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until',
        'while', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
        'it', 'we', 'they', 'what', 'which', 'who', 'whom', 'its', 'his',
        'her', 'their', 'our', 'my', 'your', 'me', 'him', 'us', 'them',
    }
    tokens = tokenize(text)
    keywords = {t for t in tokens if t not in stopwords and len(t) > 1}
    return keywords


def calculate_similarity(query1: str, query2: str) -> float:
    """Calculate similarity between two queries using Jaccard similarity."""
    set1 = set(tokenize(query1))
    set2 = set(tokenize(query2))
    if not set1 or not set2:
        return 0.0
    intersection = len(set1 & set2)
    union = len(set1 | set2)
    return intersection / union if union > 0 else 0.0