50 lines
1.9 KiB
Python
50 lines
1.9 KiB
Python
"""NLP preprocessing and tokenization module."""
|
|
|
|
import re
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""Normalize text for matching."""
|
|
text = text.lower().strip()
|
|
text = re.sub(r'\s+', ' ', text)
|
|
return text
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
"""Tokenize text into words."""
|
|
text = normalize_text(text)
|
|
tokens = re.findall(r'\b\w+\b', text)
|
|
return tokens
|
|
|
|
|
|
def extract_keywords(text: str) -> set[str]:
|
|
"""Extract important keywords from text."""
|
|
stopwords = {
|
|
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
|
|
'should', 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in',
|
|
'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through',
|
|
'during', 'before', 'after', 'above', 'below', 'between', 'under',
|
|
'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',
|
|
'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some',
|
|
'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than',
|
|
'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until',
|
|
'while', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
|
|
'it', 'we', 'they', 'what', 'which', 'who', 'whom', 'its', 'his',
|
|
'her', 'their', 'our', 'my', 'your', 'me', 'him', 'us', 'them',
|
|
}
|
|
tokens = tokenize(text)
|
|
keywords = {t for t in tokens if t not in stopwords and len(t) > 1}
|
|
return keywords
|
|
|
|
|
|
def calculate_similarity(query1: str, query2: str) -> float:
|
|
"""Calculate similarity between two queries using Jaccard similarity."""
|
|
set1 = set(tokenize(query1))
|
|
set2 = set(tokenize(query2))
|
|
if not set1 or not set2:
|
|
return 0.0
|
|
intersection = len(set1 & set2)
|
|
union = len(set1 | set2)
|
|
return intersection / union if union > 0 else 0.0
|