Initial upload: shell-speak CLI tool with natural language to shell command conversion

shell_speak/nlp.py

@@ -0,0 +1,50 @@
+"""NLP preprocessing and tokenization module."""
+
+import re
+from typing import List, Set
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for matching."""
+    text = text.lower().strip()
+    text = re.sub(r'\s+', ' ', text)
+    return text
+
+
+def tokenize(text: str) -> List[str]:
+    """Tokenize text into words."""
+    text = normalize_text(text)
+    tokens = re.findall(r'\b\w+\b', text)
+    return tokens
+
+
+def extract_keywords(text: str) -> Set[str]:
+    """Extract important keywords from text."""
+    stopwords = {
+        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+        'should', 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in',
+        'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through',
+        'during', 'before', 'after', 'above', 'below', 'between', 'under',
+        'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',
+        'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some',
+        'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than',
+        'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until',
+        'while', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
+        'it', 'we', 'they', 'what', 'which', 'who', 'whom', 'its', 'his',
+        'her', 'their', 'our', 'my', 'your', 'me', 'him', 'us', 'them',
+    }
+    tokens = tokenize(text)
+    keywords = {t for t in tokens if t not in stopwords and len(t) > 1}
+    return keywords
+
+
+def calculate_similarity(query1: str, query2: str) -> float:
+    """Calculate similarity between two queries using Jaccard similarity."""
+    set1 = set(tokenize(query1))
+    set2 = set(tokenize(query2))
+    if not set1 or not set2:
+        return 0.0
+    intersection = len(set1 & set2)
+    union = len(set1 | set2)
+    return intersection / union if union > 0 else 0.0