Add CI/CD and parser modules
This commit is contained in:
214
depaudit/parsers/python.py
Normal file
214
depaudit/parsers/python.py
Normal file
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
import tomli as tomllib
|
||||
|
||||
from depaudit.parsers import Parser, ParsedManifest, Dependency
|
||||
|
||||
|
||||
class PythonParser(Parser):
|
||||
language = "python"
|
||||
|
||||
def can_parse(self, file_path: Path) -> bool:
|
||||
return file_path.name in (
|
||||
"requirements.txt",
|
||||
"requirements-dev.txt",
|
||||
"requirements.txt",
|
||||
"setup.py",
|
||||
"setup.cfg",
|
||||
"pyproject.toml",
|
||||
"Pipfile",
|
||||
"Pipfile.lock",
|
||||
)
|
||||
|
||||
def parse(self, file_path: Path) -> ParsedManifest:
|
||||
manifest = ParsedManifest(
|
||||
language=self.language,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
if file_path.name == "pyproject.toml":
|
||||
self._parse_pyproject_toml(file_path, manifest)
|
||||
elif file_path.name == "requirements.txt":
|
||||
self._parse_requirements_txt(file_path, manifest)
|
||||
elif file_path.name == "setup.py":
|
||||
self._parse_setup_py(file_path, manifest)
|
||||
elif file_path.name == "setup.cfg":
|
||||
self._parse_setup_cfg(file_path, manifest)
|
||||
elif file_path.name == "Pipfile":
|
||||
self._parse_pipfile(file_path, manifest)
|
||||
elif file_path.name == "Pipfile.lock":
|
||||
self._parse_pipfile_lock(file_path, manifest)
|
||||
|
||||
return manifest
|
||||
|
||||
def _parse_pyproject_toml(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
with open(file_path, "rb") as f:
|
||||
data = tomllib.load(f)
|
||||
|
||||
project = data.get("project", {})
|
||||
manifest.project_name = project.get("name")
|
||||
manifest.project_version = project.get("version")
|
||||
|
||||
dependencies = project.get("dependencies", [])
|
||||
for dep in dependencies:
|
||||
name, version = self._parse_requirement_string(dep)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version)
|
||||
)
|
||||
|
||||
optional_dependencies = project.get("optional-dependencies", {})
|
||||
for group, deps in optional_dependencies.items():
|
||||
for dep in deps:
|
||||
name, version = self._parse_requirement_string(dep)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(
|
||||
file_path, name, version, optional=(group != "dev")
|
||||
)
|
||||
)
|
||||
|
||||
build = data.get("build-system", {})
|
||||
if "requires" in build:
|
||||
for req in build["requires"]:
|
||||
if isinstance(req, str) and not req.startswith("setuptools"):
|
||||
name, version = self._parse_requirement_string(req)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version)
|
||||
)
|
||||
|
||||
manifest.raw_data = data
|
||||
|
||||
def _parse_requirements_txt(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
for line in content.split("\n"):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
name, version = self._parse_requirement_string(line)
|
||||
if name:
|
||||
dev = "-dev" in line or "-dev" in name.lower()
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version, dev=dev)
|
||||
)
|
||||
|
||||
def _parse_setup_py(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
|
||||
install_requires_match = re.search(
|
||||
r"install_requires\s*=\s*\[([^\]]*)\]", content, re.DOTALL
|
||||
)
|
||||
if install_requires_match:
|
||||
deps_str = install_requires_match.group(1)
|
||||
for dep in re.findall(r'"([^"]+)"', deps_str):
|
||||
name, version = self._parse_requirement_string(dep)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version)
|
||||
)
|
||||
|
||||
name_match = re.search(r'name\s*=\s*["']([^"\']+)["']', content)
|
||||
if name_match:
|
||||
manifest.project_name = name_match.group(1)
|
||||
|
||||
version_match = re.search(r'version\s*=\s*["']([^"\']+)["']', content)
|
||||
if version_match:
|
||||
manifest.project_version = version_match.group(1)
|
||||
|
||||
def _parse_setup_cfg(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
|
||||
install_requires_match = re.search(
|
||||
r"install_requires\s*=\s*([^\n]+)", content
|
||||
)
|
||||
if install_requires_match:
|
||||
deps = install_requires_match.group(1).split()
|
||||
for dep in deps:
|
||||
if dep.startswith("-e"):
|
||||
continue
|
||||
name, version = self._parse_requirement_string(dep)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version)
|
||||
)
|
||||
|
||||
def _parse_pipfile(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
|
||||
name_match = re.search(r'name\s*=\s*["']([^"\']+)["']', content)
|
||||
if name_match:
|
||||
manifest.project_name = name_match.group(1)
|
||||
|
||||
for section in ["packages", "dev-packages"]:
|
||||
section_match = re.search(
|
||||
rf'{section}\s*=\s*\{{([^}}]+)\}}', content, re.DOTALL
|
||||
)
|
||||
if section_match:
|
||||
deps_block = section_match.group(1)
|
||||
for match in re.finditer(r'"([^"]+)"', deps_block):
|
||||
dep_str = match.group(1)
|
||||
name, version = self._parse_requirement_string(dep_str)
|
||||
if name:
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(
|
||||
file_path, name, version, dev=(section == "dev-packages")
|
||||
)
|
||||
)
|
||||
|
||||
def _parse_pipfile_lock(self, file_path: Path, manifest: ParsedManifest) -> None:
|
||||
import json
|
||||
|
||||
data = json.loads(file_path.read_text(encoding="utf-8"))
|
||||
|
||||
manifest.project_name = data.get("name")
|
||||
manifest.project_version = data.get("version")
|
||||
|
||||
for name, info in data.get("default", {}).items():
|
||||
version = info.get("version", "").replace("==", "")
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version)
|
||||
)
|
||||
|
||||
for name, info in data.get("develop", {}).items():
|
||||
version = info.get("version", "").replace("==", "")
|
||||
manifest.dependencies.append(
|
||||
self._create_dependency(file_path, name, version, dev=True)
|
||||
)
|
||||
|
||||
def _parse_requirement_string(self, req: str) -> tuple[str, str]:
|
||||
req = req.strip()
|
||||
if not req:
|
||||
return "", ""
|
||||
|
||||
extras_match = re.match(r"([^\[]+)(?:\[([^\]]+)\])?", req)
|
||||
if extras_match:
|
||||
name = extras_match.group(1)
|
||||
extras = extras_match.group(2)
|
||||
else:
|
||||
name = req
|
||||
extras = None
|
||||
|
||||
for op in ["==", ">=", "<=", "~=", "!=", ">", "<", "==="]:
|
||||
if op in name:
|
||||
parts = name.split(op)
|
||||
name = parts[0].strip()
|
||||
version = parts[1].strip() if len(parts) > 1 else ""
|
||||
break
|
||||
else:
|
||||
version = ""
|
||||
|
||||
name = name.lower().replace("_", "-")
|
||||
|
||||
return name, version
|
||||
Reference in New Issue
Block a user