From c37b15bb0d7b6bd6c19bd3a23e318ddf6b962829 Mon Sep 17 00:00:00 2001 From: NotMyMainUser <208290826+NotMyMainUser@users.noreply.github.com> Date: Wed, 28 May 2025 13:21:09 +0200 Subject: [PATCH] Add PythonDepManager plugin (#563) --- plugins/PythonDepManager/PythonDepManager.yml | 12 + plugins/PythonDepManager/README.md | 119 ++++ plugins/PythonDepManager/__init__.py | 3 + plugins/PythonDepManager/deps.py | 540 ++++++++++++++++++ plugins/PythonDepManager/flush.py | 15 + plugins/PythonDepManager/log.py | 30 + 6 files changed, 719 insertions(+) create mode 100644 plugins/PythonDepManager/PythonDepManager.yml create mode 100644 plugins/PythonDepManager/README.md create mode 100644 plugins/PythonDepManager/__init__.py create mode 100644 plugins/PythonDepManager/deps.py create mode 100644 plugins/PythonDepManager/flush.py create mode 100644 plugins/PythonDepManager/log.py diff --git a/plugins/PythonDepManager/PythonDepManager.yml b/plugins/PythonDepManager/PythonDepManager.yml new file mode 100644 index 0000000..acfe2fa --- /dev/null +++ b/plugins/PythonDepManager/PythonDepManager.yml @@ -0,0 +1,12 @@ +name: PythonDepManager +description: Manage Python dependencies for CommunityScripts +version: 0.1.0 +url: https://github.com/stashapp/CommunityScripts/ +exec: + - python + - "{pluginDir}/flush.py" +interface: raw + +tasks: + - name: "Flush Dependencies" + description: Flush all cached dependencies diff --git a/plugins/PythonDepManager/README.md b/plugins/PythonDepManager/README.md new file mode 100644 index 0000000..0c48832 --- /dev/null +++ b/plugins/PythonDepManager/README.md @@ -0,0 +1,119 @@ +# PythonDepManager + +Python dependency management system for CommunityScripts plugins. + +This plugin provides an easy way to install and manage Python package dependencies in your plugins without manual user interaction. + +Don't worry about missing dependencies and wrong or conflicting versions anymore. + +## Features + +- ๐Ÿš€ Automatic dependency installation and management + - Users won't have to manually install dependencies + +- ๐Ÿ”’ Isolated dependency versions + - Specify exact version of your dependencies without worrying about conflicts with other plugin installs + +- ๐Ÿ“ฆ Support for multiple package sources: + - PyPI packages with version constraints + - Git repositories (with branch/tag/commit support) + - Custom import names for metapackages +- ๐Ÿ”„ Automatic version resolution and compatibility checking +- ๐Ÿงน Easy dependency cleanup and flushing + +## Installation + +1. Add PythonDepManager as a requirement in your plugin's YAML file: + +```yaml +name: YourPlugin +# requires: PythonDepManager +description: Your plugin description +``` + +## Usage + +### Basic Usage + +In your plugin's Python code, import and use the dependency manager: + +```python +from PythonDepManager import ensure_import +# Install and import a package with specific version +ensure_import("requests==2.26.0") + +# Afterwards imports will use only the requested versions +import requests +``` + +### Advanced Usage + +#### Minimum Versions + +Define a minimum version to use. This will either use any cached version +which matches or install the latest + +```python +from PythonDepManager import ensure_import +ensure_import("requests>=2.26.0") +``` + +#### Custom Import Names/Meta Packages + +Use custom import names for packages with different import names or meta packages + +```python +from PythonDepManager import ensure_import +# Install beautifulsoup4 but import as bs4 +ensure_import("bs4:beautifulsoup4==4.9.3") +``` + +```python +from PythonDepManager import ensure_import +# Install stashapp-tools but import as stashapi +ensure_import("stashapi:stashapp-tools==0.2.58") +``` + +#### Git Repository Dependencies + +Install packages directly from Git repositories: + +```python +from PythonDepManager import ensure_import +# Install from a Git repository +ensure_import("stashapi@git+https://github.com/user/repo.git") + +# Install specific branch/tag +ensure_import("stashapi@git+https://github.com/user/repo.git@main") + +# Install specific commit +ensure_import("stashapi@git+https://github.com/user/repo.git@ad483dc") +``` + +### Multiple Imports + +Handle multiple different requirements for imports: + +```python +from PythonDepManager import ensure_import +ensure_import( + "requests" + "bs4:beautifulsoup4==4.9.3" + "stashapi:stashapp-tools==0.2.58" + "someothermodule>=0.1" +) +``` + +### Managing Dependencies + +To flush all cached dependencies: + +```python +from PythonDepManager import flush_dependencies +flush_dependencies() +``` + +## Requirements + +- Git (for Git repository dependencies) +- pip (Python package installer) diff --git a/plugins/PythonDepManager/__init__.py b/plugins/PythonDepManager/__init__.py new file mode 100644 index 0000000..329d579 --- /dev/null +++ b/plugins/PythonDepManager/__init__.py @@ -0,0 +1,3 @@ +from .deps import ensure_import + +__all__ = ["ensure_import"] \ No newline at end of file diff --git a/plugins/PythonDepManager/deps.py b/plugins/PythonDepManager/deps.py new file mode 100644 index 0000000..ac78fd1 --- /dev/null +++ b/plugins/PythonDepManager/deps.py @@ -0,0 +1,540 @@ +""" +๐Ÿ Simple dependency management for Python projects. + +Automatically installs and manages dependencies in isolated folders. +Supports regular packages, git repositories, and version constraints. + +Usage: + Add a dependency to PythonDepManager into your plugin.yml file so it gets installed automatically: + #requires: PythonDepManager + + Then, in your python code, you can use the "ensure_import" function to install and manage dependencies: + # Example usage: + from PythonDepManager import ensure_import + + ensure_import("requests==2.26.0") # Specific version + ensure_import("requests>=2.25.0") # Minimum version + ensure_import("bs4:beautifulsoup4==4.9.3") # Custom import name/Metapackage Imports + ensure_import("stashapi@git+https://github.com/user/repo.git") # Git repo + ensure_import("stashapi@git+https://github.com/user/repo.git@main") # Git branch/tag + ensure_import("stashapi@git+https://github.com/user/repo.git@abc123") # Git commit + ensure_import("bs4:beautifulsoup4==4.9.3", "requests==2.26.0") # Multiple packages + + # If you want to flush all dependencies, you can use the flush_dependencies function: + from PythonDepManager import flush_dependencies + flush_dependencies() +""" + +import sys +import subprocess +import re +import importlib +import importlib.metadata +import hashlib +import os +from pathlib import Path +from inspect import stack +from typing import Optional, List, Set, Tuple +from dataclasses import dataclass +from PythonDepManager import log + + +@dataclass(frozen=True) +class PackageInfo: + """Immutable representation of a package specification.""" + + import_name: str + pip_name: str + version: Optional[str] = None + min_version: Optional[str] = None + git_url: Optional[str] = None + git_ref: Optional[str] = None + + @property + def is_git(self) -> bool: + return self.git_url is not None + + @property + def is_min_version(self) -> bool: + return self.min_version is not None + + def __str__(self) -> str: + if self.is_git: + ref = f"@{self.git_ref}" if self.git_ref else "" + return f"{self.import_name} (git{ref})" + elif self.is_min_version: + return f"{self.pip_name}>={self.min_version}" + elif self.version: + return f"{self.pip_name}=={self.version}" + else: + return self.pip_name + + +def check_system_requirements() -> None: + """Ensure git and pip are available.""" + for cmd, name in [ + (["git", "--version"], "git"), + ([sys.executable, "-m", "pip", "--version"], "pip"), + ]: + try: + subprocess.run( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True + ) + except (FileNotFoundError, subprocess.CalledProcessError): + log.throw(f"PythonDepManager: โŒ {name} is required but not available") + + +def run_git_command(args: List[str]) -> Optional[str]: + """Run a git command and return the first 7 characters of output.""" + try: + result = subprocess.run( + ["git"] + args, capture_output=True, text=True, timeout=10, check=True + ) + return result.stdout.split()[0][:7] if result.stdout.strip() else None + except ( + subprocess.TimeoutExpired, + subprocess.CalledProcessError, + FileNotFoundError, + IndexError, + ): + return None + + +def parse_package_spec(spec: str) -> PackageInfo: + """Parse a package specification into structured information.""" + # Split custom import name from package spec + if ":" in spec and not spec.startswith("git+") and "@git+" not in spec: + import_name, package_spec = spec.split(":", 1) + else: + import_name, package_spec = "", spec + + # Handle git packages + if "@git+" in package_spec: + import_name = import_name or package_spec.split("@")[0] + git_url = package_spec.split("@git+", 1)[1] + + if "@" in git_url: + git_url, git_ref = git_url.rsplit("@", 1) + else: + git_ref = None + + return PackageInfo( + import_name=import_name, + pip_name="", + git_url=f"git+{git_url}", + git_ref=git_ref, + ) + + # Handle version constraints + if ">=" in package_spec: + match = re.match(r"^([^>=]+)>=(.+)$", package_spec) + if not match: + log.throw( + f"PythonDepManager: โŒ Invalid version constraint: {package_spec}" + ) + + pip_name, min_version = match.groups() + return PackageInfo( + import_name=import_name or pip_name, + pip_name=pip_name, + min_version=min_version, + ) + + # Handle exact version or no version + match = re.match(r"^([^=@]+)(?:==(.+))?$", package_spec) + if not match: + log.throw(f"PythonDepManager: โŒ Invalid package specification: {package_spec}") + + pip_name, version = match.groups() + return PackageInfo( + import_name=import_name or pip_name, pip_name=pip_name, version=version + ) + + +def compare_versions(v1: str, v2: str) -> int: + """Compare version strings. Returns -1, 0, or 1.""" + try: + # Try using packaging library if available + from packaging import version + + ver1, ver2 = version.parse(v1), version.parse(v2) + return -1 if ver1 < ver2 else (1 if ver1 > ver2 else 0) + except ImportError: + # Fallback to simple numeric comparison + try: + + def normalize(v: str) -> List[int]: + return [int(x) for x in v.split(".")] + + parts1, parts2 = normalize(v1), normalize(v2) + max_len = max(len(parts1), len(parts2)) + parts1.extend([0] * (max_len - len(parts1))) + parts2.extend([0] * (max_len - len(parts2))) + + for a, b in zip(parts1, parts2): + if a < b: + return -1 + if a > b: + return 1 + return 0 + except ValueError: + return -1 if v1 < v2 else (1 if v1 > v2 else 0) + + +def find_compatible_version(pkg: PackageInfo, base_folder: Path) -> Optional[str]: + """Find the best compatible version already installed.""" + if not pkg.is_min_version or not base_folder.exists(): + return None + + compatible_versions = [] + prefix = f"{pkg.import_name}_" + + for folder in base_folder.iterdir(): + if not (folder.is_dir() and folder.name.startswith(prefix)): + continue + + version_part = folder.name[len(prefix) :] + if version_part and "git_" not in version_part: + try: + if compare_versions(version_part, pkg.min_version) >= 0: + compatible_versions.append(version_part) + except ValueError: + continue + + if compatible_versions: + try: + return max( + compatible_versions, key=lambda v: [int(x) for x in v.split(".")] + ) + except ValueError: + return max(compatible_versions) + + return None + + +def get_git_commit_hash(git_url: str, ref: Optional[str] = None) -> Optional[str]: + """Get commit hash from git remote.""" + clean_url = git_url[4:] if git_url.startswith("git+") else git_url + clean_url = clean_url.split("@")[0] + + if ref: + for ref_type in ["heads", "tags"]: + result = run_git_command(["ls-remote", clean_url, f"refs/{ref_type}/{ref}"]) + if result: + return result + else: + return run_git_command(["ls-remote", clean_url, "HEAD"]) + + return None + + +def get_folder_name(pkg: PackageInfo, base_folder: Path) -> str: + """Generate folder name for package installation.""" + if pkg.is_git: + if pkg.git_ref and re.match(r"^[a-f0-9]{7,40}$", pkg.git_ref): + commit_hash = pkg.git_ref[:7] + else: + commit_hash = get_git_commit_hash(pkg.git_url, pkg.git_ref) + + if commit_hash: + return f"{pkg.import_name}_git_{commit_hash}" + else: + url_hash = hashlib.md5(pkg.git_url.encode()).hexdigest()[:7] + return f"{pkg.import_name}_git_{url_hash}" + + elif pkg.is_min_version: + compatible_version = find_compatible_version(pkg, base_folder) + return ( + f"{pkg.import_name}_{compatible_version}" + if compatible_version + else f"{pkg.import_name}_latest" + ) + else: + return f"{pkg.import_name}_{pkg.version}" if pkg.version else pkg.import_name + + +def get_base_folder() -> Path: + """Get the base folder for automatic dependencies.""" + caller_file = stack()[2].filename + + if caller_file.startswith("<") or not caller_file: + log.throw( + "PythonDepManager: โŒ Cannot determine caller location", e_type=RuntimeError + ) + + caller_path = Path(caller_file).resolve() + deps_folder = caller_path.parent.parent / "py_dependencies" + + try: + deps_folder.mkdir(parents=True, exist_ok=True) + # Test write permissions + test_file = deps_folder / ".write_test" + test_file.touch() + test_file.unlink() + except (OSError, PermissionError) as e: + log.throw( + f"PythonDepManager: โŒ Cannot access dependencies folder '{deps_folder}': {e}", + e_type=RuntimeError, + e_from=e, + ) + + return deps_folder + + +def is_package_available(pkg: PackageInfo, base_folder: Path) -> bool: + """Check if managed package is already available and satisfies requirements.""" + # For ensure_import, we only care about our managed dependencies + # System-installed packages are ignored to ensure we use the managed version + + folder = base_folder / get_folder_name(pkg, base_folder) + if not folder.exists(): + return False + + # Check if the managed package folder is in sys.path + folder_str = os.path.normpath(str(folder.resolve())) + if folder_str not in sys.path: + return False + + # Try importing from the managed location + try: + # Temporarily prioritize our managed path + original_path = sys.path[:] + sys.path.insert(0, folder_str) + + # Clear any existing module to force reload from managed location + if pkg.import_name in sys.modules: + del sys.modules[pkg.import_name] + + # Clear import caches + importlib.invalidate_caches() + + # Try importing + importlib.import_module(pkg.import_name) + + # Restore original path order (our managed paths should already be at front) + sys.path[:] = original_path + if folder_str not in sys.path: + sys.path.insert(0, folder_str) + + return True + except ImportError: + # Restore original path + sys.path[:] = original_path + if folder_str not in sys.path: + sys.path.insert(0, folder_str) + return False + + +def get_install_spec(pkg: PackageInfo) -> str: + """Get the pip install specification for a package.""" + if pkg.is_git: + return f"{pkg.git_url}@{pkg.git_ref}" if pkg.git_ref else pkg.git_url + return f"{pkg.pip_name}=={pkg.version}" if pkg.version else pkg.pip_name + + +def install_package(pkg: PackageInfo, folder: Path) -> None: + """Install package to specified folder.""" + folder.mkdir(parents=True, exist_ok=True) + install_spec = get_install_spec(pkg) + + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-input", + "--upgrade", + "--force-reinstall", + "--quiet", + f"--target={folder.resolve()}", + install_spec, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + + +def add_to_path(folder: Path, current_paths: Set[str]) -> None: + """Add folder to front of sys.path to ensure managed dependencies are prioritized.""" + folder_str = os.path.normpath(str(folder.resolve())) + + # Remove from current position if it exists to avoid duplicates + if folder_str in sys.path: + sys.path.remove(folder_str) + current_paths.discard(folder_str) + + # Always add to front to ensure priority over system packages + sys.path.insert(0, folder_str) + current_paths.add(folder_str) + + +def clear_import_caches() -> None: + """Clear Python import caches.""" + importlib.invalidate_caches() + try: + importlib.metadata._cache.clear() + except AttributeError: + pass + + +def remove_existing_modules(packages: List[PackageInfo]) -> None: + """Remove existing modules for managed packages to ensure we use managed versions.""" + for pkg in packages: + # Remove the main module + if pkg.import_name in sys.modules: + log.debug( + f"PythonDepManager: ๐Ÿ”„ Removing existing module '{pkg.import_name}' to use managed version" + ) + del sys.modules[pkg.import_name] + + # Also remove any submodules that might be cached + modules_to_remove = [] + for module_name in sys.modules: + if module_name.startswith(f"{pkg.import_name}."): + modules_to_remove.append(module_name) + + for module_name in modules_to_remove: + log.debug( + f"PythonDepManager: ๐Ÿ”„ Removing existing submodule '{module_name}' to use managed version" + ) + del sys.modules[module_name] + + +def process_packages(deps: Tuple[str, ...]) -> Tuple[List[PackageInfo], Path]: + """Parse dependencies and prepare base folder.""" + check_system_requirements() + base_folder = get_base_folder() + + packages = [] + for dep in deps: + try: + packages.append(parse_package_spec(dep)) + except ValueError as e: + log.throw( + f"PythonDepManager: โŒ Invalid package spec '{dep}': {e}", + e_type=ValueError, + e_from=e, + ) + + if not packages: + log.throw( + "PythonDepManager: โŒ No valid package specifications found", + e_type=ValueError, + ) + + return packages, base_folder + + +def setup_existing_packages(packages: List[PackageInfo], base_folder: Path) -> Set[str]: + """Add existing package folders to sys.path and ensure managed packages are prioritized.""" + # First, remove any existing modules for packages we're managing + # This ensures we use the managed version instead of system-installed ones + remove_existing_modules(packages) + + current_paths = set(sys.path) + managed_paths = [] + + for pkg in packages: + folder = base_folder / get_folder_name(pkg, base_folder) + if folder.exists(): + folder_str = os.path.normpath(str(folder.resolve())) + managed_paths.append(folder_str) + + # Remove from current position if it exists + if folder_str in sys.path: + sys.path.remove(folder_str) + current_paths.discard(folder_str) + + # Add all managed paths to the front of sys.path to ensure priority + for folder_str in reversed(managed_paths): # Reverse to maintain order + sys.path.insert(0, folder_str) + current_paths.add(folder_str) + + clear_import_caches() + return current_paths + + +def install_missing_packages( + packages: List[PackageInfo], base_folder: Path, current_paths: Set[str] +) -> None: + """Install packages that aren't already satisfied.""" + # Handle minimum version packages by finding compatible versions + resolved_packages = [] + for pkg in packages: + if pkg.is_min_version: + compatible_version = find_compatible_version(pkg, base_folder) + if compatible_version: + # Create new package info with resolved version + resolved_pkg = PackageInfo( + import_name=pkg.import_name, + pip_name=pkg.pip_name, + version=compatible_version, + ) + resolved_packages.append(resolved_pkg) + else: + resolved_packages.append(pkg) + else: + resolved_packages.append(pkg) + + to_install = [ + pkg for pkg in resolved_packages if not is_package_available(pkg, base_folder) + ] + + if not to_install: + log.debug("PythonDepManger: โœ… All dependencies satisfied") + return + + # Remove existing modules for packages we're about to install + # This ensures we use the newly installed managed version + remove_existing_modules(to_install) + + for pkg in to_install: + folder_name = get_folder_name(pkg, base_folder) + folder = base_folder / folder_name + + log.info(f"PythonDepManager: ๐Ÿ“ฆ Installing {pkg} โ†’ {folder_name}") + + try: + install_package(pkg, folder) + add_to_path(folder, current_paths) + log.info(f"PythonDepManager: โœ… Successfully installed {pkg.import_name}") + except Exception as e: + log.throw( + f"PythonDepManager: โŒ Failed to install {pkg.import_name}: {e}", + e_type=RuntimeError, + e_from=e, + ) + + clear_import_caches() + + +def ensure_import(*deps: str) -> None: + """ + ๐ŸŽฏ Install and import dependencies automatically. + + โš ๏ธ IMPORTANT: This function always prioritizes managed dependencies over system-installed ones. + When you use ensure_import, any existing system-installed versions of the specified packages + will be ignored in favor of the managed versions in the py_dependencies folder. + + Supported formats: + โ€ข Regular: "requests", "requests==2.26.0" + โ€ข Version ranges: "requests>=2.25.0" + โ€ข Custom import name/Metapackage Imports: "bs4:beautifulsoup4==4.9.3" + โ€ข Git: "stashapi@git+https://github.com/user/repo.git" + โ€ข Git with ref: "stashapi@git+https://github.com/user/repo.git@main" + """ + if not deps: + return + + try: + packages, base_folder = process_packages(deps) + current_paths = setup_existing_packages(packages, base_folder) + install_missing_packages(packages, base_folder, current_paths) + + # Final cache clear to ensure all imports use managed versions + clear_import_caches() + + except (RuntimeError, ValueError) as e: + log.throw(f"PythonDepManager: โŒ {e}", e_type=RuntimeError, e_from=e) diff --git a/plugins/PythonDepManager/flush.py b/plugins/PythonDepManager/flush.py new file mode 100644 index 0000000..bd71271 --- /dev/null +++ b/plugins/PythonDepManager/flush.py @@ -0,0 +1,15 @@ +import log +import shutil +from deps import get_base_folder + + +def flush_dependencies() -> None: + """Delete all dependencies in the base folder""" + # get working directory + plugin_folder = get_base_folder() + log.info(f"Flushing dependencies from {plugin_folder}") + shutil.rmtree(plugin_folder) + + +if __name__ == "__main__": + flush_dependencies() diff --git a/plugins/PythonDepManager/log.py b/plugins/PythonDepManager/log.py new file mode 100644 index 0000000..e8d4191 --- /dev/null +++ b/plugins/PythonDepManager/log.py @@ -0,0 +1,30 @@ +import sys +import re +from functools import partial + + +def _log(level_char: str, s): + lvl_char = "\x01{}\x02".format(level_char) + s = re.sub(r"data:.+?;base64[^'\"]+", "[...]", str(s)) + for line in s.splitlines(): + print(lvl_char, line, file=sys.stderr, flush=True) + + +trace = partial(_log, "t") +debug = partial(_log, "d") +info = partial(_log, "i") +warning = partial(_log, "w") +error = partial(_log, "e") + + +def throw(s, e_type=None, e_from=None): + error(s) + + if e_type and e_from: + raise e_type(s) from e_from + elif e_type and not e_from: + raise e_type(s) + elif not e_type and e_from: + raise Exception(s) from e_from + else: + raise Exception(s)