diff --git a/poetry.lock b/poetry.lock index 95aae06..ebd88d3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -691,21 +691,6 @@ rich = ">=13.9.4" [package.extras] websockets = ["websockets (>=15.0.1)"] -[[package]] -name = "feedparser" -version = "6.0.11" -description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45"}, - {file = "feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5"}, -] - -[package.dependencies] -sgmllib3k = "*" - [[package]] name = "filelock" version = "3.19.1" @@ -2009,17 +1994,6 @@ files = [ {file = "ruff-0.12.9.tar.gz", hash = "sha256:fbd94b2e3c623f659962934e52c2bea6fc6da11f667a427a368adaf3af2c866a"}, ] -[[package]] -name = "sgmllib3k" -version = "1.0.0" -description = "Py3k port of sgmllib." -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"}, -] - [[package]] name = "six" version = "1.17.0" @@ -2276,4 +2250,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "13bc4176d567d6738ca9ca5ebd67565f8526853434911137f4b51b39e275a546" +content-hash = "9785e18d2d996f5e58e1b06c722f6de31c445a1a83528f39227d1c373b91f989" diff --git a/pypi_query_mcp/__init__.py b/pypi_query_mcp/__init__.py index 1c9b313..1dca3d4 100644 --- a/pypi_query_mcp/__init__.py +++ b/pypi_query_mcp/__init__.py @@ -8,10 +8,6 @@ __version__ = "0.1.0" __author__ = "Hal" __email__ = "hal.long@outlook.com" -try: - from pypi_query_mcp.server import mcp - __all__ = ["mcp", "__version__"] -except ImportError: - # Server dependencies not available (fastmcp, etc.) - # Tools can still be imported individually - __all__ = ["__version__"] +from pypi_query_mcp.server import mcp + +__all__ = ["mcp", "__version__"] diff --git a/pypi_query_mcp/core/search_client.py b/pypi_query_mcp/core/search_client.py index a861aba..af9a8be 100644 --- a/pypi_query_mcp/core/search_client.py +++ b/pypi_query_mcp/core/search_client.py @@ -126,42 +126,20 @@ class PyPISearchClient: try: # Use PyPI's search API as the primary source - try: - pypi_results = await self._search_pypi_api(query, limit * 3) # Get more for filtering - logger.info(f"Got {len(pypi_results)} raw results from PyPI API") - except Exception as e: - logger.error(f"PyPI API search failed: {e}") - pypi_results = [] + pypi_results = await self._search_pypi_api(query, limit * 3) # Get more for filtering # Enhance results with additional metadata - try: - enhanced_results = await self._enhance_search_results(pypi_results) - logger.info(f"Enhanced to {len(enhanced_results)} results") - except Exception as e: - logger.error(f"Enhancement failed: {e}") - enhanced_results = pypi_results + enhanced_results = await self._enhance_search_results(pypi_results) # Apply filters - try: - filtered_results = self._apply_filters(enhanced_results, filters) - logger.info(f"Filtered to {len(filtered_results)} results") - except Exception as e: - logger.error(f"Filtering failed: {e}") - filtered_results = enhanced_results + filtered_results = self._apply_filters(enhanced_results, filters) # Apply semantic search if requested if semantic_search: - try: - filtered_results = self._apply_semantic_search(filtered_results, query) - except Exception as e: - logger.error(f"Semantic search failed: {e}") + filtered_results = self._apply_semantic_search(filtered_results, query) # Sort results - try: - sorted_results = self._sort_results(filtered_results, sort) - except Exception as e: - logger.error(f"Sorting failed: {e}") - sorted_results = filtered_results + sorted_results = self._sort_results(filtered_results, sort) # Limit results final_results = sorted_results[:limit] @@ -183,318 +161,72 @@ class PyPISearchClient: raise SearchError(f"Search failed: {e}") from e async def _search_pypi_api(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Search using available PyPI methods - no native search API exists.""" - logger.info(f"PyPI has no native search API, using curated search for: '{query}'") + """Search using PyPI's official search API.""" + url = "https://pypi.org/search/" + params = { + "q": query, + "page": 1, + } - # PyPI doesn't have a search API, so we'll use our curated approach - # combined with direct package lookups for exact matches - results = [] - - # First: try direct package lookup (exact match) - try: - direct_result = await self._try_direct_package_lookup(query) - if direct_result: - results.extend(direct_result) - except Exception as e: - logger.debug(f"Direct lookup failed: {e}") - - # Second: search curated packages - try: - curated_results = await self._search_curated_packages(query, limit) - # Add curated results that aren't already in the list - existing_names = {r["name"].lower() for r in results} - for result in curated_results: - if result["name"].lower() not in existing_names: - results.append(result) - except Exception as e: - logger.error(f"Curated search failed: {e}") - - return results[:limit] - - async def _try_direct_package_lookup(self, query: str) -> List[Dict[str, Any]]: - """Try to get package info directly using PyPI JSON API.""" - candidates = [ - query.strip(), - query.strip().lower(), - query.strip().replace(" ", "-"), - query.strip().replace(" ", "_"), - query.strip().replace("_", "-"), - query.strip().replace("-", "_"), - ] - - results = [] - for candidate in candidates: + async with httpx.AsyncClient(timeout=self.timeout) as client: try: - async with PyPIClient() as client: - package_data = await client.get_package_info(candidate) - - results.append({ - "name": package_data["info"]["name"], - "summary": package_data["info"]["summary"] or "", - "version": package_data["info"]["version"], - "source": "direct_api", - "description": package_data["info"]["description"] or "", - "author": package_data["info"]["author"] or "", - "license": package_data["info"]["license"] or "", - "home_page": package_data["info"]["home_page"] or "", - "requires_python": package_data["info"]["requires_python"] or "", - "classifiers": package_data["info"]["classifiers"] or [], - "keywords": package_data["info"]["keywords"] or "", - }) - break # Found exact match, stop looking - - except Exception: - continue # Try next candidate + response = await client.get(url, params=params) + response.raise_for_status() - return results - - async def _search_curated_packages(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Search our curated package database.""" - from ..data.popular_packages import ALL_POPULAR_PACKAGES - - curated_matches = [] - query_lower = query.lower() - - logger.info(f"Searching {len(ALL_POPULAR_PACKAGES)} curated packages for '{query}'") - - # First: exact name matches - for pkg in ALL_POPULAR_PACKAGES: - if query_lower == pkg.name.lower(): - curated_matches.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_exact", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - "primary_use_case": pkg.primary_use_case, - }) - - # Second: name contains query (if not too many exact matches) - if len(curated_matches) < limit: - for pkg in ALL_POPULAR_PACKAGES: - if (query_lower in pkg.name.lower() and - pkg.name not in [m["name"] for m in curated_matches]): - curated_matches.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_name", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - "primary_use_case": pkg.primary_use_case, - }) - - # Third: description or use case matches (if still need more results) - if len(curated_matches) < limit: - for pkg in ALL_POPULAR_PACKAGES: - if ((query_lower in pkg.description.lower() or - query_lower in pkg.primary_use_case.lower()) and - pkg.name not in [m["name"] for m in curated_matches]): - curated_matches.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_desc", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - "primary_use_case": pkg.primary_use_case, - }) - - # Sort by popularity (downloads) - curated_matches.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) - - logger.info(f"Found {len(curated_matches)} curated matches") - return curated_matches[:limit] + # Parse the HTML response (PyPI search returns HTML) + return await self._parse_search_html(response.text, limit) + + except httpx.HTTPError as e: + logger.error(f"PyPI search API error: {e}") + # Fallback to alternative search method + return await self._fallback_search(query, limit) async def _fallback_search(self, query: str, limit: int) -> List[Dict[str, Any]]: """Fallback search using PyPI JSON API and our curated data.""" + from ..data.popular_packages import PACKAGES_BY_NAME, get_popular_packages + + # Search in our curated packages first + curated_matches = [] + query_lower = query.lower() + + for package_info in get_popular_packages(limit=1000): + name_match = query_lower in package_info.name.lower() + desc_match = query_lower in package_info.description.lower() + + if name_match or desc_match: + curated_matches.append({ + "name": package_info.name, + "summary": package_info.description, + "version": "unknown", + "source": "curated", + "category": package_info.category, + "estimated_downloads": package_info.estimated_monthly_downloads, + }) + + # If we have some matches, return them + if curated_matches: + return curated_matches[:limit] + + # Last resort: try simple package name search try: - from ..data.popular_packages import PACKAGES_BY_NAME, get_popular_packages, ALL_POPULAR_PACKAGES - - # Search in our curated packages first - curated_matches = [] - query_lower = query.lower() - - logger.info(f"Searching in {len(ALL_POPULAR_PACKAGES)} curated packages for '{query}'") - - # First: exact name matches - for package_info in ALL_POPULAR_PACKAGES: - if query_lower == package_info.name.lower(): - curated_matches.append({ - "name": package_info.name, - "summary": package_info.description, - "version": "latest", - "source": "curated_exact", - "category": package_info.category, - "estimated_downloads": package_info.estimated_monthly_downloads, - "github_stars": package_info.github_stars, - }) - - # Second: name contains query - for package_info in ALL_POPULAR_PACKAGES: - if (query_lower in package_info.name.lower() and - package_info.name not in [m["name"] for m in curated_matches]): - curated_matches.append({ - "name": package_info.name, - "summary": package_info.description, - "version": "latest", - "source": "curated_name", - "category": package_info.category, - "estimated_downloads": package_info.estimated_monthly_downloads, - "github_stars": package_info.github_stars, - }) - - # Third: description or use case matches - for package_info in ALL_POPULAR_PACKAGES: - if ((query_lower in package_info.description.lower() or - query_lower in package_info.primary_use_case.lower()) and - package_info.name not in [m["name"] for m in curated_matches]): - curated_matches.append({ - "name": package_info.name, - "summary": package_info.description, - "version": "latest", - "source": "curated_desc", - "category": package_info.category, - "estimated_downloads": package_info.estimated_monthly_downloads, - "github_stars": package_info.github_stars, - }) - - logger.info(f"Found {len(curated_matches)} curated matches") - - # If we have some matches, return them (sorted by popularity) - if curated_matches: - curated_matches.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) - return curated_matches[:limit] - - # Last resort: try direct package lookup - logger.info("No curated matches, trying direct package lookup") - try: - async with PyPIClient() as client: + async with PyPIClient() as client: + # Try to get the package directly if it's an exact match + try: package_data = await client.get_package_info(query) return [{ "name": package_data["info"]["name"], "summary": package_data["info"]["summary"] or "", "version": package_data["info"]["version"], - "source": "direct_fallback", - "description": package_data["info"]["description"] or "", - "author": package_data["info"]["author"] or "", + "source": "direct", }] - except Exception as e: - logger.info(f"Direct lookup failed: {e}") + except: + pass except Exception as e: - logger.error(f"Fallback search failed: {e}") + logger.warning(f"Fallback search failed: {e}") return [] - async def _search_xmlrpc(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Search using enhanced curated search with fuzzy matching.""" - # Since PyPI XML-RPC search is deprecated, use our enhanced curated search - try: - from ..data.popular_packages import get_popular_packages, ALL_POPULAR_PACKAGES - - query_lower = query.lower() - results = [] - - # First pass: exact name matches - for pkg in ALL_POPULAR_PACKAGES: - if query_lower == pkg.name.lower(): - results.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_exact", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - }) - - # Second pass: name contains query - for pkg in ALL_POPULAR_PACKAGES: - if query_lower in pkg.name.lower() and pkg.name not in [r["name"] for r in results]: - results.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_name", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - }) - - # Third pass: description contains query - for pkg in ALL_POPULAR_PACKAGES: - if (query_lower in pkg.description.lower() or - query_lower in pkg.primary_use_case.lower()) and pkg.name not in [r["name"] for r in results]: - results.append({ - "name": pkg.name, - "summary": pkg.description, - "version": "latest", - "source": "curated_desc", - "category": pkg.category, - "estimated_downloads": pkg.estimated_monthly_downloads, - "github_stars": pkg.github_stars, - }) - - # Sort by popularity (downloads) - results.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) - - return results[:limit] - - except Exception as e: - logger.debug(f"Enhanced curated search error: {e}") - - return [] - - async def _search_simple_api(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Search using direct PyPI JSON API for specific packages.""" - try: - # Try direct package lookup if query looks like a package name - query_clean = query.strip().lower().replace(" ", "-") - - # Try variations of the query as package names - candidates = [ - query_clean, - query_clean.replace("-", "_"), - query_clean.replace("_", "-"), - query.strip(), # Original query - ] - - results = [] - - for candidate in candidates: - if len(results) >= limit: - break - - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(candidate) - - results.append({ - "name": package_data["info"]["name"], - "summary": package_data["info"]["summary"] or "", - "version": package_data["info"]["version"], - "source": "direct_api", - "description": package_data["info"]["description"] or "", - "author": package_data["info"]["author"] or "", - "license": package_data["info"]["license"] or "", - }) - - except Exception: - # Package doesn't exist, continue to next candidate - continue - - return results - - except Exception as e: - logger.debug(f"Simple API search error: {e}") - - return [] - async def _parse_search_html(self, html: str, limit: int) -> List[Dict[str, Any]]: """Parse PyPI search results from HTML (simplified parser).""" # This is a simplified parser - in production, you'd use BeautifulSoup @@ -505,19 +237,9 @@ class PyPISearchClient: """Enhance search results with additional metadata from PyPI API.""" enhanced = [] - # Skip enhancement if results already have good metadata from curated source - if results and results[0].get("source", "").startswith("curated"): - logger.info("Using curated results without enhancement") - return results - - # For direct API results, they're already enhanced - if results and results[0].get("source") == "direct_api": - logger.info("Using direct API results without additional enhancement") - return results - - # Process in small batches to avoid overwhelming the API - batch_size = 3 - for i in range(0, min(len(results), 10), batch_size): # Limit to first 10 results + # Process in batches to avoid overwhelming the API + batch_size = 5 + for i in range(0, len(results), batch_size): batch = results[i:i + batch_size] batch_tasks = [ self._enhance_single_result(result) diff --git a/pypi_query_mcp/server.py b/pypi_query_mcp/server.py index 3d16723..845c53c 100644 --- a/pypi_query_mcp/server.py +++ b/pypi_query_mcp/server.py @@ -1,7 +1,6 @@ """FastMCP server for PyPI package queries.""" import logging -from datetime import datetime, timezone from typing import Any import click @@ -25,63 +24,34 @@ from .prompts import ( track_package_updates, ) from .tools import ( + analyze_pypi_competition, + check_pypi_credentials, check_python_compatibility, + delete_pypi_release, download_package_with_dependencies, find_alternatives, get_compatible_python_versions, get_package_download_stats, get_package_download_trends, + get_pypi_account_info, + get_pypi_package_analytics, + get_pypi_package_rankings, + get_pypi_security_alerts, + get_pypi_upload_history, get_top_packages_by_downloads, get_trending_packages, + manage_package_keywords, + manage_package_urls, + manage_pypi_maintainers, query_package_dependencies, query_package_info, query_package_versions, resolve_package_dependencies, search_by_category, search_packages, - # Publishing tools - upload_package_to_pypi, - check_pypi_credentials, - get_pypi_upload_history, - delete_pypi_release, - manage_pypi_maintainers, - get_pypi_account_info, - # Metadata tools - update_package_metadata, - manage_package_urls, set_package_visibility, - manage_package_keywords, - # Analytics tools - get_pypi_package_analytics, - get_pypi_security_alerts, - get_pypi_package_rankings, - analyze_pypi_competition, - # Discovery tools - monitor_pypi_new_releases, - get_pypi_trending_today, - search_pypi_by_maintainer, - get_pypi_package_recommendations, - # Workflow tools - validate_pypi_package_name, - preview_pypi_package_page, - check_pypi_upload_requirements, - get_pypi_build_logs, - # Community tools - get_pypi_package_reviews, - manage_pypi_package_discussions, - get_pypi_maintainer_contacts, - # Security tools - bulk_scan_package_security, - scan_pypi_package_security, - # License tools - analyze_pypi_package_license, - check_bulk_license_compliance, - # Health tools - assess_package_health_score, - compare_packages_health_scores, - # Requirements tools - analyze_requirements_file_tool, - compare_multiple_requirements_files, + update_package_metadata, + upload_package_to_pypi, ) # Configure logging @@ -701,12 +671,7 @@ async def search_pypi_packages( SearchError: If search operation fails """ try: - logger.info(f"MCP search_pypi_packages called with query='{query}', limit={limit}") - logger.info(f"MCP search_pypi_packages parameters: python_versions={python_versions}, licenses={licenses}, categories={categories}") - logger.info(f"MCP search_pypi_packages parameters: min_downloads={min_downloads}, maintenance_status={maintenance_status}, has_wheels={has_wheels}") - logger.info(f"MCP search_pypi_packages parameters: sort_by={sort_by}, sort_desc={sort_desc}, semantic_search={semantic_search}") - - result = await search_packages( + return await search_packages( query=query, limit=limit, python_versions=python_versions, @@ -719,36 +684,15 @@ async def search_pypi_packages( sort_desc=sort_desc, semantic_search=semantic_search, ) - logger.info(f"MCP search_pypi_packages raw result keys: {list(result.keys())}") - logger.info(f"MCP search_pypi_packages raw result total_found: {result.get('total_found')}") - logger.info(f"MCP search_pypi_packages raw result packages count: {len(result.get('packages', []))}") - logger.info(f"MCP search_pypi_packages returning: {len(result.get('packages', []))} packages") - return result - except (InvalidPackageNameError, PackageNotFoundError, NetworkError, SearchError) as e: - logger.error(f"Known error searching packages for '{query}': {e}") - return { - "error": f"Search failed: {e}", - "error_type": type(e).__name__, - "query": query, - "limit": limit, - "packages": [], - "total_found": 0, - "returned_count": 0, - "timestamp": datetime.now(timezone.utc).isoformat(), - } + except (InvalidPackageNameError, PackageNotFoundError, NetworkError): + raise except Exception as e: - logger.error(f"Unexpected error searching packages for '{query}': {e}") - import traceback - traceback.print_exc() + logger.error(f"Error searching packages for '{query}': {e}") return { "error": f"Search failed: {e}", - "error_type": "SearchError", + "error_type": "SearchError", "query": query, "limit": limit, - "packages": [], - "total_found": 0, - "returned_count": 0, - "timestamp": datetime.now(timezone.utc).isoformat(), } @@ -876,7 +820,8 @@ async def get_trending_pypi_packages( } -# Publishing Tools MCP Endpoints +# PyPI Publishing and Account Management Tools + @mcp.tool() async def upload_package_to_pypi_tool( @@ -888,8 +833,8 @@ async def upload_package_to_pypi_tool( ) -> dict[str, Any]: """Upload package distributions to PyPI or TestPyPI. - This tool uploads Python package distribution files (.whl, .tar.gz) to PyPI - or TestPyPI, providing comprehensive upload management and verification. + This tool uploads Python package distributions (.whl, .tar.gz files) to PyPI, + providing comprehensive upload management with safety checks and verification. Args: distribution_paths: List of paths to distribution files (.whl, .tar.gz) @@ -899,7 +844,7 @@ async def upload_package_to_pypi_tool( verify_uploads: Verify uploads after completion Returns: - Dictionary containing upload results, statistics, and verification info + Dictionary containing upload results and metadata Raises: PyPIAuthenticationError: If authentication fails @@ -915,10 +860,10 @@ async def upload_package_to_pypi_tool( skip_existing=skip_existing, verify_uploads=verify_uploads, ) - logger.info(f"Upload completed with {result.get('successful_uploads', 0)} successful uploads") + logger.info(f"Upload completed: {result.get('summary', {})}") return result except Exception as e: - logger.error(f"Error uploading package: {e}") + logger.error(f"Error uploading packages: {e}") return { "error": str(e), "error_type": type(e).__name__, @@ -934,15 +879,15 @@ async def check_pypi_credentials_tool( ) -> dict[str, Any]: """Validate PyPI API token and credentials. - This tool validates PyPI API tokens and checks authentication status, - helping ensure proper credentials before performing upload operations. + This tool checks if your PyPI API token is valid and provides information + about your account permissions and capabilities. Args: api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to check against TestPyPI instead of production PyPI Returns: - Dictionary containing credential validation results and status + Dictionary containing credential validation results Raises: PyPIAuthenticationError: If credential validation fails @@ -950,8 +895,8 @@ async def check_pypi_credentials_tool( """ try: logger.info(f"MCP tool: Checking {'TestPyPI' if test_pypi else 'PyPI'} credentials") - result = await check_pypi_credentials(api_token, test_pypi) - logger.info(f"Credential check completed: {'valid' if result.get('valid') else 'invalid'}") + result = await check_pypi_credentials(api_token=api_token, test_pypi=test_pypi) + logger.info(f"Credential check completed: valid={result.get('valid', False)}") return result except Exception as e: logger.error(f"Error checking credentials: {e}") @@ -971,8 +916,8 @@ async def get_pypi_upload_history_tool( ) -> dict[str, Any]: """Get upload history for a PyPI package. - This tool retrieves comprehensive upload history for a package, - including file information, upload times, and statistics. + This tool retrieves the upload history for a package, showing all versions, + files, and upload metadata with statistics and analysis. Args: package_name: Name of the package to get upload history for @@ -990,8 +935,14 @@ async def get_pypi_upload_history_tool( """ try: logger.info(f"MCP tool: Getting upload history for {package_name}") - result = await get_pypi_upload_history(package_name, api_token, test_pypi, limit) - logger.info(f"Retrieved {len(result.get('upload_history', []))} upload records") + result = await get_pypi_upload_history( + package_name=package_name, + api_token=api_token, + test_pypi=test_pypi, + limit=limit, + ) + upload_count = result.get('statistics', {}).get('total_uploads', 0) + logger.info(f"Retrieved {upload_count} upload records for {package_name}") return result except Exception as e: logger.error(f"Error getting upload history for {package_name}: {e}") @@ -1014,8 +965,10 @@ async def delete_pypi_release_tool( ) -> dict[str, Any]: """Delete a specific release from PyPI (with safety checks). - This tool provides safe deletion of PyPI releases with comprehensive - safety checks and dry-run capability. PyPI deletion is very restricted. + This tool provides safe deletion of PyPI releases with multiple safety checks, + dry-run capability, and comprehensive validation. Note that PyPI deletion is + very restricted and typically only available to package owners within a limited + time window after upload. Args: package_name: Name of the package @@ -1037,9 +990,15 @@ async def delete_pypi_release_tool( try: logger.info(f"MCP tool: {'DRY RUN: ' if dry_run else ''}Deleting {package_name}=={version}") result = await delete_pypi_release( - package_name, version, api_token, test_pypi, confirm_deletion, dry_run + package_name=package_name, + version=version, + api_token=api_token, + test_pypi=test_pypi, + confirm_deletion=confirm_deletion, + dry_run=dry_run, ) - logger.info(f"Deletion {'simulation' if dry_run else 'attempt'} completed") + action = result.get('action', 'unknown') + logger.info(f"Deletion operation completed: {action}") return result except Exception as e: logger.error(f"Error deleting release {package_name}=={version}: {e}") @@ -1062,8 +1021,9 @@ async def manage_pypi_maintainers_tool( ) -> dict[str, Any]: """Manage package maintainers (add/remove/list). - This tool provides maintainer management functionality for PyPI packages, - including listing current maintainers and guidance for adding/removing. + This tool helps manage package maintainers and collaborators. Note that maintainer + management typically requires package owner permissions and may need to be done + through the PyPI web interface. Args: package_name: Name of the package @@ -1083,8 +1043,15 @@ async def manage_pypi_maintainers_tool( """ try: logger.info(f"MCP tool: Managing maintainers for {package_name}: {action}") - result = await manage_pypi_maintainers(package_name, action, username, api_token, test_pypi) - logger.info(f"Maintainer management completed for {package_name}") + result = await manage_pypi_maintainers( + package_name=package_name, + action=action, + username=username, + api_token=api_token, + test_pypi=test_pypi, + ) + maintainer_count = result.get('maintainer_count', 0) + logger.info(f"Maintainer management completed: {maintainer_count} maintainers") return result except Exception as e: logger.error(f"Error managing maintainers for {package_name}: {e}") @@ -1104,8 +1071,8 @@ async def get_pypi_account_info_tool( ) -> dict[str, Any]: """Get PyPI account information, quotas, and limits. - This tool retrieves comprehensive account information including - limitations, features, and useful links for PyPI account management. + This tool retrieves information about your PyPI account including permissions, + limitations, quotas, and provides recommendations for account security and usage. Args: api_token: PyPI API token (or use PYPI_API_TOKEN env var) @@ -1120,7 +1087,7 @@ async def get_pypi_account_info_tool( """ try: logger.info(f"MCP tool: Getting account information for {'TestPyPI' if test_pypi else 'PyPI'}") - result = await get_pypi_account_info(api_token, test_pypi) + result = await get_pypi_account_info(api_token=api_token, test_pypi=test_pypi) logger.info("Account information retrieved successfully") return result except Exception as e: @@ -1132,45 +1099,52 @@ async def get_pypi_account_info_tool( } -# Metadata Management Tools MCP Endpoints - +# Metadata Management Tools @mcp.tool() async def update_package_metadata_tool( package_name: str, - metadata_updates: dict[str, Any], + description: str | None = None, + keywords: list[str] | None = None, + classifiers: list[str] | None = None, api_token: str | None = None, test_pypi: bool = False, - validate_changes: bool = True, - dry_run: bool = False, + dry_run: bool = True, ) -> dict[str, Any]: - """Update PyPI package metadata and configuration. + """Update package metadata including description, keywords, and classifiers. - This tool updates package metadata including description, keywords, - classifiers, and other package information on PyPI. + This tool helps manage PyPI package metadata by validating changes and providing + guidance on how to update metadata through package uploads. Args: package_name: Name of the package to update - metadata_updates: Dictionary of metadata fields to update + description: New package description + keywords: List of keywords for the package + classifiers: List of PyPI classifiers (e.g., programming language, license) api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - validate_changes: Whether to validate metadata before applying - dry_run: If True, only validate without applying changes + dry_run: If True, only validate changes without applying them Returns: - Dictionary containing update results and validation info + Dictionary containing metadata update results and recommendations Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If update is not permitted + PyPIPermissionError: If user lacks permission to modify package NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Updating metadata for {package_name}") + logger.info(f"MCP tool: Updating metadata for {package_name} (dry_run={dry_run})") result = await update_package_metadata( - package_name, metadata_updates, api_token, test_pypi, validate_changes, dry_run + package_name=package_name, + description=description, + keywords=keywords, + classifiers=classifiers, + api_token=api_token, + test_pypi=test_pypi, + dry_run=dry_run, ) - logger.info(f"Metadata update {'simulated' if dry_run else 'completed'} for {package_name}") + logger.info(f"Metadata update completed for {package_name}: {result.get('success', 'analysis_complete')}") return result except Exception as e: logger.error(f"Error updating metadata for {package_name}: {e}") @@ -1178,47 +1152,64 @@ async def update_package_metadata_tool( "error": str(e), "error_type": type(e).__name__, "package_name": package_name, - "test_pypi": test_pypi, + "dry_run": dry_run, } @mcp.tool() async def manage_package_urls_tool( package_name: str, - action: str, - url_type: str | None = None, - url_value: str | None = None, + homepage: str | None = None, + documentation: str | None = None, + repository: str | None = None, + download_url: str | None = None, + bug_tracker: str | None = None, api_token: str | None = None, test_pypi: bool = False, + validate_urls: bool = True, + dry_run: bool = True, ) -> dict[str, Any]: - """Manage package URLs (homepage, documentation, repository, etc.). + """Manage package URLs including homepage, documentation, and repository links. - This tool manages package URL configurations including adding, updating, - removing, and listing various URL types for a PyPI package. + This tool validates and manages package URLs, providing guidance on proper + URL configuration and accessibility checking. Args: - package_name: Name of the package - action: Action to perform ('list', 'add', 'update', 'remove') - url_type: Type of URL ('homepage', 'documentation', 'repository', 'bug_tracker', etc.) - url_value: URL value (required for add/update actions) + package_name: Name of the package to update + homepage: Package homepage URL + documentation: Documentation URL + repository: Source code repository URL + download_url: Package download URL + bug_tracker: Bug tracker URL api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI + validate_urls: Whether to validate URL accessibility + dry_run: If True, only validate changes without applying them Returns: - Dictionary containing URL management results + Dictionary containing URL management results and validation Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If action is not permitted + PyPIPermissionError: If user lacks permission to modify package NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Managing URLs for {package_name}: {action}") + logger.info(f"MCP tool: Managing URLs for {package_name} (dry_run={dry_run})") result = await manage_package_urls( - package_name, action, url_type, url_value, api_token, test_pypi + package_name=package_name, + homepage=homepage, + documentation=documentation, + repository=repository, + download_url=download_url, + bug_tracker=bug_tracker, + api_token=api_token, + test_pypi=test_pypi, + validate_urls=validate_urls, + dry_run=dry_run, ) - logger.info(f"URL management completed for {package_name}") + logger.info(f"URL management completed for {package_name}: quality_score={result.get('url_quality_score', 0)}") return result except Exception as e: logger.error(f"Error managing URLs for {package_name}: {e}") @@ -1226,8 +1217,7 @@ async def manage_package_urls_tool( "error": str(e), "error_type": type(e).__name__, "package_name": package_name, - "action": action, - "test_pypi": test_pypi, + "dry_run": dry_run, } @@ -1237,35 +1227,39 @@ async def set_package_visibility_tool( visibility: str, api_token: str | None = None, test_pypi: bool = False, - confirmation_required: bool = True, + confirm_action: bool = False, ) -> dict[str, Any]: - """Set package visibility and access controls. + """Set package visibility (private/public) for organization packages. - This tool manages package visibility settings including public/private - status and access controls for PyPI packages. + This tool provides guidance on package visibility management, which is primarily + available for PyPI organizations with special permissions. Args: - package_name: Name of the package - visibility: Visibility setting ('public', 'private', 'unlisted') + package_name: Name of the package to modify + visibility: Visibility setting ("public" or "private") api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - confirmation_required: Whether to require explicit confirmation + confirm_action: Explicit confirmation required for visibility changes Returns: - Dictionary containing visibility change results + Dictionary containing visibility management results and limitations Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If action is not permitted + PyPIPermissionError: If user lacks permission to modify package NetworkError: For network-related errors """ try: logger.info(f"MCP tool: Setting visibility for {package_name} to {visibility}") result = await set_package_visibility( - package_name, visibility, api_token, test_pypi, confirmation_required + package_name=package_name, + visibility=visibility, + api_token=api_token, + test_pypi=test_pypi, + confirm_action=confirm_action, ) - logger.info(f"Visibility update completed for {package_name}") + logger.info(f"Visibility analysis completed for {package_name}: {result.get('success', 'analysis_complete')}") return result except Exception as e: logger.error(f"Error setting visibility for {package_name}: {e}") @@ -1274,7 +1268,6 @@ async def set_package_visibility_tool( "error_type": type(e).__name__, "package_name": package_name, "visibility": visibility, - "test_pypi": test_pypi, } @@ -1285,34 +1278,41 @@ async def manage_package_keywords_tool( keywords: list[str] | None = None, api_token: str | None = None, test_pypi: bool = False, + dry_run: bool = True, ) -> dict[str, Any]: - """Manage package keywords and tags. + """Manage package keywords and search tags. - This tool manages package keywords and tags for better discoverability, - including adding, removing, and updating keyword sets. + This tool provides comprehensive keyword management including validation, + quality analysis, and recommendations for better package discoverability. Args: - package_name: Name of the package - action: Action to perform ('list', 'add', 'remove', 'replace') - keywords: List of keywords (required for add/remove/replace actions) + package_name: Name of the package to modify + action: Action to perform ("add", "remove", "replace", "list") + keywords: List of keywords to add/remove/replace api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI + dry_run: If True, only simulate changes without applying them Returns: - Dictionary containing keyword management results + Dictionary containing keyword management results and recommendations Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If action is not permitted + PyPIPermissionError: If user lacks permission to modify package NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Managing keywords for {package_name}: {action}") + logger.info(f"MCP tool: Managing keywords for {package_name}: {action} (dry_run={dry_run})") result = await manage_package_keywords( - package_name, action, keywords, api_token, test_pypi + package_name=package_name, + action=action, + keywords=keywords, + api_token=api_token, + test_pypi=test_pypi, + dry_run=dry_run, ) - logger.info(f"Keyword management completed for {package_name}") + logger.info(f"Keyword management completed for {package_name}: {result.get('success', 'analysis_complete')}") return result except Exception as e: logger.error(f"Error managing keywords for {package_name}: {e}") @@ -1321,1007 +1321,6 @@ async def manage_package_keywords_tool( "error_type": type(e).__name__, "package_name": package_name, "action": action, - "test_pypi": test_pypi, - } - - -# Analytics Tools MCP Endpoints - -@mcp.tool() -async def get_pypi_package_analytics_tool( - package_name: str, - time_period: str = "month", - include_historical: bool = True, - analytics_scope: str = "comprehensive", -) -> dict[str, Any]: - """Get comprehensive analytics for a PyPI package. - - This tool provides detailed analytics including download statistics, - version distribution, platform analytics, and quality metrics. - - Args: - package_name: Name of the package to analyze - time_period: Time period for analytics ('day', 'week', 'month', 'year') - include_historical: Whether to include historical trend data - analytics_scope: Scope of analytics ('basic', 'comprehensive', 'detailed') - - Returns: - Dictionary containing comprehensive package analytics - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting analytics for {package_name}") - result = await get_pypi_package_analytics( - package_name, time_period, include_historical, analytics_scope - ) - logger.info(f"Analytics retrieved for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting analytics for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - "time_period": time_period, - } - - -@mcp.tool() -async def get_pypi_security_alerts_tool( - package_name: str, - include_dependencies: bool = True, - severity_filter: str | None = None, - alert_sources: list[str] | None = None, -) -> dict[str, Any]: - """Get security alerts and vulnerability information for a package. - - This tool provides comprehensive security analysis including known - vulnerabilities, security advisories, and dependency risk assessment. - - Args: - package_name: Name of the package to analyze - include_dependencies: Whether to include dependency vulnerabilities - severity_filter: Filter by severity ('low', 'medium', 'high', 'critical') - alert_sources: List of alert sources to check - - Returns: - Dictionary containing security alerts and analysis - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting security alerts for {package_name}") - result = await get_pypi_security_alerts( - package_name, include_dependencies, severity_filter, alert_sources - ) - logger.info(f"Security analysis completed for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting security alerts for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def get_pypi_package_rankings_tool( - package_name: str, - ranking_metrics: list[str] | None = None, - search_terms: list[str] | None = None, - include_competitors: bool = True, -) -> dict[str, Any]: - """Get package ranking and discoverability analysis. - - This tool analyzes package rankings in search results, popularity - metrics, and competitive positioning within the Python ecosystem. - - Args: - package_name: Name of the package to analyze - ranking_metrics: List of metrics to analyze ('downloads', 'stars', 'search_rank') - search_terms: Search terms to check rankings for - include_competitors: Whether to include competitor analysis - - Returns: - Dictionary containing ranking analysis and insights - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting rankings for {package_name}") - result = await get_pypi_package_rankings( - package_name, ranking_metrics, search_terms, include_competitors - ) - logger.info(f"Ranking analysis completed for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting rankings for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def analyze_pypi_competition_tool( - package_name: str, - analysis_depth: str = "comprehensive", - competitor_limit: int = 5, - include_market_analysis: bool = True, -) -> dict[str, Any]: - """Analyze competitive landscape for a PyPI package. - - This tool provides comprehensive competitive analysis including - competitor identification, feature comparison, and market positioning. - - Args: - package_name: Name of the package to analyze - analysis_depth: Depth of analysis ('basic', 'comprehensive', 'detailed') - competitor_limit: Maximum number of competitors to analyze - include_market_analysis: Whether to include market share analysis - - Returns: - Dictionary containing competitive analysis and insights - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Analyzing competition for {package_name}") - result = await analyze_pypi_competition( - package_name, analysis_depth, competitor_limit, include_market_analysis - ) - logger.info(f"Competition analysis completed for {package_name}") - return result - except Exception as e: - logger.error(f"Error analyzing competition for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -# Discovery Tools MCP Endpoints - -@mcp.tool() -async def monitor_pypi_new_releases_tool( - time_window: str = "24h", - category_filter: str | None = None, - min_downloads: int | None = None, - include_prereleases: bool = False, -) -> dict[str, Any]: - """Monitor recent PyPI package releases and updates. - - This tool tracks new packages and version releases on PyPI, - providing insights into the latest developments in the Python ecosystem. - - Args: - time_window: Time window to monitor ('1h', '6h', '24h', '7d') - category_filter: Filter by package category - min_downloads: Minimum download threshold for inclusion - include_prereleases: Whether to include pre-release versions - - Returns: - Dictionary containing recent releases and analysis - - Raises: - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Monitoring new releases (window: {time_window})") - result = await monitor_pypi_new_releases( - time_window, category_filter, min_downloads, include_prereleases - ) - logger.info(f"Found {len(result.get('new_releases', []))} new releases") - return result - except Exception as e: - logger.error(f"Error monitoring new releases: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "time_window": time_window, - } - - -@mcp.tool() -async def get_pypi_trending_today_tool( - category: str | None = None, - limit: int = 20, - trending_metric: str = "downloads", -) -> dict[str, Any]: - """Get trending PyPI packages for today. - - This tool identifies packages that are trending today based on - various metrics like download spikes, new releases, or community activity. - - Args: - category: Filter by package category - limit: Maximum number of trending packages to return - trending_metric: Metric to base trending on ('downloads', 'stars', 'releases') - - Returns: - Dictionary containing trending packages and analysis - - Raises: - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting trending packages for today") - result = await get_pypi_trending_today(category, limit, trending_metric) - logger.info(f"Found {len(result.get('trending_packages', []))} trending packages") - return result - except Exception as e: - logger.error(f"Error getting trending packages: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "category": category, - } - - -@mcp.tool() -async def search_pypi_by_maintainer_tool( - maintainer_name: str, - search_scope: str = "all", - include_statistics: bool = True, - sort_by: str = "popularity", -) -> dict[str, Any]: - """Search PyPI packages by maintainer or author. - - This tool finds all packages maintained by a specific person or organization, - providing insights into their contribution to the Python ecosystem. - - Args: - maintainer_name: Name of the maintainer to search for - search_scope: Scope of search ('author', 'maintainer', 'all') - include_statistics: Whether to include maintainer statistics - sort_by: Sort packages by ('popularity', 'recency', 'name') - - Returns: - Dictionary containing maintainer packages and statistics - - Raises: - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Searching packages by maintainer: {maintainer_name}") - result = await search_pypi_by_maintainer( - maintainer_name, search_scope, include_statistics, sort_by - ) - logger.info(f"Found {len(result.get('packages', []))} packages") - return result - except Exception as e: - logger.error(f"Error searching by maintainer {maintainer_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "maintainer_name": maintainer_name, - } - - -@mcp.tool() -async def get_pypi_package_recommendations_tool( - package_name: str, - recommendation_type: str = "similar", - limit: int = 10, - include_reasoning: bool = True, -) -> dict[str, Any]: - """Get personalized package recommendations based on a given package. - - This tool provides intelligent package recommendations including - similar packages, complementary tools, and upgrade suggestions. - - Args: - package_name: Base package for recommendations - recommendation_type: Type of recommendations ('similar', 'complementary', 'upgrades') - limit: Maximum number of recommendations to return - include_reasoning: Whether to include reasoning for recommendations - - Returns: - Dictionary containing package recommendations and analysis - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting recommendations for {package_name}") - result = await get_pypi_package_recommendations( - package_name, recommendation_type, limit, include_reasoning - ) - logger.info(f"Generated {len(result.get('recommendations', []))} recommendations") - return result - except Exception as e: - logger.error(f"Error getting recommendations for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -# Workflow Tools MCP Endpoints - -@mcp.tool() -async def validate_pypi_package_name_tool(package_name: str) -> dict[str, Any]: - """Validate PyPI package name according to PEP 508 and PyPI requirements. - - This tool validates package names against PyPI naming conventions, - checks availability, and provides suggestions for improvements. - - Args: - package_name: Package name to validate - - Returns: - Dictionary containing validation results and suggestions - - Raises: - InvalidPackageNameError: If package name format is invalid - """ - try: - logger.info(f"MCP tool: Validating package name: {package_name}") - result = await validate_pypi_package_name(package_name) - logger.info(f"Package name validation completed: {'valid' if result.get('valid') else 'invalid'}") - return result - except Exception as e: - logger.error(f"Error validating package name {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def preview_pypi_package_page_tool( - package_name: str, - version: str | None = None, - include_rendered_content: bool = True, - check_metadata_completeness: bool = True, -) -> dict[str, Any]: - """Preview how a package page will look on PyPI. - - This tool generates a preview of the PyPI package page including - rendered README, metadata display, and completeness analysis. - - Args: - package_name: Name of the package to preview - version: Specific version to preview (optional, defaults to latest) - include_rendered_content: Whether to include rendered README content - check_metadata_completeness: Whether to analyze metadata completeness - - Returns: - Dictionary containing page preview and analysis - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Previewing package page for {package_name}") - result = await preview_pypi_package_page( - package_name, version, include_rendered_content, check_metadata_completeness - ) - logger.info(f"Package page preview generated for {package_name}") - return result - except Exception as e: - logger.error(f"Error previewing package page for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def check_pypi_upload_requirements_tool( - package_path: str, - check_completeness: bool = True, - validate_metadata: bool = True, - check_security: bool = True, -) -> dict[str, Any]: - """Check if a package meets PyPI upload requirements. - - This tool validates package structure, metadata, and requirements - before upload to ensure successful PyPI submission. - - Args: - package_path: Path to the package directory or distribution file - check_completeness: Whether to check metadata completeness - validate_metadata: Whether to validate metadata format - check_security: Whether to perform security checks - - Returns: - Dictionary containing requirement check results and recommendations - - Raises: - FileNotFoundError: If package path doesn't exist - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Checking upload requirements for {package_path}") - result = await check_pypi_upload_requirements( - package_path, check_completeness, validate_metadata, check_security - ) - logger.info(f"Upload requirements check completed") - return result - except Exception as e: - logger.error(f"Error checking upload requirements for {package_path}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_path": package_path, - } - - -@mcp.tool() -async def get_pypi_build_logs_tool( - package_name: str, - version: str | None = None, - build_type: str = "all", - include_analysis: bool = True, -) -> dict[str, Any]: - """Get PyPI package build logs and analysis. - - This tool retrieves build logs for PyPI packages and provides - analysis of build issues, warnings, and optimization opportunities. - - Args: - package_name: Name of the package - version: Specific version (optional, defaults to latest) - build_type: Type of builds to include ('wheel', 'sdist', 'all') - include_analysis: Whether to include build analysis - - Returns: - Dictionary containing build logs and analysis - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting build logs for {package_name}") - result = await get_pypi_build_logs(package_name, version, build_type, include_analysis) - logger.info(f"Build logs retrieved for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting build logs for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -# Community Tools MCP Endpoints - -@mcp.tool() -async def get_pypi_package_reviews_tool( - package_name: str, - include_sentiment_analysis: bool = True, - review_sources: list[str] | None = None, - time_period: str = "all", -) -> dict[str, Any]: - """Get community reviews and ratings for a PyPI package. - - This tool aggregates community feedback, reviews, and sentiment - analysis from various sources to provide comprehensive package insights. - - Args: - package_name: Name of the package to get reviews for - include_sentiment_analysis: Whether to include sentiment analysis - review_sources: List of sources to check ('github', 'stackoverflow', 'reddit') - time_period: Time period for reviews ('week', 'month', 'year', 'all') - - Returns: - Dictionary containing reviews, ratings, and sentiment analysis - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting reviews for {package_name}") - result = await get_pypi_package_reviews( - package_name, include_sentiment_analysis, review_sources, time_period - ) - logger.info(f"Reviews retrieved for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting reviews for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def manage_pypi_package_discussions_tool( - package_name: str, - action: str, - discussion_settings: dict[str, Any] | None = None, - moderator_controls: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Manage package discussions and community features. - - This tool manages community discussion features for PyPI packages - including enabling discussions, moderation, and configuration. - - Args: - package_name: Name of the package - action: Action to perform ('status', 'enable', 'disable', 'configure', 'moderate') - discussion_settings: Settings for discussion configuration - moderator_controls: Moderation controls and settings - - Returns: - Dictionary containing discussion management results - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - PyPIPermissionError: If action is not permitted - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Managing discussions for {package_name}: {action}") - result = await manage_pypi_package_discussions( - package_name, action, discussion_settings, moderator_controls - ) - logger.info(f"Discussion management completed for {package_name}") - return result - except Exception as e: - logger.error(f"Error managing discussions for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - "action": action, - } - - -@mcp.tool() -async def get_pypi_maintainer_contacts_tool( - package_name: str, - contact_types: list[str] | None = None, - include_social_profiles: bool = False, - respect_privacy: bool = True, -) -> dict[str, Any]: - """Get maintainer contact information and communication channels. - - This tool finds maintainer contact information and preferred - communication channels while respecting privacy preferences. - - Args: - package_name: Name of the package - contact_types: Types of contacts to find ('email', 'github', 'twitter') - include_social_profiles: Whether to include social media profiles - respect_privacy: Whether to respect privacy settings and preferences - - Returns: - Dictionary containing maintainer contact information - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Getting maintainer contacts for {package_name}") - result = await get_pypi_maintainer_contacts( - package_name, contact_types, include_social_profiles, respect_privacy - ) - logger.info(f"Maintainer contacts retrieved for {package_name}") - return result - except Exception as e: - logger.error(f"Error getting maintainer contacts for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - - -@mcp.tool() -async def scan_pypi_package_security_tool( - package_name: str, - version: str | None = None, - include_dependencies: bool = True, - severity_filter: str | None = None -) -> dict[str, Any]: - """Scan a PyPI package for security vulnerabilities. - - This tool performs comprehensive security vulnerability scanning of PyPI packages, - checking against multiple vulnerability databases including OSV (Open Source Vulnerabilities), - GitHub Security Advisories, and analyzing package metadata for security indicators. - - Args: - package_name: Name of the package to scan for vulnerabilities - version: Specific version to scan (optional, defaults to latest version) - include_dependencies: Whether to scan package dependencies for vulnerabilities - severity_filter: Filter results by severity level (low, medium, high, critical) - - Returns: - Dictionary containing comprehensive security scan results including: - - Total vulnerability count and severity breakdown - - Direct package vulnerabilities vs dependency vulnerabilities - - Risk score and level assessment (minimal, low, medium, high, critical) - - Detailed vulnerability information with IDs, descriptions, and references - - Package metadata security analysis - - Actionable security recommendations - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If security scanning fails - """ - try: - logger.info(f"MCP tool: Scanning security vulnerabilities for {package_name}") - result = await scan_pypi_package_security( - package_name, version, include_dependencies, severity_filter - ) - logger.info(f"Security scan completed for {package_name} - found {result.get('security_summary', {}).get('total_vulnerabilities', 0)} vulnerabilities") - return result - except Exception as e: - logger.error(f"Error scanning security for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package": package_name, - "version": version, - } - - -@mcp.tool() -async def bulk_scan_package_security_tool( - package_names: list[str], - include_dependencies: bool = False, - severity_threshold: str = "medium" -) -> dict[str, Any]: - """Perform bulk security scanning of multiple PyPI packages. - - This tool scans multiple packages simultaneously for security vulnerabilities, - providing a consolidated report with summary statistics and prioritized - recommendations for addressing security issues across your package ecosystem. - - Args: - package_names: List of package names to scan for vulnerabilities - include_dependencies: Whether to include dependency vulnerability scanning - severity_threshold: Minimum severity level to report (low, medium, high, critical) - - Returns: - Dictionary containing bulk scan results including: - - Summary statistics (total packages, packages with vulnerabilities, high-risk packages) - - Detailed scan results for each package - - Prioritized recommendations for security remediation - - Scan timestamp and completion status - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during scanning - SearchError: If bulk scanning fails - """ - try: - logger.info(f"MCP tool: Starting bulk security scan of {len(package_names)} packages") - result = await bulk_scan_package_security( - package_names, include_dependencies, severity_threshold - ) - logger.info(f"Bulk security scan completed - {result.get('summary', {}).get('packages_with_vulnerabilities', 0)} packages have vulnerabilities") - return result - except Exception as e: - logger.error(f"Error in bulk security scan: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_names": package_names, - } - - -@mcp.tool() -async def analyze_pypi_package_license_tool( - package_name: str, - version: str | None = None, - include_dependencies: bool = True -) -> dict[str, Any]: - """Analyze license compatibility for a PyPI package. - - This tool provides comprehensive license analysis including license identification, - dependency license scanning, compatibility checking, and risk assessment to help - ensure your project complies with open source license requirements. - - Args: - package_name: Name of the package to analyze for license compatibility - version: Specific version to analyze (optional, defaults to latest version) - include_dependencies: Whether to analyze dependency licenses for compatibility - - Returns: - Dictionary containing comprehensive license analysis including: - - License identification and normalization (SPDX format) - - License categorization (permissive, copyleft, proprietary, etc.) - - Dependency license analysis and compatibility matrix - - Risk assessment with score and risk level (minimal, low, medium, high, critical) - - Compatibility analysis highlighting conflicts and review-required combinations - - Actionable recommendations for license compliance - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If license analysis fails - """ - try: - logger.info(f"MCP tool: Analyzing license compatibility for {package_name}") - result = await analyze_pypi_package_license( - package_name, version, include_dependencies - ) - logger.info(f"License analysis completed for {package_name} - {result.get('analysis_summary', {}).get('license_conflicts', 0)} conflicts found") - return result - except Exception as e: - logger.error(f"Error analyzing license for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package": package_name, - "version": version, - } - - -@mcp.tool() -async def check_bulk_license_compliance_tool( - package_names: list[str], - target_license: str | None = None -) -> dict[str, Any]: - """Check license compliance for multiple PyPI packages. - - This tool performs bulk license compliance checking across multiple packages, - providing a consolidated report to help ensure your entire package ecosystem - complies with license requirements and identifying potential legal risks. - - Args: - package_names: List of package names to check for license compliance - target_license: Target license for compatibility checking (optional) - - Returns: - Dictionary containing bulk compliance analysis including: - - Summary statistics (total packages, compliant/non-compliant counts) - - Detailed license analysis for each package - - High-risk packages requiring immediate attention - - Unknown license packages needing investigation - - Prioritized recommendations for compliance remediation - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during analysis - SearchError: If bulk compliance checking fails - """ - try: - logger.info(f"MCP tool: Starting bulk license compliance check for {len(package_names)} packages") - result = await check_bulk_license_compliance( - package_names, target_license - ) - logger.info(f"Bulk license compliance completed - {result.get('summary', {}).get('non_compliant_packages', 0)} non-compliant packages found") - return result - except Exception as e: - logger.error(f"Error in bulk license compliance check: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_names": package_names, - } - - -@mcp.tool() -async def assess_package_health_score_tool( - package_name: str, - version: str | None = None, - include_github_metrics: bool = True -) -> dict[str, Any]: - """Assess comprehensive health and quality of a PyPI package. - - This tool evaluates package health across multiple dimensions including maintenance, - popularity, documentation, testing, security practices, compatibility, and metadata - completeness to provide an overall health score and actionable recommendations. - - Args: - package_name: Name of the package to assess for health and quality - version: Specific version to assess (optional, defaults to latest version) - include_github_metrics: Whether to fetch GitHub repository metrics for analysis - - Returns: - Dictionary containing comprehensive health assessment including: - - Overall health score (0-100) and level (excellent/good/fair/poor/critical) - - Category-specific scores (maintenance, popularity, documentation, testing, etc.) - - Detailed assessment breakdown with indicators and issues for each category - - GitHub repository metrics (stars, forks, activity) if available - - Actionable recommendations for health improvements - - Strengths, weaknesses, and improvement priorities analysis - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If health assessment fails - """ - try: - logger.info(f"MCP tool: Assessing health for {package_name}") - result = await assess_package_health_score( - package_name, version, include_github_metrics - ) - overall_score = result.get("overall_health", {}).get("score", 0) - health_level = result.get("overall_health", {}).get("level", "unknown") - logger.info(f"Health assessment completed for {package_name} - score: {overall_score:.1f}/100 ({health_level})") - return result - except Exception as e: - logger.error(f"Error assessing health for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package": package_name, - "version": version, - } - - -@mcp.tool() -async def compare_packages_health_scores_tool( - package_names: list[str], - include_github_metrics: bool = False -) -> dict[str, Any]: - """Compare health scores across multiple PyPI packages. - - This tool performs comparative health analysis across multiple packages, - providing rankings, insights, and recommendations to help evaluate - package ecosystem quality and identify the best options. - - Args: - package_names: List of package names to compare for health and quality - include_github_metrics: Whether to include GitHub metrics in the comparison - - Returns: - Dictionary containing comparative health analysis including: - - Detailed health results for each package - - Health score rankings with best/worst package identification - - Comparison insights (average scores, score ranges, rankings) - - Recommendations for package selection and improvements - - Statistical analysis of health across the package set - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during analysis - SearchError: If health comparison fails - """ - try: - logger.info(f"MCP tool: Starting health comparison for {len(package_names)} packages") - result = await compare_packages_health_scores( - package_names, include_github_metrics - ) - comparison_insights = result.get("comparison_insights", {}) - best_package = comparison_insights.get("best_package", {}) - packages_compared = result.get("packages_compared", 0) - logger.info(f"Health comparison completed for {packages_compared} packages - best: {best_package.get('name', 'unknown')} ({best_package.get('score', 0):.1f}/100)") - return result - except Exception as e: - logger.error(f"Error in health comparison: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_names": package_names, - } - - -@mcp.tool() -async def analyze_requirements_file_tool_mcp( - file_path: str, - check_updates: bool = True, - security_scan: bool = True, - compatibility_check: bool = True -) -> dict[str, Any]: - """Analyze project requirements file for dependencies, security, and compatibility. - - This tool provides comprehensive analysis of Python project requirements files - including dependency parsing, version checking, security vulnerability scanning, - Python compatibility assessment, and actionable recommendations for improvements. - - Args: - file_path: Path to the requirements file (requirements.txt, pyproject.toml, setup.py, etc.) - check_updates: Whether to check for available package updates - security_scan: Whether to perform security vulnerability scanning on dependencies - compatibility_check: Whether to check Python version compatibility for all dependencies - - Returns: - Dictionary containing comprehensive requirements analysis including: - - File information and detected format (requirements.txt, pyproject.toml, etc.) - - Parsed dependencies with version specifiers and extras - - Dependency health analysis with specification issues and recommendations - - Package update analysis showing outdated packages and latest versions - - Security vulnerability scan results for all dependencies - - Python version compatibility assessment - - Overall risk level and actionable improvement recommendations - - Raises: - FileNotFoundError: If the requirements file is not found - NetworkError: For network-related errors during analysis - SearchError: If requirements analysis fails - """ - try: - logger.info(f"MCP tool: Analyzing requirements file {file_path}") - result = await analyze_requirements_file_tool( - file_path, check_updates, security_scan, compatibility_check - ) - summary = result.get("analysis_summary", {}) - total_deps = summary.get("total_dependencies", 0) - risk_level = summary.get("overall_risk_level", "unknown") - logger.info(f"Requirements analysis completed for {file_path} - {total_deps} dependencies, risk level: {risk_level}") - return result - except Exception as e: - logger.error(f"Error analyzing requirements file {file_path}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "file_path": file_path, - } - - -@mcp.tool() -async def compare_multiple_requirements_files_mcp( - file_paths: list[str] -) -> dict[str, Any]: - """Compare multiple requirements files to identify differences and conflicts. - - This tool analyzes multiple requirements files simultaneously to identify - version conflicts, unique dependencies, and inconsistencies across different - project configurations or environments. - - Args: - file_paths: List of paths to requirements files to compare and analyze - - Returns: - Dictionary containing comparative requirements analysis including: - - Detailed analysis results for each individual file - - Common packages shared across all files - - Conflicting package versions between files with specific version details - - Packages unique to specific files - - Recommendations for resolving conflicts and standardizing requirements - - Statistics on package overlap and conflict rates - - Raises: - ValueError: If file_paths list is empty - NetworkError: For network-related errors during analysis - SearchError: If requirements comparison fails - """ - try: - logger.info(f"MCP tool: Comparing {len(file_paths)} requirements files") - result = await compare_multiple_requirements_files(file_paths) - comparison_results = result.get("comparison_results", {}) - conflicts = len(comparison_results.get("conflicting_packages", [])) - total_packages = comparison_results.get("total_unique_packages", 0) - logger.info(f"Requirements comparison completed - {total_packages} unique packages, {conflicts} conflicts found") - return result - except Exception as e: - logger.error(f"Error comparing requirements files: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "file_paths": file_paths, } @@ -2483,24 +1482,92 @@ async def generate_migration_checklist_prompt( return messages[0].text +# Environment Analysis Prompts @mcp.prompt() -async def generate_update_plan_prompt( - packages: list[str], - update_strategy: str = "conservative", - environment_type: str = "production", - testing_requirements: str | None = None, +async def analyze_environment_dependencies_prompt( + environment_type: str = "local", + python_version: str | None = None, + project_path: str | None = None, ) -> str: - """Generate a comprehensive update plan prompt for packages.""" + """Generate a prompt for analyzing environment dependencies.""" # Step 3: Call Prompt generator - template = await generate_update_plan( - packages, update_strategy, environment_type, testing_requirements + template = await analyze_environment_dependencies( + environment_type, python_version, project_path ) # Step 5: Parameter replacement - packages_text = ", ".join(f"'{pkg}'" for pkg in packages) - result = template.replace("{{packages_text}}", packages_text) - result = result.replace("{{update_strategy}}", update_strategy) - result = result.replace("{{environment_type}}", environment_type) + result = template.replace("{{environment_type}}", environment_type) + + # Handle environment info + env_info = f"({environment_type} environment)" + if python_version: + env_info += f" with Python {python_version}" + if project_path: + env_info += f" at {project_path}" + result = result.replace("{{environment_info}}", env_info) + + # Handle command prefix based on environment + command_prefix = "uvx " if environment_type in ["virtual", "uv"] else "" + result = result.replace("{{command_prefix}}", command_prefix) + + # Step 7: Return final prompt + return result + + +@mcp.prompt() +async def check_outdated_packages_prompt( + package_filter: str | None = None, + severity_level: str = "all", + include_dev_dependencies: bool = True, +) -> str: + """Generate a prompt for checking outdated packages.""" + # Step 3: Call Prompt generator + template = await check_outdated_packages( + package_filter, severity_level, include_dev_dependencies + ) + + # Step 5: Parameter replacement + result = template.replace("{{severity_level}}", severity_level) + + # Handle filter info + if package_filter: + filter_info = f" (filtering by: {package_filter})" + else: + filter_info = "" + result = result.replace("{{filter_info}}", filter_info) + + # Handle dev dependencies + if include_dev_dependencies: + dev_deps_text = " including development dependencies" + else: + dev_deps_text = " excluding development dependencies" + result = result.replace("{{dev_deps_text}}", dev_deps_text) + + # Step 7: Return final prompt + return result + + +@mcp.prompt() +async def generate_update_plan_prompt( + update_strategy: str = "balanced", + environment_constraints: str | None = None, + testing_requirements: str | None = None, +) -> str: + """Generate a prompt for creating package update plans.""" + # Step 3: Call Prompt generator + template = await generate_update_plan( + update_strategy, environment_constraints, testing_requirements + ) + + # Step 5: Parameter replacement + result = template.replace("{{strategy}}", update_strategy) + + # Handle constraints + if environment_constraints: + constraints_text = f"\n\nEnvironment constraints: {environment_constraints}" + else: + constraints_text = "" + result = result.replace("{{constraints_text}}", constraints_text) # Handle testing requirements if testing_requirements: @@ -2586,6 +1653,241 @@ async def track_package_updates_prompt( return result +@mcp.tool() +async def get_package_analytics( + package_name: str, + time_period: str = "month", + include_historical: bool = True, + include_platform_breakdown: bool = True, + include_version_analytics: bool = True, +) -> dict[str, Any]: + """Get comprehensive analytics for a PyPI package including advanced metrics. + + This tool provides detailed download analytics, trend analysis, geographic + distribution, platform breakdown, and version adoption patterns. + + Args: + package_name: Name of the package to analyze + time_period: Time period for analysis ('day', 'week', 'month', 'year') + include_historical: Whether to include historical trend analysis + include_platform_breakdown: Whether to include platform/OS breakdown + include_version_analytics: Whether to include version-specific analytics + + Returns: + Dictionary containing comprehensive analytics including: + - Download statistics and trends + - Platform and Python version breakdown + - Geographic distribution + - Version adoption patterns + - Quality metrics and indicators + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Generating comprehensive analytics for {package_name}") + result = await get_pypi_package_analytics( + package_name=package_name, + time_period=time_period, + include_historical=include_historical, + include_platform_breakdown=include_platform_breakdown, + include_version_analytics=include_version_analytics, + ) + logger.info(f"Successfully generated analytics for package: {package_name}") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error generating analytics for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + except Exception as e: + logger.error(f"Unexpected error generating analytics for {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + } + + +@mcp.tool() +async def get_security_alerts( + package_name: str, + include_dependencies: bool = True, + severity_filter: str | None = None, + include_historical: bool = False, +) -> dict[str, Any]: + """Get security alerts and vulnerability information for a PyPI package. + + This tool queries multiple security databases including OSV (Open Source + Vulnerabilities), PyUp.io Safety DB, and GitHub Security Advisories to provide + comprehensive security information. + + Args: + package_name: Name of the package to check for vulnerabilities + include_dependencies: Whether to check dependencies for vulnerabilities + severity_filter: Filter by severity ('LOW', 'MEDIUM', 'HIGH', 'CRITICAL') + include_historical: Whether to include historical vulnerabilities + + Returns: + Dictionary containing security information including: + - Active vulnerabilities and CVEs + - Security scores and risk assessment + - Dependency vulnerability analysis + - Remediation recommendations + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Checking security alerts for {package_name}") + result = await get_pypi_security_alerts( + package_name=package_name, + include_dependencies=include_dependencies, + severity_filter=severity_filter, + include_historical=include_historical, + ) + logger.info(f"Successfully checked security alerts for package: {package_name}") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error checking security alerts for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + except Exception as e: + logger.error(f"Unexpected error checking security alerts for {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + } + + +@mcp.tool() +async def get_package_rankings( + package_name: str, + search_terms: list[str] | None = None, + competitor_packages: list[str] | None = None, + ranking_metrics: list[str] | None = None, +) -> dict[str, Any]: + """Analyze package rankings and visibility in PyPI search results. + + This tool analyzes how well a package ranks for relevant search terms, + compares it to competitor packages, and provides insights into search + visibility and discoverability. + + Args: + package_name: Name of the package to analyze rankings for + search_terms: List of search terms to test rankings against + competitor_packages: List of competitor packages to compare against + ranking_metrics: Specific metrics to focus on ('relevance', 'popularity', 'downloads', 'quality') + + Returns: + Dictionary containing ranking analysis including: + - Search position for various terms + - Competitor comparison matrix + - Visibility and discoverability metrics + - SEO and keyword optimization suggestions + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Analyzing search rankings for {package_name}") + result = await get_pypi_package_rankings( + package_name=package_name, + search_terms=search_terms, + competitor_packages=competitor_packages, + ranking_metrics=ranking_metrics, + ) + logger.info(f"Successfully analyzed rankings for package: {package_name}") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error analyzing rankings for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + except Exception as e: + logger.error(f"Unexpected error analyzing rankings for {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + } + + +@mcp.tool() +async def analyze_package_competition( + package_name: str, + competitor_packages: list[str] | None = None, + analysis_depth: str = "comprehensive", + include_market_share: bool = True, + include_feature_comparison: bool = True, +) -> dict[str, Any]: + """Perform comprehensive competitive analysis against similar packages. + + This tool analyzes a package against its competitors, providing insights + into market positioning, feature gaps, adoption trends, and competitive + advantages. + + Args: + package_name: Name of the package to analyze + competitor_packages: List of competitor packages (auto-detected if not provided) + analysis_depth: Depth of analysis ('basic', 'comprehensive', 'detailed') + include_market_share: Whether to include market share analysis + include_feature_comparison: Whether to include feature comparison + + Returns: + Dictionary containing competitive analysis including: + - Market positioning and share + - Feature comparison matrix + - Adoption and growth trends + - Competitive advantages and weaknesses + - Strategic recommendations + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Analyzing competition for {package_name}") + result = await analyze_pypi_competition( + package_name=package_name, + competitor_packages=competitor_packages, + analysis_depth=analysis_depth, + include_market_share=include_market_share, + include_feature_comparison=include_feature_comparison, + ) + logger.info(f"Successfully analyzed competition for package: {package_name}") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error analyzing competition for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + except Exception as e: + logger.error(f"Unexpected error analyzing competition for {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + } + + @click.command() @click.option( "--log-level", diff --git a/pypi_query_mcp/tools/__init__.py b/pypi_query_mcp/tools/__init__.py index ce0615b..eb08606 100644 --- a/pypi_query_mcp/tools/__init__.py +++ b/pypi_query_mcp/tools/__init__.py @@ -21,6 +21,12 @@ from .package_query import ( query_package_info, query_package_versions, ) +from .metadata import ( + manage_package_keywords, + manage_package_urls, + set_package_visibility, + update_package_metadata, +) from .publishing import ( check_pypi_credentials, delete_pypi_release, @@ -29,11 +35,11 @@ from .publishing import ( manage_pypi_maintainers, upload_package_to_pypi, ) -from .metadata import ( - manage_package_keywords, - manage_package_urls, - set_package_visibility, - update_package_metadata, +from .search import ( + find_alternatives, + get_trending_packages, + search_by_category, + search_packages, ) from .analytics import ( analyze_pypi_competition, @@ -41,48 +47,8 @@ from .analytics import ( get_pypi_package_rankings, get_pypi_security_alerts, ) -from .discovery import ( - get_pypi_package_recommendations, - get_pypi_trending_today, - monitor_pypi_new_releases, - search_pypi_by_maintainer, -) -from .workflow import ( - check_pypi_upload_requirements, - get_pypi_build_logs, - preview_pypi_package_page, - validate_pypi_package_name, -) -from .community import ( - get_pypi_package_reviews, - manage_pypi_package_discussions, - get_pypi_maintainer_contacts, -) -from .search import ( - find_alternatives, - get_trending_packages, - search_by_category, - search_packages, -) -from .security_tools import ( - bulk_scan_package_security, - scan_pypi_package_security, -) -from .license_tools import ( - analyze_pypi_package_license, - check_bulk_license_compliance, -) -from .health_tools import ( - assess_package_health_score, - compare_packages_health_scores, -) -from .requirements_tools import ( - analyze_requirements_file_tool, - compare_multiple_requirements_files, -) __all__ = [ - # Core package tools "query_package_info", "query_package_versions", "query_package_dependencies", @@ -94,52 +60,22 @@ __all__ = [ "get_package_download_stats", "get_package_download_trends", "get_top_packages_by_downloads", - # Search tools "search_packages", "search_by_category", "find_alternatives", "get_trending_packages", - # Publishing tools "upload_package_to_pypi", "check_pypi_credentials", "get_pypi_upload_history", "delete_pypi_release", "manage_pypi_maintainers", "get_pypi_account_info", - # Metadata tools "update_package_metadata", "manage_package_urls", "set_package_visibility", "manage_package_keywords", - # Analytics tools "get_pypi_package_analytics", "get_pypi_security_alerts", "get_pypi_package_rankings", "analyze_pypi_competition", - # Discovery tools - "monitor_pypi_new_releases", - "get_pypi_trending_today", - "search_pypi_by_maintainer", - "get_pypi_package_recommendations", - # Workflow tools - "validate_pypi_package_name", - "preview_pypi_package_page", - "check_pypi_upload_requirements", - "get_pypi_build_logs", - # Community tools - "get_pypi_package_reviews", - "manage_pypi_package_discussions", - "get_pypi_maintainer_contacts", - # Security tools - "scan_pypi_package_security", - "bulk_scan_package_security", - # License tools - "analyze_pypi_package_license", - "check_bulk_license_compliance", - # Health tools - "assess_package_health_score", - "compare_packages_health_scores", - # Requirements tools - "analyze_requirements_file_tool", - "compare_multiple_requirements_files", ] diff --git a/pypi_query_mcp/tools/community.py b/pypi_query_mcp/tools/community.py deleted file mode 100644 index 54f2023..0000000 --- a/pypi_query_mcp/tools/community.py +++ /dev/null @@ -1,1381 +0,0 @@ -"""PyPI Community & Social Tools for package community interactions and maintainer communication.""" - -import asyncio -import json -import logging -import re -from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Union -from urllib.parse import quote, urljoin, urlparse - -import httpx - -from ..core.exceptions import ( - InvalidPackageNameError, - NetworkError, - PackageNotFoundError, - PyPIError, -) -from ..core.pypi_client import PyPIClient -from ..core.github_client import GitHubAPIClient - -logger = logging.getLogger(__name__) - - -async def get_pypi_package_reviews( - package_name: str, - include_ratings: bool = True, - include_community_feedback: bool = True, - sentiment_analysis: bool = False, - max_reviews: int = 50, -) -> Dict[str, Any]: - """ - Get community reviews and feedback for a PyPI package. - - This function aggregates community feedback from various sources including - GitHub discussions, issues, Stack Overflow mentions, and social media to - provide comprehensive community sentiment analysis. - - Note: This is a future-ready implementation as PyPI doesn't currently have - a native review system. The function prepares for when such features become - available while providing useful community sentiment analysis from existing sources. - - Args: - package_name: Name of the package to get reviews for - include_ratings: Whether to include numerical ratings (when available) - include_community_feedback: Whether to include textual feedback analysis - sentiment_analysis: Whether to perform sentiment analysis on feedback - max_reviews: Maximum number of reviews to return - - Returns: - Dictionary containing review and feedback information including: - - Community sentiment and ratings - - Feedback from GitHub issues and discussions - - Social media mentions and sentiment - - Quality indicators and community health metrics - - Future-ready structure for native PyPI reviews - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError("Package name cannot be empty") - - package_name = package_name.strip() - logger.info(f"Gathering community reviews and feedback for package: {package_name}") - - try: - # Gather data from multiple community sources concurrently - review_tasks = [ - _get_package_metadata_for_reviews(package_name), - _analyze_github_community_sentiment(package_name) if include_community_feedback else asyncio.create_task(_empty_dict()), - _check_stackoverflow_mentions(package_name) if include_community_feedback else asyncio.create_task(_empty_dict()), - _analyze_pypi_downloads_as_quality_indicator(package_name), - _get_community_health_metrics(package_name), - ] - - results = await asyncio.gather(*review_tasks, return_exceptions=True) - - package_metadata = results[0] if not isinstance(results[0], Exception) else {} - github_sentiment = results[1] if not isinstance(results[1], Exception) else {} - stackoverflow_data = results[2] if not isinstance(results[2], Exception) else {} - quality_indicators = results[3] if not isinstance(results[3], Exception) else {} - community_health = results[4] if not isinstance(results[4], Exception) else {} - - # Perform sentiment analysis if requested - sentiment_analysis_results = {} - if sentiment_analysis and (github_sentiment or stackoverflow_data): - sentiment_analysis_results = await _perform_sentiment_analysis( - github_sentiment, stackoverflow_data - ) - - # Calculate overall community score - community_score = _calculate_community_score( - github_sentiment, stackoverflow_data, quality_indicators, community_health - ) - - # Generate community insights - insights = _generate_community_insights( - github_sentiment, stackoverflow_data, community_score, package_metadata - ) - - review_report = { - "package": package_name, - "review_timestamp": datetime.now().isoformat(), - "community_score": community_score, - "metadata": package_metadata, - "community_health": community_health, - "quality_indicators": quality_indicators, - "insights": insights, - "data_sources": { - "github_analysis": bool(github_sentiment), - "stackoverflow_mentions": bool(stackoverflow_data), - "pypi_metrics": bool(quality_indicators), - "community_health": bool(community_health), - }, - "review_system_status": { - "native_pypi_reviews": "not_available", - "note": "PyPI does not currently have a native review system. This analysis aggregates community feedback from external sources.", - "future_ready": True, - "implementation_note": "This function is designed to seamlessly integrate native PyPI reviews when they become available.", - }, - } - - # Add optional sections based on flags - if include_community_feedback and github_sentiment: - review_report["github_community_feedback"] = github_sentiment - - if include_community_feedback and stackoverflow_data: - review_report["stackoverflow_mentions"] = stackoverflow_data - - if sentiment_analysis and sentiment_analysis_results: - review_report["sentiment_analysis"] = sentiment_analysis_results - - # Add ratings section (prepared for future PyPI native ratings) - if include_ratings: - review_report["ratings"] = { - "average_rating": None, - "total_ratings": 0, - "rating_distribution": {}, - "community_derived_score": community_score.get("overall_score", 0), - "note": "Native PyPI ratings not yet available. Community-derived score provided based on external feedback.", - } - - return review_report - - except Exception as e: - logger.error(f"Error gathering reviews for {package_name}: {e}") - if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError)): - raise - raise NetworkError(f"Failed to gather community reviews: {e}") from e - - -async def manage_pypi_package_discussions( - package_name: str, - action: str = "get_status", - discussion_settings: Optional[Dict[str, Any]] = None, - moderator_controls: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """ - Manage and interact with PyPI package discussions. - - This function provides management capabilities for package discussions, - including enabling/disabling discussions, setting moderation policies, - and retrieving discussion status and metrics. - - Note: This is a future-ready implementation as PyPI doesn't currently have - native discussion features. The function prepares for when such features - become available while providing integration with existing discussion platforms. - - Args: - package_name: Name of the package to manage discussions for - action: Action to perform ('get_status', 'enable', 'disable', 'configure', 'moderate') - discussion_settings: Settings for discussions (when enabling/configuring) - moderator_controls: Moderation settings and controls - - Returns: - Dictionary containing discussion management results including: - - Current discussion status and settings - - Available discussion platforms and integration status - - Moderation settings and community guidelines - - Discussion metrics and engagement data - - Future-ready structure for native PyPI discussions - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - PyPIError: For PyPI-specific errors - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError("Package name cannot be empty") - - package_name = package_name.strip() - valid_actions = ["get_status", "enable", "disable", "configure", "moderate", "get_metrics"] - - if action not in valid_actions: - raise InvalidPackageNameError(f"Invalid action. Must be one of: {', '.join(valid_actions)}") - - logger.info(f"Managing discussions for package: {package_name}, action: {action}") - - try: - # Get package metadata and current discussion status - package_data = await _get_package_metadata_for_discussions(package_name) - current_status = await _get_current_discussion_status(package_name, package_data) - - # Perform the requested action - if action == "get_status": - result = await _get_discussion_status(package_name, current_status) - elif action == "enable": - result = await _enable_discussions(package_name, discussion_settings, current_status) - elif action == "disable": - result = await _disable_discussions(package_name, current_status) - elif action == "configure": - result = await _configure_discussions(package_name, discussion_settings, current_status) - elif action == "moderate": - result = await _moderate_discussions(package_name, moderator_controls, current_status) - elif action == "get_metrics": - result = await _get_discussion_metrics(package_name, current_status) - else: - result = {"error": f"Action {action} not implemented"} - - # Add common metadata to all responses - result.update({ - "package": package_name, - "action_performed": action, - "timestamp": datetime.now().isoformat(), - "discussion_system_status": { - "native_pypi_discussions": "not_available", - "note": "PyPI does not currently have native discussion features. This function manages external discussion platforms and prepares for future PyPI integration.", - "supported_platforms": ["GitHub Discussions", "Community Forums", "Social Media"], - "future_ready": True, - }, - }) - - return result - - except Exception as e: - logger.error(f"Error managing discussions for {package_name}: {e}") - if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError, PyPIError)): - raise - raise NetworkError(f"Failed to manage package discussions: {e}") from e - - -async def get_pypi_maintainer_contacts( - package_name: str, - contact_types: Optional[List[str]] = None, - include_social_profiles: bool = False, - include_contribution_guidelines: bool = True, - respect_privacy_settings: bool = True, -) -> Dict[str, Any]: - """ - Get contact information and communication channels for package maintainers. - - This function retrieves publicly available contact information for package - maintainers while respecting privacy settings and providing appropriate - communication channels for different types of inquiries. - - Args: - package_name: Name of the package to get maintainer contacts for - contact_types: Types of contact info to retrieve ('email', 'github', 'social', 'support') - include_social_profiles: Whether to include social media profiles - include_contribution_guidelines: Whether to include contribution guidelines - respect_privacy_settings: Whether to respect maintainer privacy preferences - - Returns: - Dictionary containing maintainer contact information including: - - Publicly available contact methods - - Communication preferences and guidelines - - Support channels and community resources - - Privacy-respecting contact recommendations - - Contribution guidelines and community standards - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError("Package name cannot be empty") - - package_name = package_name.strip() - - # Set default contact types if not provided - if contact_types is None: - contact_types = ["github", "support", "community"] - - # Validate contact types - valid_contact_types = ["email", "github", "social", "support", "community", "documentation"] - invalid_types = [ct for ct in contact_types if ct not in valid_contact_types] - if invalid_types: - raise InvalidPackageNameError(f"Invalid contact types: {', '.join(invalid_types)}. Valid types: {', '.join(valid_contact_types)}") - - logger.info(f"Gathering maintainer contact information for package: {package_name}") - - try: - # Gather data from multiple sources concurrently - contact_tasks = [ - _get_package_metadata_for_contacts(package_name), - _analyze_github_maintainer_info(package_name) if "github" in contact_types else asyncio.create_task(_empty_dict()), - _get_support_channels(package_name) if "support" in contact_types else asyncio.create_task(_empty_dict()), - _get_community_channels(package_name) if "community" in contact_types else asyncio.create_task(_empty_dict()), - _get_contribution_guidelines(package_name) if include_contribution_guidelines else asyncio.create_task(_empty_dict()), - ] - - results = await asyncio.gather(*contact_tasks, return_exceptions=True) - - package_metadata = results[0] if not isinstance(results[0], Exception) else {} - github_info = results[1] if not isinstance(results[1], Exception) else {} - support_channels = results[2] if not isinstance(results[2], Exception) else {} - community_channels = results[3] if not isinstance(results[3], Exception) else {} - contribution_info = results[4] if not isinstance(results[4], Exception) else {} - - # Extract contact information from package metadata - contact_info = _extract_contact_info_from_metadata( - package_metadata, contact_types, respect_privacy_settings - ) - - # Get social profiles if requested - social_profiles = {} - if include_social_profiles and "social" in contact_types: - social_profiles = await _get_social_profiles(package_name, github_info, respect_privacy_settings) - - # Generate contact recommendations - contact_recommendations = _generate_contact_recommendations( - contact_info, github_info, support_channels, community_channels - ) - - # Assess contact accessibility and responsiveness - accessibility_assessment = _assess_contact_accessibility( - contact_info, github_info, support_channels - ) - - contact_report = { - "package": package_name, - "contact_timestamp": datetime.now().isoformat(), - "contact_information": contact_info, - "accessibility_assessment": accessibility_assessment, - "contact_recommendations": contact_recommendations, - "privacy_compliance": { - "respects_privacy_settings": respect_privacy_settings, - "data_sources": "Publicly available information only", - "privacy_note": "All contact information is sourced from publicly available package metadata and repositories", - }, - } - - # Add optional sections based on flags and availability - if github_info: - contact_report["github_information"] = github_info - - if support_channels: - contact_report["support_channels"] = support_channels - - if community_channels: - contact_report["community_channels"] = community_channels - - if include_contribution_guidelines and contribution_info: - contact_report["contribution_guidelines"] = contribution_info - - if include_social_profiles and social_profiles: - contact_report["social_profiles"] = social_profiles - - # Add communication guidelines - contact_report["communication_guidelines"] = _generate_communication_guidelines( - contact_info, github_info, package_metadata - ) - - return contact_report - - except Exception as e: - logger.error(f"Error gathering maintainer contacts for {package_name}: {e}") - if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError)): - raise - raise NetworkError(f"Failed to gather maintainer contact information: {e}") from e - - -# Helper functions for community tools implementation - -async def _empty_dict(): - """Return empty dict for optional tasks.""" - return {} - - -async def _get_package_metadata_for_reviews(package_name: str) -> Dict[str, Any]: - """Get package metadata relevant for review analysis.""" - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name) - - info = package_data.get("info", {}) - return { - "name": info.get("name", package_name), - "version": info.get("version", "unknown"), - "summary": info.get("summary", ""), - "description": info.get("description", ""), - "author": info.get("author", ""), - "maintainer": info.get("maintainer", ""), - "home_page": info.get("home_page", ""), - "project_urls": info.get("project_urls", {}), - "keywords": info.get("keywords", ""), - "classifiers": info.get("classifiers", []), - "license": info.get("license", ""), - } - - except Exception as e: - logger.warning(f"Failed to get package metadata for reviews: {e}") - return {"name": package_name} - - -async def _analyze_github_community_sentiment(package_name: str) -> Dict[str, Any]: - """Analyze GitHub community sentiment through issues and discussions.""" - try: - # Get GitHub repository information - github_info = await _find_github_repository(package_name) - if not github_info.get("repository_url"): - return {"status": "no_github_repository"} - - # Use GitHub client to get community data - async with GitHubAPIClient() as github_client: - # Get recent issues for sentiment analysis - issues_data = await github_client.get_repository_issues( - github_info["owner"], - github_info["repo"], - state="all", - limit=50 - ) - - # Analyze sentiment from issue titles and comments - sentiment_analysis = _analyze_issue_sentiment(issues_data) - - # Get repository stats for community health - repo_stats = await github_client.get_repository_stats( - github_info["owner"], - github_info["repo"] - ) - - return { - "repository": github_info["repository_url"], - "sentiment_analysis": sentiment_analysis, - "repository_stats": repo_stats, - "issues_analyzed": len(issues_data.get("issues", [])), - "analysis_timestamp": datetime.now().isoformat(), - } - - except Exception as e: - logger.warning(f"Failed to analyze GitHub sentiment for {package_name}: {e}") - return {"error": str(e), "status": "analysis_failed"} - - -async def _check_stackoverflow_mentions(package_name: str) -> Dict[str, Any]: - """Check Stack Overflow for package mentions and sentiment.""" - try: - # Use Stack Exchange API to search for package mentions - async with httpx.AsyncClient(timeout=30.0) as client: - # Search for questions mentioning the package - stackoverflow_url = "https://api.stackexchange.com/2.3/search/advanced" - params = { - "site": "stackoverflow", - "q": package_name, - "sort": "relevance", - "order": "desc", - "pagesize": 20, - } - - response = await client.get(stackoverflow_url, params=params) - - if response.status_code == 200: - data = response.json() - questions = data.get("items", []) - - # Analyze question sentiment and topics - stackoverflow_analysis = _analyze_stackoverflow_sentiment(questions, package_name) - - return { - "questions_found": len(questions), - "sentiment_analysis": stackoverflow_analysis, - "search_timestamp": datetime.now().isoformat(), - "data_source": "Stack Overflow API", - } - else: - logger.warning(f"Stack Overflow API returned status {response.status_code}") - return {"status": "api_unavailable", "questions_found": 0} - - except Exception as e: - logger.warning(f"Failed to check Stack Overflow mentions for {package_name}: {e}") - return {"error": str(e), "status": "analysis_failed"} - - -async def _analyze_pypi_downloads_as_quality_indicator(package_name: str) -> Dict[str, Any]: - """Use download statistics as a quality and adoption indicator.""" - try: - # Use existing download stats functionality - from .download_stats import get_package_download_stats - - download_stats = await get_package_download_stats(package_name) - - # Convert download numbers to quality indicators - downloads = download_stats.get("downloads", {}) - last_month = downloads.get("last_month", 0) - - # Categorize adoption level - if last_month > 1000000: - adoption_level = "very_high" - elif last_month > 100000: - adoption_level = "high" - elif last_month > 10000: - adoption_level = "moderate" - elif last_month > 1000: - adoption_level = "growing" - else: - adoption_level = "emerging" - - return { - "download_stats": downloads, - "adoption_level": adoption_level, - "quality_indicator_score": min(100, (last_month / 10000) * 10), # Scale to 0-100 - "analysis_timestamp": datetime.now().isoformat(), - } - - except Exception as e: - logger.warning(f"Failed to analyze download stats for {package_name}: {e}") - return {"error": str(e), "adoption_level": "unknown"} - - -async def _get_community_health_metrics(package_name: str) -> Dict[str, Any]: - """Get community health metrics from various sources.""" - try: - # Get GitHub repository for community metrics - github_info = await _find_github_repository(package_name) - - if github_info.get("repository_url"): - async with GitHubAPIClient() as github_client: - # Get community health data - community_profile = await github_client.get_community_profile( - github_info["owner"], - github_info["repo"] - ) - - return { - "github_community_health": community_profile, - "has_repository": True, - "repository_url": github_info["repository_url"], - } - else: - return { - "has_repository": False, - "note": "No GitHub repository found for community health analysis", - } - - except Exception as e: - logger.warning(f"Failed to get community health metrics for {package_name}: {e}") - return {"error": str(e), "has_repository": False} - - -async def _perform_sentiment_analysis(github_sentiment: Dict, stackoverflow_data: Dict) -> Dict[str, Any]: - """Perform sentiment analysis on community feedback.""" - try: - # Simple sentiment analysis based on available data - sentiment_scores = [] - - # Analyze GitHub sentiment - if github_sentiment.get("sentiment_analysis"): - github_score = github_sentiment["sentiment_analysis"].get("overall_sentiment_score", 50) - sentiment_scores.append(("github", github_score)) - - # Analyze Stack Overflow sentiment - if stackoverflow_data.get("sentiment_analysis"): - so_score = stackoverflow_data["sentiment_analysis"].get("overall_sentiment_score", 50) - sentiment_scores.append(("stackoverflow", so_score)) - - # Calculate overall sentiment - if sentiment_scores: - overall_score = sum(score for _, score in sentiment_scores) / len(sentiment_scores) - sentiment_level = "positive" if overall_score > 60 else "neutral" if overall_score > 40 else "negative" - else: - overall_score = 50 - sentiment_level = "neutral" - - return { - "overall_sentiment_score": round(overall_score, 1), - "sentiment_level": sentiment_level, - "source_scores": dict(sentiment_scores), - "confidence": "medium" if len(sentiment_scores) > 1 else "low", - "analysis_timestamp": datetime.now().isoformat(), - } - - except Exception as e: - logger.warning(f"Failed to perform sentiment analysis: {e}") - return {"error": str(e), "sentiment_level": "unknown"} - - -def _calculate_community_score( - github_sentiment: Dict, - stackoverflow_data: Dict, - quality_indicators: Dict, - community_health: Dict -) -> Dict[str, Any]: - """Calculate overall community score based on multiple factors.""" - score_components = {} - total_weight = 0 - weighted_score = 0 - - # GitHub sentiment component (weight: 30%) - if github_sentiment.get("sentiment_analysis"): - github_score = github_sentiment["sentiment_analysis"].get("overall_sentiment_score", 50) - score_components["github_sentiment"] = github_score - weighted_score += github_score * 0.3 - total_weight += 0.3 - - # Stack Overflow mentions component (weight: 20%) - if stackoverflow_data.get("sentiment_analysis"): - so_score = stackoverflow_data["sentiment_analysis"].get("overall_sentiment_score", 50) - score_components["stackoverflow_sentiment"] = so_score - weighted_score += so_score * 0.2 - total_weight += 0.2 - - # Download/adoption component (weight: 25%) - if quality_indicators.get("quality_indicator_score"): - adoption_score = quality_indicators["quality_indicator_score"] - score_components["adoption_level"] = adoption_score - weighted_score += adoption_score * 0.25 - total_weight += 0.25 - - # Community health component (weight: 25%) - if community_health.get("github_community_health"): - # Simplified health score based on repository activity - health_score = 70 # Default moderate score - score_components["community_health"] = health_score - weighted_score += health_score * 0.25 - total_weight += 0.25 - - # Calculate final score - overall_score = weighted_score / total_weight if total_weight > 0 else 50 - - # Determine community status - if overall_score >= 80: - status = "excellent" - elif overall_score >= 65: - status = "good" - elif overall_score >= 50: - status = "moderate" - elif overall_score >= 35: - status = "limited" - else: - status = "poor" - - return { - "overall_score": round(overall_score, 1), - "community_status": status, - "score_components": score_components, - "data_reliability": total_weight, - "calculation_timestamp": datetime.now().isoformat(), - } - - -def _generate_community_insights( - github_sentiment: Dict, - stackoverflow_data: Dict, - community_score: Dict, - package_metadata: Dict -) -> Dict[str, Any]: - """Generate insights from community analysis.""" - insights = { - "key_insights": [], - "recommendations": [], - "community_strengths": [], - "areas_for_improvement": [], - } - - score = community_score.get("overall_score", 50) - - # Generate insights based on score - if score >= 70: - insights["key_insights"].append("Package has strong community support and positive sentiment") - insights["community_strengths"].append("Active and engaged community") - elif score >= 50: - insights["key_insights"].append("Package has moderate community engagement") - insights["recommendations"].append("Consider increasing community engagement initiatives") - else: - insights["key_insights"].append("Package has limited community feedback available") - insights["areas_for_improvement"].append("Community engagement and visibility") - - # Add specific insights from GitHub - if github_sentiment.get("repository_stats"): - stars = github_sentiment["repository_stats"].get("stargazers_count", 0) - if stars > 1000: - insights["community_strengths"].append("High GitHub stars indicating community appreciation") - elif stars > 100: - insights["community_strengths"].append("Moderate GitHub community following") - - # Add Stack Overflow insights - if stackoverflow_data.get("questions_found", 0) > 10: - insights["community_strengths"].append("Active discussion on Stack Overflow") - elif stackoverflow_data.get("questions_found", 0) > 0: - insights["key_insights"].append("Some Stack Overflow discussion available") - else: - insights["areas_for_improvement"].append("Limited presence in developer Q&A platforms") - - return insights - - -# Helper functions for discussion management - -async def _get_package_metadata_for_discussions(package_name: str) -> Dict[str, Any]: - """Get package metadata relevant for discussion management.""" - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name) - return package_data - except Exception as e: - logger.warning(f"Failed to get package metadata for discussions: {e}") - return {} - - -async def _get_current_discussion_status(package_name: str, package_data: Dict) -> Dict[str, Any]: - """Get current discussion status from various platforms.""" - try: - # Check GitHub Discussions - github_status = await _check_github_discussions_status(package_name, package_data) - - # Check other community platforms - community_platforms = await _check_community_platforms(package_name, package_data) - - return { - "github_discussions": github_status, - "community_platforms": community_platforms, - "native_pypi_discussions": { - "available": False, - "note": "PyPI does not currently support native discussions", - }, - } - - except Exception as e: - logger.warning(f"Failed to get discussion status for {package_name}: {e}") - return {"error": str(e)} - - -async def _get_discussion_status(package_name: str, current_status: Dict) -> Dict[str, Any]: - """Get comprehensive discussion status.""" - return { - "status": "retrieved", - "current_discussion_status": current_status, - "available_platforms": [ - { - "platform": "GitHub Discussions", - "status": current_status.get("github_discussions", {}).get("enabled", False), - "url": current_status.get("github_discussions", {}).get("url"), - } - ], - "management_options": [ - "Enable/disable GitHub Discussions", - "Configure community guidelines", - "Set up moderation policies", - "Monitor discussion metrics", - ], - } - - -async def _enable_discussions(package_name: str, settings: Optional[Dict], current_status: Dict) -> Dict[str, Any]: - """Enable discussions for the package.""" - # This is a placeholder for future implementation - # Would integrate with GitHub API to enable discussions - - return { - "status": "configured", - "action": "enable_discussions", - "message": "Discussion enabling configured (requires repository admin access)", - "settings_applied": settings or {}, - "next_steps": [ - "Verify repository admin access", - "Enable GitHub Discussions in repository settings", - "Configure community guidelines", - "Set up moderation policies", - ], - "note": "Actual enabling requires repository admin privileges and GitHub API integration", - } - - -async def _disable_discussions(package_name: str, current_status: Dict) -> Dict[str, Any]: - """Disable discussions for the package.""" - return { - "status": "configured", - "action": "disable_discussions", - "message": "Discussion disabling configured (requires repository admin access)", - "current_status": current_status, - "next_steps": [ - "Verify repository admin access", - "Disable GitHub Discussions in repository settings", - "Archive existing discussions if needed", - ], - "note": "Actual disabling requires repository admin privileges", - } - - -async def _configure_discussions(package_name: str, settings: Optional[Dict], current_status: Dict) -> Dict[str, Any]: - """Configure discussion settings.""" - return { - "status": "configured", - "action": "configure_discussions", - "settings": settings or {}, - "current_status": current_status, - "configuration_options": { - "categories": ["General", "Q&A", "Ideas", "Show and Tell"], - "moderation": ["Auto-moderation", "Manual review", "Community moderation"], - "notifications": ["Email notifications", "Web notifications", "Digest emails"], - }, - "note": "Configuration changes require repository admin access", - } - - -async def _moderate_discussions(package_name: str, moderator_controls: Optional[Dict], current_status: Dict) -> Dict[str, Any]: - """Apply moderation controls to discussions.""" - return { - "status": "moderation_configured", - "action": "moderate_discussions", - "moderator_controls": moderator_controls or {}, - "current_status": current_status, - "moderation_features": { - "content_filtering": "Automatic filtering of inappropriate content", - "user_management": "Block/unblock users, assign moderator roles", - "discussion_management": "Lock/unlock, pin/unpin discussions", - "reporting": "Community reporting and review systems", - }, - "note": "Moderation requires appropriate permissions and platform integration", - } - - -async def _get_discussion_metrics(package_name: str, current_status: Dict) -> Dict[str, Any]: - """Get discussion engagement metrics.""" - try: - github_metrics = {} - if current_status.get("github_discussions", {}).get("enabled"): - github_metrics = await _get_github_discussion_metrics(package_name) - - return { - "status": "metrics_retrieved", - "github_metrics": github_metrics, - "overall_engagement": _calculate_discussion_engagement(github_metrics), - "metrics_timestamp": datetime.now().isoformat(), - } - - except Exception as e: - logger.warning(f"Failed to get discussion metrics: {e}") - return {"error": str(e), "status": "metrics_unavailable"} - - -# Helper functions for maintainer contacts - -async def _get_package_metadata_for_contacts(package_name: str) -> Dict[str, Any]: - """Get package metadata relevant for contact information.""" - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name) - - info = package_data.get("info", {}) - return { - "name": info.get("name", package_name), - "author": info.get("author", ""), - "author_email": info.get("author_email", ""), - "maintainer": info.get("maintainer", ""), - "maintainer_email": info.get("maintainer_email", ""), - "home_page": info.get("home_page", ""), - "project_urls": info.get("project_urls", {}), - "download_url": info.get("download_url", ""), - } - - except Exception as e: - logger.warning(f"Failed to get package metadata for contacts: {e}") - return {"name": package_name} - - -async def _analyze_github_maintainer_info(package_name: str) -> Dict[str, Any]: - """Analyze GitHub repository for maintainer information.""" - try: - github_info = await _find_github_repository(package_name) - if not github_info.get("repository_url"): - return {"status": "no_github_repository"} - - async with GitHubAPIClient() as github_client: - # Get repository information - repo_data = await github_client.get_repository_info( - github_info["owner"], - github_info["repo"] - ) - - # Get contributors - contributors = await github_client.get_repository_contributors( - github_info["owner"], - github_info["repo"], - limit=10 - ) - - return { - "repository": github_info["repository_url"], - "owner": github_info["owner"], - "repository_data": repo_data, - "contributors": contributors, - "primary_maintainer": repo_data.get("owner", {}), - } - - except Exception as e: - logger.warning(f"Failed to analyze GitHub maintainer info: {e}") - return {"error": str(e), "status": "analysis_failed"} - - -async def _get_support_channels(package_name: str) -> Dict[str, Any]: - """Get available support channels for the package.""" - try: - # Check common support channels - support_channels = { - "issue_tracker": None, - "documentation": None, - "community_forum": None, - "chat_channels": [], - } - - # Get GitHub repository for issue tracker - github_info = await _find_github_repository(package_name) - if github_info.get("repository_url"): - support_channels["issue_tracker"] = f"{github_info['repository_url']}/issues" - - # Check for documentation links - async with GitHubAPIClient() as github_client: - repo_data = await github_client.get_repository_info( - github_info["owner"], - github_info["repo"] - ) - - if repo_data.get("has_pages"): - support_channels["documentation"] = f"https://{github_info['owner']}.github.io/{github_info['repo']}/" - - return support_channels - - except Exception as e: - logger.warning(f"Failed to get support channels: {e}") - return {} - - -async def _get_community_channels(package_name: str) -> Dict[str, Any]: - """Get available community channels for the package.""" - try: - community_channels = { - "github_discussions": None, - "stackoverflow_tag": None, - "reddit_community": None, - "discord_server": None, - } - - # Check for GitHub Discussions - github_info = await _find_github_repository(package_name) - if github_info.get("repository_url"): - # Check if discussions are enabled - discussions_enabled = await _check_github_discussions_enabled( - github_info["owner"], - github_info["repo"] - ) - if discussions_enabled: - community_channels["github_discussions"] = f"{github_info['repository_url']}/discussions" - - # Check for Stack Overflow tag - stackoverflow_tag = await _check_stackoverflow_tag(package_name) - if stackoverflow_tag: - community_channels["stackoverflow_tag"] = f"https://stackoverflow.com/questions/tagged/{stackoverflow_tag}" - - return community_channels - - except Exception as e: - logger.warning(f"Failed to get community channels: {e}") - return {} - - -async def _get_contribution_guidelines(package_name: str) -> Dict[str, Any]: - """Get contribution guidelines for the package.""" - try: - github_info = await _find_github_repository(package_name) - if not github_info.get("repository_url"): - return {"status": "no_repository"} - - async with GitHubAPIClient() as github_client: - # Check for common contribution files - contribution_files = await github_client.get_community_files( - github_info["owner"], - github_info["repo"] - ) - - return { - "repository": github_info["repository_url"], - "contribution_files": contribution_files, - "guidelines_available": bool(contribution_files), - } - - except Exception as e: - logger.warning(f"Failed to get contribution guidelines: {e}") - return {"error": str(e)} - - -def _extract_contact_info_from_metadata( - package_metadata: Dict, - contact_types: List[str], - respect_privacy: bool -) -> Dict[str, Any]: - """Extract contact information from package metadata.""" - contact_info = { - "available_contacts": {}, - "project_urls": {}, - "privacy_compliant": respect_privacy, - } - - # Extract email contacts if requested and available - if "email" in contact_types: - if package_metadata.get("author_email") and not respect_privacy: - contact_info["available_contacts"]["author_email"] = package_metadata["author_email"] - if package_metadata.get("maintainer_email") and not respect_privacy: - contact_info["available_contacts"]["maintainer_email"] = package_metadata["maintainer_email"] - if respect_privacy: - contact_info["available_contacts"]["email_note"] = "Email addresses hidden due to privacy settings" - - # Extract project URLs - project_urls = package_metadata.get("project_urls", {}) - for url_type, url in project_urls.items(): - if _is_relevant_project_url(url_type, contact_types): - contact_info["project_urls"][url_type] = url - - # Add homepage if available - if package_metadata.get("home_page"): - contact_info["project_urls"]["Homepage"] = package_metadata["home_page"] - - return contact_info - - -async def _get_social_profiles(package_name: str, github_info: Dict, respect_privacy: bool) -> Dict[str, Any]: - """Get social media profiles for maintainers.""" - if respect_privacy: - return { - "status": "privacy_protected", - "note": "Social profiles hidden due to privacy settings", - } - - # This would require additional API integrations - # For now, return a placeholder structure - return { - "status": "limited_data", - "note": "Social profile discovery requires additional API integrations", - "available_platforms": ["GitHub", "Twitter", "LinkedIn"], - } - - -def _generate_contact_recommendations( - contact_info: Dict, - github_info: Dict, - support_channels: Dict, - community_channels: Dict -) -> List[str]: - """Generate recommendations for contacting maintainers.""" - recommendations = [] - - # Recommend GitHub issues for bugs and features - if github_info.get("repository"): - recommendations.append("Use GitHub issues for bug reports and feature requests") - - # Recommend GitHub discussions for general questions - if community_channels.get("github_discussions"): - recommendations.append("Use GitHub Discussions for general questions and community interaction") - - # Recommend proper channels based on inquiry type - recommendations.extend([ - "Check existing issues before creating new ones", - "Follow contribution guidelines when proposing changes", - "Be respectful and patient when seeking support", - "Provide detailed information when reporting issues", - ]) - - return recommendations - - -def _assess_contact_accessibility( - contact_info: Dict, - github_info: Dict, - support_channels: Dict -) -> Dict[str, Any]: - """Assess how accessible maintainers are for contact.""" - accessibility_score = 0 - factors = [] - - # GitHub repository increases accessibility - if github_info.get("repository"): - accessibility_score += 40 - factors.append("GitHub repository available") - - # Issue tracker increases accessibility - if support_channels.get("issue_tracker"): - accessibility_score += 30 - factors.append("Issue tracker available") - - # Project URLs increase accessibility - if contact_info.get("project_urls"): - accessibility_score += 20 - factors.append("Project URLs provided") - - # Documentation increases accessibility - if support_channels.get("documentation"): - accessibility_score += 10 - factors.append("Documentation available") - - # Determine accessibility level - if accessibility_score >= 80: - level = "excellent" - elif accessibility_score >= 60: - level = "good" - elif accessibility_score >= 40: - level = "moderate" - else: - level = "limited" - - return { - "accessibility_score": accessibility_score, - "accessibility_level": level, - "contributing_factors": factors, - "assessment_timestamp": datetime.now().isoformat(), - } - - -def _generate_communication_guidelines( - contact_info: Dict, - github_info: Dict, - package_metadata: Dict -) -> Dict[str, Any]: - """Generate communication guidelines for contacting maintainers.""" - return { - "best_practices": [ - "Be clear and concise in your communication", - "Provide reproducible examples for bug reports", - "Search existing issues before creating new ones", - "Follow the project's code of conduct", - "Be patient and respectful in all interactions", - ], - "communication_channels": { - "bug_reports": "GitHub Issues (if available)", - "feature_requests": "GitHub Issues or Discussions", - "general_questions": "GitHub Discussions or community forums", - "security_issues": "Private communication via email or security contact", - }, - "response_expectations": { - "bug_reports": "Response within 1-7 days (varies by project)", - "feature_requests": "Response time varies significantly", - "general_questions": "Community may respond faster than maintainers", - "note": "Response times depend on maintainer availability and project activity", - }, - } - - -# Additional helper functions for GitHub and community analysis - -async def _find_github_repository(package_name: str) -> Dict[str, Any]: - """Find GitHub repository for a package.""" - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name) - - info = package_data.get("info", {}) - - # Check project URLs for GitHub - project_urls = info.get("project_urls", {}) - for url_type, url in project_urls.items(): - if "github.com" in url.lower(): - return _parse_github_url(url) - - # Check homepage for GitHub - home_page = info.get("home_page", "") - if "github.com" in home_page.lower(): - return _parse_github_url(home_page) - - return {"status": "no_github_repository"} - - except Exception as e: - logger.warning(f"Failed to find GitHub repository: {e}") - return {"error": str(e)} - - -def _parse_github_url(url: str) -> Dict[str, str]: - """Parse GitHub URL to extract owner and repo.""" - try: - # Remove .git suffix and clean URL - clean_url = url.replace(".git", "").rstrip("/") - - # Parse URL - parsed = urlparse(clean_url) - if parsed.netloc != "github.com": - return {"status": "not_github"} - - # Extract owner and repo from path - path_parts = parsed.path.strip("/").split("/") - if len(path_parts) >= 2: - owner = path_parts[0] - repo = path_parts[1] - - return { - "repository_url": f"https://github.com/{owner}/{repo}", - "owner": owner, - "repo": repo, - } - else: - return {"status": "invalid_github_url"} - - except Exception as e: - logger.warning(f"Failed to parse GitHub URL {url}: {e}") - return {"error": str(e)} - - -def _analyze_issue_sentiment(issues_data: Dict) -> Dict[str, Any]: - """Analyze sentiment from GitHub issues.""" - issues = issues_data.get("issues", []) - if not issues: - return {"status": "no_issues", "overall_sentiment_score": 50} - - # Simple sentiment analysis based on issue characteristics - positive_indicators = 0 - negative_indicators = 0 - total_issues = len(issues) - - for issue in issues: - title = issue.get("title", "").lower() - labels = [label.get("name", "").lower() for label in issue.get("labels", [])] - state = issue.get("state", "") - - # Count positive indicators - if state == "closed": - positive_indicators += 1 - if any(label in ["enhancement", "feature", "good first issue"] for label in labels): - positive_indicators += 1 - - # Count negative indicators - if any(label in ["bug", "critical", "high priority"] for label in labels): - negative_indicators += 1 - if any(word in title for word in ["crash", "broken", "error", "fail"]): - negative_indicators += 1 - - # Calculate sentiment score (0-100) - if total_issues > 0: - positive_ratio = positive_indicators / (total_issues * 2) # Max 2 positive per issue - negative_ratio = negative_indicators / (total_issues * 2) # Max 2 negative per issue - sentiment_score = max(0, min(100, 50 + (positive_ratio - negative_ratio) * 100)) - else: - sentiment_score = 50 - - return { - "overall_sentiment_score": round(sentiment_score, 1), - "issues_analyzed": total_issues, - "positive_indicators": positive_indicators, - "negative_indicators": negative_indicators, - "sentiment_factors": { - "closed_issues": sum(1 for issue in issues if issue.get("state") == "closed"), - "open_issues": sum(1 for issue in issues if issue.get("state") == "open"), - "enhancement_requests": len([issue for issue in issues if any("enhancement" in label.get("name", "").lower() for label in issue.get("labels", []))]), - "bug_reports": len([issue for issue in issues if any("bug" in label.get("name", "").lower() for label in issue.get("labels", []))]), - }, - } - - -def _analyze_stackoverflow_sentiment(questions: List[Dict], package_name: str) -> Dict[str, Any]: - """Analyze sentiment from Stack Overflow questions.""" - if not questions: - return {"status": "no_questions", "overall_sentiment_score": 50} - - # Simple sentiment analysis based on question characteristics - positive_indicators = 0 - negative_indicators = 0 - total_questions = len(questions) - - for question in questions: - title = question.get("title", "").lower() - tags = question.get("tags", []) - score = question.get("score", 0) - is_answered = question.get("is_answered", False) - - # Count positive indicators - if is_answered: - positive_indicators += 1 - if score > 0: - positive_indicators += 1 - if any(word in title for word in ["how to", "tutorial", "example", "best practice"]): - positive_indicators += 1 - - # Count negative indicators - if any(word in title for word in ["error", "problem", "issue", "broken", "not working"]): - negative_indicators += 1 - if score < 0: - negative_indicators += 1 - - # Calculate sentiment score - if total_questions > 0: - positive_ratio = positive_indicators / (total_questions * 3) # Max 3 positive per question - negative_ratio = negative_indicators / (total_questions * 2) # Max 2 negative per question - sentiment_score = max(0, min(100, 50 + (positive_ratio - negative_ratio) * 100)) - else: - sentiment_score = 50 - - return { - "overall_sentiment_score": round(sentiment_score, 1), - "questions_analyzed": total_questions, - "positive_indicators": positive_indicators, - "negative_indicators": negative_indicators, - "question_characteristics": { - "answered_questions": sum(1 for q in questions if q.get("is_answered")), - "unanswered_questions": sum(1 for q in questions if not q.get("is_answered")), - "average_score": sum(q.get("score", 0) for q in questions) / total_questions if total_questions > 0 else 0, - }, - } - - -# Additional placeholder functions for future implementation - -async def _check_github_discussions_status(package_name: str, package_data: Dict) -> Dict[str, Any]: - """Check if GitHub Discussions are enabled for the repository.""" - github_info = await _find_github_repository(package_name) - if not github_info.get("repository_url"): - return {"enabled": False, "reason": "no_github_repository"} - - # This would require GitHub API integration to check discussions status - return { - "enabled": False, - "reason": "requires_github_api_integration", - "repository": github_info["repository_url"], - "note": "Checking discussions status requires GitHub API integration", - } - - -async def _check_community_platforms(package_name: str, package_data: Dict) -> Dict[str, Any]: - """Check other community platforms for discussions.""" - return { - "discord": {"available": False, "note": "Discord integration not implemented"}, - "reddit": {"available": False, "note": "Reddit integration not implemented"}, - "forums": {"available": False, "note": "Forum integration not implemented"}, - } - - -async def _get_github_discussion_metrics(package_name: str) -> Dict[str, Any]: - """Get GitHub discussion engagement metrics.""" - return { - "discussions_count": 0, - "participants": 0, - "note": "Requires GitHub API integration for actual metrics", - } - - -def _calculate_discussion_engagement(github_metrics: Dict) -> Dict[str, Any]: - """Calculate overall discussion engagement score.""" - return { - "engagement_score": 0, - "engagement_level": "unknown", - "note": "Engagement calculation requires actual discussion data", - } - - -async def _check_github_discussions_enabled(owner: str, repo: str) -> bool: - """Check if GitHub Discussions are enabled for a repository.""" - # This would require GitHub GraphQL API to check discussions - return False - - -async def _check_stackoverflow_tag(package_name: str) -> Optional[str]: - """Check if there's a Stack Overflow tag for the package.""" - # This would require Stack Exchange API integration - return None - - -def _is_relevant_project_url(url_type: str, contact_types: List[str]) -> bool: - """Check if a project URL is relevant for the requested contact types.""" - url_type_lower = url_type.lower() - - if "github" in contact_types and any(keyword in url_type_lower for keyword in ["repository", "source", "github"]): - return True - if "support" in contact_types and any(keyword in url_type_lower for keyword in ["support", "help", "issues", "bug"]): - return True - if "documentation" in contact_types and any(keyword in url_type_lower for keyword in ["documentation", "docs", "wiki"]): - return True - if "community" in contact_types and any(keyword in url_type_lower for keyword in ["community", "forum", "chat", "discussion"]): - return True - - return False \ No newline at end of file diff --git a/pypi_query_mcp/tools/compatibility_check.py b/pypi_query_mcp/tools/compatibility_check.py index 3422e9a..b559035 100644 --- a/pypi_query_mcp/tools/compatibility_check.py +++ b/pypi_query_mcp/tools/compatibility_check.py @@ -39,7 +39,7 @@ async def check_python_compatibility( try: async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, use_cache=use_cache) + package_data = await client.get_package_info(package_name, use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") @@ -103,7 +103,7 @@ async def get_compatible_python_versions( try: async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, use_cache=use_cache) + package_data = await client.get_package_info(package_name, use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") @@ -177,7 +177,7 @@ async def suggest_python_version_for_packages( async with PyPIClient() as client: for package_name in package_names: try: - package_data = await client.get_package_info(package_name, use_cache=use_cache) + package_data = await client.get_package_info(package_name, use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") diff --git a/pypi_query_mcp/tools/discovery.py b/pypi_query_mcp/tools/discovery.py deleted file mode 100644 index 21a7c40..0000000 --- a/pypi_query_mcp/tools/discovery.py +++ /dev/null @@ -1,1081 +0,0 @@ -"""PyPI Discovery & Monitoring Tools for tracking new releases and trending packages.""" - -import asyncio -import json -import logging -import time -from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Set -from urllib.parse import urlencode - -import httpx - -try: - from feedparser import parse as parse_feed - HAS_FEEDPARSER = True -except ImportError: - HAS_FEEDPARSER = False - def parse_feed(url_or_content): - """Fallback when feedparser is not available.""" - return {"entries": []} - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..core.pypi_client import PyPIClient - -logger = logging.getLogger(__name__) - - -class DiscoveryCache: - """Simple in-memory cache for discovery data with TTL.""" - - def __init__(self, default_ttl: int = 300): # 5 minutes default - self._cache: Dict[str, Dict[str, Any]] = {} - self._default_ttl = default_ttl - - def get(self, key: str) -> Optional[Any]: - """Get cached value if not expired.""" - if key in self._cache: - entry = self._cache[key] - if time.time() < entry["expires_at"]: - return entry["data"] - else: - del self._cache[key] - return None - - def set(self, key: str, data: Any, ttl: Optional[int] = None) -> None: - """Cache data with TTL.""" - expires_at = time.time() + (ttl or self._default_ttl) - self._cache[key] = {"data": data, "expires_at": expires_at} - - def clear(self) -> None: - """Clear all cached data.""" - self._cache.clear() - - -# Global cache instance -_discovery_cache = DiscoveryCache() - - -async def monitor_pypi_new_releases( - categories: Optional[List[str]] = None, - hours: int = 24, - min_downloads: Optional[int] = None, - maintainer_filter: Optional[str] = None, - enable_notifications: bool = False, - cache_ttl: int = 300, -) -> Dict[str, Any]: - """ - Track new releases in specified categories over a time period. - - Args: - categories: List of categories to monitor (e.g., ["web", "data-science", "ai", "cli"]) - hours: Number of hours to look back for new releases (default: 24) - min_downloads: Minimum monthly downloads to include (filters out very new packages) - maintainer_filter: Filter releases by specific maintainer names - enable_notifications: Whether to enable alert system for monitoring - cache_ttl: Cache time-to-live in seconds (default: 300) - - Returns: - Dictionary containing new releases with metadata and analysis - - Raises: - NetworkError: If unable to fetch release data - SearchError: If category filtering fails - """ - logger.info(f"Monitoring new PyPI releases for last {hours}h, categories: {categories}") - - # Generate cache key based on parameters - cache_key = f"new_releases_{categories}_{hours}_{min_downloads}_{maintainer_filter}" - cached_result = _discovery_cache.get(cache_key) - if cached_result: - logger.info("Returning cached new releases data") - return cached_result - - try: - # Use PyPI RSS feeds for recent releases - releases_data = await _fetch_recent_releases_from_rss(hours) - - # Enhance with package metadata - enhanced_releases = [] - async with PyPIClient() as client: - for release in releases_data: - try: - # Get full package info for filtering and categorization - package_info = await client.get_package_info(release["name"]) - info = package_info["info"] - - # Apply filters - if min_downloads: - # Skip packages that might not have download stats yet - try: - from .download_stats import get_package_download_stats - stats = await get_package_download_stats(release["name"], "month", use_cache=True) - if stats.get("recent_downloads", {}).get("last_month", 0) < min_downloads: - continue - except: - # If we can't get stats, assume it's a new package and include it - pass - - if maintainer_filter and maintainer_filter.lower() not in info.get("author", "").lower(): - continue - - # Categorize package - package_categories = await _categorize_package(info) - - # Apply category filter - if categories: - if not any(cat.lower() in [pc.lower() for pc in package_categories] for cat in categories): - continue - - enhanced_release = { - **release, - "summary": info.get("summary", ""), - "author": info.get("author", ""), - "license": info.get("license", ""), - "home_page": info.get("home_page", ""), - "keywords": info.get("keywords", ""), - "categories": package_categories, - "python_requires": info.get("requires_python", ""), - "project_urls": info.get("project_urls", {}), - "classifiers": info.get("classifiers", []), - } - - enhanced_releases.append(enhanced_release) - - except Exception as e: - logger.warning(f"Failed to enhance release data for {release['name']}: {e}") - # Include basic release info even if enhancement fails - enhanced_releases.append(release) - - # Sort by release time (most recent first) - enhanced_releases.sort(key=lambda x: x.get("release_time", ""), reverse=True) - - # Generate alerts if monitoring is enabled - alerts = [] - if enable_notifications: - alerts = _generate_release_alerts(enhanced_releases, categories, min_downloads) - - result = { - "new_releases": enhanced_releases, - "monitoring_period_hours": hours, - "categories_monitored": categories or ["all"], - "total_releases_found": len(enhanced_releases), - "filters_applied": { - "categories": categories, - "min_downloads": min_downloads, - "maintainer_filter": maintainer_filter, - }, - "alerts": alerts, - "monitoring_enabled": enable_notifications, - "analysis": { - "most_active_categories": _analyze_category_activity(enhanced_releases), - "trending_maintainers": _analyze_maintainer_activity(enhanced_releases), - "release_frequency": _analyze_release_frequency(enhanced_releases, hours), - }, - "timestamp": datetime.utcnow().isoformat() + "Z", - } - - # Cache the result - _discovery_cache.set(cache_key, result, cache_ttl) - - return result - - except Exception as e: - logger.error(f"Error monitoring new releases: {e}") - raise NetworkError(f"Failed to monitor new releases: {e}") from e - - -async def get_pypi_trending_today( - category: Optional[str] = None, - min_downloads: int = 1000, - limit: int = 50, - include_new_packages: bool = True, - trending_threshold: float = 1.5, -) -> Dict[str, Any]: - """ - Get packages that are trending on PyPI right now based on recent activity. - - Args: - category: Optional category filter ("web", "ai", "data-science", etc.) - min_downloads: Minimum daily downloads to be considered trending - limit: Maximum number of trending packages to return - include_new_packages: Include recently released packages in trending analysis - trending_threshold: Multiplier for determining trending status (1.5 = 50% increase) - - Returns: - Dictionary containing trending packages with activity metrics - - Raises: - SearchError: If trending analysis fails - NetworkError: If unable to fetch trending data - """ - logger.info(f"Analyzing today's PyPI trends, category: {category}, limit: {limit}") - - try: - # Get recent release activity as a proxy for trending - recent_releases = await monitor_pypi_new_releases( - categories=[category] if category else None, - hours=24, - min_downloads=min_downloads if not include_new_packages else None - ) - - # Use our existing trending functionality as a baseline - from .search import get_trending_packages - trending_base = await get_trending_packages( - category=category, - time_period="day", - limit=limit * 2 # Get more to analyze - ) - - # Combine and analyze trending signals - trending_packages = [] - seen_packages = set() - - # Add packages from recent releases (high activity signal) - for release in recent_releases["new_releases"][:limit // 2]: - if release["name"] not in seen_packages: - trending_packages.append({ - "name": release["name"], - "version": release["version"], - "summary": release.get("summary", ""), - "trending_score": 10.0, # High score for new releases - "trending_reason": "new_release", - "release_time": release.get("release_time"), - "categories": release.get("categories", []), - "download_trend": "rising", - }) - seen_packages.add(release["name"]) - - # Add packages from download-based trending - for pkg in trending_base.get("trending_packages", []): - if pkg["package"] not in seen_packages and len(trending_packages) < limit: - trending_packages.append({ - "name": pkg["package"], - "version": pkg.get("version", "unknown"), - "summary": pkg.get("summary", ""), - "trending_score": 8.0, # High score for download trending - "trending_reason": "download_surge", - "downloads": pkg.get("downloads", {}), - "download_trend": "rising", - }) - seen_packages.add(pkg["package"]) - - # Enhance with real-time popularity signals - enhanced_trending = await _enhance_trending_analysis(trending_packages, category) - - # Sort by trending score - enhanced_trending.sort(key=lambda x: x["trending_score"], reverse=True) - - result = { - "trending_today": enhanced_trending[:limit], - "analysis_date": datetime.utcnow().strftime("%Y-%m-%d"), - "category": category, - "total_trending": len(enhanced_trending), - "filters_applied": { - "category": category, - "min_downloads": min_downloads, - "include_new_packages": include_new_packages, - "trending_threshold": trending_threshold, - }, - "trending_analysis": { - "methodology": "Combined release activity and download patterns", - "signals_used": ["new_releases", "download_surges", "community_activity"], - "confidence_level": "high" if len(enhanced_trending) > 10 else "medium", - }, - "market_insights": { - "hot_categories": _analyze_trending_categories(enhanced_trending), - "emerging_patterns": _identify_emerging_patterns(enhanced_trending), - "recommendation": _generate_trending_recommendations(enhanced_trending, category), - }, - "timestamp": datetime.utcnow().isoformat() + "Z", - } - - return result - - except Exception as e: - logger.error(f"Error analyzing trending packages: {e}") - raise SearchError(f"Failed to analyze trending packages: {e}") from e - - -async def search_pypi_by_maintainer( - maintainer: str, - include_email: bool = False, - sort_by: str = "popularity", - limit: int = 50, - include_stats: bool = True, -) -> Dict[str, Any]: - """ - Find all packages maintained by a specific maintainer or organization. - - Args: - maintainer: Maintainer name or email to search for - include_email: Whether to search by email addresses too - sort_by: Sort results by ("popularity", "recent", "name", "downloads") - limit: Maximum number of packages to return - include_stats: Include download and popularity statistics - - Returns: - Dictionary containing packages by the maintainer with detailed analysis - - Raises: - InvalidPackageNameError: If maintainer name is invalid - SearchError: If maintainer search fails - """ - if not maintainer or not maintainer.strip(): - raise InvalidPackageNameError("Maintainer name cannot be empty") - - maintainer = maintainer.strip() - logger.info(f"Searching packages by maintainer: '{maintainer}'") - - try: - # Search PyPI using maintainer name in various ways - maintainer_packages = [] - - # Method 1: Search by author name - from .search import search_packages - author_results = await search_packages( - query=f"author:{maintainer}", - limit=limit * 2, - sort_by="popularity" - ) - - # Method 2: Full-text search including maintainer name - text_results = await search_packages( - query=maintainer, - limit=limit, - sort_by="popularity", - semantic_search=True - ) - - # Collect potential packages and verify maintainer - candidate_packages = set() - - # Add packages from author search - for pkg in author_results.get("packages", []): - candidate_packages.add(pkg["name"]) - - # Add packages from text search (need to verify) - for pkg in text_results.get("packages", []): - candidate_packages.add(pkg["name"]) - - # Verify maintainer for each package and collect detailed info - verified_packages = [] - async with PyPIClient() as client: - for package_name in candidate_packages: - if len(verified_packages) >= limit: - break - - try: - package_info = await client.get_package_info(package_name) - info = package_info["info"] - - # Check if maintainer matches - is_maintainer = _is_package_maintainer(info, maintainer, include_email) - - if is_maintainer: - package_data = { - "name": info["name"], - "version": info["version"], - "summary": info.get("summary", ""), - "author": info.get("author", ""), - "author_email": info.get("author_email", ""), - "maintainer": info.get("maintainer", ""), - "maintainer_email": info.get("maintainer_email", ""), - "license": info.get("license", ""), - "home_page": info.get("home_page", ""), - "project_urls": info.get("project_urls", {}), - "keywords": info.get("keywords", ""), - "classifiers": info.get("classifiers", []), - "requires_python": info.get("requires_python", ""), - "upload_time": package_info.get("releases", {}).get(info["version"], [{}])[-1].get("upload_time", ""), - } - - # Add download statistics if requested - if include_stats: - try: - from .download_stats import get_package_download_stats - stats = await get_package_download_stats(package_name, "month", use_cache=True) - package_data["download_stats"] = stats.get("recent_downloads", {}) - except: - package_data["download_stats"] = None - - # Categorize package - package_data["categories"] = await _categorize_package(info) - - verified_packages.append(package_data) - - except Exception as e: - logger.warning(f"Failed to verify maintainer for {package_name}: {e}") - continue - - # Sort packages based on sort criteria - sorted_packages = _sort_maintainer_packages(verified_packages, sort_by) - - # Analyze maintainer's package portfolio - portfolio_analysis = _analyze_maintainer_portfolio(sorted_packages, maintainer) - - result = { - "maintainer": maintainer, - "packages": sorted_packages, - "total_packages": len(sorted_packages), - "search_parameters": { - "include_email": include_email, - "sort_by": sort_by, - "limit": limit, - "include_stats": include_stats, - }, - "portfolio_analysis": portfolio_analysis, - "maintainer_profile": { - "active_categories": list(portfolio_analysis["category_distribution"].keys()), - "package_count": len(sorted_packages), - "total_downloads": portfolio_analysis.get("total_downloads", 0), - "average_quality": portfolio_analysis.get("average_quality", 0), - "activity_level": portfolio_analysis.get("activity_level", "unknown"), - }, - "timestamp": datetime.utcnow().isoformat() + "Z", - } - - return result - - except Exception as e: - logger.error(f"Error searching packages by maintainer {maintainer}: {e}") - raise SearchError(f"Failed to search by maintainer: {e}") from e - - -async def get_pypi_package_recommendations( - package_name: str, - recommendation_type: str = "similar", - limit: int = 20, - include_alternatives: bool = True, - user_context: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """ - Get PyPI's algorithm-based package recommendations and suggestions. - - Args: - package_name: Base package to get recommendations for - recommendation_type: Type of recommendations ("similar", "complementary", "upgrades", "alternatives") - limit: Maximum number of recommendations to return - include_alternatives: Include alternative packages that serve similar purposes - user_context: Optional user context for personalized recommendations (use_case, experience_level, etc.) - - Returns: - Dictionary containing personalized package recommendations with reasoning - - Raises: - PackageNotFoundError: If base package is not found - SearchError: If recommendation generation fails - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError("Package name cannot be empty") - - logger.info(f"Generating recommendations for package: '{package_name}', type: {recommendation_type}") - - try: - # Get base package information - async with PyPIClient() as client: - base_package = await client.get_package_info(package_name) - - base_info = base_package["info"] - - # Generate different types of recommendations - recommendations = [] - - if recommendation_type in ["similar", "complementary"]: - # Find packages with similar functionality - similar_packages = await _find_similar_packages(base_info, limit) - recommendations.extend(similar_packages) - - if recommendation_type in ["alternatives", "similar"]: - # Find alternative packages - from .search import find_alternatives - alternatives_result = await find_alternatives( - package_name=package_name, - limit=limit, - include_similar=True - ) - - for alt in alternatives_result["alternatives"]: - recommendations.append({ - "name": alt["name"], - "type": "alternative", - "reason": "Similar functionality and purpose", - "summary": alt.get("summary", ""), - "confidence": 0.8, - "metadata": alt, - }) - - if recommendation_type == "complementary": - # Find packages that work well together - complementary = await _find_complementary_packages(base_info, limit) - recommendations.extend(complementary) - - if recommendation_type == "upgrades": - # Find newer or better versions/alternatives - upgrades = await _find_upgrade_recommendations(base_info, limit) - recommendations.extend(upgrades) - - # Apply user context if provided - if user_context: - recommendations = _personalize_recommendations(recommendations, user_context) - - # Remove duplicates and limit results - seen_packages = set() - filtered_recommendations = [] - for rec in recommendations: - if rec["name"] not in seen_packages and rec["name"] != package_name: - filtered_recommendations.append(rec) - seen_packages.add(rec["name"]) - if len(filtered_recommendations) >= limit: - break - - # Sort by confidence score - filtered_recommendations.sort(key=lambda x: x.get("confidence", 0), reverse=True) - - # Enhance recommendations with additional data - enhanced_recommendations = await _enhance_recommendations(filtered_recommendations) - - result = { - "base_package": { - "name": package_name, - "version": base_info["version"], - "summary": base_info.get("summary", ""), - "categories": await _categorize_package(base_info), - }, - "recommendations": enhanced_recommendations, - "recommendation_type": recommendation_type, - "total_recommendations": len(enhanced_recommendations), - "parameters": { - "limit": limit, - "include_alternatives": include_alternatives, - "user_context": user_context, - }, - "algorithm_insights": { - "methodology": "Hybrid content-based and collaborative filtering", - "signals_used": ["keywords", "categories", "dependencies", "usage_patterns"], - "personalization_applied": user_context is not None, - }, - "recommendation_summary": { - "by_type": _summarize_recommendations_by_type(enhanced_recommendations), - "confidence_distribution": _analyze_confidence_distribution(enhanced_recommendations), - "category_coverage": _analyze_category_coverage(enhanced_recommendations), - }, - "timestamp": datetime.utcnow().isoformat() + "Z", - } - - return result - - except Exception as e: - logger.error(f"Error generating recommendations for {package_name}: {e}") - raise SearchError(f"Failed to generate recommendations: {e}") from e - - -# Helper functions for internal processing - -async def _fetch_recent_releases_from_rss(hours: int) -> List[Dict[str, Any]]: - """Fetch recent releases from PyPI RSS feeds.""" - releases = [] - - try: - # PyPI RSS feed for recent updates - rss_url = "https://pypi.org/rss/updates.xml" - - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.get(rss_url) - response.raise_for_status() - - # Parse RSS feed - if not HAS_FEEDPARSER: - logger.warning("feedparser not available - RSS monitoring limited") - return { - "new_releases": [], - "time_period": f"last {hours} hours", - "note": "RSS parsing unavailable - feedparser dependency missing", - "fallback_used": True, - "total_found": 0, - "category": category, - "timestamp": datetime.utcnow().isoformat() - } - - feed = parse_feed(response.content) - cutoff_time = datetime.utcnow() - timedelta(hours=hours) - - for entry in feed.entries: - # Parse release time - try: - release_time = datetime.strptime(entry.published, "%a, %d %b %Y %H:%M:%S %Z") - if release_time < cutoff_time: - continue - except: - # If we can't parse time, include it anyway - release_time = None - - # Extract package name and version from title - title_parts = entry.title.split() - if len(title_parts) >= 2: - package_name = title_parts[0] - version = title_parts[1] - - releases.append({ - "name": package_name, - "version": version, - "release_time": release_time.isoformat() + "Z" if release_time else None, - "description": entry.description, - "link": entry.link, - }) - - except Exception as e: - logger.warning(f"Failed to fetch RSS releases: {e}") - # Fallback: return empty list but don't fail - - return releases - - -async def _categorize_package(package_info: Dict[str, Any]) -> List[str]: - """Categorize a package based on its metadata.""" - categories = [] - - # Extract text for analysis - text_data = " ".join([ - package_info.get("summary") or "", - package_info.get("description") or "", - package_info.get("keywords") or "", - ]).lower() - - # Classifier-based categorization - classifiers = package_info.get("classifiers", []) - for classifier in classifiers: - if "Topic ::" in classifier: - topic = classifier.split("Topic ::")[-1].strip() - if topic: - categories.append(topic.lower().replace(" ", "-")) - - # Content-based categorization - category_keywords = { - "web": ["web", "http", "flask", "django", "fastapi", "server", "wsgi", "asgi", "rest", "api"], - "data-science": ["data", "science", "analytics", "pandas", "numpy", "machine learning", "ml", "ai"], - "database": ["database", "sql", "orm", "sqlite", "postgres", "mysql", "mongodb"], - "testing": ["test", "testing", "pytest", "unittest", "mock", "coverage"], - "cli": ["command", "cli", "terminal", "argparse", "click", "console"], - "security": ["security", "crypto", "encryption", "ssl", "auth", "oauth"], - "networking": ["network", "socket", "requests", "urllib", "http", "tcp", "udp"], - "gui": ["gui", "interface", "tkinter", "qt", "desktop", "ui"], - "dev-tools": ["development", "build", "deploy", "packaging", "tools"], - "ai": ["artificial intelligence", "ai", "neural", "deep learning", "tensorflow", "pytorch"], - } - - for category, keywords in category_keywords.items(): - if any(keyword in text_data for keyword in keywords): - if category not in categories: - categories.append(category) - - return categories if categories else ["general"] - - -def _generate_release_alerts(releases: List[Dict[str, Any]], categories: Optional[List[str]], min_downloads: Optional[int]) -> List[Dict[str, Any]]: - """Generate alerts for monitored releases.""" - alerts = [] - - # Alert for high-activity categories - if categories: - category_counts = {} - for release in releases: - for cat in release.get("categories", []): - category_counts[cat] = category_counts.get(cat, 0) + 1 - - for cat, count in category_counts.items(): - if count >= 5: # 5+ releases in category - alerts.append({ - "type": "high_activity", - "category": cat, - "message": f"High activity in {cat} category: {count} new releases", - "severity": "info", - "package_count": count, - }) - - # Alert for notable new packages - for release in releases: - if "ai" in release.get("categories", []) or "machine-learning" in release.get("categories", []): - alerts.append({ - "type": "trending_category", - "package": release["name"], - "message": f"New AI/ML package released: {release['name']}", - "severity": "info", - "category": "ai", - }) - - return alerts - - -def _analyze_category_activity(releases: List[Dict[str, Any]]) -> Dict[str, int]: - """Analyze release activity by category.""" - category_counts = {} - for release in releases: - for category in release.get("categories", []): - category_counts[category] = category_counts.get(category, 0) + 1 - - # Return top 5 most active categories - return dict(sorted(category_counts.items(), key=lambda x: x[1], reverse=True)[:5]) - - -def _analyze_maintainer_activity(releases: List[Dict[str, Any]]) -> Dict[str, int]: - """Analyze release activity by maintainer.""" - maintainer_counts = {} - for release in releases: - author = release.get("author", "").strip() - if author: - maintainer_counts[author] = maintainer_counts.get(author, 0) + 1 - - # Return top 5 most active maintainers - return dict(sorted(maintainer_counts.items(), key=lambda x: x[1], reverse=True)[:5]) - - -def _analyze_release_frequency(releases: List[Dict[str, Any]], hours: int) -> Dict[str, Any]: - """Analyze release frequency patterns.""" - total_releases = len(releases) - releases_per_hour = total_releases / hours if hours > 0 else 0 - - return { - "total_releases": total_releases, - "releases_per_hour": round(releases_per_hour, 2), - "activity_level": "high" if releases_per_hour > 10 else "medium" if releases_per_hour > 2 else "low", - } - - -async def _enhance_trending_analysis(packages: List[Dict[str, Any]], category: Optional[str]) -> List[Dict[str, Any]]: - """Enhance trending analysis with additional signals.""" - enhanced = [] - - for pkg in packages: - enhanced_pkg = pkg.copy() - - # Add trending signals - if "new_release" in pkg.get("trending_reason", ""): - enhanced_pkg["trending_score"] += 2.0 # Boost for new releases - - # Category relevance boost - if category and category.lower() in [c.lower() for c in pkg.get("categories", [])]: - enhanced_pkg["trending_score"] += 1.0 - - # Add confidence level - score = enhanced_pkg["trending_score"] - if score >= 9.0: - enhanced_pkg["confidence"] = "high" - elif score >= 7.0: - enhanced_pkg["confidence"] = "medium" - else: - enhanced_pkg["confidence"] = "low" - - enhanced.append(enhanced_pkg) - - return enhanced - - -def _analyze_trending_categories(packages: List[Dict[str, Any]]) -> Dict[str, int]: - """Analyze which categories are trending.""" - category_counts = {} - for pkg in packages: - for category in pkg.get("categories", []): - category_counts[category] = category_counts.get(category, 0) + 1 - - return dict(sorted(category_counts.items(), key=lambda x: x[1], reverse=True)[:5]) - - -def _identify_emerging_patterns(packages: List[Dict[str, Any]]) -> List[str]: - """Identify emerging patterns in trending packages.""" - patterns = [] - - # Analyze package names and descriptions for patterns - names = [pkg["name"].lower() for pkg in packages] - - # Look for common prefixes/suffixes - if sum(1 for name in names if "ai" in name) >= 3: - patterns.append("AI-related packages are trending") - - if sum(1 for name in names if any(web in name for web in ["api", "web", "http"])) >= 3: - patterns.append("Web development packages are popular") - - if sum(1 for name in names if "async" in name) >= 2: - patterns.append("Async/concurrent programming tools are emerging") - - return patterns - - -def _generate_trending_recommendations(packages: List[Dict[str, Any]], category: Optional[str]) -> str: - """Generate recommendations based on trending analysis.""" - if not packages: - return "No significant trending packages found at this time." - - top_package = packages[0] - recommendations = [ - f"Consider exploring '{top_package['name']}' - it's showing strong trending signals." - ] - - if category: - category_packages = [p for p in packages if category.lower() in [c.lower() for c in p.get("categories", [])]] - if category_packages: - recommendations.append(f"The {category} category is particularly active today.") - - return " ".join(recommendations) - - -def _is_package_maintainer(package_info: Dict[str, Any], maintainer: str, include_email: bool) -> bool: - """Check if the given maintainer matches the package maintainer.""" - maintainer_lower = maintainer.lower() - - # Check author field - author = package_info.get("author", "").lower() - if maintainer_lower in author: - return True - - # Check maintainer field - package_maintainer = package_info.get("maintainer", "").lower() - if maintainer_lower in package_maintainer: - return True - - # Check email fields if enabled - if include_email: - author_email = package_info.get("author_email", "").lower() - maintainer_email = package_info.get("maintainer_email", "").lower() - - if maintainer_lower in author_email or maintainer_lower in maintainer_email: - return True - - return False - - -def _sort_maintainer_packages(packages: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]: - """Sort maintainer packages by specified criteria.""" - if sort_by == "popularity": - # Sort by download stats if available - return sorted( - packages, - key=lambda x: x.get("download_stats", {}).get("last_month", 0), - reverse=True - ) - elif sort_by == "recent": - # Sort by upload time - return sorted( - packages, - key=lambda x: x.get("upload_time", ""), - reverse=True - ) - elif sort_by == "name": - # Sort alphabetically - return sorted(packages, key=lambda x: x["name"].lower()) - elif sort_by == "downloads": - # Sort by downloads - return sorted( - packages, - key=lambda x: x.get("download_stats", {}).get("last_month", 0), - reverse=True - ) - else: - return packages - - -def _analyze_maintainer_portfolio(packages: List[Dict[str, Any]], maintainer: str) -> Dict[str, Any]: - """Analyze a maintainer's package portfolio.""" - total_downloads = 0 - categories = {} - upload_times = [] - - for pkg in packages: - # Count downloads - downloads = pkg.get("download_stats", {}).get("last_month", 0) - if downloads: - total_downloads += downloads - - # Count categories - for category in pkg.get("categories", []): - categories[category] = categories.get(category, 0) + 1 - - # Collect upload times - if pkg.get("upload_time"): - upload_times.append(pkg["upload_time"]) - - # Determine activity level - if len(packages) >= 10: - activity_level = "high" - elif len(packages) >= 3: - activity_level = "medium" - else: - activity_level = "low" - - return { - "total_downloads": total_downloads, - "category_distribution": dict(sorted(categories.items(), key=lambda x: x[1], reverse=True)), - "activity_level": activity_level, - "package_count": len(packages), - "average_quality": 8.0, # Placeholder - could be enhanced with quality metrics - } - - -async def _find_similar_packages(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: - """Find packages similar to the base package.""" - similar_packages = [] - - # Use keywords and categories for similarity - keywords = (base_info.get("keywords") or "").split() - summary = base_info.get("summary", "") - - if keywords or summary: - from .search import search_packages - search_query = " ".join(keywords[:3]) + " " + summary[:50] - - results = await search_packages( - query=search_query, - limit=limit, - semantic_search=True, - sort_by="relevance" - ) - - for pkg in results.get("packages", []): - similar_packages.append({ - "name": pkg["name"], - "type": "similar", - "reason": "Similar keywords and functionality", - "summary": pkg.get("summary", ""), - "confidence": 0.7, - "metadata": pkg, - }) - - return similar_packages - - -async def _find_complementary_packages(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: - """Find packages that complement the base package.""" - complementary = [] - - # Map packages to common complementary packages - package_name = base_info["name"].lower() - - complement_map = { - "flask": ["flask-sqlalchemy", "flask-login", "flask-wtf"], - "django": ["djangorestframework", "django-cors-headers", "celery"], - "fastapi": ["uvicorn", "pydantic", "sqlalchemy"], - "pandas": ["numpy", "matplotlib", "seaborn", "jupyter"], - "numpy": ["scipy", "matplotlib", "pandas"], - "requests": ["urllib3", "httpx", "aiohttp"], - } - - complements = complement_map.get(package_name, []) - - for comp_name in complements[:limit]: - complementary.append({ - "name": comp_name, - "type": "complementary", - "reason": f"Commonly used with {package_name}", - "confidence": 0.8, - }) - - return complementary - - -async def _find_upgrade_recommendations(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: - """Find upgrade recommendations for the base package.""" - upgrades = [] - - # Suggest newer alternatives for older packages - package_name = base_info["name"].lower() - - upgrade_map = { - "urllib": ["requests", "httpx"], - "optparse": ["argparse", "click"], - "unittest": ["pytest"], - "PIL": ["pillow"], - } - - upgrade_suggestions = upgrade_map.get(package_name, []) - - for upgrade_name in upgrade_suggestions[:limit]: - upgrades.append({ - "name": upgrade_name, - "type": "upgrade", - "reason": f"Modern alternative to {package_name}", - "confidence": 0.9, - }) - - return upgrades - - -def _personalize_recommendations(recommendations: List[Dict[str, Any]], user_context: Dict[str, Any]) -> List[Dict[str, Any]]: - """Personalize recommendations based on user context.""" - experience_level = user_context.get("experience_level", "intermediate") - use_case = user_context.get("use_case", "") - - # Adjust confidence based on experience level - for rec in recommendations: - if experience_level == "beginner": - # Prefer well-documented, stable packages - if "flask" in rec["name"].lower() or "requests" in rec["name"].lower(): - rec["confidence"] += 0.1 - elif experience_level == "advanced": - # Prefer cutting-edge packages - if "async" in rec["name"].lower() or "fast" in rec["name"].lower(): - rec["confidence"] += 0.1 - - return recommendations - - -async def _enhance_recommendations(recommendations: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Enhance recommendations with additional metadata.""" - enhanced = [] - - async with PyPIClient() as client: - for rec in recommendations: - try: - package_info = await client.get_package_info(rec["name"]) - info = package_info["info"] - - enhanced_rec = rec.copy() - enhanced_rec.update({ - "version": info["version"], - "summary": info.get("summary", ""), - "license": info.get("license", ""), - "requires_python": info.get("requires_python", ""), - "categories": await _categorize_package(info), - }) - - enhanced.append(enhanced_rec) - - except Exception as e: - logger.warning(f"Failed to enhance recommendation for {rec['name']}: {e}") - enhanced.append(rec) - - return enhanced - - -def _summarize_recommendations_by_type(recommendations: List[Dict[str, Any]]) -> Dict[str, int]: - """Summarize recommendations by type.""" - type_counts = {} - for rec in recommendations: - rec_type = rec.get("type", "unknown") - type_counts[rec_type] = type_counts.get(rec_type, 0) + 1 - - return type_counts - - -def _analyze_confidence_distribution(recommendations: List[Dict[str, Any]]) -> Dict[str, int]: - """Analyze confidence score distribution.""" - distribution = {"high": 0, "medium": 0, "low": 0} - - for rec in recommendations: - confidence = rec.get("confidence", 0) - if confidence >= 0.8: - distribution["high"] += 1 - elif confidence >= 0.6: - distribution["medium"] += 1 - else: - distribution["low"] += 1 - - return distribution - - -def _analyze_category_coverage(recommendations: List[Dict[str, Any]]) -> List[str]: - """Analyze category coverage in recommendations.""" - categories = set() - for rec in recommendations: - categories.update(rec.get("categories", [])) - - return list(categories) \ No newline at end of file diff --git a/pypi_query_mcp/tools/health_scorer.py b/pypi_query_mcp/tools/health_scorer.py deleted file mode 100644 index 838abe7..0000000 --- a/pypi_query_mcp/tools/health_scorer.py +++ /dev/null @@ -1,974 +0,0 @@ -"""Package health scoring and quality assessment tools for PyPI packages.""" - -import asyncio -import logging -import re -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from urllib.parse import urlparse - -import httpx - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..core.pypi_client import PyPIClient - -logger = logging.getLogger(__name__) - - -class PackageHealthScorer: - """Comprehensive health and quality scorer for PyPI packages.""" - - def __init__(self): - self.timeout = 30.0 - - # Health scoring weights (total = 100) - self.weights = { - "maintenance": 25, # Maintenance indicators - "popularity": 20, # Download stats, stars, usage - "documentation": 15, # Documentation quality - "testing": 15, # Testing and CI indicators - "security": 10, # Security practices - "compatibility": 10, # Python version support - "metadata": 5, # Metadata completeness - } - - # Quality metrics thresholds - self.thresholds = { - "downloads_monthly_excellent": 1000000, - "downloads_monthly_good": 100000, - "downloads_monthly_fair": 10000, - "version_age_days_fresh": 90, - "version_age_days_good": 365, - "version_age_days_stale": 730, - "python_versions_excellent": 4, - "python_versions_good": 3, - "python_versions_fair": 2, - } - - async def assess_package_health( - self, - package_name: str, - version: Optional[str] = None, - include_github_metrics: bool = True - ) -> Dict[str, Any]: - """ - Assess comprehensive health and quality of a PyPI package. - - Args: - package_name: Name of the package to assess - version: Specific version to assess (optional) - include_github_metrics: Whether to fetch GitHub repository metrics - - Returns: - Dictionary containing health assessment results - """ - logger.info(f"Starting health assessment for package: {package_name}") - - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, version) - - package_version = version or package_data["info"]["version"] - - # Run parallel health assessments - assessment_tasks = [ - self._assess_maintenance_health(package_data), - self._assess_popularity_metrics(package_data), - self._assess_documentation_quality(package_data), - self._assess_testing_indicators(package_data), - self._assess_security_practices(package_data), - self._assess_compatibility_support(package_data), - self._assess_metadata_completeness(package_data), - ] - - if include_github_metrics: - github_url = self._extract_github_url(package_data) - if github_url: - assessment_tasks.append(self._fetch_github_metrics(github_url)) - else: - assessment_tasks.append(asyncio.create_task(self._empty_github_metrics())) - else: - assessment_tasks.append(asyncio.create_task(self._empty_github_metrics())) - - results = await asyncio.gather(*assessment_tasks, return_exceptions=True) - - # Unpack results - (maintenance, popularity, documentation, testing, - security, compatibility, metadata, github_metrics) = results - - # Handle exceptions - if isinstance(github_metrics, Exception): - github_metrics = self._empty_github_metrics() - - # Calculate overall health score - health_scores = { - "maintenance": maintenance.get("score", 0) if not isinstance(maintenance, Exception) else 0, - "popularity": popularity.get("score", 0) if not isinstance(popularity, Exception) else 0, - "documentation": documentation.get("score", 0) if not isinstance(documentation, Exception) else 0, - "testing": testing.get("score", 0) if not isinstance(testing, Exception) else 0, - "security": security.get("score", 0) if not isinstance(security, Exception) else 0, - "compatibility": compatibility.get("score", 0) if not isinstance(compatibility, Exception) else 0, - "metadata": metadata.get("score", 0) if not isinstance(metadata, Exception) else 0, - } - - overall_score = sum( - health_scores[category] * (self.weights[category] / 100) - for category in health_scores - ) - - health_level = self._calculate_health_level(overall_score) - - # Generate recommendations - recommendations = self._generate_health_recommendations( - health_scores, maintenance, popularity, documentation, - testing, security, compatibility, metadata, github_metrics - ) - - return { - "package": package_name, - "version": package_version, - "assessment_timestamp": datetime.now(timezone.utc).isoformat(), - "overall_health": { - "score": round(overall_score, 2), - "level": health_level, - "max_score": 100, - }, - "category_scores": health_scores, - "detailed_assessment": { - "maintenance": maintenance if not isinstance(maintenance, Exception) else {"score": 0, "indicators": [], "issues": [str(maintenance)]}, - "popularity": popularity if not isinstance(popularity, Exception) else {"score": 0, "metrics": {}, "issues": [str(popularity)]}, - "documentation": documentation if not isinstance(documentation, Exception) else {"score": 0, "indicators": [], "issues": [str(documentation)]}, - "testing": testing if not isinstance(testing, Exception) else {"score": 0, "indicators": [], "issues": [str(testing)]}, - "security": security if not isinstance(security, Exception) else {"score": 0, "practices": [], "issues": [str(security)]}, - "compatibility": compatibility if not isinstance(compatibility, Exception) else {"score": 0, "support": [], "issues": [str(compatibility)]}, - "metadata": metadata if not isinstance(metadata, Exception) else {"score": 0, "completeness": {}, "issues": [str(metadata)]}, - "github_metrics": github_metrics, - }, - "recommendations": recommendations, - "health_summary": { - "strengths": self._identify_strengths(health_scores), - "weaknesses": self._identify_weaknesses(health_scores), - "improvement_priority": self._prioritize_improvements(health_scores), - } - } - - except Exception as e: - logger.error(f"Health assessment failed for {package_name}: {e}") - raise SearchError(f"Health assessment failed: {e}") from e - - async def _assess_maintenance_health(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess package maintenance health indicators.""" - info = package_data.get("info", {}) - releases = package_data.get("releases", {}) - - score = 0 - indicators = [] - issues = [] - - # Check release frequency - if releases: - release_dates = [] - for version_releases in releases.values(): - for release in version_releases: - upload_time = release.get("upload_time_iso_8601") - if upload_time: - try: - release_dates.append(datetime.fromisoformat(upload_time.replace('Z', '+00:00'))) - except: - pass - - if release_dates: - release_dates.sort(reverse=True) - latest_release = release_dates[0] - days_since_release = (datetime.now(timezone.utc) - latest_release).days - - if days_since_release <= self.thresholds["version_age_days_fresh"]: - score += 25 - indicators.append(f"Recent release ({days_since_release} days ago)") - elif days_since_release <= self.thresholds["version_age_days_good"]: - score += 20 - indicators.append(f"Moderately recent release ({days_since_release} days ago)") - elif days_since_release <= self.thresholds["version_age_days_stale"]: - score += 10 - indicators.append(f"Older release ({days_since_release} days ago)") - else: - issues.append(f"Very old release ({days_since_release} days ago)") - - # Check release consistency (last 5 releases) - if len(release_dates) >= 5: - recent_releases = release_dates[:5] - intervals = [] - for i in range(len(recent_releases) - 1): - interval = (recent_releases[i] - recent_releases[i + 1]).days - intervals.append(interval) - - avg_interval = sum(intervals) / len(intervals) - if avg_interval <= 180: # Releases every 6 months or less - score += 15 - indicators.append(f"Regular releases (avg {avg_interval:.0f} days)") - elif avg_interval <= 365: - score += 10 - indicators.append(f"Periodic releases (avg {avg_interval:.0f} days)") - else: - issues.append(f"Infrequent releases (avg {avg_interval:.0f} days)") - else: - issues.append("No release history available") - - # Check for development indicators - if "dev" in info.get("version", "").lower() or "alpha" in info.get("version", "").lower(): - issues.append("Development/alpha version") - elif "beta" in info.get("version", "").lower(): - score += 5 - indicators.append("Beta version (active development)") - else: - score += 10 - indicators.append("Stable version") - - # Check for author/maintainer info - if info.get("author") or info.get("maintainer"): - score += 10 - indicators.append("Active maintainer information") - else: - issues.append("No maintainer information") - - return { - "score": min(score, 100), - "indicators": indicators, - "issues": issues, - "metrics": { - "days_since_last_release": days_since_release if 'days_since_release' in locals() else None, - "total_releases": len(releases), - } - } - - async def _assess_popularity_metrics(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess package popularity and usage metrics.""" - info = package_data.get("info", {}) - - score = 0 - metrics = {} - - # Estimate download popularity (since we don't have direct access) - # Use proxy indicators: project URLs, description length, classifiers - - # Check for GitHub stars indicator - project_urls = info.get("project_urls", {}) or {} - github_url = None - for key, url in project_urls.items(): - if "github.com" in (url or "").lower(): - github_url = url - break - - if not github_url: - home_page = info.get("home_page", "") - if "github.com" in home_page: - github_url = home_page - - if github_url: - score += 15 - metrics["has_github_repo"] = True - else: - metrics["has_github_repo"] = False - - # Check description quality as popularity indicator - description = info.get("description", "") or "" - summary = info.get("summary", "") or "" - - if len(description) > 1000: - score += 20 - metrics["description_quality"] = "excellent" - elif len(description) > 500: - score += 15 - metrics["description_quality"] = "good" - elif len(description) > 100: - score += 10 - metrics["description_quality"] = "fair" - else: - metrics["description_quality"] = "poor" - - # Check for comprehensive metadata (popularity indicator) - if info.get("keywords"): - score += 10 - if len(info.get("classifiers", [])) > 5: - score += 15 - if info.get("project_urls") and len(info.get("project_urls", {})) > 2: - score += 10 - - # Check for documentation links - docs_indicators = ["documentation", "docs", "readthedocs", "github.io"] - has_docs = any( - any(indicator in (url or "").lower() for indicator in docs_indicators) - for url in project_urls.values() - ) - if has_docs: - score += 15 - metrics["has_documentation"] = True - else: - metrics["has_documentation"] = False - - # Check for community indicators - community_urls = ["issues", "bug", "tracker", "discussion", "forum"] - has_community = any( - any(indicator in key.lower() for indicator in community_urls) - for key in project_urls.keys() - ) - if has_community: - score += 15 - metrics["has_community_links"] = True - else: - metrics["has_community_links"] = False - - return { - "score": min(score, 100), - "metrics": metrics, - } - - async def _assess_documentation_quality(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess documentation quality indicators.""" - info = package_data.get("info", {}) - - score = 0 - indicators = [] - issues = [] - - # Check description completeness - description = info.get("description", "") or "" - summary = info.get("summary", "") or "" - - if len(description) > 2000: - score += 30 - indicators.append("Comprehensive description") - elif len(description) > 1000: - score += 25 - indicators.append("Good description length") - elif len(description) > 500: - score += 15 - indicators.append("Adequate description") - elif len(description) > 100: - score += 10 - indicators.append("Basic description") - else: - issues.append("Very short or missing description") - - # Check for README indicators in description - readme_indicators = ["## ", "### ", "```", "# Installation", "# Usage", "# Examples"] - if any(indicator in description for indicator in readme_indicators): - score += 20 - indicators.append("Structured documentation (README-style)") - - # Check for documentation URLs - project_urls = info.get("project_urls", {}) or {} - docs_urls = [] - for key, url in project_urls.items(): - if any(term in key.lower() for term in ["doc", "guide", "manual", "wiki"]): - docs_urls.append(url) - - if docs_urls: - score += 25 - indicators.append(f"Documentation links ({len(docs_urls)} found)") - else: - issues.append("No dedicated documentation links") - - # Check for example code in description - if "```" in description or " " in description: # Code blocks - score += 15 - indicators.append("Contains code examples") - - # Check for installation instructions - install_keywords = ["install", "pip install", "setup.py", "requirements"] - if any(keyword in description.lower() for keyword in install_keywords): - score += 10 - indicators.append("Installation instructions provided") - else: - issues.append("No clear installation instructions") - - return { - "score": min(score, 100), - "indicators": indicators, - "issues": issues, - } - - async def _assess_testing_indicators(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess testing and CI/CD indicators.""" - info = package_data.get("info", {}) - - score = 0 - indicators = [] - issues = [] - - # Check for testing-related classifiers - classifiers = info.get("classifiers", []) - testing_classifiers = [c for c in classifiers if "testing" in c.lower()] - if testing_classifiers: - score += 15 - indicators.append("Testing framework classifiers") - - # Check for CI/CD indicators in URLs - project_urls = info.get("project_urls", {}) or {} - ci_indicators = ["travis", "circleci", "appveyor", "azure", "github", "actions", "ci", "build"] - ci_urls = [] - for key, url in project_urls.items(): - if any(indicator in key.lower() or indicator in (url or "").lower() for indicator in ci_indicators): - ci_urls.append(key) - - if ci_urls: - score += 25 - indicators.append(f"CI/CD indicators ({len(ci_urls)} found)") - - # Check description for testing mentions - description = (info.get("description", "") or "").lower() - testing_keywords = ["test", "pytest", "unittest", "nose", "coverage", "tox", "ci/cd", "continuous integration"] - testing_mentions = [kw for kw in testing_keywords if kw in description] - - if testing_mentions: - score += 20 - indicators.append(f"Testing framework mentions ({len(testing_mentions)} found)") - else: - issues.append("No testing framework mentions") - - # Check for test dependencies (common patterns) - requires_dist = info.get("requires_dist", []) or [] - test_deps = [] - for req in requires_dist: - req_lower = req.lower() - if any(test_pkg in req_lower for test_pkg in ["pytest", "unittest", "nose", "coverage", "tox", "test"]): - test_deps.append(req.split()[0]) - - if test_deps: - score += 20 - indicators.append(f"Test dependencies ({len(test_deps)} found)") - else: - issues.append("No test dependencies found") - - # Check for badges (often indicate CI/testing) - badge_indicators = ["[![", "https://img.shields.io", "badge", "build status", "coverage"] - if any(indicator in description for indicator in badge_indicators): - score += 20 - indicators.append("Status badges (likely CI integration)") - - return { - "score": min(score, 100), - "indicators": indicators, - "issues": issues, - } - - async def _assess_security_practices(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess security practices and indicators.""" - info = package_data.get("info", {}) - - score = 0 - practices = [] - issues = [] - - # Check for security-related URLs - project_urls = info.get("project_urls", {}) or {} - security_urls = [] - for key, url in project_urls.items(): - if any(term in key.lower() for term in ["security", "vulnerability", "report", "bug"]): - security_urls.append(key) - - if security_urls: - score += 25 - practices.append(f"Security reporting channels ({len(security_urls)} found)") - else: - issues.append("No security reporting channels") - - # Check for HTTPS URLs - https_urls = [url for url in project_urls.values() if (url or "").startswith("https://")] - if len(https_urls) == len([url for url in project_urls.values() if url]): - score += 15 - practices.append("All URLs use HTTPS") - elif https_urls: - score += 10 - practices.append("Some URLs use HTTPS") - else: - issues.append("No HTTPS URLs found") - - # Check for security mentions in description - description = (info.get("description", "") or "").lower() - security_keywords = ["security", "secure", "vulnerability", "encryption", "authentication", "authorization"] - security_mentions = [kw for kw in security_keywords if kw in description] - - if security_mentions: - score += 20 - practices.append(f"Security awareness ({len(security_mentions)} mentions)") - - # Check for license (security practice) - if info.get("license") or any("license" in c.lower() for c in info.get("classifiers", [])): - score += 15 - practices.append("Clear license information") - else: - issues.append("No clear license information") - - # Check for author/maintainer email (security contact) - if info.get("author_email") or info.get("maintainer_email"): - score += 10 - practices.append("Maintainer contact information") - else: - issues.append("No maintainer contact information") - - # Check for requirements specification (dependency security) - requires_dist = info.get("requires_dist", []) - if requires_dist: - # Check for version pinning (security practice) - pinned_deps = [req for req in requires_dist if any(op in req for op in ["==", ">=", "~="])] - if pinned_deps: - score += 15 - practices.append(f"Version-pinned dependencies ({len(pinned_deps)}/{len(requires_dist)})") - else: - issues.append("No version-pinned dependencies") - - return { - "score": min(score, 100), - "practices": practices, - "issues": issues, - } - - async def _assess_compatibility_support(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess Python version and platform compatibility.""" - info = package_data.get("info", {}) - - score = 0 - support = [] - issues = [] - - # Check Python version support from classifiers - classifiers = info.get("classifiers", []) - python_versions = [] - for classifier in classifiers: - if "Programming Language :: Python ::" in classifier: - version_part = classifier.split("::")[-1].strip() - if re.match(r'^\d+\.\d+$', version_part): # Like "3.8", "3.9" - python_versions.append(version_part) - - if len(python_versions) >= self.thresholds["python_versions_excellent"]: - score += 30 - support.append(f"Excellent Python version support ({len(python_versions)} versions)") - elif len(python_versions) >= self.thresholds["python_versions_good"]: - score += 25 - support.append(f"Good Python version support ({len(python_versions)} versions)") - elif len(python_versions) >= self.thresholds["python_versions_fair"]: - score += 15 - support.append(f"Fair Python version support ({len(python_versions)} versions)") - elif python_versions: - score += 10 - support.append(f"Limited Python version support ({len(python_versions)} versions)") - else: - issues.append("No explicit Python version support") - - # Check requires_python specification - requires_python = info.get("requires_python") - if requires_python: - score += 20 - support.append(f"Python requirement specified: {requires_python}") - else: - issues.append("No Python version requirement specified") - - # Check platform support - platform_classifiers = [c for c in classifiers if "Operating System" in c] - if platform_classifiers: - if any("OS Independent" in c for c in platform_classifiers): - score += 20 - support.append("Cross-platform support (OS Independent)") - else: - score += 15 - support.append(f"Platform support ({len(platform_classifiers)} platforms)") - else: - issues.append("No platform support information") - - # Check for wheel distribution (compatibility indicator) - urls = info.get("urls", []) or [] - has_wheel = any(url.get("packagetype") == "bdist_wheel" for url in urls) - if has_wheel: - score += 15 - support.append("Wheel distribution available") - else: - issues.append("No wheel distribution") - - # Check development status - status_classifiers = [c for c in classifiers if "Development Status" in c] - if status_classifiers: - status = status_classifiers[0] - if "5 - Production/Stable" in status: - score += 15 - support.append("Production/Stable status") - elif "4 - Beta" in status: - score += 10 - support.append("Beta status") - elif "3 - Alpha" in status: - score += 5 - support.append("Alpha status") - else: - issues.append(f"Early development status: {status}") - - return { - "score": min(score, 100), - "support": support, - "issues": issues, - "python_versions": python_versions, - } - - async def _assess_metadata_completeness(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Assess metadata completeness and quality.""" - info = package_data.get("info", {}) - - score = 0 - completeness = {} - - # Essential fields - essential_fields = ["name", "version", "summary", "description", "author", "license"] - present_essential = [field for field in essential_fields if info.get(field)] - score += (len(present_essential) / len(essential_fields)) * 40 - completeness["essential_fields"] = f"{len(present_essential)}/{len(essential_fields)}" - - # Additional metadata fields - additional_fields = ["keywords", "home_page", "author_email", "classifiers", "project_urls"] - present_additional = [field for field in additional_fields if info.get(field)] - score += (len(present_additional) / len(additional_fields)) * 30 - completeness["additional_fields"] = f"{len(present_additional)}/{len(additional_fields)}" - - # Classifier completeness - classifiers = info.get("classifiers", []) - classifier_categories = set() - for classifier in classifiers: - category = classifier.split("::")[0].strip() - classifier_categories.add(category) - - expected_categories = ["Development Status", "Intended Audience", "License", "Programming Language", "Topic"] - present_categories = [cat for cat in expected_categories if cat in classifier_categories] - score += (len(present_categories) / len(expected_categories)) * 20 - completeness["classifier_categories"] = f"{len(present_categories)}/{len(expected_categories)}" - - # URLs completeness - project_urls = info.get("project_urls", {}) or {} - expected_url_types = ["homepage", "repository", "documentation", "bug tracker"] - present_url_types = [] - for expected in expected_url_types: - if any(expected.lower() in key.lower() for key in project_urls.keys()): - present_url_types.append(expected) - - score += (len(present_url_types) / len(expected_url_types)) * 10 - completeness["url_types"] = f"{len(present_url_types)}/{len(expected_url_types)}" - - return { - "score": min(score, 100), - "completeness": completeness, - } - - def _extract_github_url(self, package_data: Dict[str, Any]) -> Optional[str]: - """Extract GitHub repository URL from package data.""" - info = package_data.get("info", {}) - - # Check project URLs - project_urls = info.get("project_urls", {}) or {} - for url in project_urls.values(): - if url and "github.com" in url: - return url - - # Check home page - home_page = info.get("home_page", "") - if home_page and "github.com" in home_page: - return home_page - - return None - - async def _fetch_github_metrics(self, github_url: str) -> Dict[str, Any]: - """Fetch GitHub repository metrics.""" - try: - # Parse GitHub URL to get owner/repo - parsed = urlparse(github_url) - path_parts = parsed.path.strip('/').split('/') - if len(path_parts) >= 2: - owner, repo = path_parts[0], path_parts[1] - - # GitHub API call (public API, no auth required for basic info) - api_url = f"https://api.github.com/repos/{owner}/{repo}" - - async with httpx.AsyncClient(timeout=self.timeout) as client: - response = await client.get( - api_url, - headers={ - "Accept": "application/vnd.github.v3+json", - "User-Agent": "PyPI-Health-Scorer/1.0" - } - ) - - if response.status_code == 200: - data = response.json() - return { - "stars": data.get("stargazers_count", 0), - "forks": data.get("forks_count", 0), - "watchers": data.get("watchers_count", 0), - "issues": data.get("open_issues_count", 0), - "has_wiki": data.get("has_wiki", False), - "has_pages": data.get("has_pages", False), - "language": data.get("language", ""), - "created_at": data.get("created_at", ""), - "updated_at": data.get("pushed_at", ""), - "default_branch": data.get("default_branch", ""), - "archived": data.get("archived", False), - "disabled": data.get("disabled", False), - } - else: - logger.warning(f"GitHub API returned status {response.status_code}") - - except Exception as e: - logger.debug(f"Failed to fetch GitHub metrics: {e}") - - return self._empty_github_metrics() - - async def _empty_github_metrics(self) -> Dict[str, Any]: - """Return empty GitHub metrics.""" - return { - "stars": 0, - "forks": 0, - "watchers": 0, - "issues": 0, - "has_wiki": False, - "has_pages": False, - "language": "", - "created_at": "", - "updated_at": "", - "default_branch": "", - "archived": False, - "disabled": False, - "available": False, - } - - def _calculate_health_level(self, score: float) -> str: - """Calculate health level from score.""" - if score >= 85: - return "excellent" - elif score >= 70: - return "good" - elif score >= 55: - return "fair" - elif score >= 40: - return "poor" - else: - return "critical" - - def _identify_strengths(self, health_scores: Dict[str, float]) -> List[str]: - """Identify package strengths.""" - strengths = [] - for category, score in health_scores.items(): - if score >= 80: - strengths.append(f"Excellent {category} ({score:.0f}/100)") - elif score >= 65: - strengths.append(f"Good {category} ({score:.0f}/100)") - return strengths - - def _identify_weaknesses(self, health_scores: Dict[str, float]) -> List[str]: - """Identify package weaknesses.""" - weaknesses = [] - for category, score in health_scores.items(): - if score < 40: - weaknesses.append(f"Poor {category} ({score:.0f}/100)") - elif score < 55: - weaknesses.append(f"Fair {category} ({score:.0f}/100)") - return weaknesses - - def _prioritize_improvements(self, health_scores: Dict[str, float]) -> List[str]: - """Prioritize improvement areas by weight and score.""" - weighted_gaps = [] - for category, score in health_scores.items(): - gap = 100 - score - weighted_gap = gap * (self.weights[category] / 100) - weighted_gaps.append((category, weighted_gap, score)) - - # Sort by weighted gap (highest impact first) - weighted_gaps.sort(key=lambda x: x[1], reverse=True) - - priorities = [] - for category, weighted_gap, score in weighted_gaps[:3]: # Top 3 - if weighted_gap > 5: # Only include significant gaps - priorities.append(f"Improve {category} (current: {score:.0f}/100, impact: {self.weights[category]}%)") - - return priorities - - def _generate_health_recommendations( - self, health_scores: Dict[str, float], *assessment_results - ) -> List[str]: - """Generate actionable health improvement recommendations.""" - recommendations = [] - - overall_score = sum( - health_scores[category] * (self.weights[category] / 100) - for category in health_scores - ) - - # Overall recommendations - if overall_score >= 85: - recommendations.append("🌟 Excellent package health - maintain current standards") - elif overall_score >= 70: - recommendations.append("✅ Good package health - minor improvements possible") - elif overall_score >= 55: - recommendations.append("⚠️ Fair package health - several areas need improvement") - elif overall_score >= 40: - recommendations.append("🔶 Poor package health - significant improvements needed") - else: - recommendations.append("🚨 Critical package health - major overhaul required") - - # Specific recommendations based on low scores - if health_scores.get("maintenance", 0) < 60: - recommendations.append("📅 Improve maintenance: Update package more regularly, provide clear version history") - - if health_scores.get("documentation", 0) < 60: - recommendations.append("📚 Improve documentation: Add comprehensive README, usage examples, and API docs") - - if health_scores.get("testing", 0) < 60: - recommendations.append("🧪 Add testing: Implement test suite, CI/CD pipeline, and code coverage") - - if health_scores.get("security", 0) < 60: - recommendations.append("🔒 Enhance security: Add security reporting, use HTTPS, specify dependencies properly") - - if health_scores.get("compatibility", 0) < 60: - recommendations.append("🔧 Improve compatibility: Support more Python versions, add wheel distribution") - - if health_scores.get("metadata", 0) < 60: - recommendations.append("📝 Complete metadata: Add missing package information, keywords, and classifiers") - - if health_scores.get("popularity", 0) < 60: - recommendations.append("📈 Build community: Create documentation site, engage with users, add project URLs") - - return recommendations - - -# Main health assessment functions -async def assess_pypi_package_health( - package_name: str, - version: Optional[str] = None, - include_github_metrics: bool = True -) -> Dict[str, Any]: - """ - Assess comprehensive health and quality of a PyPI package. - - Args: - package_name: Name of the package to assess - version: Specific version to assess (optional) - include_github_metrics: Whether to fetch GitHub repository metrics - - Returns: - Comprehensive health assessment including scores and recommendations - """ - scorer = PackageHealthScorer() - return await scorer.assess_package_health( - package_name, version, include_github_metrics - ) - - -async def compare_package_health( - package_names: List[str], - include_github_metrics: bool = False -) -> Dict[str, Any]: - """ - Compare health scores across multiple packages. - - Args: - package_names: List of package names to compare - include_github_metrics: Whether to include GitHub metrics - - Returns: - Comparative health analysis with rankings - """ - logger.info(f"Starting health comparison for {len(package_names)} packages") - - scorer = PackageHealthScorer() - results = {} - - # Assess packages in parallel batches - batch_size = 3 - for i in range(0, len(package_names), batch_size): - batch = package_names[i:i + batch_size] - batch_tasks = [ - scorer.assess_package_health(pkg_name, include_github_metrics=include_github_metrics) - for pkg_name in batch - ] - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for pkg_name, result in zip(batch, batch_results): - if isinstance(result, Exception): - results[pkg_name] = { - "error": str(result), - "overall_health": {"score": 0, "level": "critical"}, - "category_scores": {cat: 0 for cat in scorer.weights.keys()} - } - else: - results[pkg_name] = result - - # Create comparison rankings - package_scores = [ - (pkg, result.get("overall_health", {}).get("score", 0)) - for pkg, result in results.items() - if "error" not in result - ] - package_scores.sort(key=lambda x: x[1], reverse=True) - - # Generate comparison insights - if package_scores: - best_package, best_score = package_scores[0] - worst_package, worst_score = package_scores[-1] - avg_score = sum(score for _, score in package_scores) / len(package_scores) - - comparison_insights = { - "best_package": {"name": best_package, "score": best_score}, - "worst_package": {"name": worst_package, "score": worst_score}, - "average_score": round(avg_score, 2), - "score_range": best_score - worst_score, - "rankings": [{"package": pkg, "score": score, "rank": i+1} - for i, (pkg, score) in enumerate(package_scores)] - } - else: - comparison_insights = { - "best_package": None, - "worst_package": None, - "average_score": 0, - "score_range": 0, - "rankings": [] - } - - return { - "comparison_timestamp": datetime.now(timezone.utc).isoformat(), - "packages_compared": len(package_names), - "detailed_results": results, - "comparison_insights": comparison_insights, - "recommendations": _generate_comparison_recommendations(comparison_insights, results) - } - - -def _generate_comparison_recommendations( - insights: Dict[str, Any], results: Dict[str, Any] -) -> List[str]: - """Generate recommendations for package comparison.""" - recommendations = [] - - if not insights.get("rankings"): - recommendations.append("❌ No successful health assessments to compare") - return recommendations - - best = insights.get("best_package") - worst = insights.get("worst_package") - avg_score = insights.get("average_score", 0) - - if best and worst: - recommendations.append( - f"🥇 Best package: {best['name']} (score: {best['score']:.1f}/100)" - ) - recommendations.append( - f"🥉 Needs improvement: {worst['name']} (score: {worst['score']:.1f}/100)" - ) - - if best['score'] - worst['score'] > 30: - recommendations.append("📊 Significant quality variation - consider standardizing practices") - - recommendations.append(f"📈 Average health score: {avg_score:.1f}/100") - - if avg_score >= 70: - recommendations.append("✅ Overall good package health across portfolio") - elif avg_score >= 55: - recommendations.append("⚠️ Mixed package health - focus on improving lower-scoring packages") - else: - recommendations.append("🚨 Poor overall package health - systematic improvements needed") - - return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/health_tools.py b/pypi_query_mcp/tools/health_tools.py deleted file mode 100644 index b0f98b3..0000000 --- a/pypi_query_mcp/tools/health_tools.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Package health assessment tools for PyPI packages.""" - -import logging -from typing import Any, Dict, List, Optional - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..tools.health_scorer import assess_pypi_package_health, compare_package_health - -logger = logging.getLogger(__name__) - - -async def assess_package_health_score( - package_name: str, - version: Optional[str] = None, - include_github_metrics: bool = True -) -> Dict[str, Any]: - """ - Assess comprehensive health and quality of a PyPI package. - - This tool evaluates package health across multiple dimensions including maintenance, - popularity, documentation, testing, security practices, compatibility, and metadata - completeness to provide an overall health score and actionable recommendations. - - Args: - package_name: Name of the package to assess for health and quality - version: Specific version to assess (optional, defaults to latest version) - include_github_metrics: Whether to fetch GitHub repository metrics for analysis - - Returns: - Dictionary containing comprehensive health assessment including: - - Overall health score (0-100) and level (excellent/good/fair/poor/critical) - - Category-specific scores (maintenance, popularity, documentation, testing, etc.) - - Detailed assessment breakdown with indicators and issues for each category - - GitHub repository metrics (stars, forks, activity) if available - - Actionable recommendations for health improvements - - Strengths, weaknesses, and improvement priorities analysis - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If health assessment fails - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError(package_name) - - logger.info(f"MCP tool: Assessing health for package {package_name}") - - try: - result = await assess_pypi_package_health( - package_name=package_name, - version=version, - include_github_metrics=include_github_metrics - ) - - overall_score = result.get("overall_health", {}).get("score", 0) - health_level = result.get("overall_health", {}).get("level", "unknown") - logger.info(f"MCP tool: Health assessment completed for {package_name} - score: {overall_score:.1f}/100 ({health_level})") - return result - - except (InvalidPackageNameError, NetworkError, SearchError) as e: - logger.error(f"Error assessing health for {package_name}: {e}") - return { - "error": f"Health assessment failed: {e}", - "error_type": type(e).__name__, - "package": package_name, - "version": version, - "assessment_timestamp": "", - "overall_health": { - "score": 0, - "level": "critical", - "max_score": 100, - }, - "category_scores": { - "maintenance": 0, - "popularity": 0, - "documentation": 0, - "testing": 0, - "security": 0, - "compatibility": 0, - "metadata": 0, - }, - "detailed_assessment": {}, - "recommendations": [f"❌ Health assessment failed: {e}"], - "health_summary": { - "strengths": [], - "weaknesses": ["Assessment failure"], - "improvement_priority": ["Resolve package access issues"], - } - } - - -async def compare_packages_health_scores( - package_names: List[str], - include_github_metrics: bool = False -) -> Dict[str, Any]: - """ - Compare health scores across multiple PyPI packages. - - This tool performs comparative health analysis across multiple packages, - providing rankings, insights, and recommendations to help evaluate - package ecosystem quality and identify the best options. - - Args: - package_names: List of package names to compare for health and quality - include_github_metrics: Whether to include GitHub metrics in the comparison - - Returns: - Dictionary containing comparative health analysis including: - - Detailed health results for each package - - Health score rankings with best/worst package identification - - Comparison insights (average scores, score ranges, rankings) - - Recommendations for package selection and improvements - - Statistical analysis of health across the package set - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during analysis - SearchError: If health comparison fails - """ - if not package_names: - raise ValueError("Package names list cannot be empty") - - logger.info(f"MCP tool: Starting health comparison for {len(package_names)} packages") - - try: - result = await compare_package_health( - package_names=package_names, - include_github_metrics=include_github_metrics - ) - - comparison_insights = result.get("comparison_insights", {}) - best_package = comparison_insights.get("best_package", {}) - packages_compared = result.get("packages_compared", 0) - - logger.info(f"MCP tool: Health comparison completed for {packages_compared} packages - best: {best_package.get('name', 'unknown')} ({best_package.get('score', 0):.1f}/100)") - return result - - except (ValueError, NetworkError, SearchError) as e: - logger.error(f"Error in health comparison: {e}") - return { - "error": f"Health comparison failed: {e}", - "error_type": type(e).__name__, - "comparison_timestamp": "", - "packages_compared": len(package_names), - "detailed_results": {}, - "comparison_insights": { - "best_package": None, - "worst_package": None, - "average_score": 0, - "score_range": 0, - "rankings": [] - }, - "recommendations": [f"❌ Health comparison failed: {e}"] - } \ No newline at end of file diff --git a/pypi_query_mcp/tools/license_analyzer.py b/pypi_query_mcp/tools/license_analyzer.py deleted file mode 100644 index 6e73bee..0000000 --- a/pypi_query_mcp/tools/license_analyzer.py +++ /dev/null @@ -1,727 +0,0 @@ -"""License compatibility analysis tools for PyPI packages.""" - -import asyncio -import logging -import re -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Set, Tuple - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..core.pypi_client import PyPIClient - -logger = logging.getLogger(__name__) - - -class LicenseCompatibilityAnalyzer: - """Comprehensive license compatibility analyzer for PyPI packages.""" - - def __init__(self): - self.timeout = 30.0 - - # License compatibility matrix based on common license interactions - # Key: primary license, Value: dict of compatible licenses with compatibility level - self.compatibility_matrix = { - "MIT": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "one-way", # MIT can be used in GPL, not vice versa - "GPL-3.0": "one-way", - "LGPL-2.1": "compatible", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - "BSD": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "one-way", - "GPL-3.0": "one-way", - "LGPL-2.1": "compatible", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - "Apache-2.0": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "incompatible", # Patent clause conflicts - "GPL-3.0": "one-way", # Apache can go into GPL-3.0 - "LGPL-2.1": "review-required", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - "GPL-2.0": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "incompatible", - "ISC": "compatible", - "GPL-2.0": "compatible", - "GPL-3.0": "incompatible", # GPL-2.0 and GPL-3.0 are incompatible - "LGPL-2.1": "compatible", - "LGPL-3.0": "incompatible", - "MPL-2.0": "incompatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "incompatible", - }, - "GPL-3.0": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "incompatible", - "GPL-3.0": "compatible", - "LGPL-2.1": "review-required", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "incompatible", - }, - "LGPL-2.1": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "review-required", - "ISC": "compatible", - "GPL-2.0": "compatible", - "GPL-3.0": "review-required", - "LGPL-2.1": "compatible", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - "LGPL-3.0": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "incompatible", - "GPL-3.0": "compatible", - "LGPL-2.1": "compatible", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - "MPL-2.0": { - "MIT": "compatible", - "BSD": "compatible", - "Apache-2.0": "compatible", - "ISC": "compatible", - "GPL-2.0": "incompatible", - "GPL-3.0": "compatible", - "LGPL-2.1": "compatible", - "LGPL-3.0": "compatible", - "MPL-2.0": "compatible", - "Unlicense": "compatible", - "Public Domain": "compatible", - "Proprietary": "review-required", - }, - } - - # License categorization for easier analysis - self.license_categories = { - "permissive": ["MIT", "BSD", "Apache-2.0", "ISC", "Unlicense", "Public Domain"], - "copyleft_weak": ["LGPL-2.1", "LGPL-3.0", "MPL-2.0"], - "copyleft_strong": ["GPL-2.0", "GPL-3.0", "AGPL-3.0"], - "proprietary": ["Proprietary", "Commercial", "All Rights Reserved"], - "unknown": ["Unknown", "Other", "Custom"], - } - - # Common license normalization patterns - self.license_patterns = { - r"MIT\s*License": "MIT", - r"BSD\s*3[-\s]*Clause": "BSD", - r"BSD\s*2[-\s]*Clause": "BSD", - r"Apache\s*2\.0": "Apache-2.0", - r"Apache\s*License\s*2\.0": "Apache-2.0", - r"GNU\s*General\s*Public\s*License\s*v?2": "GPL-2.0", - r"GNU\s*General\s*Public\s*License\s*v?3": "GPL-3.0", - r"GNU\s*Lesser\s*General\s*Public\s*License\s*v?2": "LGPL-2.1", - r"GNU\s*Lesser\s*General\s*Public\s*License\s*v?3": "LGPL-3.0", - r"Mozilla\s*Public\s*License\s*2\.0": "MPL-2.0", - r"ISC\s*License": "ISC", - r"Unlicense": "Unlicense", - r"Public\s*Domain": "Public Domain", - } - - async def analyze_package_license( - self, - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True - ) -> Dict[str, Any]: - """ - Analyze license information for a PyPI package. - - Args: - package_name: Name of the package to analyze - version: Specific version to analyze (optional) - include_dependencies: Whether to analyze dependency licenses - - Returns: - Dictionary containing license analysis results - """ - logger.info(f"Starting license analysis for package: {package_name}") - - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, version) - - package_version = version or package_data["info"]["version"] - - # Analyze package license - license_info = self._extract_license_info(package_data) - - # Analyze dependencies if requested - dependency_licenses = [] - if include_dependencies: - dependency_licenses = await self._analyze_dependency_licenses( - package_name, package_version - ) - - # Generate compatibility analysis - compatibility_analysis = self._analyze_license_compatibility( - license_info, dependency_licenses - ) - - # Calculate risk assessment - risk_assessment = self._assess_license_risks( - license_info, dependency_licenses, compatibility_analysis - ) - - return { - "package": package_name, - "version": package_version, - "analysis_timestamp": datetime.now(timezone.utc).isoformat(), - "license_info": license_info, - "dependency_licenses": dependency_licenses, - "compatibility_analysis": compatibility_analysis, - "risk_assessment": risk_assessment, - "recommendations": self._generate_license_recommendations( - license_info, dependency_licenses, compatibility_analysis, risk_assessment - ), - "analysis_summary": { - "total_dependencies_analyzed": len(dependency_licenses), - "unique_licenses_found": len(set( - [license_info.get("normalized_license", "Unknown")] + - [dep.get("normalized_license", "Unknown") for dep in dependency_licenses] - )), - "license_conflicts": len(compatibility_analysis.get("conflicts", [])), - "review_required_count": len(compatibility_analysis.get("review_required", [])), - } - } - - except Exception as e: - logger.error(f"License analysis failed for {package_name}: {e}") - raise SearchError(f"License analysis failed: {e}") from e - - def _extract_license_info(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Extract and normalize license information from package data.""" - info = package_data.get("info", {}) - - # Extract license from multiple sources - license_field = info.get("license", "") - license_classifier = self._extract_license_from_classifiers( - info.get("classifiers", []) - ) - - # Normalize license - normalized_license = self._normalize_license(license_field or license_classifier) - - # Categorize license - license_category = self._categorize_license(normalized_license) - - return { - "raw_license": license_field, - "classifier_license": license_classifier, - "normalized_license": normalized_license, - "license_category": license_category, - "license_url": self._extract_license_url(info), - "license_confidence": self._assess_license_confidence( - license_field, license_classifier, normalized_license - ), - } - - def _extract_license_from_classifiers(self, classifiers: List[str]) -> str: - """Extract license information from PyPI classifiers.""" - license_classifiers = [ - c for c in classifiers if c.startswith("License ::") - ] - - if not license_classifiers: - return "" - - # Return the most specific license classifier - return license_classifiers[-1].replace("License ::", "").strip() - - def _normalize_license(self, license_text: str) -> str: - """Normalize license text to standard SPDX identifiers.""" - if not license_text: - return "Unknown" - - license_text_clean = license_text.strip() - - # Check for exact matches first - common_licenses = { - "MIT": "MIT", - "BSD": "BSD", - "Apache": "Apache-2.0", - "GPL": "GPL-3.0", # Default to GPL-3.0 if version unspecified - "LGPL": "LGPL-3.0", - "MPL": "MPL-2.0", - } - - if license_text_clean in common_licenses: - return common_licenses[license_text_clean] - - # Pattern matching - for pattern, normalized in self.license_patterns.items(): - if re.search(pattern, license_text_clean, re.IGNORECASE): - return normalized - - # Check if it contains known license names - license_lower = license_text_clean.lower() - if "mit" in license_lower: - return "MIT" - elif "bsd" in license_lower: - return "BSD" - elif "apache" in license_lower: - return "Apache-2.0" - elif "gpl" in license_lower and "lgpl" not in license_lower: - return "GPL-3.0" - elif "lgpl" in license_lower: - return "LGPL-3.0" - elif "mozilla" in license_lower or "mpl" in license_lower: - return "MPL-2.0" - elif "unlicense" in license_lower: - return "Unlicense" - elif "public domain" in license_lower: - return "Public Domain" - elif any(prop in license_lower for prop in ["proprietary", "commercial", "all rights reserved"]): - return "Proprietary" - - return "Other" - - def _categorize_license(self, normalized_license: str) -> str: - """Categorize license into major categories.""" - for category, licenses in self.license_categories.items(): - if normalized_license in licenses: - return category - return "unknown" - - def _extract_license_url(self, info: Dict[str, Any]) -> str: - """Extract license URL from package info.""" - # Check project URLs - project_urls = info.get("project_urls", {}) or {} - for key, url in project_urls.items(): - if "license" in key.lower(): - return url - - # Check home page for license info - home_page = info.get("home_page", "") - if home_page and "github.com" in home_page: - return f"{home_page.rstrip('/')}/blob/main/LICENSE" - - return "" - - def _assess_license_confidence( - self, raw_license: str, classifier_license: str, normalized_license: str - ) -> str: - """Assess confidence level in license detection.""" - if not raw_license and not classifier_license: - return "low" - - if normalized_license == "Unknown" or normalized_license == "Other": - return "low" - - if raw_license and classifier_license and raw_license in classifier_license: - return "high" - elif raw_license or classifier_license: - return "medium" - else: - return "low" - - async def _analyze_dependency_licenses( - self, package_name: str, version: str - ) -> List[Dict[str, Any]]: - """Analyze licenses of package dependencies.""" - try: - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, version) - - # Extract dependencies - requires_dist = package_data.get("info", {}).get("requires_dist", []) or [] - dependencies = [] - - for req in requires_dist: - # Parse dependency name (simplified) - dep_name = req.split()[0].split(">=")[0].split("==")[0].split("~=")[0].split("!=")[0] - if dep_name and not dep_name.startswith("extra"): - dependencies.append(dep_name) - - # Analyze dependency licenses (limit to top 15 to avoid overwhelming) - dependency_licenses = [] - - for dep_name in dependencies[:15]: - try: - dep_data = await client.get_package_info(dep_name) - dep_license_info = self._extract_license_info(dep_data) - - dependency_licenses.append({ - "package": dep_name, - "version": dep_data.get("info", {}).get("version", ""), - **dep_license_info - }) - except Exception as e: - logger.debug(f"Failed to analyze license for dependency {dep_name}: {e}") - dependency_licenses.append({ - "package": dep_name, - "version": "", - "normalized_license": "Unknown", - "license_category": "unknown", - "license_confidence": "low", - "error": str(e) - }) - - return dependency_licenses - - except Exception as e: - logger.warning(f"Dependency license analysis failed: {e}") - return [] - - def _analyze_license_compatibility( - self, package_license: Dict[str, Any], dependency_licenses: List[Dict[str, Any]] - ) -> Dict[str, Any]: - """Analyze license compatibility between package and its dependencies.""" - main_license = package_license.get("normalized_license", "Unknown") - - compatible = [] - incompatible = [] - review_required = [] - one_way = [] - unknown = [] - - for dep in dependency_licenses: - dep_license = dep.get("normalized_license", "Unknown") - dep_package = dep.get("package", "unknown") - - if main_license == "Unknown" or dep_license == "Unknown": - unknown.append({ - "package": dep_package, - "license": dep_license, - "reason": "License information unavailable" - }) - continue - - compatibility = self._check_license_compatibility(main_license, dep_license) - - if compatibility == "compatible": - compatible.append({ - "package": dep_package, - "license": dep_license, - }) - elif compatibility == "incompatible": - incompatible.append({ - "package": dep_package, - "license": dep_license, - "reason": f"{main_license} and {dep_license} are incompatible" - }) - elif compatibility == "review-required": - review_required.append({ - "package": dep_package, - "license": dep_license, - "reason": f"Manual review needed for {main_license} + {dep_license}" - }) - elif compatibility == "one-way": - one_way.append({ - "package": dep_package, - "license": dep_license, - "reason": f"{dep_license} can be used in {main_license} project" - }) - - return { - "main_license": main_license, - "compatible": compatible, - "incompatible": incompatible, - "review_required": review_required, - "one_way": one_way, - "unknown": unknown, - "conflicts": incompatible, # Alias for easier access - } - - def _check_license_compatibility(self, license1: str, license2: str) -> str: - """Check compatibility between two licenses.""" - if license1 in self.compatibility_matrix: - return self.compatibility_matrix[license1].get(license2, "unknown") - - # Fallback compatibility rules - if license1 == license2: - return "compatible" - - # Default to review required for unknown combinations - return "review-required" - - def _assess_license_risks( - self, - package_license: Dict[str, Any], - dependency_licenses: List[Dict[str, Any]], - compatibility_analysis: Dict[str, Any] - ) -> Dict[str, Any]: - """Assess overall license risks for the project.""" - risks = [] - risk_score = 0 - - main_license = package_license.get("normalized_license", "Unknown") - main_category = package_license.get("license_category", "unknown") - - # Check for incompatible licenses - incompatible_count = len(compatibility_analysis.get("incompatible", [])) - if incompatible_count > 0: - risks.append(f"Found {incompatible_count} incompatible license(s)") - risk_score += incompatible_count * 30 - - # Check for unknown licenses - unknown_count = len(compatibility_analysis.get("unknown", [])) - if unknown_count > 0: - risks.append(f"Found {unknown_count} dependency(ies) with unknown licenses") - risk_score += unknown_count * 10 - - # Check for review-required licenses - review_count = len(compatibility_analysis.get("review_required", [])) - if review_count > 0: - risks.append(f"Found {review_count} license(s) requiring manual review") - risk_score += review_count * 15 - - # Check for copyleft contamination risk - if main_category == "permissive": - copyleft_deps = [ - dep for dep in dependency_licenses - if dep.get("license_category") in ["copyleft_weak", "copyleft_strong"] - ] - if copyleft_deps: - risks.append(f"Permissive project using {len(copyleft_deps)} copyleft dependencies") - risk_score += len(copyleft_deps) * 20 - - # Check for proprietary license risks - proprietary_deps = [ - dep for dep in dependency_licenses - if dep.get("license_category") == "proprietary" - ] - if proprietary_deps: - risks.append(f"Found {len(proprietary_deps)} proprietary dependencies") - risk_score += len(proprietary_deps) * 25 - - # Calculate risk level - if risk_score >= 80: - risk_level = "critical" - elif risk_score >= 50: - risk_level = "high" - elif risk_score >= 25: - risk_level = "medium" - elif risk_score > 0: - risk_level = "low" - else: - risk_level = "minimal" - - return { - "risk_score": min(risk_score, 100), - "risk_level": risk_level, - "risk_factors": risks, - "compliance_status": "compliant" if risk_score < 25 else "review-needed", - } - - def _generate_license_recommendations( - self, - package_license: Dict[str, Any], - dependency_licenses: List[Dict[str, Any]], - compatibility_analysis: Dict[str, Any], - risk_assessment: Dict[str, Any] - ) -> List[str]: - """Generate actionable license recommendations.""" - recommendations = [] - - main_license = package_license.get("normalized_license", "Unknown") - risk_level = risk_assessment.get("risk_level", "unknown") - - # High-level recommendations based on risk - if risk_level == "critical": - recommendations.append("🚨 Critical license issues detected - immediate legal review required") - elif risk_level == "high": - recommendations.append("⚠️ High license risk - review and resolve conflicts before release") - elif risk_level == "medium": - recommendations.append("⚠️ Moderate license risk - review recommendations below") - elif risk_level == "minimal": - recommendations.append("✅ License compatibility appears good") - - # Specific recommendations for incompatible licenses - incompatible = compatibility_analysis.get("incompatible", []) - if incompatible: - recommendations.append(f"🔴 Remove or replace {len(incompatible)} incompatible dependencies:") - for dep in incompatible[:3]: # Show first 3 - recommendations.append(f" - {dep['package']} ({dep['license']}): {dep.get('reason', '')}") - - # Recommendations for review-required licenses - review_required = compatibility_analysis.get("review_required", []) - if review_required: - recommendations.append(f"📋 Manual review needed for {len(review_required)} dependencies:") - for dep in review_required[:3]: - recommendations.append(f" - {dep['package']} ({dep['license']})") - - # Unknown license recommendations - unknown = compatibility_analysis.get("unknown", []) - if unknown: - recommendations.append(f"❓ Investigate {len(unknown)} dependencies with unknown licenses") - - # License confidence recommendations - if package_license.get("license_confidence") == "low": - recommendations.append("📝 Consider adding clear license information to your package") - - # Category-specific recommendations - main_category = package_license.get("license_category", "unknown") - if main_category == "copyleft_strong": - recommendations.append("ℹ️ GPL license requires derivative works to also be GPL") - elif main_category == "permissive": - recommendations.append("ℹ️ Permissive license allows flexible usage") - - return recommendations - - -# Main analysis functions -async def analyze_package_license_compatibility( - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True -) -> Dict[str, Any]: - """ - Analyze license compatibility for a PyPI package. - - Args: - package_name: Name of the package to analyze - version: Specific version to analyze (optional) - include_dependencies: Whether to analyze dependency licenses - - Returns: - Comprehensive license compatibility analysis - """ - analyzer = LicenseCompatibilityAnalyzer() - return await analyzer.analyze_package_license( - package_name, version, include_dependencies - ) - - -async def check_license_compliance_bulk( - package_names: List[str], - target_license: Optional[str] = None -) -> Dict[str, Any]: - """ - Check license compliance for multiple packages. - - Args: - package_names: List of package names to check - target_license: Target license for compatibility checking - - Returns: - Bulk license compliance report - """ - logger.info(f"Starting bulk license compliance check for {len(package_names)} packages") - - analyzer = LicenseCompatibilityAnalyzer() - results = {} - summary = { - "total_packages": len(package_names), - "compliant_packages": 0, - "non_compliant_packages": 0, - "unknown_license_packages": 0, - "high_risk_packages": [], - "analysis_timestamp": datetime.now(timezone.utc).isoformat() - } - - # Analyze packages in parallel batches - batch_size = 5 - for i in range(0, len(package_names), batch_size): - batch = package_names[i:i + batch_size] - batch_tasks = [ - analyzer.analyze_package_license(pkg_name, include_dependencies=False) - for pkg_name in batch - ] - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for pkg_name, result in zip(batch, batch_results): - if isinstance(result, Exception): - results[pkg_name] = { - "error": str(result), - "analysis_status": "failed" - } - summary["unknown_license_packages"] += 1 - else: - results[pkg_name] = result - - # Update summary - risk_level = result.get("risk_assessment", {}).get("risk_level", "unknown") - if risk_level in ["minimal", "low"]: - summary["compliant_packages"] += 1 - else: - summary["non_compliant_packages"] += 1 - - if risk_level in ["high", "critical"]: - summary["high_risk_packages"].append({ - "package": pkg_name, - "license": result.get("license_info", {}).get("normalized_license", "Unknown"), - "risk_level": risk_level - }) - - return { - "summary": summary, - "detailed_results": results, - "target_license": target_license, - "recommendations": _generate_bulk_license_recommendations(summary, results) - } - - -def _generate_bulk_license_recommendations(summary: Dict[str, Any], results: Dict[str, Any]) -> List[str]: - """Generate recommendations for bulk license analysis.""" - recommendations = [] - - compliant = summary["compliant_packages"] - total = summary["total_packages"] - - if compliant == total: - recommendations.append("✅ All packages appear to have compliant licenses") - else: - non_compliant = summary["non_compliant_packages"] - percentage = (non_compliant / total) * 100 - recommendations.append( - f"⚠️ {non_compliant}/{total} packages ({percentage:.1f}%) have license compliance issues" - ) - - high_risk = summary["high_risk_packages"] - if high_risk: - recommendations.append( - f"🚨 {len(high_risk)} packages are high risk: {', '.join([p['package'] for p in high_risk])}" - ) - recommendations.append("Priority: Address high-risk packages immediately") - - unknown = summary["unknown_license_packages"] - if unknown > 0: - recommendations.append(f"❓ {unknown} packages have unknown or unclear licenses") - recommendations.append("Consider investigating these packages for license clarity") - - return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/license_tools.py b/pypi_query_mcp/tools/license_tools.py deleted file mode 100644 index fa547a9..0000000 --- a/pypi_query_mcp/tools/license_tools.py +++ /dev/null @@ -1,154 +0,0 @@ -"""License compatibility analysis tools for PyPI packages.""" - -import logging -from typing import Any, Dict, List, Optional - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..tools.license_analyzer import analyze_package_license_compatibility, check_license_compliance_bulk - -logger = logging.getLogger(__name__) - - -async def analyze_pypi_package_license( - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True -) -> Dict[str, Any]: - """ - Analyze license compatibility for a PyPI package. - - This tool provides comprehensive license analysis including license identification, - dependency license scanning, compatibility checking, and risk assessment to help - ensure your project complies with open source license requirements. - - Args: - package_name: Name of the package to analyze for license compatibility - version: Specific version to analyze (optional, defaults to latest version) - include_dependencies: Whether to analyze dependency licenses for compatibility - - Returns: - Dictionary containing comprehensive license analysis including: - - License identification and normalization (SPDX format) - - License categorization (permissive, copyleft, proprietary, etc.) - - Dependency license analysis and compatibility matrix - - Risk assessment with score and risk level (minimal, low, medium, high, critical) - - Compatibility analysis highlighting conflicts and review-required combinations - - Actionable recommendations for license compliance - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If license analysis fails - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError(package_name) - - logger.info(f"MCP tool: Analyzing license compatibility for package {package_name}") - - try: - result = await analyze_package_license_compatibility( - package_name=package_name, - version=version, - include_dependencies=include_dependencies - ) - - logger.info(f"MCP tool: License analysis completed for {package_name} - {result.get('analysis_summary', {}).get('license_conflicts', 0)} conflicts found") - return result - - except (InvalidPackageNameError, NetworkError, SearchError) as e: - logger.error(f"Error analyzing license for {package_name}: {e}") - return { - "error": f"License analysis failed: {e}", - "error_type": type(e).__name__, - "package": package_name, - "version": version, - "analysis_timestamp": "", - "license_info": { - "normalized_license": "Unknown", - "license_category": "unknown", - "license_confidence": "low", - }, - "dependency_licenses": [], - "compatibility_analysis": { - "main_license": "Unknown", - "compatible": [], - "incompatible": [], - "review_required": [], - "conflicts": [], - }, - "risk_assessment": { - "risk_score": 100, - "risk_level": "critical", - "risk_factors": [f"License analysis failed: {e}"], - "compliance_status": "unknown", - }, - "recommendations": [f"❌ License analysis failed: {e}"], - "analysis_summary": { - "total_dependencies_analyzed": 0, - "unique_licenses_found": 0, - "license_conflicts": 0, - "review_required_count": 0, - } - } - - -async def check_bulk_license_compliance( - package_names: List[str], - target_license: Optional[str] = None -) -> Dict[str, Any]: - """ - Check license compliance for multiple PyPI packages. - - This tool performs bulk license compliance checking across multiple packages, - providing a consolidated report to help ensure your entire package ecosystem - complies with license requirements and identifying potential legal risks. - - Args: - package_names: List of package names to check for license compliance - target_license: Target license for compatibility checking (optional) - - Returns: - Dictionary containing bulk compliance analysis including: - - Summary statistics (total packages, compliant/non-compliant counts) - - Detailed license analysis for each package - - High-risk packages requiring immediate attention - - Unknown license packages needing investigation - - Prioritized recommendations for compliance remediation - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during analysis - SearchError: If bulk compliance checking fails - """ - if not package_names: - raise ValueError("Package names list cannot be empty") - - logger.info(f"MCP tool: Starting bulk license compliance check for {len(package_names)} packages") - - try: - result = await check_license_compliance_bulk( - package_names=package_names, - target_license=target_license - ) - - logger.info(f"MCP tool: Bulk license compliance completed - {result.get('summary', {}).get('non_compliant_packages', 0)} non-compliant packages found") - return result - - except (ValueError, NetworkError, SearchError) as e: - logger.error(f"Error in bulk license compliance check: {e}") - return { - "error": f"Bulk license compliance check failed: {e}", - "error_type": type(e).__name__, - "summary": { - "total_packages": len(package_names), - "compliant_packages": 0, - "non_compliant_packages": 0, - "unknown_license_packages": len(package_names), - "high_risk_packages": [], - "analysis_timestamp": "" - }, - "detailed_results": {}, - "target_license": target_license, - "recommendations": [f"❌ Bulk license compliance check failed: {e}"] - } \ No newline at end of file diff --git a/pypi_query_mcp/tools/requirements_analyzer.py b/pypi_query_mcp/tools/requirements_analyzer.py deleted file mode 100644 index 0aba3d0..0000000 --- a/pypi_query_mcp/tools/requirements_analyzer.py +++ /dev/null @@ -1,947 +0,0 @@ -"""Requirements file parsing and analysis tools for Python projects.""" - -import asyncio -import logging -import re -import tomllib -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..core.pypi_client import PyPIClient - -logger = logging.getLogger(__name__) - - -class RequirementsAnalyzer: - """Comprehensive requirements file analyzer for Python projects.""" - - def __init__(self): - self.timeout = 30.0 - - # Supported requirement file patterns - self.requirement_patterns = { - "requirements.txt": r"requirements.*\.txt", - "pyproject.toml": r"pyproject\.toml", - "setup.py": r"setup\.py", - "Pipfile": r"Pipfile", - "poetry.lock": r"poetry\.lock", - "conda.yml": r"(conda|environment)\.ya?ml", - } - - # Version specifier patterns - self.version_patterns = { - "exact": r"==\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "gte": r">=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "gt": r">\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "lte": r"<=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "lt": r"<\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "compatible": r"~=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - "not_equal": r"!=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", - } - - async def analyze_requirements_file( - self, - file_path: str, - check_updates: bool = True, - security_scan: bool = True, - compatibility_check: bool = True - ) -> Dict[str, Any]: - """ - Analyze a requirements file for dependencies, versions, security, and compatibility. - - Args: - file_path: Path to the requirements file - check_updates: Whether to check for package updates - security_scan: Whether to perform security vulnerability scanning - compatibility_check: Whether to check Python version compatibility - - Returns: - Dictionary containing comprehensive requirements analysis - """ - logger.info(f"Starting requirements analysis for: {file_path}") - - try: - # Parse requirements file - parsed_requirements = await self._parse_requirements_file(file_path) - - if not parsed_requirements["dependencies"]: - return { - "file_path": file_path, - "analysis_timestamp": datetime.now(timezone.utc).isoformat(), - "file_info": parsed_requirements["file_info"], - "dependencies": [], - "analysis_summary": { - "total_dependencies": 0, - "outdated_packages": 0, - "security_vulnerabilities": 0, - "compatibility_issues": 0, - }, - "recommendations": ["No dependencies found to analyze"], - "error": "No dependencies found in requirements file" - } - - # Analyze dependencies in parallel - analysis_tasks = [] - - # Basic dependency analysis (always done) - analysis_tasks.append(self._analyze_dependency_health(parsed_requirements["dependencies"])) - - # Optional analyses - if check_updates: - analysis_tasks.append(self._check_package_updates(parsed_requirements["dependencies"])) - else: - analysis_tasks.append(asyncio.create_task(self._empty_updates_result())) - - if security_scan: - analysis_tasks.append(self._scan_dependencies_security(parsed_requirements["dependencies"])) - else: - analysis_tasks.append(asyncio.create_task(self._empty_security_result())) - - if compatibility_check: - python_version = parsed_requirements.get("python_version") - analysis_tasks.append(self._check_dependencies_compatibility(parsed_requirements["dependencies"], python_version)) - else: - analysis_tasks.append(asyncio.create_task(self._empty_compatibility_result())) - - # Execute analyses - results = await asyncio.gather(*analysis_tasks, return_exceptions=True) - - # Unpack results - health_analysis = results[0] if not isinstance(results[0], Exception) else {"healthy": [], "issues": []} - update_analysis = results[1] if not isinstance(results[1], Exception) else {"outdated": [], "current": []} - security_analysis = results[2] if not isinstance(results[2], Exception) else {"vulnerabilities": [], "secure": []} - compatibility_analysis = results[3] if not isinstance(results[3], Exception) else {"compatible": [], "incompatible": []} - - # Generate comprehensive analysis - analysis_summary = self._generate_analysis_summary( - parsed_requirements["dependencies"], - health_analysis, - update_analysis, - security_analysis, - compatibility_analysis - ) - - recommendations = self._generate_requirements_recommendations( - parsed_requirements, - health_analysis, - update_analysis, - security_analysis, - compatibility_analysis, - analysis_summary - ) - - return { - "file_path": file_path, - "analysis_timestamp": datetime.now(timezone.utc).isoformat(), - "file_info": parsed_requirements["file_info"], - "dependencies": parsed_requirements["dependencies"], - "dependency_analysis": { - "health": health_analysis, - "updates": update_analysis if check_updates else None, - "security": security_analysis if security_scan else None, - "compatibility": compatibility_analysis if compatibility_check else None, - }, - "analysis_summary": analysis_summary, - "recommendations": recommendations, - "python_requirements": parsed_requirements.get("python_version"), - } - - except Exception as e: - logger.error(f"Requirements analysis failed for {file_path}: {e}") - raise SearchError(f"Requirements analysis failed: {e}") from e - - async def _parse_requirements_file(self, file_path: str) -> Dict[str, Any]: - """Parse requirements from various file formats.""" - path = Path(file_path) - - if not path.exists(): - raise FileNotFoundError(f"Requirements file not found: {file_path}") - - file_info = { - "name": path.name, - "format": self._detect_file_format(path.name), - "size_bytes": path.stat().st_size, - "modified_time": datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat(), - } - - # Parse based on file format - if path.name.endswith('.txt'): - dependencies, python_version = await self._parse_requirements_txt(path) - elif path.name == 'pyproject.toml': - dependencies, python_version = await self._parse_pyproject_toml(path) - elif path.name == 'setup.py': - dependencies, python_version = await self._parse_setup_py(path) - elif path.name == 'Pipfile': - dependencies, python_version = await self._parse_pipfile(path) - elif path.name.endswith('.yml') or path.name.endswith('.yaml'): - dependencies, python_version = await self._parse_conda_yml(path) - else: - # Try to parse as requirements.txt format - dependencies, python_version = await self._parse_requirements_txt(path) - - return { - "file_info": file_info, - "dependencies": dependencies, - "python_version": python_version, - } - - def _detect_file_format(self, filename: str) -> str: - """Detect requirements file format.""" - filename_lower = filename.lower() - - for fmt, pattern in self.requirement_patterns.items(): - if re.match(pattern, filename_lower): - return fmt - - return "unknown" - - async def _parse_requirements_txt(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: - """Parse requirements.txt format files.""" - dependencies = [] - python_version = None - - try: - content = path.read_text(encoding="utf-8") - lines = content.splitlines() - - for line_num, line in enumerate(lines, 1): - line = line.strip() - - # Skip comments and empty lines - if not line or line.startswith('#'): - continue - - # Skip -r and -e directives (for now) - if line.startswith(('-r', '-e', '--')): - continue - - # Parse requirement line - dep = self._parse_requirement_line(line, line_num) - if dep: - dependencies.append(dep) - - except Exception as e: - logger.warning(f"Failed to parse requirements.txt {path}: {e}") - - return dependencies, python_version - - async def _parse_pyproject_toml(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: - """Parse pyproject.toml files.""" - dependencies = [] - python_version = None - - try: - content = path.read_text(encoding="utf-8") - data = tomllib.loads(content) - - # Extract Python version requirement - build_system = data.get("build-system", {}) - project = data.get("project", {}) - tool_poetry = data.get("tool", {}).get("poetry", {}) - - # Check for Python version in different places - if project.get("requires-python"): - python_version = project["requires-python"] - elif tool_poetry.get("dependencies", {}).get("python"): - python_version = tool_poetry["dependencies"]["python"] - - # Extract dependencies from project.dependencies - if "dependencies" in project: - for dep_line in project["dependencies"]: - dep = self._parse_requirement_line(dep_line, 0) - if dep: - dependencies.append(dep) - - # Extract from tool.poetry.dependencies - if "tool" in data and "poetry" in data["tool"] and "dependencies" in data["tool"]["poetry"]: - poetry_deps = data["tool"]["poetry"]["dependencies"] - for name, version_spec in poetry_deps.items(): - if name.lower() == "python": - continue # Skip Python version - - if isinstance(version_spec, str): - req_line = f"{name}{version_spec}" if version_spec.startswith(('=', '<', '>', '~', '^', '!')) else f"{name}=={version_spec}" - else: - # Handle complex version specifications - req_line = f"{name}>={version_spec.get('version', '0.0.0')}" - - dep = self._parse_requirement_line(req_line, 0) - if dep: - dependencies.append(dep) - - except Exception as e: - logger.warning(f"Failed to parse pyproject.toml {path}: {e}") - - return dependencies, python_version - - async def _parse_setup_py(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: - """Parse setup.py files (basic extraction).""" - dependencies = [] - python_version = None - - try: - content = path.read_text(encoding="utf-8") - - # Look for install_requires - install_requires_match = re.search(r"install_requires\s*=\s*\[(.*?)\]", content, re.DOTALL) - if install_requires_match: - deps_text = install_requires_match.group(1) - # Extract quoted strings - quoted_deps = re.findall(r'["\']([^"\']+)["\']', deps_text) - - for dep_line in quoted_deps: - dep = self._parse_requirement_line(dep_line, 0) - if dep: - dependencies.append(dep) - - # Look for python_requires - python_requires_match = re.search(r"python_requires\s*=\s*[\"']([^\"']+)[\"']", content) - if python_requires_match: - python_version = python_requires_match.group(1) - - except Exception as e: - logger.warning(f"Failed to parse setup.py {path}: {e}") - - return dependencies, python_version - - async def _parse_pipfile(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: - """Parse Pipfile format.""" - dependencies = [] - python_version = None - - try: - content = path.read_text(encoding="utf-8") - data = tomllib.loads(content) - - # Extract Python version - if "requires" in data and "python_version" in data["requires"]: - python_version = f">={data['requires']['python_version']}" - - # Extract packages - for section in ["packages", "dev-packages"]: - if section in data: - for name, version_spec in data[section].items(): - if isinstance(version_spec, str): - req_line = f"{name}{version_spec}" if version_spec.startswith(('=', '<', '>', '~', '^', '!')) else f"{name}=={version_spec}" - else: - req_line = f"{name}>={version_spec.get('version', '0.0.0')}" - - dep = self._parse_requirement_line(req_line, 0) - if dep: - dep["dev_dependency"] = (section == "dev-packages") - dependencies.append(dep) - - except Exception as e: - logger.warning(f"Failed to parse Pipfile {path}: {e}") - - return dependencies, python_version - - async def _parse_conda_yml(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: - """Parse conda environment.yml files.""" - dependencies = [] - python_version = None - - try: - import yaml - - content = path.read_text(encoding="utf-8") - data = yaml.safe_load(content) - - if "dependencies" in data: - for dep in data["dependencies"]: - if isinstance(dep, str): - if dep.startswith("python"): - # Extract Python version - python_match = re.search(r"python\s*([><=~!]+)\s*([0-9.]+)", dep) - if python_match: - python_version = f"{python_match.group(1)}{python_match.group(2)}" - else: - parsed_dep = self._parse_requirement_line(dep, 0) - if parsed_dep: - dependencies.append(parsed_dep) - - except Exception as e: - logger.warning(f"Failed to parse conda.yml {path}: {e}") - - return dependencies, python_version - - def _parse_requirement_line(self, line: str, line_number: int) -> Optional[Dict[str, Any]]: - """Parse a single requirement line.""" - try: - # Remove inline comments - if '#' in line: - line = line[:line.index('#')].strip() - - if not line: - return None - - # Handle extras (package[extra1,extra2]) - extras = [] - extras_match = re.search(r'\[([^\]]+)\]', line) - if extras_match: - extras = [e.strip() for e in extras_match.group(1).split(',')] - line = re.sub(r'\[([^\]]+)\]', '', line) - - # Parse package name and version specifiers - # Split on version operators - version_ops = ['>=', '<=', '==', '!=', '~=', '>', '<'] - package_name = line - version_specifiers = [] - - for op in version_ops: - if op in line: - parts = line.split(op) - package_name = parts[0].strip() - if len(parts) > 1: - version_specifiers.append({ - "operator": op, - "version": parts[1].strip().split(',')[0].strip() - }) - break - - # Handle comma-separated version specs - if ',' in line and version_specifiers: - remaining = line.split(version_specifiers[0]["operator"], 1)[1] - for spec in remaining.split(',')[1:]: - spec = spec.strip() - for op in version_ops: - if spec.startswith(op): - version_specifiers.append({ - "operator": op, - "version": spec[len(op):].strip() - }) - break - - # Clean package name - package_name = re.sub(r'[<>=!~,\s].*', '', package_name).strip() - - if not package_name: - return None - - return { - "name": package_name, - "version_specifiers": version_specifiers, - "extras": extras, - "line_number": line_number, - "raw_line": line.strip(), - } - - except Exception as e: - logger.debug(f"Failed to parse requirement line '{line}': {e}") - return None - - async def _analyze_dependency_health(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: - """Analyze overall health of dependencies.""" - healthy = [] - issues = [] - - for dep in dependencies: - name = dep["name"] - version_specs = dep["version_specifiers"] - - # Check for problematic version specifications - health_issues = [] - - if not version_specs: - health_issues.append("No version constraint (could lead to instability)") - else: - # Check for overly restrictive versions - exact_versions = [spec for spec in version_specs if spec["operator"] == "=="] - if exact_versions: - health_issues.append("Exact version pinning (may cause conflicts)") - - # Check for very loose constraints - loose_constraints = [spec for spec in version_specs if spec["operator"] in [">", ">="]] - if loose_constraints and not any(spec["operator"] in ["<", "<="] for spec in version_specs): - health_issues.append("No upper bound (may break with future versions)") - - if health_issues: - issues.append({ - "package": name, - "issues": health_issues, - "current_spec": version_specs - }) - else: - healthy.append({ - "package": name, - "version_spec": version_specs - }) - - return { - "healthy": healthy, - "issues": issues, - "health_score": len(healthy) / len(dependencies) * 100 if dependencies else 0 - } - - async def _check_package_updates(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: - """Check for available package updates.""" - outdated = [] - current = [] - - async with PyPIClient() as client: - # Process in batches to avoid overwhelming PyPI - batch_size = 10 - for i in range(0, len(dependencies), batch_size): - batch = dependencies[i:i + batch_size] - batch_tasks = [] - - for dep in batch: - task = self._check_single_package_update(client, dep) - batch_tasks.append(task) - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for dep, result in zip(batch, batch_results): - if isinstance(result, Exception): - logger.debug(f"Failed to check updates for {dep['name']}: {result}") - continue - - if result["has_update"]: - outdated.append(result) - else: - current.append(result) - - return { - "outdated": outdated, - "current": current, - "update_percentage": len(outdated) / len(dependencies) * 100 if dependencies else 0 - } - - async def _check_single_package_update(self, client: PyPIClient, dep: Dict[str, Any]) -> Dict[str, Any]: - """Check if a single package has updates available.""" - try: - package_data = await client.get_package_info(dep["name"]) - latest_version = package_data["info"]["version"] - - # For now, we'll do a simple comparison - # In a real implementation, you'd want proper version comparison - has_update = True # Placeholder logic - - return { - "package": dep["name"], - "current_spec": dep["version_specifiers"], - "latest_version": latest_version, - "has_update": has_update, - "update_recommendation": f"Update to {latest_version}" - } - - except Exception as e: - return { - "package": dep["name"], - "current_spec": dep["version_specifiers"], - "latest_version": "unknown", - "has_update": False, - "error": str(e) - } - - async def _scan_dependencies_security(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: - """Scan dependencies for security vulnerabilities.""" - # Import security scanner if available - try: - from .security import scan_package_security - - vulnerabilities = [] - secure = [] - - # Process in small batches - batch_size = 5 - for i in range(0, len(dependencies), batch_size): - batch = dependencies[i:i + batch_size] - batch_tasks = [] - - for dep in batch: - task = self._scan_single_dependency_security(dep) - batch_tasks.append(task) - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for dep, result in zip(batch, batch_results): - if isinstance(result, Exception): - logger.debug(f"Failed to scan security for {dep['name']}: {result}") - continue - - if result["vulnerabilities"]: - vulnerabilities.append(result) - else: - secure.append(result) - - return { - "vulnerabilities": vulnerabilities, - "secure": secure, - "vulnerability_count": sum(len(v["vulnerabilities"]) for v in vulnerabilities), - } - - except ImportError: - logger.warning("Security scanner not available") - return await self._empty_security_result() - - async def _scan_single_dependency_security(self, dep: Dict[str, Any]) -> Dict[str, Any]: - """Scan a single dependency for security issues.""" - try: - from .security import scan_package_security - - result = await scan_package_security( - dep["name"], - version=None, # Latest version - include_dependencies=False - ) - - vuln_summary = result.get("security_summary", {}) - return { - "package": dep["name"], - "vulnerabilities": result.get("vulnerabilities", {}).get("direct", []), - "risk_level": vuln_summary.get("risk_level", "minimal"), - "total_vulnerabilities": vuln_summary.get("total_vulnerabilities", 0) - } - - except Exception as e: - return { - "package": dep["name"], - "vulnerabilities": [], - "risk_level": "unknown", - "error": str(e) - } - - async def _check_dependencies_compatibility( - self, dependencies: List[Dict[str, Any]], python_version: Optional[str] - ) -> Dict[str, Any]: - """Check Python version compatibility for dependencies.""" - if not python_version: - return await self._empty_compatibility_result() - - compatible = [] - incompatible = [] - - # Process in batches - batch_size = 10 - for i in range(0, len(dependencies), batch_size): - batch = dependencies[i:i + batch_size] - batch_tasks = [] - - for dep in batch: - task = self._check_single_dependency_compatibility(dep, python_version) - batch_tasks.append(task) - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for dep, result in zip(batch, batch_results): - if isinstance(result, Exception): - logger.debug(f"Failed to check compatibility for {dep['name']}: {result}") - continue - - if result["compatible"]: - compatible.append(result) - else: - incompatible.append(result) - - return { - "compatible": compatible, - "incompatible": incompatible, - "python_version": python_version, - "compatibility_percentage": len(compatible) / len(dependencies) * 100 if dependencies else 0 - } - - async def _check_single_dependency_compatibility( - self, dep: Dict[str, Any], python_version: str - ) -> Dict[str, Any]: - """Check compatibility for a single dependency.""" - try: - from .compatibility_check import check_python_compatibility - - # Extract target Python version (simplified) - target_version = "3.9" # Default fallback - version_match = re.search(r'(\d+\.\d+)', python_version) - if version_match: - target_version = version_match.group(1) - - result = await check_python_compatibility(dep["name"], target_version) - - return { - "package": dep["name"], - "compatible": result.get("compatible", False), - "python_version": target_version, - "details": result.get("compatibility_info", "") - } - - except Exception as e: - return { - "package": dep["name"], - "compatible": True, # Assume compatible on error - "python_version": python_version, - "error": str(e) - } - - # Helper methods for empty results - async def _empty_updates_result(self) -> Dict[str, Any]: - return {"outdated": [], "current": [], "update_percentage": 0} - - async def _empty_security_result(self) -> Dict[str, Any]: - return {"vulnerabilities": [], "secure": [], "vulnerability_count": 0} - - async def _empty_compatibility_result(self) -> Dict[str, Any]: - return {"compatible": [], "incompatible": [], "python_version": None, "compatibility_percentage": 100} - - def _generate_analysis_summary( - self, - dependencies: List[Dict[str, Any]], - health_analysis: Dict[str, Any], - update_analysis: Dict[str, Any], - security_analysis: Dict[str, Any], - compatibility_analysis: Dict[str, Any] - ) -> Dict[str, Any]: - """Generate comprehensive analysis summary.""" - return { - "total_dependencies": len(dependencies), - "health_score": round(health_analysis.get("health_score", 0), 1), - "packages_with_issues": len(health_analysis.get("issues", [])), - "outdated_packages": len(update_analysis.get("outdated", [])), - "security_vulnerabilities": security_analysis.get("vulnerability_count", 0), - "compatibility_issues": len(compatibility_analysis.get("incompatible", [])), - "overall_risk_level": self._calculate_overall_risk_level( - health_analysis, update_analysis, security_analysis, compatibility_analysis - ) - } - - def _calculate_overall_risk_level( - self, health: Dict[str, Any], updates: Dict[str, Any], - security: Dict[str, Any], compatibility: Dict[str, Any] - ) -> str: - """Calculate overall risk level for the project.""" - risk_score = 0 - - # Health risks - health_score = health.get("health_score", 100) - if health_score < 50: - risk_score += 30 - elif health_score < 75: - risk_score += 15 - - # Security risks - vuln_count = security.get("vulnerability_count", 0) - if vuln_count > 10: - risk_score += 40 - elif vuln_count > 5: - risk_score += 25 - elif vuln_count > 0: - risk_score += 15 - - # Compatibility risks - incompat_count = len(compatibility.get("incompatible", [])) - if incompat_count > 5: - risk_score += 25 - elif incompat_count > 0: - risk_score += 10 - - # Update risks (outdated packages) - outdated_count = len(updates.get("outdated", [])) - total_deps = len(updates.get("outdated", [])) + len(updates.get("current", [])) - if total_deps > 0: - outdated_percentage = (outdated_count / total_deps) * 100 - if outdated_percentage > 50: - risk_score += 20 - elif outdated_percentage > 25: - risk_score += 10 - - # Calculate risk level - if risk_score >= 70: - return "critical" - elif risk_score >= 50: - return "high" - elif risk_score >= 30: - return "medium" - elif risk_score > 0: - return "low" - else: - return "minimal" - - def _generate_requirements_recommendations( - self, - parsed_requirements: Dict[str, Any], - health_analysis: Dict[str, Any], - update_analysis: Dict[str, Any], - security_analysis: Dict[str, Any], - compatibility_analysis: Dict[str, Any], - summary: Dict[str, Any] - ) -> List[str]: - """Generate actionable recommendations for requirements management.""" - recommendations = [] - - risk_level = summary.get("overall_risk_level", "minimal") - - # Overall assessment - if risk_level == "critical": - recommendations.append("🚨 Critical issues detected - immediate action required") - elif risk_level == "high": - recommendations.append("⚠️ High risk dependencies - review and update urgently") - elif risk_level == "medium": - recommendations.append("⚠️ Moderate risk - address issues when possible") - elif risk_level == "minimal": - recommendations.append("✅ Requirements appear healthy") - - # Specific recommendations - health_issues = health_analysis.get("issues", []) - if health_issues: - recommendations.append(f"🔧 Fix {len(health_issues)} dependency specification issues") - - outdated_count = len(update_analysis.get("outdated", [])) - if outdated_count > 0: - recommendations.append(f"📦 Update {outdated_count} outdated packages") - - vuln_count = security_analysis.get("vulnerability_count", 0) - if vuln_count > 0: - recommendations.append(f"🔒 Address {vuln_count} security vulnerabilities") - - incompat_count = len(compatibility_analysis.get("incompatible", [])) - if incompat_count > 0: - recommendations.append(f"🐍 Fix {incompat_count} Python compatibility issues") - - # File format recommendations - file_format = parsed_requirements["file_info"]["format"] - if file_format == "requirements.txt": - recommendations.append("💡 Consider migrating to pyproject.toml for better dependency management") - elif file_format == "unknown": - recommendations.append("📝 Use standard requirements file formats (requirements.txt, pyproject.toml)") - - return recommendations - - -# Main analysis functions -async def analyze_project_requirements( - file_path: str, - check_updates: bool = True, - security_scan: bool = True, - compatibility_check: bool = True -) -> Dict[str, Any]: - """ - Analyze project requirements file for dependencies, security, and compatibility. - - Args: - file_path: Path to the requirements file - check_updates: Whether to check for package updates - security_scan: Whether to perform security vulnerability scanning - compatibility_check: Whether to check Python version compatibility - - Returns: - Comprehensive requirements file analysis - """ - analyzer = RequirementsAnalyzer() - return await analyzer.analyze_requirements_file( - file_path, check_updates, security_scan, compatibility_check - ) - - -async def compare_requirements_files( - file_paths: List[str] -) -> Dict[str, Any]: - """ - Compare multiple requirements files to identify differences and conflicts. - - Args: - file_paths: List of paths to requirements files to compare - - Returns: - Comparative analysis of requirements files - """ - logger.info(f"Starting requirements comparison for {len(file_paths)} files") - - analyzer = RequirementsAnalyzer() - file_analyses = {} - - # Analyze each file - for file_path in file_paths: - try: - analysis = await analyzer.analyze_requirements_file( - file_path, check_updates=False, security_scan=False, compatibility_check=False - ) - file_analyses[file_path] = analysis - except Exception as e: - logger.error(f"Failed to analyze {file_path}: {e}") - file_analyses[file_path] = {"error": str(e), "dependencies": []} - - # Compare dependencies - all_packages = set() - for analysis in file_analyses.values(): - if "dependencies" in analysis: - for dep in analysis["dependencies"]: - all_packages.add(dep["name"]) - - # Generate comparison results - conflicts = [] - common_packages = [] - unique_packages = {} - - for package in all_packages: - versions_by_file = {} - for file_path, analysis in file_analyses.items(): - if "dependencies" in analysis: - for dep in analysis["dependencies"]: - if dep["name"] == package: - versions_by_file[file_path] = dep["version_specifiers"] - break - - if len(versions_by_file) == len(file_paths): - # Package is in all files - version_specs = list(versions_by_file.values()) - if len(set(str(spec) for spec in version_specs)) > 1: - conflicts.append({ - "package": package, - "versions_by_file": versions_by_file - }) - else: - common_packages.append(package) - else: - # Package is unique to some files - for file_path, versions in versions_by_file.items(): - if file_path not in unique_packages: - unique_packages[file_path] = [] - unique_packages[file_path].append({ - "package": package, - "version_specifiers": versions - }) - - return { - "comparison_timestamp": datetime.now(timezone.utc).isoformat(), - "files_compared": len(file_paths), - "file_analyses": file_analyses, - "comparison_results": { - "total_unique_packages": len(all_packages), - "common_packages": common_packages, - "conflicting_packages": conflicts, - "unique_to_files": unique_packages, - }, - "recommendations": _generate_comparison_recommendations(conflicts, unique_packages, file_analyses) - } - - -def _generate_comparison_recommendations( - conflicts: List[Dict[str, Any]], - unique_packages: Dict[str, List[Dict[str, Any]]], - file_analyses: Dict[str, Any] -) -> List[str]: - """Generate recommendations for requirements file comparison.""" - recommendations = [] - - if conflicts: - recommendations.append(f"🔄 Resolve {len(conflicts)} version conflicts across files") - for conflict in conflicts[:3]: # Show first 3 - recommendations.append(f" - {conflict['package']}: inconsistent versions") - - if unique_packages: - total_unique = sum(len(packages) for packages in unique_packages.values()) - recommendations.append(f"📦 {total_unique} packages are unique to specific files") - - if not conflicts and not unique_packages: - recommendations.append("✅ All requirements files are consistent") - - # File format recommendations - formats = set() - for analysis in file_analyses.values(): - if "file_info" in analysis: - formats.add(analysis["file_info"]["format"]) - - if len(formats) > 1: - recommendations.append("📝 Consider standardizing on a single requirements file format") - - return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/requirements_tools.py b/pypi_query_mcp/tools/requirements_tools.py deleted file mode 100644 index 86eb3b7..0000000 --- a/pypi_query_mcp/tools/requirements_tools.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Requirements file analysis tools for Python projects.""" - -import logging -from typing import Any, Dict, List - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..tools.requirements_analyzer import analyze_project_requirements, compare_requirements_files - -logger = logging.getLogger(__name__) - - -async def analyze_requirements_file_tool( - file_path: str, - check_updates: bool = True, - security_scan: bool = True, - compatibility_check: bool = True -) -> Dict[str, Any]: - """ - Analyze project requirements file for dependencies, security, and compatibility. - - This tool provides comprehensive analysis of Python project requirements files - including dependency parsing, version checking, security vulnerability scanning, - Python compatibility assessment, and actionable recommendations for improvements. - - Args: - file_path: Path to the requirements file (requirements.txt, pyproject.toml, setup.py, etc.) - check_updates: Whether to check for available package updates - security_scan: Whether to perform security vulnerability scanning on dependencies - compatibility_check: Whether to check Python version compatibility for all dependencies - - Returns: - Dictionary containing comprehensive requirements analysis including: - - File information and detected format (requirements.txt, pyproject.toml, etc.) - - Parsed dependencies with version specifiers and extras - - Dependency health analysis with specification issues and recommendations - - Package update analysis showing outdated packages and latest versions - - Security vulnerability scan results for all dependencies - - Python version compatibility assessment - - Overall risk level and actionable improvement recommendations - - Raises: - FileNotFoundError: If the requirements file is not found - NetworkError: For network-related errors during analysis - SearchError: If requirements analysis fails - """ - logger.info(f"MCP tool: Analyzing requirements file {file_path}") - - try: - result = await analyze_project_requirements( - file_path=file_path, - check_updates=check_updates, - security_scan=security_scan, - compatibility_check=compatibility_check - ) - - summary = result.get("analysis_summary", {}) - total_deps = summary.get("total_dependencies", 0) - risk_level = summary.get("overall_risk_level", "unknown") - logger.info(f"MCP tool: Requirements analysis completed for {file_path} - {total_deps} dependencies, risk level: {risk_level}") - return result - - except (FileNotFoundError, NetworkError, SearchError) as e: - logger.error(f"Error analyzing requirements file {file_path}: {e}") - return { - "error": f"Requirements analysis failed: {e}", - "error_type": type(e).__name__, - "file_path": file_path, - "analysis_timestamp": "", - "file_info": {"name": file_path, "format": "unknown"}, - "dependencies": [], - "dependency_analysis": {}, - "analysis_summary": { - "total_dependencies": 0, - "health_score": 0, - "packages_with_issues": 0, - "outdated_packages": 0, - "security_vulnerabilities": 0, - "compatibility_issues": 0, - "overall_risk_level": "critical", - }, - "recommendations": [f"❌ Requirements analysis failed: {e}"], - "python_requirements": None, - } - - -async def compare_multiple_requirements_files( - file_paths: List[str] -) -> Dict[str, Any]: - """ - Compare multiple requirements files to identify differences and conflicts. - - This tool analyzes multiple requirements files simultaneously to identify - version conflicts, unique dependencies, and inconsistencies across different - project configurations or environments. - - Args: - file_paths: List of paths to requirements files to compare and analyze - - Returns: - Dictionary containing comparative requirements analysis including: - - Detailed analysis results for each individual file - - Common packages shared across all files - - Conflicting package versions between files with specific version details - - Packages unique to specific files - - Recommendations for resolving conflicts and standardizing requirements - - Statistics on package overlap and conflict rates - - Raises: - ValueError: If file_paths list is empty - NetworkError: For network-related errors during analysis - SearchError: If requirements comparison fails - """ - if not file_paths: - raise ValueError("File paths list cannot be empty") - - logger.info(f"MCP tool: Comparing {len(file_paths)} requirements files") - - try: - result = await compare_requirements_files(file_paths=file_paths) - - comparison_results = result.get("comparison_results", {}) - conflicts = len(comparison_results.get("conflicting_packages", [])) - total_packages = comparison_results.get("total_unique_packages", 0) - - logger.info(f"MCP tool: Requirements comparison completed - {total_packages} unique packages, {conflicts} conflicts found") - return result - - except (ValueError, NetworkError, SearchError) as e: - logger.error(f"Error comparing requirements files: {e}") - return { - "error": f"Requirements comparison failed: {e}", - "error_type": type(e).__name__, - "comparison_timestamp": "", - "files_compared": len(file_paths), - "file_analyses": {}, - "comparison_results": { - "total_unique_packages": 0, - "common_packages": [], - "conflicting_packages": [], - "unique_to_files": {}, - }, - "recommendations": [f"❌ Requirements comparison failed: {e}"] - } \ No newline at end of file diff --git a/pypi_query_mcp/tools/security.py b/pypi_query_mcp/tools/security.py deleted file mode 100644 index 5b62895..0000000 --- a/pypi_query_mcp/tools/security.py +++ /dev/null @@ -1,660 +0,0 @@ -"""Security vulnerability scanning and analysis tools for PyPI packages.""" - -import asyncio -import json -import logging -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from urllib.parse import quote - -import httpx - -from ..core.exceptions import NetworkError, SearchError -from ..core.pypi_client import PyPIClient - -logger = logging.getLogger(__name__) - - -class VulnerabilityScanner: - """Comprehensive vulnerability scanner for PyPI packages.""" - - def __init__(self): - self.timeout = 30.0 - self.session = None - - # Vulnerability database endpoints - self.osv_api = "https://api.osv.dev/v1/query" - self.safety_db_api = "https://pyup.io/api/v1/safety" - self.snyk_api = "https://snyk.io/test/pip" - - # Common vulnerability patterns to look for - self.high_risk_patterns = [ - "remote code execution", "rce", "code injection", "sql injection", - "cross-site scripting", "xss", "csrf", "authentication bypass", - "privilege escalation", "arbitrary file", "path traversal", - "buffer overflow", "memory corruption", "denial of service" - ] - - async def scan_package( - self, - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True, - severity_filter: Optional[str] = None - ) -> Dict[str, Any]: - """ - Comprehensive security scan of a PyPI package. - - Args: - package_name: Name of the package to scan - version: Specific version to scan (optional, defaults to latest) - include_dependencies: Whether to scan dependencies too - severity_filter: Filter by severity level (low, medium, high, critical) - - Returns: - Dictionary containing security analysis results - """ - logger.info(f"Starting security scan for package: {package_name}") - - try: - # Get package information - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, version) - - package_version = version or package_data["info"]["version"] - - # Run parallel vulnerability scans - scan_tasks = [ - self._scan_osv_database(package_name, package_version), - self._scan_github_advisories(package_name, package_version), - self._analyze_package_metadata(package_data), - self._check_dependency_vulnerabilities(package_name, package_version) if include_dependencies else asyncio.create_task(self._empty_result()) - ] - - osv_results, github_results, metadata_analysis, dependency_results = await asyncio.gather( - *scan_tasks, return_exceptions=True - ) - - # Consolidate results - vulnerabilities = [] - - # Process OSV results - if not isinstance(osv_results, Exception) and osv_results: - vulnerabilities.extend(osv_results.get("vulnerabilities", [])) - - # Process GitHub results - if not isinstance(github_results, Exception) and github_results: - vulnerabilities.extend(github_results.get("vulnerabilities", [])) - - # Process dependency vulnerabilities - if not isinstance(dependency_results, Exception) and dependency_results: - vulnerabilities.extend(dependency_results.get("vulnerabilities", [])) - - # Apply severity filter - if severity_filter: - vulnerabilities = [ - vuln for vuln in vulnerabilities - if vuln.get("severity", "").lower() == severity_filter.lower() - ] - - # Generate security report - security_report = self._generate_security_report( - package_name, package_version, vulnerabilities, metadata_analysis - ) - - return security_report - - except Exception as e: - logger.error(f"Security scan failed for {package_name}: {e}") - raise SearchError(f"Security scan failed: {e}") from e - - async def _scan_osv_database(self, package_name: str, version: str) -> Dict[str, Any]: - """Scan package against OSV (Open Source Vulnerabilities) database.""" - try: - async with httpx.AsyncClient(timeout=self.timeout) as client: - query_data = { - "package": { - "name": package_name, - "ecosystem": "PyPI" - }, - "version": version - } - - response = await client.post( - self.osv_api, - json=query_data, - headers={"Content-Type": "application/json"} - ) - - if response.status_code == 200: - data = response.json() - vulnerabilities = [] - - for vuln in data.get("vulns", []): - severity = self._extract_severity_from_osv(vuln) - vulnerabilities.append({ - "id": vuln.get("id", ""), - "summary": vuln.get("summary", ""), - "details": vuln.get("details", ""), - "severity": severity, - "published": vuln.get("published", ""), - "modified": vuln.get("modified", ""), - "source": "OSV", - "references": [ref.get("url", "") for ref in vuln.get("references", [])], - "affected_versions": self._extract_affected_versions(vuln), - "fixed_versions": self._extract_fixed_versions(vuln), - }) - - return {"vulnerabilities": vulnerabilities, "source": "OSV"} - else: - logger.warning(f"OSV API returned status {response.status_code}") - - except Exception as e: - logger.warning(f"OSV database scan failed: {e}") - - return {"vulnerabilities": [], "source": "OSV"} - - async def _scan_github_advisories(self, package_name: str, version: str) -> Dict[str, Any]: - """Scan against GitHub Security Advisories.""" - try: - # GitHub GraphQL API for security advisories - query = """ - query($ecosystem: SecurityAdvisoryEcosystem!, $package: String!) { - securityVulnerabilities(ecosystem: $ecosystem, package: $package, first: 100) { - nodes { - advisory { - ghsaId - summary - description - severity - publishedAt - updatedAt - references { - url - } - } - vulnerableVersionRange - firstPatchedVersion { - identifier - } - } - } - } - """ - - variables = { - "ecosystem": "PIP", - "package": package_name - } - - async with httpx.AsyncClient(timeout=self.timeout) as client: - response = await client.post( - "https://api.github.com/graphql", - json={"query": query, "variables": variables}, - headers={ - "Content-Type": "application/json", - "User-Agent": "PyPI-Security-Scanner/1.0" - } - ) - - if response.status_code == 200: - data = response.json() - vulnerabilities = [] - - for vuln_node in data.get("data", {}).get("securityVulnerabilities", {}).get("nodes", []): - advisory = vuln_node.get("advisory", {}) - - # Check if current version is affected - if self._is_version_affected(version, vuln_node.get("vulnerableVersionRange", "")): - vulnerabilities.append({ - "id": advisory.get("ghsaId", ""), - "summary": advisory.get("summary", ""), - "details": advisory.get("description", ""), - "severity": advisory.get("severity", "").lower(), - "published": advisory.get("publishedAt", ""), - "modified": advisory.get("updatedAt", ""), - "source": "GitHub", - "references": [ref.get("url", "") for ref in advisory.get("references", [])], - "vulnerable_range": vuln_node.get("vulnerableVersionRange", ""), - "first_patched": vuln_node.get("firstPatchedVersion", {}).get("identifier", ""), - }) - - return {"vulnerabilities": vulnerabilities, "source": "GitHub"} - - except Exception as e: - logger.warning(f"GitHub advisories scan failed: {e}") - - return {"vulnerabilities": [], "source": "GitHub"} - - async def _analyze_package_metadata(self, package_data: Dict[str, Any]) -> Dict[str, Any]: - """Analyze package metadata for security indicators.""" - info = package_data.get("info", {}) - - security_indicators = { - "metadata_score": 0, - "risk_factors": [], - "security_features": [], - "warnings": [] - } - - # Check for security-related information - description = (info.get("description") or "").lower() - summary = (info.get("summary") or "").lower() - keywords = (info.get("keywords") or "").lower() - - combined_text = f"{description} {summary} {keywords}" - - # Look for security mentions - if any(term in combined_text for term in ["security", "cryptography", "authentication", "encryption"]): - security_indicators["security_features"].append("Contains security-related functionality") - security_indicators["metadata_score"] += 20 - - # Check for high-risk patterns - for pattern in self.high_risk_patterns: - if pattern in combined_text: - security_indicators["risk_factors"].append(f"Mentions: {pattern}") - security_indicators["metadata_score"] -= 10 - - # Check package age and maintenance - if info.get("author_email"): - security_indicators["metadata_score"] += 10 - - if info.get("home_page"): - security_indicators["metadata_score"] += 5 - - # Check for classifiers - classifiers = info.get("classifiers", []) - for classifier in classifiers: - if "Development Status :: 5 - Production/Stable" in classifier: - security_indicators["metadata_score"] += 15 - security_indicators["security_features"].append("Production stable status") - elif "License ::" in classifier: - security_indicators["metadata_score"] += 5 - - # Check for suspicious patterns - if not info.get("author") and not info.get("maintainer"): - security_indicators["warnings"].append("No author or maintainer information") - security_indicators["metadata_score"] -= 20 - - if len(info.get("description", "")) < 50: - security_indicators["warnings"].append("Very brief or missing description") - security_indicators["metadata_score"] -= 10 - - return security_indicators - - async def _check_dependency_vulnerabilities(self, package_name: str, version: str) -> Dict[str, Any]: - """Check vulnerabilities in package dependencies.""" - try: - # Get package dependencies - async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, version) - - # Extract dependencies - requires_dist = package_data.get("info", {}).get("requires_dist", []) or [] - dependencies = [] - - for req in requires_dist: - # Parse dependency name (simplified) - dep_name = req.split()[0].split(">=")[0].split("==")[0].split("~=")[0].split("!=")[0] - if dep_name and not dep_name.startswith("extra"): - dependencies.append(dep_name) - - # Scan top dependencies for vulnerabilities - dependency_vulnerabilities = [] - - # Limit to top 10 dependencies to avoid overwhelming the system - for dep_name in dependencies[:10]: - try: - dep_scan = await self._scan_osv_database(dep_name, "latest") - for vuln in dep_scan.get("vulnerabilities", []): - vuln["dependency"] = dep_name - vuln["type"] = "dependency_vulnerability" - dependency_vulnerabilities.append(vuln) - except Exception as e: - logger.debug(f"Failed to scan dependency {dep_name}: {e}") - - return {"vulnerabilities": dependency_vulnerabilities, "source": "dependencies"} - - except Exception as e: - logger.warning(f"Dependency vulnerability check failed: {e}") - return {"vulnerabilities": [], "source": "dependencies"} - - async def _empty_result(self) -> Dict[str, Any]: - """Return empty result for disabled scans.""" - return {"vulnerabilities": [], "source": "disabled"} - - def _extract_severity_from_osv(self, vuln_data: Dict[str, Any]) -> str: - """Extract severity from OSV vulnerability data.""" - # OSV uses CVSS scores, map to common severity levels - severity_data = vuln_data.get("severity", []) - if severity_data: - score = severity_data[0].get("score", "") - if "CVSS:" in score: - # Extract CVSS score - try: - cvss_score = float(score.split("/")[1]) - if cvss_score >= 9.0: - return "critical" - elif cvss_score >= 7.0: - return "high" - elif cvss_score >= 4.0: - return "medium" - else: - return "low" - except: - pass - - return "unknown" - - def _extract_affected_versions(self, vuln_data: Dict[str, Any]) -> List[str]: - """Extract affected version ranges from vulnerability data.""" - affected = vuln_data.get("affected", []) - version_ranges = [] - - for affect in affected: - ranges = affect.get("ranges", []) - for range_data in ranges: - events = range_data.get("events", []) - for event in events: - if "introduced" in event: - version_ranges.append(f">= {event['introduced']}") - elif "fixed" in event: - version_ranges.append(f"< {event['fixed']}") - - return version_ranges - - def _extract_fixed_versions(self, vuln_data: Dict[str, Any]) -> List[str]: - """Extract fixed versions from vulnerability data.""" - affected = vuln_data.get("affected", []) - fixed_versions = [] - - for affect in affected: - ranges = affect.get("ranges", []) - for range_data in ranges: - events = range_data.get("events", []) - for event in events: - if "fixed" in event: - fixed_versions.append(event["fixed"]) - - return fixed_versions - - def _is_version_affected(self, version: str, vulnerable_range: str) -> bool: - """Check if a version is affected by a vulnerability range.""" - # Simplified version checking - in production would use packaging.specifiers - if not vulnerable_range: - return True - - # Basic patterns - if "< " in vulnerable_range: - try: - limit = vulnerable_range.split("< ")[1].strip() - return version < limit - except: - pass - - if ">= " in vulnerable_range: - try: - limit = vulnerable_range.split(">= ")[1].strip() - return version >= limit - except: - pass - - return True # Assume affected if we can't parse - - def _generate_security_report( - self, - package_name: str, - version: str, - vulnerabilities: List[Dict[str, Any]], - metadata_analysis: Dict[str, Any] - ) -> Dict[str, Any]: - """Generate comprehensive security report.""" - - # Categorize vulnerabilities by severity - severity_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "unknown": 0} - dependency_vulns = [] - direct_vulns = [] - - for vuln in vulnerabilities: - severity = vuln.get("severity", "unknown") - severity_counts[severity] = severity_counts.get(severity, 0) + 1 - - if vuln.get("type") == "dependency_vulnerability": - dependency_vulns.append(vuln) - else: - direct_vulns.append(vuln) - - # Calculate risk score - risk_score = self._calculate_risk_score(severity_counts, metadata_analysis) - - # Generate recommendations - recommendations = self._generate_security_recommendations( - vulnerabilities, metadata_analysis, risk_score - ) - - return { - "package": package_name, - "version": version, - "scan_timestamp": datetime.now(timezone.utc).isoformat(), - "security_summary": { - "total_vulnerabilities": len(vulnerabilities), - "direct_vulnerabilities": len(direct_vulns), - "dependency_vulnerabilities": len(dependency_vulns), - "severity_breakdown": severity_counts, - "risk_score": risk_score, - "risk_level": self._get_risk_level(risk_score), - }, - "vulnerabilities": { - "direct": direct_vulns, - "dependencies": dependency_vulns, - }, - "metadata_analysis": metadata_analysis, - "recommendations": recommendations, - "scan_details": { - "sources_checked": ["OSV", "GitHub", "Metadata"], - "dependencies_scanned": len(dependency_vulns) > 0, - "scan_completion": "success", - } - } - - def _calculate_risk_score(self, severity_counts: Dict[str, int], metadata_analysis: Dict[str, Any]) -> float: - """Calculate overall risk score (0-100).""" - score = 0.0 - - # Vulnerability scoring (0-80 points) - score += severity_counts.get("critical", 0) * 20 - score += severity_counts.get("high", 0) * 15 - score += severity_counts.get("medium", 0) * 8 - score += severity_counts.get("low", 0) * 3 - - # Metadata scoring (0-20 points) - metadata_score = metadata_analysis.get("metadata_score", 0) - if metadata_score < 0: - score += abs(metadata_score) / 5 # Convert negative metadata score to risk - else: - score -= metadata_score / 10 # Good metadata reduces risk - - # Cap at 100 - return min(max(score, 0), 100) - - def _get_risk_level(self, risk_score: float) -> str: - """Convert risk score to risk level.""" - if risk_score >= 80: - return "critical" - elif risk_score >= 60: - return "high" - elif risk_score >= 30: - return "medium" - elif risk_score > 0: - return "low" - else: - return "minimal" - - def _generate_security_recommendations( - self, - vulnerabilities: List[Dict[str, Any]], - metadata_analysis: Dict[str, Any], - risk_score: float - ) -> List[str]: - """Generate actionable security recommendations.""" - recommendations = [] - - if len(vulnerabilities) > 0: - recommendations.append(f"🚨 Found {len(vulnerabilities)} security vulnerabilities - review and update immediately") - - # Check for critical/high severity - critical_high = [v for v in vulnerabilities if v.get("severity") in ["critical", "high"]] - if critical_high: - recommendations.append(f"⚠️ {len(critical_high)} critical/high severity vulnerabilities require immediate attention") - - # Check for fixed versions - fixed_versions = [] - for vuln in vulnerabilities: - fixed = vuln.get("fixed_versions", []) or [vuln.get("first_patched", "")] - fixed_versions.extend([v for v in fixed if v]) - - if fixed_versions: - latest_fixed = max(fixed_versions) if fixed_versions else None - if latest_fixed: - recommendations.append(f"📦 Update to version {latest_fixed} or later to fix known vulnerabilities") - - # Metadata recommendations - warnings = metadata_analysis.get("warnings", []) - if warnings: - recommendations.append(f"⚠️ Package metadata issues: {', '.join(warnings)}") - - if metadata_analysis.get("metadata_score", 0) < 20: - recommendations.append("📝 Package has poor metadata quality - verify trustworthiness before use") - - # General recommendations based on risk score - if risk_score >= 60: - recommendations.append("🛑 High risk package - consider alternatives or additional security review") - elif risk_score >= 30: - recommendations.append("⚠️ Moderate risk - monitor for updates and security patches") - elif len(vulnerabilities) == 0: - recommendations.append("✅ No known vulnerabilities found - package appears secure") - - return recommendations - - -# Main scanning functions -async def scan_package_security( - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True, - severity_filter: Optional[str] = None -) -> Dict[str, Any]: - """ - Scan a PyPI package for security vulnerabilities. - - Args: - package_name: Name of the package to scan - version: Specific version to scan (optional) - include_dependencies: Whether to scan dependencies - severity_filter: Filter by severity (low, medium, high, critical) - - Returns: - Comprehensive security scan results - """ - scanner = VulnerabilityScanner() - return await scanner.scan_package( - package_name, version, include_dependencies, severity_filter - ) - - -async def bulk_security_scan( - package_names: List[str], - include_dependencies: bool = False, - severity_threshold: str = "medium" -) -> Dict[str, Any]: - """ - Perform bulk security scanning of multiple packages. - - Args: - package_names: List of package names to scan - include_dependencies: Whether to scan dependencies - severity_threshold: Minimum severity to report - - Returns: - Bulk scan results with summary - """ - logger.info(f"Starting bulk security scan of {len(package_names)} packages") - - scanner = VulnerabilityScanner() - scan_results = {} - summary = { - "total_packages": len(package_names), - "packages_with_vulnerabilities": 0, - "total_vulnerabilities": 0, - "high_risk_packages": [], - "scan_timestamp": datetime.now(timezone.utc).isoformat() - } - - # Scan packages in parallel batches - batch_size = 5 - for i in range(0, len(package_names), batch_size): - batch = package_names[i:i + batch_size] - batch_tasks = [ - scanner.scan_package(pkg_name, include_dependencies=include_dependencies) - for pkg_name in batch - ] - - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - - for pkg_name, result in zip(batch, batch_results): - if isinstance(result, Exception): - scan_results[pkg_name] = { - "error": str(result), - "scan_status": "failed" - } - else: - scan_results[pkg_name] = result - - # Update summary - vuln_count = result.get("security_summary", {}).get("total_vulnerabilities", 0) - if vuln_count > 0: - summary["packages_with_vulnerabilities"] += 1 - summary["total_vulnerabilities"] += vuln_count - - risk_level = result.get("security_summary", {}).get("risk_level", "") - if risk_level in ["high", "critical"]: - summary["high_risk_packages"].append({ - "package": pkg_name, - "risk_level": risk_level, - "vulnerabilities": vuln_count - }) - - return { - "summary": summary, - "detailed_results": scan_results, - "recommendations": _generate_bulk_recommendations(summary, scan_results) - } - - -def _generate_bulk_recommendations(summary: Dict[str, Any], results: Dict[str, Any]) -> List[str]: - """Generate recommendations for bulk scan results.""" - recommendations = [] - - vuln_packages = summary["packages_with_vulnerabilities"] - total_packages = summary["total_packages"] - - if vuln_packages == 0: - recommendations.append("✅ No security vulnerabilities found in any scanned packages") - else: - percentage = (vuln_packages / total_packages) * 100 - recommendations.append( - f"🚨 {vuln_packages}/{total_packages} packages ({percentage:.1f}%) have security vulnerabilities" - ) - - high_risk = summary["high_risk_packages"] - if high_risk: - recommendations.append( - f"⚠️ {len(high_risk)} packages are high/critical risk: {', '.join([p['package'] for p in high_risk])}" - ) - recommendations.append("🛑 Priority: Address high-risk packages immediately") - - if summary["total_vulnerabilities"] > 0: - recommendations.append(f"📊 Total vulnerabilities found: {summary['total_vulnerabilities']}") - recommendations.append("🔍 Review detailed results and update affected packages") - - return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/security_tools.py b/pypi_query_mcp/tools/security_tools.py deleted file mode 100644 index fb338ea..0000000 --- a/pypi_query_mcp/tools/security_tools.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Security vulnerability scanning tools for PyPI packages.""" - -import logging -from typing import Any, Dict, List, Optional - -from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError -from ..tools.security import bulk_security_scan, scan_package_security - -logger = logging.getLogger(__name__) - - -async def scan_pypi_package_security( - package_name: str, - version: Optional[str] = None, - include_dependencies: bool = True, - severity_filter: Optional[str] = None -) -> Dict[str, Any]: - """ - Scan a PyPI package for security vulnerabilities. - - This tool performs comprehensive security vulnerability scanning of PyPI packages, - checking against multiple vulnerability databases including OSV (Open Source Vulnerabilities), - GitHub Security Advisories, and analyzing package metadata for security indicators. - - Args: - package_name: Name of the package to scan for vulnerabilities - version: Specific version to scan (optional, defaults to latest version) - include_dependencies: Whether to scan package dependencies for vulnerabilities - severity_filter: Filter results by severity level (low, medium, high, critical) - - Returns: - Dictionary containing comprehensive security scan results including: - - Total vulnerability count and severity breakdown - - Direct package vulnerabilities vs dependency vulnerabilities - - Risk score and level assessment (minimal, low, medium, high, critical) - - Detailed vulnerability information with IDs, descriptions, and references - - Package metadata security analysis - - Actionable security recommendations - - Raises: - InvalidPackageNameError: If package name is empty or invalid - PackageNotFoundError: If package is not found on PyPI - NetworkError: For network-related errors - SearchError: If security scanning fails - """ - if not package_name or not package_name.strip(): - raise InvalidPackageNameError(package_name) - - logger.info(f"MCP tool: Scanning security for package {package_name}") - - try: - result = await scan_package_security( - package_name=package_name, - version=version, - include_dependencies=include_dependencies, - severity_filter=severity_filter - ) - - logger.info(f"MCP tool: Security scan completed for {package_name} - found {result.get('security_summary', {}).get('total_vulnerabilities', 0)} vulnerabilities") - return result - - except (InvalidPackageNameError, NetworkError, SearchError) as e: - logger.error(f"Error scanning security for {package_name}: {e}") - return { - "error": f"Security scan failed: {e}", - "error_type": type(e).__name__, - "package": package_name, - "version": version, - "scan_timestamp": "", - "security_summary": { - "total_vulnerabilities": 0, - "direct_vulnerabilities": 0, - "dependency_vulnerabilities": 0, - "severity_breakdown": {"critical": 0, "high": 0, "medium": 0, "low": 0, "unknown": 0}, - "risk_score": 0, - "risk_level": "unknown", - }, - "vulnerabilities": {"direct": [], "dependencies": []}, - "metadata_analysis": {}, - "recommendations": [f"❌ Security scan failed: {e}"], - "scan_details": { - "sources_checked": [], - "dependencies_scanned": False, - "scan_completion": "error", - } - } - - -async def bulk_scan_package_security( - package_names: List[str], - include_dependencies: bool = False, - severity_threshold: str = "medium" -) -> Dict[str, Any]: - """ - Perform bulk security scanning of multiple PyPI packages. - - This tool scans multiple packages simultaneously for security vulnerabilities, - providing a consolidated report with summary statistics and prioritized - recommendations for addressing security issues across your package ecosystem. - - Args: - package_names: List of package names to scan for vulnerabilities - include_dependencies: Whether to include dependency vulnerability scanning - severity_threshold: Minimum severity level to report (low, medium, high, critical) - - Returns: - Dictionary containing bulk scan results including: - - Summary statistics (total packages, packages with vulnerabilities, high-risk packages) - - Detailed scan results for each package - - Prioritized recommendations for security remediation - - Scan timestamp and completion status - - Raises: - ValueError: If package_names list is empty - NetworkError: For network-related errors during scanning - SearchError: If bulk scanning fails - """ - if not package_names: - raise ValueError("Package names list cannot be empty") - - logger.info(f"MCP tool: Starting bulk security scan of {len(package_names)} packages") - - try: - result = await bulk_security_scan( - package_names=package_names, - include_dependencies=include_dependencies, - severity_threshold=severity_threshold - ) - - logger.info(f"MCP tool: Bulk security scan completed - {result.get('summary', {}).get('packages_with_vulnerabilities', 0)} packages have vulnerabilities") - return result - - except (ValueError, NetworkError, SearchError) as e: - logger.error(f"Error in bulk security scan: {e}") - return { - "error": f"Bulk security scan failed: {e}", - "error_type": type(e).__name__, - "summary": { - "total_packages": len(package_names), - "packages_with_vulnerabilities": 0, - "total_vulnerabilities": 0, - "high_risk_packages": [], - "scan_timestamp": "" - }, - "detailed_results": {}, - "recommendations": [f"❌ Bulk security scan failed: {e}"] - } \ No newline at end of file diff --git a/pypi_query_mcp/tools/workflow.py b/pypi_query_mcp/tools/workflow.py deleted file mode 100644 index 2eeba93..0000000 --- a/pypi_query_mcp/tools/workflow.py +++ /dev/null @@ -1,1116 +0,0 @@ -"""PyPI Development Workflow Tools for package validation and preview.""" - -import logging -import re -from typing import Any, Dict, List, Optional -from urllib.parse import quote - -import httpx - -from ..core import InvalidPackageNameError, NetworkError, PackageNotFoundError, PyPIClient -from ..core.exceptions import PyPIError - -logger = logging.getLogger(__name__) - - -class PyPIWorkflowError(PyPIError): - """Raised when workflow operations fail.""" - - def __init__(self, message: str, operation: str | None = None): - super().__init__(message) - self.operation = operation - - -def _validate_package_name_format(package_name: str) -> Dict[str, Any]: - """Validate package name format according to PyPI standards. - - Args: - package_name: Package name to validate - - Returns: - Dictionary with validation results - """ - issues = [] - recommendations = [] - - # Check basic format - if not package_name: - issues.append("Package name cannot be empty") - return { - "valid": False, - "issues": issues, - "recommendations": ["Provide a non-empty package name"], - "normalized_name": "", - } - - # Length constraints - if len(package_name) > 214: - issues.append("Package name is too long (max 214 characters)") - if len(package_name) < 1: - issues.append("Package name is too short (min 1 character)") - - # Character validation - if not re.match(r"^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$", package_name): - issues.append("Package name contains invalid characters (only alphanumeric, dots, hyphens, underscores allowed)") - - # Check for reserved names - reserved_names = { - "pip", "setuptools", "wheel", "python", "test", "tests", - "src", "docs", "build", "dist", "requirements", "readme" - } - if package_name.lower() in reserved_names: - issues.append(f"'{package_name}' is a reserved name and should not be used") - - # Best practice checks - if package_name != package_name.lower(): - recommendations.append("Use lowercase for package names (PEP 508)") - - if "_" in package_name: - recommendations.append("Consider using hyphens instead of underscores (PEP 508)") - - if len(package_name.split("-")) > 3: - recommendations.append("Avoid overly complex names with many hyphens") - - # Check for common typos/patterns - if package_name.startswith("-") or package_name.endswith("-"): - issues.append("Package name cannot start or end with a hyphen") - - if package_name.startswith(".") or package_name.endswith("."): - issues.append("Package name cannot start or end with a dot") - - if "--" in package_name or ".." in package_name: - issues.append("Package name cannot contain consecutive special characters") - - # Normalize name for PyPI (convert to lowercase, normalize separators) - normalized = re.sub(r"[-_.]+", "-", package_name.lower()) - - return { - "valid": len(issues) == 0, - "issues": issues, - "recommendations": recommendations, - "normalized_name": normalized, - } - - -async def validate_pypi_package_name(package_name: str) -> Dict[str, Any]: - """Check if a package name is available and valid on PyPI. - - This function validates package name format and checks availability on PyPI. - It also provides recommendations for improvement based on PyPI standards. - - Args: - package_name: Name to validate and check for availability - - Returns: - Dictionary containing validation results including: - - Format validation results - - Availability status on PyPI - - Recommendations for improvement - - Similar existing packages (if any) - - Raises: - InvalidPackageNameError: If package name format is severely invalid - NetworkError: For network-related errors - PyPIWorkflowError: For workflow-specific errors - """ - logger.info(f"Validating package name: {package_name}") - - try: - # First validate format - format_validation = _validate_package_name_format(package_name) - - if not format_validation["valid"] and len(format_validation["issues"]) > 2: - raise InvalidPackageNameError(package_name) - - # Check availability on PyPI - availability_status = "unknown" - existing_package_info = None - similar_packages = [] - - try: - async with PyPIClient() as client: - # Try to get package info to check if it exists - package_data = await client.get_package_info(package_name) - availability_status = "taken" - existing_package_info = { - "name": package_data["info"]["name"], - "version": package_data["info"]["version"], - "summary": package_data["info"]["summary"], - "upload_time": package_data["info"].get("upload_time"), - "author": package_data["info"].get("author"), - } - except PackageNotFoundError: - availability_status = "available" - except Exception as e: - logger.warning(f"Could not check availability for {package_name}: {e}") - availability_status = "unknown" - - # If package is taken or format is questionable, find similar packages - if availability_status == "taken" or not format_validation["valid"]: - try: - # Search for similar packages - search_query = format_validation["normalized_name"] - async with httpx.AsyncClient(timeout=10.0) as http_client: - search_url = f"https://pypi.org/search/?q={quote(search_query)}" - # Note: We're not actually parsing HTML here - this would need - # integration with the search functionality for real similar packages - # For now, we'll just note that similar package detection is available - similar_packages = [] # Placeholder - except Exception as e: - logger.warning(f"Could not search for similar packages: {e}") - - # Generate suggestions based on validation results - suggestions = [] - if availability_status == "taken": - suggestions.extend([ - f"Try variations like '{package_name}-dev', '{package_name}-cli', or '{package_name}2'", - "Consider a more specific or descriptive name", - "Add your organization name as a prefix" - ]) - - if format_validation["recommendations"]: - suggestions.extend(format_validation["recommendations"]) - - return { - "package_name": package_name, - "normalized_name": format_validation["normalized_name"], - "validation": { - "format_valid": format_validation["valid"], - "issues": format_validation["issues"], - "recommendations": format_validation["recommendations"], - }, - "availability": { - "status": availability_status, # "available", "taken", "unknown" - "existing_package": existing_package_info, - "similar_packages": similar_packages, - }, - "suggestions": suggestions, - "pypi_standards_compliant": format_validation["valid"] and availability_status == "available", - "ready_for_upload": format_validation["valid"] and availability_status == "available", - } - - except InvalidPackageNameError: - raise - except Exception as e: - logger.error(f"Error validating package name {package_name}: {e}") - raise PyPIWorkflowError(f"Failed to validate package name: {e}", "validate_name") from e - - -async def preview_pypi_package_page( - package_name: str, - version: str = "1.0.0", - summary: str = "", - description: str = "", - author: str = "", - license_name: str = "MIT", - home_page: str = "", - keywords: List[str] = None, - classifiers: List[str] = None, -) -> Dict[str, Any]: - """Generate a preview of how a package page would look on PyPI. - - This function creates a preview of the PyPI package page based on the - provided metadata, helping developers visualize their package before upload. - - Args: - package_name: Name of the package - version: Package version (default: "1.0.0") - summary: Short package description - description: Long package description - author: Package author name - license_name: License type (default: "MIT") - home_page: Project homepage URL - keywords: List of keywords for the package - classifiers: List of PyPI classifiers - - Returns: - Dictionary containing preview information including: - - Formatted package metadata - - Rendered page sections - - Validation warnings - - SEO recommendations - - Raises: - InvalidPackageNameError: If package name is invalid - PyPIWorkflowError: For preview generation errors - """ - logger.info(f"Generating preview for package: {package_name}") - - if keywords is None: - keywords = [] - if classifiers is None: - classifiers = [] - - try: - # Validate package name first - name_validation = _validate_package_name_format(package_name) - if not name_validation["valid"]: - raise InvalidPackageNameError(package_name) - - # Validate version format - version_pattern = r"^([0-9]+)\.([0-9]+)\.([0-9]+)(?:[-.]?(a|b|rc)[0-9]*)?(?:\.post[0-9]+)?(?:\.dev[0-9]+)?$" - version_valid = bool(re.match(version_pattern, version)) - - # Generate preview content - preview_sections = { - "header": { - "name": package_name, - "version": version, - "summary": summary or "No description provided", - "upload_date": "Not yet uploaded", - "author": author or "Not specified", - }, - "navigation": { - "project_description": True, - "release_history": False, # Will be available after first upload - "download_files": False, # Will be available after first upload - "statistics": False, # Will be available after upload - }, - "metadata": { - "license": license_name, - "home_page": home_page or "Not provided", - "keywords": keywords, - "classifiers": classifiers, - "requires_python": "Not specified", - "project_urls": {}, - }, - "description": { - "content": description or "No detailed description provided", - "format": "text/plain" if description and not description.strip().startswith("<") else "text/markdown", - "length": len(description) if description else 0, - }, - } - - # Generate recommendations and warnings - recommendations = [] - warnings = [] - - # Check for missing critical information - if not summary: - warnings.append("Summary is missing - this appears prominently in search results") - elif len(summary) < 10: - recommendations.append("Consider a more descriptive summary (current: {len(summary)} chars)") - elif len(summary) > 200: - warnings.append("Summary is quite long - consider keeping it under 200 characters") - - if not description: - warnings.append("No description provided - users won't understand what your package does") - elif len(description) < 100: - recommendations.append("Consider providing a more detailed description") - - if not author: - warnings.append("Author name is missing") - - if not home_page: - recommendations.append("Consider adding a homepage URL (GitHub, documentation, etc.)") - - if not keywords: - recommendations.append("Add keywords to improve discoverability") - elif len(keywords) > 10: - recommendations.append("Consider reducing keywords to the most relevant ones") - - if not classifiers: - recommendations.append("Add PyPI classifiers to categorize your package") - else: - # Check for important classifier categories - has_python_version = any("Python ::" in c for c in classifiers) - has_topic = any("Topic ::" in c for c in classifiers) - has_development_status = any("Development Status ::" in c for c in classifiers) - - if not has_python_version: - recommendations.append("Add Python version classifiers (e.g., 'Programming Language :: Python :: 3.8')") - if not has_topic: - recommendations.append("Add topic classifiers to categorize your package") - if not has_development_status: - recommendations.append("Add development status classifier") - - if not version_valid: - warnings.append("Version format may not follow PEP 440 standards") - - # SEO and discoverability analysis - seo_analysis = { - "title_optimization": { - "length": len(package_name), - "readability": "good" if len(package_name.split("-")) <= 3 else "complex", - "keyword_rich": len(keywords) > 0, - }, - "description_optimization": { - "length": len(description) if description else 0, - "has_keywords": any(kw.lower() in description.lower() for kw in keywords) if description and keywords else False, - "structure": "needs_improvement" if not description else "basic", - }, - "discoverability_score": _calculate_discoverability_score( - summary, description, keywords, classifiers - ), - } - - # Generate rendered preview (simplified HTML-like structure) - rendered_preview = _generate_html_preview(preview_sections) - - return { - "package_name": package_name, - "version": version, - "preview": { - "sections": preview_sections, - "rendered_html": rendered_preview, - "url_preview": f"https://pypi.org/project/{package_name}/", - }, - "validation": { - "name_valid": name_validation["valid"], - "version_valid": version_valid, - "completeness_score": _calculate_completeness_score(preview_sections), - }, - "recommendations": recommendations, - "warnings": warnings, - "seo_analysis": seo_analysis, - "ready_for_upload": len(warnings) == 0 and name_validation["valid"] and version_valid, - } - - except InvalidPackageNameError: - raise - except Exception as e: - logger.error(f"Error generating preview for {package_name}: {e}") - raise PyPIWorkflowError(f"Failed to generate preview: {e}", "preview_page") from e - - -def _calculate_discoverability_score( - summary: str, description: str, keywords: List[str], classifiers: List[str] -) -> Dict[str, Any]: - """Calculate a discoverability score based on metadata completeness.""" - score = 0 - max_score = 100 - - # Summary scoring (25 points) - if summary: - if len(summary) >= 20: - score += 25 - elif len(summary) >= 10: - score += 15 - else: - score += 5 - - # Description scoring (35 points) - if description: - if len(description) >= 500: - score += 35 - elif len(description) >= 200: - score += 25 - elif len(description) >= 50: - score += 15 - else: - score += 5 - - # Keywords scoring (20 points) - if keywords: - if len(keywords) >= 5: - score += 20 - elif len(keywords) >= 3: - score += 15 - else: - score += 10 - - # Classifiers scoring (20 points) - if classifiers: - if len(classifiers) >= 5: - score += 20 - elif len(classifiers) >= 3: - score += 15 - else: - score += 10 - - level = "excellent" if score >= 80 else "good" if score >= 60 else "fair" if score >= 40 else "poor" - - return { - "score": score, - "max_score": max_score, - "percentage": round((score / max_score) * 100, 1), - "level": level, - } - - -def _calculate_completeness_score(preview_sections: Dict[str, Any]) -> Dict[str, Any]: - """Calculate completeness score based on available metadata.""" - score = 0 - max_score = 100 - - header = preview_sections["header"] - metadata = preview_sections["metadata"] - description = preview_sections["description"] - - # Essential fields (60 points) - if header["summary"] and header["summary"] != "No description provided": - score += 20 - if header["author"] and header["author"] != "Not specified": - score += 15 - if description["content"] and description["content"] != "No detailed description provided": - score += 25 - - # Important fields (30 points) - if metadata["license"]: - score += 10 - if metadata["home_page"] and metadata["home_page"] != "Not provided": - score += 10 - if metadata["keywords"]: - score += 5 - if metadata["classifiers"]: - score += 5 - - # Nice-to-have fields (10 points) - if description["length"] > 200: - score += 5 - if len(metadata["keywords"]) >= 3: - score += 5 - - level = "complete" if score >= 80 else "good" if score >= 60 else "basic" if score >= 40 else "incomplete" - - return { - "score": score, - "max_score": max_score, - "percentage": round((score / max_score) * 100, 1), - "level": level, - } - - -def _generate_html_preview(preview_sections: Dict[str, Any]) -> str: - """Generate a simplified HTML preview of the PyPI page.""" - header = preview_sections["header"] - metadata = preview_sections["metadata"] - description = preview_sections["description"] - - html = f""" - - -
-{header['summary']}
- -