diff --git a/poetry.lock b/poetry.lock index ebd88d3..95aae06 100644 --- a/poetry.lock +++ b/poetry.lock @@ -691,6 +691,21 @@ rich = ">=13.9.4" [package.extras] websockets = ["websockets (>=15.0.1)"] +[[package]] +name = "feedparser" +version = "6.0.11" +description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45"}, + {file = "feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5"}, +] + +[package.dependencies] +sgmllib3k = "*" + [[package]] name = "filelock" version = "3.19.1" @@ -1994,6 +2009,17 @@ files = [ {file = "ruff-0.12.9.tar.gz", hash = "sha256:fbd94b2e3c623f659962934e52c2bea6fc6da11f667a427a368adaf3af2c866a"}, ] +[[package]] +name = "sgmllib3k" +version = "1.0.0" +description = "Py3k port of sgmllib." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"}, +] + [[package]] name = "six" version = "1.17.0" @@ -2250,4 +2276,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "9785e18d2d996f5e58e1b06c722f6de31c445a1a83528f39227d1c373b91f989" +content-hash = "13bc4176d567d6738ca9ca5ebd67565f8526853434911137f4b51b39e275a546" diff --git a/pypi_query_mcp/__init__.py b/pypi_query_mcp/__init__.py index 1dca3d4..1c9b313 100644 --- a/pypi_query_mcp/__init__.py +++ b/pypi_query_mcp/__init__.py @@ -8,6 +8,10 @@ __version__ = "0.1.0" __author__ = "Hal" __email__ = "hal.long@outlook.com" -from pypi_query_mcp.server import mcp - -__all__ = ["mcp", "__version__"] +try: + from pypi_query_mcp.server import mcp + __all__ = ["mcp", "__version__"] +except ImportError: + # Server dependencies not available (fastmcp, etc.) + # Tools can still be imported individually + __all__ = ["__version__"] diff --git a/pypi_query_mcp/core/search_client.py b/pypi_query_mcp/core/search_client.py index af9a8be..a861aba 100644 --- a/pypi_query_mcp/core/search_client.py +++ b/pypi_query_mcp/core/search_client.py @@ -126,20 +126,42 @@ class PyPISearchClient: try: # Use PyPI's search API as the primary source - pypi_results = await self._search_pypi_api(query, limit * 3) # Get more for filtering + try: + pypi_results = await self._search_pypi_api(query, limit * 3) # Get more for filtering + logger.info(f"Got {len(pypi_results)} raw results from PyPI API") + except Exception as e: + logger.error(f"PyPI API search failed: {e}") + pypi_results = [] # Enhance results with additional metadata - enhanced_results = await self._enhance_search_results(pypi_results) + try: + enhanced_results = await self._enhance_search_results(pypi_results) + logger.info(f"Enhanced to {len(enhanced_results)} results") + except Exception as e: + logger.error(f"Enhancement failed: {e}") + enhanced_results = pypi_results # Apply filters - filtered_results = self._apply_filters(enhanced_results, filters) + try: + filtered_results = self._apply_filters(enhanced_results, filters) + logger.info(f"Filtered to {len(filtered_results)} results") + except Exception as e: + logger.error(f"Filtering failed: {e}") + filtered_results = enhanced_results # Apply semantic search if requested if semantic_search: - filtered_results = self._apply_semantic_search(filtered_results, query) + try: + filtered_results = self._apply_semantic_search(filtered_results, query) + except Exception as e: + logger.error(f"Semantic search failed: {e}") # Sort results - sorted_results = self._sort_results(filtered_results, sort) + try: + sorted_results = self._sort_results(filtered_results, sort) + except Exception as e: + logger.error(f"Sorting failed: {e}") + sorted_results = filtered_results # Limit results final_results = sorted_results[:limit] @@ -161,72 +183,318 @@ class PyPISearchClient: raise SearchError(f"Search failed: {e}") from e async def _search_pypi_api(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Search using PyPI's official search API.""" - url = "https://pypi.org/search/" - params = { - "q": query, - "page": 1, - } + """Search using available PyPI methods - no native search API exists.""" + logger.info(f"PyPI has no native search API, using curated search for: '{query}'") - async with httpx.AsyncClient(timeout=self.timeout) as client: - try: - response = await client.get(url, params=params) - response.raise_for_status() - - # Parse the HTML response (PyPI search returns HTML) - return await self._parse_search_html(response.text, limit) - - except httpx.HTTPError as e: - logger.error(f"PyPI search API error: {e}") - # Fallback to alternative search method - return await self._fallback_search(query, limit) + # PyPI doesn't have a search API, so we'll use our curated approach + # combined with direct package lookups for exact matches + results = [] + + # First: try direct package lookup (exact match) + try: + direct_result = await self._try_direct_package_lookup(query) + if direct_result: + results.extend(direct_result) + except Exception as e: + logger.debug(f"Direct lookup failed: {e}") + + # Second: search curated packages + try: + curated_results = await self._search_curated_packages(query, limit) + # Add curated results that aren't already in the list + existing_names = {r["name"].lower() for r in results} + for result in curated_results: + if result["name"].lower() not in existing_names: + results.append(result) + except Exception as e: + logger.error(f"Curated search failed: {e}") + + return results[:limit] - async def _fallback_search(self, query: str, limit: int) -> List[Dict[str, Any]]: - """Fallback search using PyPI JSON API and our curated data.""" - from ..data.popular_packages import PACKAGES_BY_NAME, get_popular_packages + async def _try_direct_package_lookup(self, query: str) -> List[Dict[str, Any]]: + """Try to get package info directly using PyPI JSON API.""" + candidates = [ + query.strip(), + query.strip().lower(), + query.strip().replace(" ", "-"), + query.strip().replace(" ", "_"), + query.strip().replace("_", "-"), + query.strip().replace("-", "_"), + ] + + results = [] + for candidate in candidates: + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(candidate) + + results.append({ + "name": package_data["info"]["name"], + "summary": package_data["info"]["summary"] or "", + "version": package_data["info"]["version"], + "source": "direct_api", + "description": package_data["info"]["description"] or "", + "author": package_data["info"]["author"] or "", + "license": package_data["info"]["license"] or "", + "home_page": package_data["info"]["home_page"] or "", + "requires_python": package_data["info"]["requires_python"] or "", + "classifiers": package_data["info"]["classifiers"] or [], + "keywords": package_data["info"]["keywords"] or "", + }) + break # Found exact match, stop looking + + except Exception: + continue # Try next candidate + + return results + + async def _search_curated_packages(self, query: str, limit: int) -> List[Dict[str, Any]]: + """Search our curated package database.""" + from ..data.popular_packages import ALL_POPULAR_PACKAGES - # Search in our curated packages first curated_matches = [] query_lower = query.lower() - for package_info in get_popular_packages(limit=1000): - name_match = query_lower in package_info.name.lower() - desc_match = query_lower in package_info.description.lower() - - if name_match or desc_match: + logger.info(f"Searching {len(ALL_POPULAR_PACKAGES)} curated packages for '{query}'") + + # First: exact name matches + for pkg in ALL_POPULAR_PACKAGES: + if query_lower == pkg.name.lower(): curated_matches.append({ - "name": package_info.name, - "summary": package_info.description, - "version": "unknown", - "source": "curated", - "category": package_info.category, - "estimated_downloads": package_info.estimated_monthly_downloads, + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_exact", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + "primary_use_case": pkg.primary_use_case, }) - # If we have some matches, return them - if curated_matches: - return curated_matches[:limit] + # Second: name contains query (if not too many exact matches) + if len(curated_matches) < limit: + for pkg in ALL_POPULAR_PACKAGES: + if (query_lower in pkg.name.lower() and + pkg.name not in [m["name"] for m in curated_matches]): + curated_matches.append({ + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_name", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + "primary_use_case": pkg.primary_use_case, + }) - # Last resort: try simple package name search + # Third: description or use case matches (if still need more results) + if len(curated_matches) < limit: + for pkg in ALL_POPULAR_PACKAGES: + if ((query_lower in pkg.description.lower() or + query_lower in pkg.primary_use_case.lower()) and + pkg.name not in [m["name"] for m in curated_matches]): + curated_matches.append({ + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_desc", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + "primary_use_case": pkg.primary_use_case, + }) + + # Sort by popularity (downloads) + curated_matches.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) + + logger.info(f"Found {len(curated_matches)} curated matches") + return curated_matches[:limit] + + async def _fallback_search(self, query: str, limit: int) -> List[Dict[str, Any]]: + """Fallback search using PyPI JSON API and our curated data.""" try: - async with PyPIClient() as client: - # Try to get the package directly if it's an exact match - try: + from ..data.popular_packages import PACKAGES_BY_NAME, get_popular_packages, ALL_POPULAR_PACKAGES + + # Search in our curated packages first + curated_matches = [] + query_lower = query.lower() + + logger.info(f"Searching in {len(ALL_POPULAR_PACKAGES)} curated packages for '{query}'") + + # First: exact name matches + for package_info in ALL_POPULAR_PACKAGES: + if query_lower == package_info.name.lower(): + curated_matches.append({ + "name": package_info.name, + "summary": package_info.description, + "version": "latest", + "source": "curated_exact", + "category": package_info.category, + "estimated_downloads": package_info.estimated_monthly_downloads, + "github_stars": package_info.github_stars, + }) + + # Second: name contains query + for package_info in ALL_POPULAR_PACKAGES: + if (query_lower in package_info.name.lower() and + package_info.name not in [m["name"] for m in curated_matches]): + curated_matches.append({ + "name": package_info.name, + "summary": package_info.description, + "version": "latest", + "source": "curated_name", + "category": package_info.category, + "estimated_downloads": package_info.estimated_monthly_downloads, + "github_stars": package_info.github_stars, + }) + + # Third: description or use case matches + for package_info in ALL_POPULAR_PACKAGES: + if ((query_lower in package_info.description.lower() or + query_lower in package_info.primary_use_case.lower()) and + package_info.name not in [m["name"] for m in curated_matches]): + curated_matches.append({ + "name": package_info.name, + "summary": package_info.description, + "version": "latest", + "source": "curated_desc", + "category": package_info.category, + "estimated_downloads": package_info.estimated_monthly_downloads, + "github_stars": package_info.github_stars, + }) + + logger.info(f"Found {len(curated_matches)} curated matches") + + # If we have some matches, return them (sorted by popularity) + if curated_matches: + curated_matches.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) + return curated_matches[:limit] + + # Last resort: try direct package lookup + logger.info("No curated matches, trying direct package lookup") + try: + async with PyPIClient() as client: package_data = await client.get_package_info(query) return [{ "name": package_data["info"]["name"], "summary": package_data["info"]["summary"] or "", "version": package_data["info"]["version"], - "source": "direct", + "source": "direct_fallback", + "description": package_data["info"]["description"] or "", + "author": package_data["info"]["author"] or "", }] - except: - pass + except Exception as e: + logger.info(f"Direct lookup failed: {e}") except Exception as e: - logger.warning(f"Fallback search failed: {e}") + logger.error(f"Fallback search failed: {e}") return [] + async def _search_xmlrpc(self, query: str, limit: int) -> List[Dict[str, Any]]: + """Search using enhanced curated search with fuzzy matching.""" + # Since PyPI XML-RPC search is deprecated, use our enhanced curated search + try: + from ..data.popular_packages import get_popular_packages, ALL_POPULAR_PACKAGES + + query_lower = query.lower() + results = [] + + # First pass: exact name matches + for pkg in ALL_POPULAR_PACKAGES: + if query_lower == pkg.name.lower(): + results.append({ + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_exact", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + }) + + # Second pass: name contains query + for pkg in ALL_POPULAR_PACKAGES: + if query_lower in pkg.name.lower() and pkg.name not in [r["name"] for r in results]: + results.append({ + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_name", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + }) + + # Third pass: description contains query + for pkg in ALL_POPULAR_PACKAGES: + if (query_lower in pkg.description.lower() or + query_lower in pkg.primary_use_case.lower()) and pkg.name not in [r["name"] for r in results]: + results.append({ + "name": pkg.name, + "summary": pkg.description, + "version": "latest", + "source": "curated_desc", + "category": pkg.category, + "estimated_downloads": pkg.estimated_monthly_downloads, + "github_stars": pkg.github_stars, + }) + + # Sort by popularity (downloads) + results.sort(key=lambda x: x.get("estimated_downloads", 0), reverse=True) + + return results[:limit] + + except Exception as e: + logger.debug(f"Enhanced curated search error: {e}") + + return [] + + async def _search_simple_api(self, query: str, limit: int) -> List[Dict[str, Any]]: + """Search using direct PyPI JSON API for specific packages.""" + try: + # Try direct package lookup if query looks like a package name + query_clean = query.strip().lower().replace(" ", "-") + + # Try variations of the query as package names + candidates = [ + query_clean, + query_clean.replace("-", "_"), + query_clean.replace("_", "-"), + query.strip(), # Original query + ] + + results = [] + + for candidate in candidates: + if len(results) >= limit: + break + + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(candidate) + + results.append({ + "name": package_data["info"]["name"], + "summary": package_data["info"]["summary"] or "", + "version": package_data["info"]["version"], + "source": "direct_api", + "description": package_data["info"]["description"] or "", + "author": package_data["info"]["author"] or "", + "license": package_data["info"]["license"] or "", + }) + + except Exception: + # Package doesn't exist, continue to next candidate + continue + + return results + + except Exception as e: + logger.debug(f"Simple API search error: {e}") + + return [] + async def _parse_search_html(self, html: str, limit: int) -> List[Dict[str, Any]]: """Parse PyPI search results from HTML (simplified parser).""" # This is a simplified parser - in production, you'd use BeautifulSoup @@ -237,9 +505,19 @@ class PyPISearchClient: """Enhance search results with additional metadata from PyPI API.""" enhanced = [] - # Process in batches to avoid overwhelming the API - batch_size = 5 - for i in range(0, len(results), batch_size): + # Skip enhancement if results already have good metadata from curated source + if results and results[0].get("source", "").startswith("curated"): + logger.info("Using curated results without enhancement") + return results + + # For direct API results, they're already enhanced + if results and results[0].get("source") == "direct_api": + logger.info("Using direct API results without additional enhancement") + return results + + # Process in small batches to avoid overwhelming the API + batch_size = 3 + for i in range(0, min(len(results), 10), batch_size): # Limit to first 10 results batch = results[i:i + batch_size] batch_tasks = [ self._enhance_single_result(result) diff --git a/pypi_query_mcp/server.py b/pypi_query_mcp/server.py index 845c53c..3d16723 100644 --- a/pypi_query_mcp/server.py +++ b/pypi_query_mcp/server.py @@ -1,6 +1,7 @@ """FastMCP server for PyPI package queries.""" import logging +from datetime import datetime, timezone from typing import Any import click @@ -24,34 +25,63 @@ from .prompts import ( track_package_updates, ) from .tools import ( - analyze_pypi_competition, - check_pypi_credentials, check_python_compatibility, - delete_pypi_release, download_package_with_dependencies, find_alternatives, get_compatible_python_versions, get_package_download_stats, get_package_download_trends, - get_pypi_account_info, - get_pypi_package_analytics, - get_pypi_package_rankings, - get_pypi_security_alerts, - get_pypi_upload_history, get_top_packages_by_downloads, get_trending_packages, - manage_package_keywords, - manage_package_urls, - manage_pypi_maintainers, query_package_dependencies, query_package_info, query_package_versions, resolve_package_dependencies, search_by_category, search_packages, - set_package_visibility, - update_package_metadata, + # Publishing tools upload_package_to_pypi, + check_pypi_credentials, + get_pypi_upload_history, + delete_pypi_release, + manage_pypi_maintainers, + get_pypi_account_info, + # Metadata tools + update_package_metadata, + manage_package_urls, + set_package_visibility, + manage_package_keywords, + # Analytics tools + get_pypi_package_analytics, + get_pypi_security_alerts, + get_pypi_package_rankings, + analyze_pypi_competition, + # Discovery tools + monitor_pypi_new_releases, + get_pypi_trending_today, + search_pypi_by_maintainer, + get_pypi_package_recommendations, + # Workflow tools + validate_pypi_package_name, + preview_pypi_package_page, + check_pypi_upload_requirements, + get_pypi_build_logs, + # Community tools + get_pypi_package_reviews, + manage_pypi_package_discussions, + get_pypi_maintainer_contacts, + # Security tools + bulk_scan_package_security, + scan_pypi_package_security, + # License tools + analyze_pypi_package_license, + check_bulk_license_compliance, + # Health tools + assess_package_health_score, + compare_packages_health_scores, + # Requirements tools + analyze_requirements_file_tool, + compare_multiple_requirements_files, ) # Configure logging @@ -671,7 +701,12 @@ async def search_pypi_packages( SearchError: If search operation fails """ try: - return await search_packages( + logger.info(f"MCP search_pypi_packages called with query='{query}', limit={limit}") + logger.info(f"MCP search_pypi_packages parameters: python_versions={python_versions}, licenses={licenses}, categories={categories}") + logger.info(f"MCP search_pypi_packages parameters: min_downloads={min_downloads}, maintenance_status={maintenance_status}, has_wheels={has_wheels}") + logger.info(f"MCP search_pypi_packages parameters: sort_by={sort_by}, sort_desc={sort_desc}, semantic_search={semantic_search}") + + result = await search_packages( query=query, limit=limit, python_versions=python_versions, @@ -684,15 +719,36 @@ async def search_pypi_packages( sort_desc=sort_desc, semantic_search=semantic_search, ) - except (InvalidPackageNameError, PackageNotFoundError, NetworkError): - raise - except Exception as e: - logger.error(f"Error searching packages for '{query}': {e}") + logger.info(f"MCP search_pypi_packages raw result keys: {list(result.keys())}") + logger.info(f"MCP search_pypi_packages raw result total_found: {result.get('total_found')}") + logger.info(f"MCP search_pypi_packages raw result packages count: {len(result.get('packages', []))}") + logger.info(f"MCP search_pypi_packages returning: {len(result.get('packages', []))} packages") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError, SearchError) as e: + logger.error(f"Known error searching packages for '{query}': {e}") return { "error": f"Search failed: {e}", - "error_type": "SearchError", + "error_type": type(e).__name__, "query": query, "limit": limit, + "packages": [], + "total_found": 0, + "returned_count": 0, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + except Exception as e: + logger.error(f"Unexpected error searching packages for '{query}': {e}") + import traceback + traceback.print_exc() + return { + "error": f"Search failed: {e}", + "error_type": "SearchError", + "query": query, + "limit": limit, + "packages": [], + "total_found": 0, + "returned_count": 0, + "timestamp": datetime.now(timezone.utc).isoformat(), } @@ -820,8 +876,7 @@ async def get_trending_pypi_packages( } -# PyPI Publishing and Account Management Tools - +# Publishing Tools MCP Endpoints @mcp.tool() async def upload_package_to_pypi_tool( @@ -833,8 +888,8 @@ async def upload_package_to_pypi_tool( ) -> dict[str, Any]: """Upload package distributions to PyPI or TestPyPI. - This tool uploads Python package distributions (.whl, .tar.gz files) to PyPI, - providing comprehensive upload management with safety checks and verification. + This tool uploads Python package distribution files (.whl, .tar.gz) to PyPI + or TestPyPI, providing comprehensive upload management and verification. Args: distribution_paths: List of paths to distribution files (.whl, .tar.gz) @@ -844,7 +899,7 @@ async def upload_package_to_pypi_tool( verify_uploads: Verify uploads after completion Returns: - Dictionary containing upload results and metadata + Dictionary containing upload results, statistics, and verification info Raises: PyPIAuthenticationError: If authentication fails @@ -860,10 +915,10 @@ async def upload_package_to_pypi_tool( skip_existing=skip_existing, verify_uploads=verify_uploads, ) - logger.info(f"Upload completed: {result.get('summary', {})}") + logger.info(f"Upload completed with {result.get('successful_uploads', 0)} successful uploads") return result except Exception as e: - logger.error(f"Error uploading packages: {e}") + logger.error(f"Error uploading package: {e}") return { "error": str(e), "error_type": type(e).__name__, @@ -879,15 +934,15 @@ async def check_pypi_credentials_tool( ) -> dict[str, Any]: """Validate PyPI API token and credentials. - This tool checks if your PyPI API token is valid and provides information - about your account permissions and capabilities. + This tool validates PyPI API tokens and checks authentication status, + helping ensure proper credentials before performing upload operations. Args: api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to check against TestPyPI instead of production PyPI Returns: - Dictionary containing credential validation results + Dictionary containing credential validation results and status Raises: PyPIAuthenticationError: If credential validation fails @@ -895,8 +950,8 @@ async def check_pypi_credentials_tool( """ try: logger.info(f"MCP tool: Checking {'TestPyPI' if test_pypi else 'PyPI'} credentials") - result = await check_pypi_credentials(api_token=api_token, test_pypi=test_pypi) - logger.info(f"Credential check completed: valid={result.get('valid', False)}") + result = await check_pypi_credentials(api_token, test_pypi) + logger.info(f"Credential check completed: {'valid' if result.get('valid') else 'invalid'}") return result except Exception as e: logger.error(f"Error checking credentials: {e}") @@ -916,8 +971,8 @@ async def get_pypi_upload_history_tool( ) -> dict[str, Any]: """Get upload history for a PyPI package. - This tool retrieves the upload history for a package, showing all versions, - files, and upload metadata with statistics and analysis. + This tool retrieves comprehensive upload history for a package, + including file information, upload times, and statistics. Args: package_name: Name of the package to get upload history for @@ -935,14 +990,8 @@ async def get_pypi_upload_history_tool( """ try: logger.info(f"MCP tool: Getting upload history for {package_name}") - result = await get_pypi_upload_history( - package_name=package_name, - api_token=api_token, - test_pypi=test_pypi, - limit=limit, - ) - upload_count = result.get('statistics', {}).get('total_uploads', 0) - logger.info(f"Retrieved {upload_count} upload records for {package_name}") + result = await get_pypi_upload_history(package_name, api_token, test_pypi, limit) + logger.info(f"Retrieved {len(result.get('upload_history', []))} upload records") return result except Exception as e: logger.error(f"Error getting upload history for {package_name}: {e}") @@ -965,10 +1014,8 @@ async def delete_pypi_release_tool( ) -> dict[str, Any]: """Delete a specific release from PyPI (with safety checks). - This tool provides safe deletion of PyPI releases with multiple safety checks, - dry-run capability, and comprehensive validation. Note that PyPI deletion is - very restricted and typically only available to package owners within a limited - time window after upload. + This tool provides safe deletion of PyPI releases with comprehensive + safety checks and dry-run capability. PyPI deletion is very restricted. Args: package_name: Name of the package @@ -990,15 +1037,9 @@ async def delete_pypi_release_tool( try: logger.info(f"MCP tool: {'DRY RUN: ' if dry_run else ''}Deleting {package_name}=={version}") result = await delete_pypi_release( - package_name=package_name, - version=version, - api_token=api_token, - test_pypi=test_pypi, - confirm_deletion=confirm_deletion, - dry_run=dry_run, + package_name, version, api_token, test_pypi, confirm_deletion, dry_run ) - action = result.get('action', 'unknown') - logger.info(f"Deletion operation completed: {action}") + logger.info(f"Deletion {'simulation' if dry_run else 'attempt'} completed") return result except Exception as e: logger.error(f"Error deleting release {package_name}=={version}: {e}") @@ -1021,9 +1062,8 @@ async def manage_pypi_maintainers_tool( ) -> dict[str, Any]: """Manage package maintainers (add/remove/list). - This tool helps manage package maintainers and collaborators. Note that maintainer - management typically requires package owner permissions and may need to be done - through the PyPI web interface. + This tool provides maintainer management functionality for PyPI packages, + including listing current maintainers and guidance for adding/removing. Args: package_name: Name of the package @@ -1043,15 +1083,8 @@ async def manage_pypi_maintainers_tool( """ try: logger.info(f"MCP tool: Managing maintainers for {package_name}: {action}") - result = await manage_pypi_maintainers( - package_name=package_name, - action=action, - username=username, - api_token=api_token, - test_pypi=test_pypi, - ) - maintainer_count = result.get('maintainer_count', 0) - logger.info(f"Maintainer management completed: {maintainer_count} maintainers") + result = await manage_pypi_maintainers(package_name, action, username, api_token, test_pypi) + logger.info(f"Maintainer management completed for {package_name}") return result except Exception as e: logger.error(f"Error managing maintainers for {package_name}: {e}") @@ -1071,8 +1104,8 @@ async def get_pypi_account_info_tool( ) -> dict[str, Any]: """Get PyPI account information, quotas, and limits. - This tool retrieves information about your PyPI account including permissions, - limitations, quotas, and provides recommendations for account security and usage. + This tool retrieves comprehensive account information including + limitations, features, and useful links for PyPI account management. Args: api_token: PyPI API token (or use PYPI_API_TOKEN env var) @@ -1087,7 +1120,7 @@ async def get_pypi_account_info_tool( """ try: logger.info(f"MCP tool: Getting account information for {'TestPyPI' if test_pypi else 'PyPI'}") - result = await get_pypi_account_info(api_token=api_token, test_pypi=test_pypi) + result = await get_pypi_account_info(api_token, test_pypi) logger.info("Account information retrieved successfully") return result except Exception as e: @@ -1099,52 +1132,45 @@ async def get_pypi_account_info_tool( } -# Metadata Management Tools +# Metadata Management Tools MCP Endpoints + @mcp.tool() async def update_package_metadata_tool( package_name: str, - description: str | None = None, - keywords: list[str] | None = None, - classifiers: list[str] | None = None, + metadata_updates: dict[str, Any], api_token: str | None = None, test_pypi: bool = False, - dry_run: bool = True, + validate_changes: bool = True, + dry_run: bool = False, ) -> dict[str, Any]: - """Update package metadata including description, keywords, and classifiers. + """Update PyPI package metadata and configuration. - This tool helps manage PyPI package metadata by validating changes and providing - guidance on how to update metadata through package uploads. + This tool updates package metadata including description, keywords, + classifiers, and other package information on PyPI. Args: package_name: Name of the package to update - description: New package description - keywords: List of keywords for the package - classifiers: List of PyPI classifiers (e.g., programming language, license) + metadata_updates: Dictionary of metadata fields to update api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - dry_run: If True, only validate changes without applying them + validate_changes: Whether to validate metadata before applying + dry_run: If True, only validate without applying changes Returns: - Dictionary containing metadata update results and recommendations + Dictionary containing update results and validation info Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If user lacks permission to modify package + PyPIPermissionError: If update is not permitted NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Updating metadata for {package_name} (dry_run={dry_run})") + logger.info(f"MCP tool: Updating metadata for {package_name}") result = await update_package_metadata( - package_name=package_name, - description=description, - keywords=keywords, - classifiers=classifiers, - api_token=api_token, - test_pypi=test_pypi, - dry_run=dry_run, + package_name, metadata_updates, api_token, test_pypi, validate_changes, dry_run ) - logger.info(f"Metadata update completed for {package_name}: {result.get('success', 'analysis_complete')}") + logger.info(f"Metadata update {'simulated' if dry_run else 'completed'} for {package_name}") return result except Exception as e: logger.error(f"Error updating metadata for {package_name}: {e}") @@ -1152,64 +1178,47 @@ async def update_package_metadata_tool( "error": str(e), "error_type": type(e).__name__, "package_name": package_name, - "dry_run": dry_run, + "test_pypi": test_pypi, } @mcp.tool() async def manage_package_urls_tool( package_name: str, - homepage: str | None = None, - documentation: str | None = None, - repository: str | None = None, - download_url: str | None = None, - bug_tracker: str | None = None, + action: str, + url_type: str | None = None, + url_value: str | None = None, api_token: str | None = None, test_pypi: bool = False, - validate_urls: bool = True, - dry_run: bool = True, ) -> dict[str, Any]: - """Manage package URLs including homepage, documentation, and repository links. + """Manage package URLs (homepage, documentation, repository, etc.). - This tool validates and manages package URLs, providing guidance on proper - URL configuration and accessibility checking. + This tool manages package URL configurations including adding, updating, + removing, and listing various URL types for a PyPI package. Args: - package_name: Name of the package to update - homepage: Package homepage URL - documentation: Documentation URL - repository: Source code repository URL - download_url: Package download URL - bug_tracker: Bug tracker URL + package_name: Name of the package + action: Action to perform ('list', 'add', 'update', 'remove') + url_type: Type of URL ('homepage', 'documentation', 'repository', 'bug_tracker', etc.) + url_value: URL value (required for add/update actions) api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - validate_urls: Whether to validate URL accessibility - dry_run: If True, only validate changes without applying them Returns: - Dictionary containing URL management results and validation + Dictionary containing URL management results Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If user lacks permission to modify package + PyPIPermissionError: If action is not permitted NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Managing URLs for {package_name} (dry_run={dry_run})") + logger.info(f"MCP tool: Managing URLs for {package_name}: {action}") result = await manage_package_urls( - package_name=package_name, - homepage=homepage, - documentation=documentation, - repository=repository, - download_url=download_url, - bug_tracker=bug_tracker, - api_token=api_token, - test_pypi=test_pypi, - validate_urls=validate_urls, - dry_run=dry_run, + package_name, action, url_type, url_value, api_token, test_pypi ) - logger.info(f"URL management completed for {package_name}: quality_score={result.get('url_quality_score', 0)}") + logger.info(f"URL management completed for {package_name}") return result except Exception as e: logger.error(f"Error managing URLs for {package_name}: {e}") @@ -1217,7 +1226,8 @@ async def manage_package_urls_tool( "error": str(e), "error_type": type(e).__name__, "package_name": package_name, - "dry_run": dry_run, + "action": action, + "test_pypi": test_pypi, } @@ -1227,39 +1237,35 @@ async def set_package_visibility_tool( visibility: str, api_token: str | None = None, test_pypi: bool = False, - confirm_action: bool = False, + confirmation_required: bool = True, ) -> dict[str, Any]: - """Set package visibility (private/public) for organization packages. + """Set package visibility and access controls. - This tool provides guidance on package visibility management, which is primarily - available for PyPI organizations with special permissions. + This tool manages package visibility settings including public/private + status and access controls for PyPI packages. Args: - package_name: Name of the package to modify - visibility: Visibility setting ("public" or "private") + package_name: Name of the package + visibility: Visibility setting ('public', 'private', 'unlisted') api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - confirm_action: Explicit confirmation required for visibility changes + confirmation_required: Whether to require explicit confirmation Returns: - Dictionary containing visibility management results and limitations + Dictionary containing visibility change results Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If user lacks permission to modify package + PyPIPermissionError: If action is not permitted NetworkError: For network-related errors """ try: logger.info(f"MCP tool: Setting visibility for {package_name} to {visibility}") result = await set_package_visibility( - package_name=package_name, - visibility=visibility, - api_token=api_token, - test_pypi=test_pypi, - confirm_action=confirm_action, + package_name, visibility, api_token, test_pypi, confirmation_required ) - logger.info(f"Visibility analysis completed for {package_name}: {result.get('success', 'analysis_complete')}") + logger.info(f"Visibility update completed for {package_name}") return result except Exception as e: logger.error(f"Error setting visibility for {package_name}: {e}") @@ -1268,6 +1274,7 @@ async def set_package_visibility_tool( "error_type": type(e).__name__, "package_name": package_name, "visibility": visibility, + "test_pypi": test_pypi, } @@ -1278,41 +1285,34 @@ async def manage_package_keywords_tool( keywords: list[str] | None = None, api_token: str | None = None, test_pypi: bool = False, - dry_run: bool = True, ) -> dict[str, Any]: - """Manage package keywords and search tags. + """Manage package keywords and tags. - This tool provides comprehensive keyword management including validation, - quality analysis, and recommendations for better package discoverability. + This tool manages package keywords and tags for better discoverability, + including adding, removing, and updating keyword sets. Args: - package_name: Name of the package to modify - action: Action to perform ("add", "remove", "replace", "list") - keywords: List of keywords to add/remove/replace + package_name: Name of the package + action: Action to perform ('list', 'add', 'remove', 'replace') + keywords: List of keywords (required for add/remove/replace actions) api_token: PyPI API token (or use PYPI_API_TOKEN env var) test_pypi: Whether to use TestPyPI instead of production PyPI - dry_run: If True, only simulate changes without applying them Returns: - Dictionary containing keyword management results and recommendations + Dictionary containing keyword management results Raises: InvalidPackageNameError: If package name is invalid PackageNotFoundError: If package is not found - PyPIPermissionError: If user lacks permission to modify package + PyPIPermissionError: If action is not permitted NetworkError: For network-related errors """ try: - logger.info(f"MCP tool: Managing keywords for {package_name}: {action} (dry_run={dry_run})") + logger.info(f"MCP tool: Managing keywords for {package_name}: {action}") result = await manage_package_keywords( - package_name=package_name, - action=action, - keywords=keywords, - api_token=api_token, - test_pypi=test_pypi, - dry_run=dry_run, + package_name, action, keywords, api_token, test_pypi ) - logger.info(f"Keyword management completed for {package_name}: {result.get('success', 'analysis_complete')}") + logger.info(f"Keyword management completed for {package_name}") return result except Exception as e: logger.error(f"Error managing keywords for {package_name}: {e}") @@ -1321,6 +1321,1007 @@ async def manage_package_keywords_tool( "error_type": type(e).__name__, "package_name": package_name, "action": action, + "test_pypi": test_pypi, + } + + +# Analytics Tools MCP Endpoints + +@mcp.tool() +async def get_pypi_package_analytics_tool( + package_name: str, + time_period: str = "month", + include_historical: bool = True, + analytics_scope: str = "comprehensive", +) -> dict[str, Any]: + """Get comprehensive analytics for a PyPI package. + + This tool provides detailed analytics including download statistics, + version distribution, platform analytics, and quality metrics. + + Args: + package_name: Name of the package to analyze + time_period: Time period for analytics ('day', 'week', 'month', 'year') + include_historical: Whether to include historical trend data + analytics_scope: Scope of analytics ('basic', 'comprehensive', 'detailed') + + Returns: + Dictionary containing comprehensive package analytics + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting analytics for {package_name}") + result = await get_pypi_package_analytics( + package_name, time_period, include_historical, analytics_scope + ) + logger.info(f"Analytics retrieved for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting analytics for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + "time_period": time_period, + } + + +@mcp.tool() +async def get_pypi_security_alerts_tool( + package_name: str, + include_dependencies: bool = True, + severity_filter: str | None = None, + alert_sources: list[str] | None = None, +) -> dict[str, Any]: + """Get security alerts and vulnerability information for a package. + + This tool provides comprehensive security analysis including known + vulnerabilities, security advisories, and dependency risk assessment. + + Args: + package_name: Name of the package to analyze + include_dependencies: Whether to include dependency vulnerabilities + severity_filter: Filter by severity ('low', 'medium', 'high', 'critical') + alert_sources: List of alert sources to check + + Returns: + Dictionary containing security alerts and analysis + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting security alerts for {package_name}") + result = await get_pypi_security_alerts( + package_name, include_dependencies, severity_filter, alert_sources + ) + logger.info(f"Security analysis completed for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting security alerts for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def get_pypi_package_rankings_tool( + package_name: str, + ranking_metrics: list[str] | None = None, + search_terms: list[str] | None = None, + include_competitors: bool = True, +) -> dict[str, Any]: + """Get package ranking and discoverability analysis. + + This tool analyzes package rankings in search results, popularity + metrics, and competitive positioning within the Python ecosystem. + + Args: + package_name: Name of the package to analyze + ranking_metrics: List of metrics to analyze ('downloads', 'stars', 'search_rank') + search_terms: Search terms to check rankings for + include_competitors: Whether to include competitor analysis + + Returns: + Dictionary containing ranking analysis and insights + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting rankings for {package_name}") + result = await get_pypi_package_rankings( + package_name, ranking_metrics, search_terms, include_competitors + ) + logger.info(f"Ranking analysis completed for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting rankings for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def analyze_pypi_competition_tool( + package_name: str, + analysis_depth: str = "comprehensive", + competitor_limit: int = 5, + include_market_analysis: bool = True, +) -> dict[str, Any]: + """Analyze competitive landscape for a PyPI package. + + This tool provides comprehensive competitive analysis including + competitor identification, feature comparison, and market positioning. + + Args: + package_name: Name of the package to analyze + analysis_depth: Depth of analysis ('basic', 'comprehensive', 'detailed') + competitor_limit: Maximum number of competitors to analyze + include_market_analysis: Whether to include market share analysis + + Returns: + Dictionary containing competitive analysis and insights + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Analyzing competition for {package_name}") + result = await analyze_pypi_competition( + package_name, analysis_depth, competitor_limit, include_market_analysis + ) + logger.info(f"Competition analysis completed for {package_name}") + return result + except Exception as e: + logger.error(f"Error analyzing competition for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +# Discovery Tools MCP Endpoints + +@mcp.tool() +async def monitor_pypi_new_releases_tool( + time_window: str = "24h", + category_filter: str | None = None, + min_downloads: int | None = None, + include_prereleases: bool = False, +) -> dict[str, Any]: + """Monitor recent PyPI package releases and updates. + + This tool tracks new packages and version releases on PyPI, + providing insights into the latest developments in the Python ecosystem. + + Args: + time_window: Time window to monitor ('1h', '6h', '24h', '7d') + category_filter: Filter by package category + min_downloads: Minimum download threshold for inclusion + include_prereleases: Whether to include pre-release versions + + Returns: + Dictionary containing recent releases and analysis + + Raises: + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Monitoring new releases (window: {time_window})") + result = await monitor_pypi_new_releases( + time_window, category_filter, min_downloads, include_prereleases + ) + logger.info(f"Found {len(result.get('new_releases', []))} new releases") + return result + except Exception as e: + logger.error(f"Error monitoring new releases: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "time_window": time_window, + } + + +@mcp.tool() +async def get_pypi_trending_today_tool( + category: str | None = None, + limit: int = 20, + trending_metric: str = "downloads", +) -> dict[str, Any]: + """Get trending PyPI packages for today. + + This tool identifies packages that are trending today based on + various metrics like download spikes, new releases, or community activity. + + Args: + category: Filter by package category + limit: Maximum number of trending packages to return + trending_metric: Metric to base trending on ('downloads', 'stars', 'releases') + + Returns: + Dictionary containing trending packages and analysis + + Raises: + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting trending packages for today") + result = await get_pypi_trending_today(category, limit, trending_metric) + logger.info(f"Found {len(result.get('trending_packages', []))} trending packages") + return result + except Exception as e: + logger.error(f"Error getting trending packages: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "category": category, + } + + +@mcp.tool() +async def search_pypi_by_maintainer_tool( + maintainer_name: str, + search_scope: str = "all", + include_statistics: bool = True, + sort_by: str = "popularity", +) -> dict[str, Any]: + """Search PyPI packages by maintainer or author. + + This tool finds all packages maintained by a specific person or organization, + providing insights into their contribution to the Python ecosystem. + + Args: + maintainer_name: Name of the maintainer to search for + search_scope: Scope of search ('author', 'maintainer', 'all') + include_statistics: Whether to include maintainer statistics + sort_by: Sort packages by ('popularity', 'recency', 'name') + + Returns: + Dictionary containing maintainer packages and statistics + + Raises: + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Searching packages by maintainer: {maintainer_name}") + result = await search_pypi_by_maintainer( + maintainer_name, search_scope, include_statistics, sort_by + ) + logger.info(f"Found {len(result.get('packages', []))} packages") + return result + except Exception as e: + logger.error(f"Error searching by maintainer {maintainer_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "maintainer_name": maintainer_name, + } + + +@mcp.tool() +async def get_pypi_package_recommendations_tool( + package_name: str, + recommendation_type: str = "similar", + limit: int = 10, + include_reasoning: bool = True, +) -> dict[str, Any]: + """Get personalized package recommendations based on a given package. + + This tool provides intelligent package recommendations including + similar packages, complementary tools, and upgrade suggestions. + + Args: + package_name: Base package for recommendations + recommendation_type: Type of recommendations ('similar', 'complementary', 'upgrades') + limit: Maximum number of recommendations to return + include_reasoning: Whether to include reasoning for recommendations + + Returns: + Dictionary containing package recommendations and analysis + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting recommendations for {package_name}") + result = await get_pypi_package_recommendations( + package_name, recommendation_type, limit, include_reasoning + ) + logger.info(f"Generated {len(result.get('recommendations', []))} recommendations") + return result + except Exception as e: + logger.error(f"Error getting recommendations for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +# Workflow Tools MCP Endpoints + +@mcp.tool() +async def validate_pypi_package_name_tool(package_name: str) -> dict[str, Any]: + """Validate PyPI package name according to PEP 508 and PyPI requirements. + + This tool validates package names against PyPI naming conventions, + checks availability, and provides suggestions for improvements. + + Args: + package_name: Package name to validate + + Returns: + Dictionary containing validation results and suggestions + + Raises: + InvalidPackageNameError: If package name format is invalid + """ + try: + logger.info(f"MCP tool: Validating package name: {package_name}") + result = await validate_pypi_package_name(package_name) + logger.info(f"Package name validation completed: {'valid' if result.get('valid') else 'invalid'}") + return result + except Exception as e: + logger.error(f"Error validating package name {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def preview_pypi_package_page_tool( + package_name: str, + version: str | None = None, + include_rendered_content: bool = True, + check_metadata_completeness: bool = True, +) -> dict[str, Any]: + """Preview how a package page will look on PyPI. + + This tool generates a preview of the PyPI package page including + rendered README, metadata display, and completeness analysis. + + Args: + package_name: Name of the package to preview + version: Specific version to preview (optional, defaults to latest) + include_rendered_content: Whether to include rendered README content + check_metadata_completeness: Whether to analyze metadata completeness + + Returns: + Dictionary containing page preview and analysis + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Previewing package page for {package_name}") + result = await preview_pypi_package_page( + package_name, version, include_rendered_content, check_metadata_completeness + ) + logger.info(f"Package page preview generated for {package_name}") + return result + except Exception as e: + logger.error(f"Error previewing package page for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def check_pypi_upload_requirements_tool( + package_path: str, + check_completeness: bool = True, + validate_metadata: bool = True, + check_security: bool = True, +) -> dict[str, Any]: + """Check if a package meets PyPI upload requirements. + + This tool validates package structure, metadata, and requirements + before upload to ensure successful PyPI submission. + + Args: + package_path: Path to the package directory or distribution file + check_completeness: Whether to check metadata completeness + validate_metadata: Whether to validate metadata format + check_security: Whether to perform security checks + + Returns: + Dictionary containing requirement check results and recommendations + + Raises: + FileNotFoundError: If package path doesn't exist + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Checking upload requirements for {package_path}") + result = await check_pypi_upload_requirements( + package_path, check_completeness, validate_metadata, check_security + ) + logger.info(f"Upload requirements check completed") + return result + except Exception as e: + logger.error(f"Error checking upload requirements for {package_path}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_path": package_path, + } + + +@mcp.tool() +async def get_pypi_build_logs_tool( + package_name: str, + version: str | None = None, + build_type: str = "all", + include_analysis: bool = True, +) -> dict[str, Any]: + """Get PyPI package build logs and analysis. + + This tool retrieves build logs for PyPI packages and provides + analysis of build issues, warnings, and optimization opportunities. + + Args: + package_name: Name of the package + version: Specific version (optional, defaults to latest) + build_type: Type of builds to include ('wheel', 'sdist', 'all') + include_analysis: Whether to include build analysis + + Returns: + Dictionary containing build logs and analysis + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting build logs for {package_name}") + result = await get_pypi_build_logs(package_name, version, build_type, include_analysis) + logger.info(f"Build logs retrieved for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting build logs for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +# Community Tools MCP Endpoints + +@mcp.tool() +async def get_pypi_package_reviews_tool( + package_name: str, + include_sentiment_analysis: bool = True, + review_sources: list[str] | None = None, + time_period: str = "all", +) -> dict[str, Any]: + """Get community reviews and ratings for a PyPI package. + + This tool aggregates community feedback, reviews, and sentiment + analysis from various sources to provide comprehensive package insights. + + Args: + package_name: Name of the package to get reviews for + include_sentiment_analysis: Whether to include sentiment analysis + review_sources: List of sources to check ('github', 'stackoverflow', 'reddit') + time_period: Time period for reviews ('week', 'month', 'year', 'all') + + Returns: + Dictionary containing reviews, ratings, and sentiment analysis + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting reviews for {package_name}") + result = await get_pypi_package_reviews( + package_name, include_sentiment_analysis, review_sources, time_period + ) + logger.info(f"Reviews retrieved for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting reviews for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def manage_pypi_package_discussions_tool( + package_name: str, + action: str, + discussion_settings: dict[str, Any] | None = None, + moderator_controls: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Manage package discussions and community features. + + This tool manages community discussion features for PyPI packages + including enabling discussions, moderation, and configuration. + + Args: + package_name: Name of the package + action: Action to perform ('status', 'enable', 'disable', 'configure', 'moderate') + discussion_settings: Settings for discussion configuration + moderator_controls: Moderation controls and settings + + Returns: + Dictionary containing discussion management results + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + PyPIPermissionError: If action is not permitted + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Managing discussions for {package_name}: {action}") + result = await manage_pypi_package_discussions( + package_name, action, discussion_settings, moderator_controls + ) + logger.info(f"Discussion management completed for {package_name}") + return result + except Exception as e: + logger.error(f"Error managing discussions for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + "action": action, + } + + +@mcp.tool() +async def get_pypi_maintainer_contacts_tool( + package_name: str, + contact_types: list[str] | None = None, + include_social_profiles: bool = False, + respect_privacy: bool = True, +) -> dict[str, Any]: + """Get maintainer contact information and communication channels. + + This tool finds maintainer contact information and preferred + communication channels while respecting privacy preferences. + + Args: + package_name: Name of the package + contact_types: Types of contacts to find ('email', 'github', 'twitter') + include_social_profiles: Whether to include social media profiles + respect_privacy: Whether to respect privacy settings and preferences + + Returns: + Dictionary containing maintainer contact information + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + try: + logger.info(f"MCP tool: Getting maintainer contacts for {package_name}") + result = await get_pypi_maintainer_contacts( + package_name, contact_types, include_social_profiles, respect_privacy + ) + logger.info(f"Maintainer contacts retrieved for {package_name}") + return result + except Exception as e: + logger.error(f"Error getting maintainer contacts for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + } + + +@mcp.tool() +async def scan_pypi_package_security_tool( + package_name: str, + version: str | None = None, + include_dependencies: bool = True, + severity_filter: str | None = None +) -> dict[str, Any]: + """Scan a PyPI package for security vulnerabilities. + + This tool performs comprehensive security vulnerability scanning of PyPI packages, + checking against multiple vulnerability databases including OSV (Open Source Vulnerabilities), + GitHub Security Advisories, and analyzing package metadata for security indicators. + + Args: + package_name: Name of the package to scan for vulnerabilities + version: Specific version to scan (optional, defaults to latest version) + include_dependencies: Whether to scan package dependencies for vulnerabilities + severity_filter: Filter results by severity level (low, medium, high, critical) + + Returns: + Dictionary containing comprehensive security scan results including: + - Total vulnerability count and severity breakdown + - Direct package vulnerabilities vs dependency vulnerabilities + - Risk score and level assessment (minimal, low, medium, high, critical) + - Detailed vulnerability information with IDs, descriptions, and references + - Package metadata security analysis + - Actionable security recommendations + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If security scanning fails + """ + try: + logger.info(f"MCP tool: Scanning security vulnerabilities for {package_name}") + result = await scan_pypi_package_security( + package_name, version, include_dependencies, severity_filter + ) + logger.info(f"Security scan completed for {package_name} - found {result.get('security_summary', {}).get('total_vulnerabilities', 0)} vulnerabilities") + return result + except Exception as e: + logger.error(f"Error scanning security for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package": package_name, + "version": version, + } + + +@mcp.tool() +async def bulk_scan_package_security_tool( + package_names: list[str], + include_dependencies: bool = False, + severity_threshold: str = "medium" +) -> dict[str, Any]: + """Perform bulk security scanning of multiple PyPI packages. + + This tool scans multiple packages simultaneously for security vulnerabilities, + providing a consolidated report with summary statistics and prioritized + recommendations for addressing security issues across your package ecosystem. + + Args: + package_names: List of package names to scan for vulnerabilities + include_dependencies: Whether to include dependency vulnerability scanning + severity_threshold: Minimum severity level to report (low, medium, high, critical) + + Returns: + Dictionary containing bulk scan results including: + - Summary statistics (total packages, packages with vulnerabilities, high-risk packages) + - Detailed scan results for each package + - Prioritized recommendations for security remediation + - Scan timestamp and completion status + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during scanning + SearchError: If bulk scanning fails + """ + try: + logger.info(f"MCP tool: Starting bulk security scan of {len(package_names)} packages") + result = await bulk_scan_package_security( + package_names, include_dependencies, severity_threshold + ) + logger.info(f"Bulk security scan completed - {result.get('summary', {}).get('packages_with_vulnerabilities', 0)} packages have vulnerabilities") + return result + except Exception as e: + logger.error(f"Error in bulk security scan: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_names": package_names, + } + + +@mcp.tool() +async def analyze_pypi_package_license_tool( + package_name: str, + version: str | None = None, + include_dependencies: bool = True +) -> dict[str, Any]: + """Analyze license compatibility for a PyPI package. + + This tool provides comprehensive license analysis including license identification, + dependency license scanning, compatibility checking, and risk assessment to help + ensure your project complies with open source license requirements. + + Args: + package_name: Name of the package to analyze for license compatibility + version: Specific version to analyze (optional, defaults to latest version) + include_dependencies: Whether to analyze dependency licenses for compatibility + + Returns: + Dictionary containing comprehensive license analysis including: + - License identification and normalization (SPDX format) + - License categorization (permissive, copyleft, proprietary, etc.) + - Dependency license analysis and compatibility matrix + - Risk assessment with score and risk level (minimal, low, medium, high, critical) + - Compatibility analysis highlighting conflicts and review-required combinations + - Actionable recommendations for license compliance + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If license analysis fails + """ + try: + logger.info(f"MCP tool: Analyzing license compatibility for {package_name}") + result = await analyze_pypi_package_license( + package_name, version, include_dependencies + ) + logger.info(f"License analysis completed for {package_name} - {result.get('analysis_summary', {}).get('license_conflicts', 0)} conflicts found") + return result + except Exception as e: + logger.error(f"Error analyzing license for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package": package_name, + "version": version, + } + + +@mcp.tool() +async def check_bulk_license_compliance_tool( + package_names: list[str], + target_license: str | None = None +) -> dict[str, Any]: + """Check license compliance for multiple PyPI packages. + + This tool performs bulk license compliance checking across multiple packages, + providing a consolidated report to help ensure your entire package ecosystem + complies with license requirements and identifying potential legal risks. + + Args: + package_names: List of package names to check for license compliance + target_license: Target license for compatibility checking (optional) + + Returns: + Dictionary containing bulk compliance analysis including: + - Summary statistics (total packages, compliant/non-compliant counts) + - Detailed license analysis for each package + - High-risk packages requiring immediate attention + - Unknown license packages needing investigation + - Prioritized recommendations for compliance remediation + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during analysis + SearchError: If bulk compliance checking fails + """ + try: + logger.info(f"MCP tool: Starting bulk license compliance check for {len(package_names)} packages") + result = await check_bulk_license_compliance( + package_names, target_license + ) + logger.info(f"Bulk license compliance completed - {result.get('summary', {}).get('non_compliant_packages', 0)} non-compliant packages found") + return result + except Exception as e: + logger.error(f"Error in bulk license compliance check: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_names": package_names, + } + + +@mcp.tool() +async def assess_package_health_score_tool( + package_name: str, + version: str | None = None, + include_github_metrics: bool = True +) -> dict[str, Any]: + """Assess comprehensive health and quality of a PyPI package. + + This tool evaluates package health across multiple dimensions including maintenance, + popularity, documentation, testing, security practices, compatibility, and metadata + completeness to provide an overall health score and actionable recommendations. + + Args: + package_name: Name of the package to assess for health and quality + version: Specific version to assess (optional, defaults to latest version) + include_github_metrics: Whether to fetch GitHub repository metrics for analysis + + Returns: + Dictionary containing comprehensive health assessment including: + - Overall health score (0-100) and level (excellent/good/fair/poor/critical) + - Category-specific scores (maintenance, popularity, documentation, testing, etc.) + - Detailed assessment breakdown with indicators and issues for each category + - GitHub repository metrics (stars, forks, activity) if available + - Actionable recommendations for health improvements + - Strengths, weaknesses, and improvement priorities analysis + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If health assessment fails + """ + try: + logger.info(f"MCP tool: Assessing health for {package_name}") + result = await assess_package_health_score( + package_name, version, include_github_metrics + ) + overall_score = result.get("overall_health", {}).get("score", 0) + health_level = result.get("overall_health", {}).get("level", "unknown") + logger.info(f"Health assessment completed for {package_name} - score: {overall_score:.1f}/100 ({health_level})") + return result + except Exception as e: + logger.error(f"Error assessing health for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package": package_name, + "version": version, + } + + +@mcp.tool() +async def compare_packages_health_scores_tool( + package_names: list[str], + include_github_metrics: bool = False +) -> dict[str, Any]: + """Compare health scores across multiple PyPI packages. + + This tool performs comparative health analysis across multiple packages, + providing rankings, insights, and recommendations to help evaluate + package ecosystem quality and identify the best options. + + Args: + package_names: List of package names to compare for health and quality + include_github_metrics: Whether to include GitHub metrics in the comparison + + Returns: + Dictionary containing comparative health analysis including: + - Detailed health results for each package + - Health score rankings with best/worst package identification + - Comparison insights (average scores, score ranges, rankings) + - Recommendations for package selection and improvements + - Statistical analysis of health across the package set + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during analysis + SearchError: If health comparison fails + """ + try: + logger.info(f"MCP tool: Starting health comparison for {len(package_names)} packages") + result = await compare_packages_health_scores( + package_names, include_github_metrics + ) + comparison_insights = result.get("comparison_insights", {}) + best_package = comparison_insights.get("best_package", {}) + packages_compared = result.get("packages_compared", 0) + logger.info(f"Health comparison completed for {packages_compared} packages - best: {best_package.get('name', 'unknown')} ({best_package.get('score', 0):.1f}/100)") + return result + except Exception as e: + logger.error(f"Error in health comparison: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_names": package_names, + } + + +@mcp.tool() +async def analyze_requirements_file_tool_mcp( + file_path: str, + check_updates: bool = True, + security_scan: bool = True, + compatibility_check: bool = True +) -> dict[str, Any]: + """Analyze project requirements file for dependencies, security, and compatibility. + + This tool provides comprehensive analysis of Python project requirements files + including dependency parsing, version checking, security vulnerability scanning, + Python compatibility assessment, and actionable recommendations for improvements. + + Args: + file_path: Path to the requirements file (requirements.txt, pyproject.toml, setup.py, etc.) + check_updates: Whether to check for available package updates + security_scan: Whether to perform security vulnerability scanning on dependencies + compatibility_check: Whether to check Python version compatibility for all dependencies + + Returns: + Dictionary containing comprehensive requirements analysis including: + - File information and detected format (requirements.txt, pyproject.toml, etc.) + - Parsed dependencies with version specifiers and extras + - Dependency health analysis with specification issues and recommendations + - Package update analysis showing outdated packages and latest versions + - Security vulnerability scan results for all dependencies + - Python version compatibility assessment + - Overall risk level and actionable improvement recommendations + + Raises: + FileNotFoundError: If the requirements file is not found + NetworkError: For network-related errors during analysis + SearchError: If requirements analysis fails + """ + try: + logger.info(f"MCP tool: Analyzing requirements file {file_path}") + result = await analyze_requirements_file_tool( + file_path, check_updates, security_scan, compatibility_check + ) + summary = result.get("analysis_summary", {}) + total_deps = summary.get("total_dependencies", 0) + risk_level = summary.get("overall_risk_level", "unknown") + logger.info(f"Requirements analysis completed for {file_path} - {total_deps} dependencies, risk level: {risk_level}") + return result + except Exception as e: + logger.error(f"Error analyzing requirements file {file_path}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "file_path": file_path, + } + + +@mcp.tool() +async def compare_multiple_requirements_files_mcp( + file_paths: list[str] +) -> dict[str, Any]: + """Compare multiple requirements files to identify differences and conflicts. + + This tool analyzes multiple requirements files simultaneously to identify + version conflicts, unique dependencies, and inconsistencies across different + project configurations or environments. + + Args: + file_paths: List of paths to requirements files to compare and analyze + + Returns: + Dictionary containing comparative requirements analysis including: + - Detailed analysis results for each individual file + - Common packages shared across all files + - Conflicting package versions between files with specific version details + - Packages unique to specific files + - Recommendations for resolving conflicts and standardizing requirements + - Statistics on package overlap and conflict rates + + Raises: + ValueError: If file_paths list is empty + NetworkError: For network-related errors during analysis + SearchError: If requirements comparison fails + """ + try: + logger.info(f"MCP tool: Comparing {len(file_paths)} requirements files") + result = await compare_multiple_requirements_files(file_paths) + comparison_results = result.get("comparison_results", {}) + conflicts = len(comparison_results.get("conflicting_packages", [])) + total_packages = comparison_results.get("total_unique_packages", 0) + logger.info(f"Requirements comparison completed - {total_packages} unique packages, {conflicts} conflicts found") + return result + except Exception as e: + logger.error(f"Error comparing requirements files: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "file_paths": file_paths, } @@ -1482,92 +2483,24 @@ async def generate_migration_checklist_prompt( return messages[0].text -# Environment Analysis Prompts -@mcp.prompt() -async def analyze_environment_dependencies_prompt( - environment_type: str = "local", - python_version: str | None = None, - project_path: str | None = None, -) -> str: - """Generate a prompt for analyzing environment dependencies.""" - # Step 3: Call Prompt generator - template = await analyze_environment_dependencies( - environment_type, python_version, project_path - ) - - # Step 5: Parameter replacement - result = template.replace("{{environment_type}}", environment_type) - - # Handle environment info - env_info = f"({environment_type} environment)" - if python_version: - env_info += f" with Python {python_version}" - if project_path: - env_info += f" at {project_path}" - result = result.replace("{{environment_info}}", env_info) - - # Handle command prefix based on environment - command_prefix = "uvx " if environment_type in ["virtual", "uv"] else "" - result = result.replace("{{command_prefix}}", command_prefix) - - # Step 7: Return final prompt - return result - - -@mcp.prompt() -async def check_outdated_packages_prompt( - package_filter: str | None = None, - severity_level: str = "all", - include_dev_dependencies: bool = True, -) -> str: - """Generate a prompt for checking outdated packages.""" - # Step 3: Call Prompt generator - template = await check_outdated_packages( - package_filter, severity_level, include_dev_dependencies - ) - - # Step 5: Parameter replacement - result = template.replace("{{severity_level}}", severity_level) - - # Handle filter info - if package_filter: - filter_info = f" (filtering by: {package_filter})" - else: - filter_info = "" - result = result.replace("{{filter_info}}", filter_info) - - # Handle dev dependencies - if include_dev_dependencies: - dev_deps_text = " including development dependencies" - else: - dev_deps_text = " excluding development dependencies" - result = result.replace("{{dev_deps_text}}", dev_deps_text) - - # Step 7: Return final prompt - return result - - @mcp.prompt() async def generate_update_plan_prompt( - update_strategy: str = "balanced", - environment_constraints: str | None = None, + packages: list[str], + update_strategy: str = "conservative", + environment_type: str = "production", testing_requirements: str | None = None, ) -> str: - """Generate a prompt for creating package update plans.""" + """Generate a comprehensive update plan prompt for packages.""" # Step 3: Call Prompt generator template = await generate_update_plan( - update_strategy, environment_constraints, testing_requirements + packages, update_strategy, environment_type, testing_requirements ) # Step 5: Parameter replacement - result = template.replace("{{strategy}}", update_strategy) - - # Handle constraints - if environment_constraints: - constraints_text = f"\n\nEnvironment constraints: {environment_constraints}" - else: - constraints_text = "" - result = result.replace("{{constraints_text}}", constraints_text) + packages_text = ", ".join(f"'{pkg}'" for pkg in packages) + result = template.replace("{{packages_text}}", packages_text) + result = result.replace("{{update_strategy}}", update_strategy) + result = result.replace("{{environment_type}}", environment_type) # Handle testing requirements if testing_requirements: @@ -1653,241 +2586,6 @@ async def track_package_updates_prompt( return result -@mcp.tool() -async def get_package_analytics( - package_name: str, - time_period: str = "month", - include_historical: bool = True, - include_platform_breakdown: bool = True, - include_version_analytics: bool = True, -) -> dict[str, Any]: - """Get comprehensive analytics for a PyPI package including advanced metrics. - - This tool provides detailed download analytics, trend analysis, geographic - distribution, platform breakdown, and version adoption patterns. - - Args: - package_name: Name of the package to analyze - time_period: Time period for analysis ('day', 'week', 'month', 'year') - include_historical: Whether to include historical trend analysis - include_platform_breakdown: Whether to include platform/OS breakdown - include_version_analytics: Whether to include version-specific analytics - - Returns: - Dictionary containing comprehensive analytics including: - - Download statistics and trends - - Platform and Python version breakdown - - Geographic distribution - - Version adoption patterns - - Quality metrics and indicators - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Generating comprehensive analytics for {package_name}") - result = await get_pypi_package_analytics( - package_name=package_name, - time_period=time_period, - include_historical=include_historical, - include_platform_breakdown=include_platform_breakdown, - include_version_analytics=include_version_analytics, - ) - logger.info(f"Successfully generated analytics for package: {package_name}") - return result - except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: - logger.error(f"Error generating analytics for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - except Exception as e: - logger.error(f"Unexpected error generating analytics for {package_name}: {e}") - return { - "error": f"Unexpected error: {e}", - "error_type": "UnexpectedError", - "package_name": package_name, - } - - -@mcp.tool() -async def get_security_alerts( - package_name: str, - include_dependencies: bool = True, - severity_filter: str | None = None, - include_historical: bool = False, -) -> dict[str, Any]: - """Get security alerts and vulnerability information for a PyPI package. - - This tool queries multiple security databases including OSV (Open Source - Vulnerabilities), PyUp.io Safety DB, and GitHub Security Advisories to provide - comprehensive security information. - - Args: - package_name: Name of the package to check for vulnerabilities - include_dependencies: Whether to check dependencies for vulnerabilities - severity_filter: Filter by severity ('LOW', 'MEDIUM', 'HIGH', 'CRITICAL') - include_historical: Whether to include historical vulnerabilities - - Returns: - Dictionary containing security information including: - - Active vulnerabilities and CVEs - - Security scores and risk assessment - - Dependency vulnerability analysis - - Remediation recommendations - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Checking security alerts for {package_name}") - result = await get_pypi_security_alerts( - package_name=package_name, - include_dependencies=include_dependencies, - severity_filter=severity_filter, - include_historical=include_historical, - ) - logger.info(f"Successfully checked security alerts for package: {package_name}") - return result - except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: - logger.error(f"Error checking security alerts for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - except Exception as e: - logger.error(f"Unexpected error checking security alerts for {package_name}: {e}") - return { - "error": f"Unexpected error: {e}", - "error_type": "UnexpectedError", - "package_name": package_name, - } - - -@mcp.tool() -async def get_package_rankings( - package_name: str, - search_terms: list[str] | None = None, - competitor_packages: list[str] | None = None, - ranking_metrics: list[str] | None = None, -) -> dict[str, Any]: - """Analyze package rankings and visibility in PyPI search results. - - This tool analyzes how well a package ranks for relevant search terms, - compares it to competitor packages, and provides insights into search - visibility and discoverability. - - Args: - package_name: Name of the package to analyze rankings for - search_terms: List of search terms to test rankings against - competitor_packages: List of competitor packages to compare against - ranking_metrics: Specific metrics to focus on ('relevance', 'popularity', 'downloads', 'quality') - - Returns: - Dictionary containing ranking analysis including: - - Search position for various terms - - Competitor comparison matrix - - Visibility and discoverability metrics - - SEO and keyword optimization suggestions - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Analyzing search rankings for {package_name}") - result = await get_pypi_package_rankings( - package_name=package_name, - search_terms=search_terms, - competitor_packages=competitor_packages, - ranking_metrics=ranking_metrics, - ) - logger.info(f"Successfully analyzed rankings for package: {package_name}") - return result - except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: - logger.error(f"Error analyzing rankings for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - except Exception as e: - logger.error(f"Unexpected error analyzing rankings for {package_name}: {e}") - return { - "error": f"Unexpected error: {e}", - "error_type": "UnexpectedError", - "package_name": package_name, - } - - -@mcp.tool() -async def analyze_package_competition( - package_name: str, - competitor_packages: list[str] | None = None, - analysis_depth: str = "comprehensive", - include_market_share: bool = True, - include_feature_comparison: bool = True, -) -> dict[str, Any]: - """Perform comprehensive competitive analysis against similar packages. - - This tool analyzes a package against its competitors, providing insights - into market positioning, feature gaps, adoption trends, and competitive - advantages. - - Args: - package_name: Name of the package to analyze - competitor_packages: List of competitor packages (auto-detected if not provided) - analysis_depth: Depth of analysis ('basic', 'comprehensive', 'detailed') - include_market_share: Whether to include market share analysis - include_feature_comparison: Whether to include feature comparison - - Returns: - Dictionary containing competitive analysis including: - - Market positioning and share - - Feature comparison matrix - - Adoption and growth trends - - Competitive advantages and weaknesses - - Strategic recommendations - - Raises: - InvalidPackageNameError: If package name is invalid - PackageNotFoundError: If package is not found - NetworkError: For network-related errors - """ - try: - logger.info(f"MCP tool: Analyzing competition for {package_name}") - result = await analyze_pypi_competition( - package_name=package_name, - competitor_packages=competitor_packages, - analysis_depth=analysis_depth, - include_market_share=include_market_share, - include_feature_comparison=include_feature_comparison, - ) - logger.info(f"Successfully analyzed competition for package: {package_name}") - return result - except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: - logger.error(f"Error analyzing competition for {package_name}: {e}") - return { - "error": str(e), - "error_type": type(e).__name__, - "package_name": package_name, - } - except Exception as e: - logger.error(f"Unexpected error analyzing competition for {package_name}: {e}") - return { - "error": f"Unexpected error: {e}", - "error_type": "UnexpectedError", - "package_name": package_name, - } - - @click.command() @click.option( "--log-level", diff --git a/pypi_query_mcp/tools/__init__.py b/pypi_query_mcp/tools/__init__.py index eb08606..ce0615b 100644 --- a/pypi_query_mcp/tools/__init__.py +++ b/pypi_query_mcp/tools/__init__.py @@ -21,12 +21,6 @@ from .package_query import ( query_package_info, query_package_versions, ) -from .metadata import ( - manage_package_keywords, - manage_package_urls, - set_package_visibility, - update_package_metadata, -) from .publishing import ( check_pypi_credentials, delete_pypi_release, @@ -35,11 +29,11 @@ from .publishing import ( manage_pypi_maintainers, upload_package_to_pypi, ) -from .search import ( - find_alternatives, - get_trending_packages, - search_by_category, - search_packages, +from .metadata import ( + manage_package_keywords, + manage_package_urls, + set_package_visibility, + update_package_metadata, ) from .analytics import ( analyze_pypi_competition, @@ -47,8 +41,48 @@ from .analytics import ( get_pypi_package_rankings, get_pypi_security_alerts, ) +from .discovery import ( + get_pypi_package_recommendations, + get_pypi_trending_today, + monitor_pypi_new_releases, + search_pypi_by_maintainer, +) +from .workflow import ( + check_pypi_upload_requirements, + get_pypi_build_logs, + preview_pypi_package_page, + validate_pypi_package_name, +) +from .community import ( + get_pypi_package_reviews, + manage_pypi_package_discussions, + get_pypi_maintainer_contacts, +) +from .search import ( + find_alternatives, + get_trending_packages, + search_by_category, + search_packages, +) +from .security_tools import ( + bulk_scan_package_security, + scan_pypi_package_security, +) +from .license_tools import ( + analyze_pypi_package_license, + check_bulk_license_compliance, +) +from .health_tools import ( + assess_package_health_score, + compare_packages_health_scores, +) +from .requirements_tools import ( + analyze_requirements_file_tool, + compare_multiple_requirements_files, +) __all__ = [ + # Core package tools "query_package_info", "query_package_versions", "query_package_dependencies", @@ -60,22 +94,52 @@ __all__ = [ "get_package_download_stats", "get_package_download_trends", "get_top_packages_by_downloads", + # Search tools "search_packages", "search_by_category", "find_alternatives", "get_trending_packages", + # Publishing tools "upload_package_to_pypi", "check_pypi_credentials", "get_pypi_upload_history", "delete_pypi_release", "manage_pypi_maintainers", "get_pypi_account_info", + # Metadata tools "update_package_metadata", "manage_package_urls", "set_package_visibility", "manage_package_keywords", + # Analytics tools "get_pypi_package_analytics", "get_pypi_security_alerts", "get_pypi_package_rankings", "analyze_pypi_competition", + # Discovery tools + "monitor_pypi_new_releases", + "get_pypi_trending_today", + "search_pypi_by_maintainer", + "get_pypi_package_recommendations", + # Workflow tools + "validate_pypi_package_name", + "preview_pypi_package_page", + "check_pypi_upload_requirements", + "get_pypi_build_logs", + # Community tools + "get_pypi_package_reviews", + "manage_pypi_package_discussions", + "get_pypi_maintainer_contacts", + # Security tools + "scan_pypi_package_security", + "bulk_scan_package_security", + # License tools + "analyze_pypi_package_license", + "check_bulk_license_compliance", + # Health tools + "assess_package_health_score", + "compare_packages_health_scores", + # Requirements tools + "analyze_requirements_file_tool", + "compare_multiple_requirements_files", ] diff --git a/pypi_query_mcp/tools/community.py b/pypi_query_mcp/tools/community.py new file mode 100644 index 0000000..54f2023 --- /dev/null +++ b/pypi_query_mcp/tools/community.py @@ -0,0 +1,1381 @@ +"""PyPI Community & Social Tools for package community interactions and maintainer communication.""" + +import asyncio +import json +import logging +import re +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional, Union +from urllib.parse import quote, urljoin, urlparse + +import httpx + +from ..core.exceptions import ( + InvalidPackageNameError, + NetworkError, + PackageNotFoundError, + PyPIError, +) +from ..core.pypi_client import PyPIClient +from ..core.github_client import GitHubAPIClient + +logger = logging.getLogger(__name__) + + +async def get_pypi_package_reviews( + package_name: str, + include_ratings: bool = True, + include_community_feedback: bool = True, + sentiment_analysis: bool = False, + max_reviews: int = 50, +) -> Dict[str, Any]: + """ + Get community reviews and feedback for a PyPI package. + + This function aggregates community feedback from various sources including + GitHub discussions, issues, Stack Overflow mentions, and social media to + provide comprehensive community sentiment analysis. + + Note: This is a future-ready implementation as PyPI doesn't currently have + a native review system. The function prepares for when such features become + available while providing useful community sentiment analysis from existing sources. + + Args: + package_name: Name of the package to get reviews for + include_ratings: Whether to include numerical ratings (when available) + include_community_feedback: Whether to include textual feedback analysis + sentiment_analysis: Whether to perform sentiment analysis on feedback + max_reviews: Maximum number of reviews to return + + Returns: + Dictionary containing review and feedback information including: + - Community sentiment and ratings + - Feedback from GitHub issues and discussions + - Social media mentions and sentiment + - Quality indicators and community health metrics + - Future-ready structure for native PyPI reviews + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError("Package name cannot be empty") + + package_name = package_name.strip() + logger.info(f"Gathering community reviews and feedback for package: {package_name}") + + try: + # Gather data from multiple community sources concurrently + review_tasks = [ + _get_package_metadata_for_reviews(package_name), + _analyze_github_community_sentiment(package_name) if include_community_feedback else asyncio.create_task(_empty_dict()), + _check_stackoverflow_mentions(package_name) if include_community_feedback else asyncio.create_task(_empty_dict()), + _analyze_pypi_downloads_as_quality_indicator(package_name), + _get_community_health_metrics(package_name), + ] + + results = await asyncio.gather(*review_tasks, return_exceptions=True) + + package_metadata = results[0] if not isinstance(results[0], Exception) else {} + github_sentiment = results[1] if not isinstance(results[1], Exception) else {} + stackoverflow_data = results[2] if not isinstance(results[2], Exception) else {} + quality_indicators = results[3] if not isinstance(results[3], Exception) else {} + community_health = results[4] if not isinstance(results[4], Exception) else {} + + # Perform sentiment analysis if requested + sentiment_analysis_results = {} + if sentiment_analysis and (github_sentiment or stackoverflow_data): + sentiment_analysis_results = await _perform_sentiment_analysis( + github_sentiment, stackoverflow_data + ) + + # Calculate overall community score + community_score = _calculate_community_score( + github_sentiment, stackoverflow_data, quality_indicators, community_health + ) + + # Generate community insights + insights = _generate_community_insights( + github_sentiment, stackoverflow_data, community_score, package_metadata + ) + + review_report = { + "package": package_name, + "review_timestamp": datetime.now().isoformat(), + "community_score": community_score, + "metadata": package_metadata, + "community_health": community_health, + "quality_indicators": quality_indicators, + "insights": insights, + "data_sources": { + "github_analysis": bool(github_sentiment), + "stackoverflow_mentions": bool(stackoverflow_data), + "pypi_metrics": bool(quality_indicators), + "community_health": bool(community_health), + }, + "review_system_status": { + "native_pypi_reviews": "not_available", + "note": "PyPI does not currently have a native review system. This analysis aggregates community feedback from external sources.", + "future_ready": True, + "implementation_note": "This function is designed to seamlessly integrate native PyPI reviews when they become available.", + }, + } + + # Add optional sections based on flags + if include_community_feedback and github_sentiment: + review_report["github_community_feedback"] = github_sentiment + + if include_community_feedback and stackoverflow_data: + review_report["stackoverflow_mentions"] = stackoverflow_data + + if sentiment_analysis and sentiment_analysis_results: + review_report["sentiment_analysis"] = sentiment_analysis_results + + # Add ratings section (prepared for future PyPI native ratings) + if include_ratings: + review_report["ratings"] = { + "average_rating": None, + "total_ratings": 0, + "rating_distribution": {}, + "community_derived_score": community_score.get("overall_score", 0), + "note": "Native PyPI ratings not yet available. Community-derived score provided based on external feedback.", + } + + return review_report + + except Exception as e: + logger.error(f"Error gathering reviews for {package_name}: {e}") + if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError)): + raise + raise NetworkError(f"Failed to gather community reviews: {e}") from e + + +async def manage_pypi_package_discussions( + package_name: str, + action: str = "get_status", + discussion_settings: Optional[Dict[str, Any]] = None, + moderator_controls: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Manage and interact with PyPI package discussions. + + This function provides management capabilities for package discussions, + including enabling/disabling discussions, setting moderation policies, + and retrieving discussion status and metrics. + + Note: This is a future-ready implementation as PyPI doesn't currently have + native discussion features. The function prepares for when such features + become available while providing integration with existing discussion platforms. + + Args: + package_name: Name of the package to manage discussions for + action: Action to perform ('get_status', 'enable', 'disable', 'configure', 'moderate') + discussion_settings: Settings for discussions (when enabling/configuring) + moderator_controls: Moderation settings and controls + + Returns: + Dictionary containing discussion management results including: + - Current discussion status and settings + - Available discussion platforms and integration status + - Moderation settings and community guidelines + - Discussion metrics and engagement data + - Future-ready structure for native PyPI discussions + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + PyPIError: For PyPI-specific errors + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError("Package name cannot be empty") + + package_name = package_name.strip() + valid_actions = ["get_status", "enable", "disable", "configure", "moderate", "get_metrics"] + + if action not in valid_actions: + raise InvalidPackageNameError(f"Invalid action. Must be one of: {', '.join(valid_actions)}") + + logger.info(f"Managing discussions for package: {package_name}, action: {action}") + + try: + # Get package metadata and current discussion status + package_data = await _get_package_metadata_for_discussions(package_name) + current_status = await _get_current_discussion_status(package_name, package_data) + + # Perform the requested action + if action == "get_status": + result = await _get_discussion_status(package_name, current_status) + elif action == "enable": + result = await _enable_discussions(package_name, discussion_settings, current_status) + elif action == "disable": + result = await _disable_discussions(package_name, current_status) + elif action == "configure": + result = await _configure_discussions(package_name, discussion_settings, current_status) + elif action == "moderate": + result = await _moderate_discussions(package_name, moderator_controls, current_status) + elif action == "get_metrics": + result = await _get_discussion_metrics(package_name, current_status) + else: + result = {"error": f"Action {action} not implemented"} + + # Add common metadata to all responses + result.update({ + "package": package_name, + "action_performed": action, + "timestamp": datetime.now().isoformat(), + "discussion_system_status": { + "native_pypi_discussions": "not_available", + "note": "PyPI does not currently have native discussion features. This function manages external discussion platforms and prepares for future PyPI integration.", + "supported_platforms": ["GitHub Discussions", "Community Forums", "Social Media"], + "future_ready": True, + }, + }) + + return result + + except Exception as e: + logger.error(f"Error managing discussions for {package_name}: {e}") + if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError, PyPIError)): + raise + raise NetworkError(f"Failed to manage package discussions: {e}") from e + + +async def get_pypi_maintainer_contacts( + package_name: str, + contact_types: Optional[List[str]] = None, + include_social_profiles: bool = False, + include_contribution_guidelines: bool = True, + respect_privacy_settings: bool = True, +) -> Dict[str, Any]: + """ + Get contact information and communication channels for package maintainers. + + This function retrieves publicly available contact information for package + maintainers while respecting privacy settings and providing appropriate + communication channels for different types of inquiries. + + Args: + package_name: Name of the package to get maintainer contacts for + contact_types: Types of contact info to retrieve ('email', 'github', 'social', 'support') + include_social_profiles: Whether to include social media profiles + include_contribution_guidelines: Whether to include contribution guidelines + respect_privacy_settings: Whether to respect maintainer privacy preferences + + Returns: + Dictionary containing maintainer contact information including: + - Publicly available contact methods + - Communication preferences and guidelines + - Support channels and community resources + - Privacy-respecting contact recommendations + - Contribution guidelines and community standards + + Raises: + InvalidPackageNameError: If package name is invalid + PackageNotFoundError: If package is not found + NetworkError: For network-related errors + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError("Package name cannot be empty") + + package_name = package_name.strip() + + # Set default contact types if not provided + if contact_types is None: + contact_types = ["github", "support", "community"] + + # Validate contact types + valid_contact_types = ["email", "github", "social", "support", "community", "documentation"] + invalid_types = [ct for ct in contact_types if ct not in valid_contact_types] + if invalid_types: + raise InvalidPackageNameError(f"Invalid contact types: {', '.join(invalid_types)}. Valid types: {', '.join(valid_contact_types)}") + + logger.info(f"Gathering maintainer contact information for package: {package_name}") + + try: + # Gather data from multiple sources concurrently + contact_tasks = [ + _get_package_metadata_for_contacts(package_name), + _analyze_github_maintainer_info(package_name) if "github" in contact_types else asyncio.create_task(_empty_dict()), + _get_support_channels(package_name) if "support" in contact_types else asyncio.create_task(_empty_dict()), + _get_community_channels(package_name) if "community" in contact_types else asyncio.create_task(_empty_dict()), + _get_contribution_guidelines(package_name) if include_contribution_guidelines else asyncio.create_task(_empty_dict()), + ] + + results = await asyncio.gather(*contact_tasks, return_exceptions=True) + + package_metadata = results[0] if not isinstance(results[0], Exception) else {} + github_info = results[1] if not isinstance(results[1], Exception) else {} + support_channels = results[2] if not isinstance(results[2], Exception) else {} + community_channels = results[3] if not isinstance(results[3], Exception) else {} + contribution_info = results[4] if not isinstance(results[4], Exception) else {} + + # Extract contact information from package metadata + contact_info = _extract_contact_info_from_metadata( + package_metadata, contact_types, respect_privacy_settings + ) + + # Get social profiles if requested + social_profiles = {} + if include_social_profiles and "social" in contact_types: + social_profiles = await _get_social_profiles(package_name, github_info, respect_privacy_settings) + + # Generate contact recommendations + contact_recommendations = _generate_contact_recommendations( + contact_info, github_info, support_channels, community_channels + ) + + # Assess contact accessibility and responsiveness + accessibility_assessment = _assess_contact_accessibility( + contact_info, github_info, support_channels + ) + + contact_report = { + "package": package_name, + "contact_timestamp": datetime.now().isoformat(), + "contact_information": contact_info, + "accessibility_assessment": accessibility_assessment, + "contact_recommendations": contact_recommendations, + "privacy_compliance": { + "respects_privacy_settings": respect_privacy_settings, + "data_sources": "Publicly available information only", + "privacy_note": "All contact information is sourced from publicly available package metadata and repositories", + }, + } + + # Add optional sections based on flags and availability + if github_info: + contact_report["github_information"] = github_info + + if support_channels: + contact_report["support_channels"] = support_channels + + if community_channels: + contact_report["community_channels"] = community_channels + + if include_contribution_guidelines and contribution_info: + contact_report["contribution_guidelines"] = contribution_info + + if include_social_profiles and social_profiles: + contact_report["social_profiles"] = social_profiles + + # Add communication guidelines + contact_report["communication_guidelines"] = _generate_communication_guidelines( + contact_info, github_info, package_metadata + ) + + return contact_report + + except Exception as e: + logger.error(f"Error gathering maintainer contacts for {package_name}: {e}") + if isinstance(e, (InvalidPackageNameError, PackageNotFoundError, NetworkError)): + raise + raise NetworkError(f"Failed to gather maintainer contact information: {e}") from e + + +# Helper functions for community tools implementation + +async def _empty_dict(): + """Return empty dict for optional tasks.""" + return {} + + +async def _get_package_metadata_for_reviews(package_name: str) -> Dict[str, Any]: + """Get package metadata relevant for review analysis.""" + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + return { + "name": info.get("name", package_name), + "version": info.get("version", "unknown"), + "summary": info.get("summary", ""), + "description": info.get("description", ""), + "author": info.get("author", ""), + "maintainer": info.get("maintainer", ""), + "home_page": info.get("home_page", ""), + "project_urls": info.get("project_urls", {}), + "keywords": info.get("keywords", ""), + "classifiers": info.get("classifiers", []), + "license": info.get("license", ""), + } + + except Exception as e: + logger.warning(f"Failed to get package metadata for reviews: {e}") + return {"name": package_name} + + +async def _analyze_github_community_sentiment(package_name: str) -> Dict[str, Any]: + """Analyze GitHub community sentiment through issues and discussions.""" + try: + # Get GitHub repository information + github_info = await _find_github_repository(package_name) + if not github_info.get("repository_url"): + return {"status": "no_github_repository"} + + # Use GitHub client to get community data + async with GitHubAPIClient() as github_client: + # Get recent issues for sentiment analysis + issues_data = await github_client.get_repository_issues( + github_info["owner"], + github_info["repo"], + state="all", + limit=50 + ) + + # Analyze sentiment from issue titles and comments + sentiment_analysis = _analyze_issue_sentiment(issues_data) + + # Get repository stats for community health + repo_stats = await github_client.get_repository_stats( + github_info["owner"], + github_info["repo"] + ) + + return { + "repository": github_info["repository_url"], + "sentiment_analysis": sentiment_analysis, + "repository_stats": repo_stats, + "issues_analyzed": len(issues_data.get("issues", [])), + "analysis_timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.warning(f"Failed to analyze GitHub sentiment for {package_name}: {e}") + return {"error": str(e), "status": "analysis_failed"} + + +async def _check_stackoverflow_mentions(package_name: str) -> Dict[str, Any]: + """Check Stack Overflow for package mentions and sentiment.""" + try: + # Use Stack Exchange API to search for package mentions + async with httpx.AsyncClient(timeout=30.0) as client: + # Search for questions mentioning the package + stackoverflow_url = "https://api.stackexchange.com/2.3/search/advanced" + params = { + "site": "stackoverflow", + "q": package_name, + "sort": "relevance", + "order": "desc", + "pagesize": 20, + } + + response = await client.get(stackoverflow_url, params=params) + + if response.status_code == 200: + data = response.json() + questions = data.get("items", []) + + # Analyze question sentiment and topics + stackoverflow_analysis = _analyze_stackoverflow_sentiment(questions, package_name) + + return { + "questions_found": len(questions), + "sentiment_analysis": stackoverflow_analysis, + "search_timestamp": datetime.now().isoformat(), + "data_source": "Stack Overflow API", + } + else: + logger.warning(f"Stack Overflow API returned status {response.status_code}") + return {"status": "api_unavailable", "questions_found": 0} + + except Exception as e: + logger.warning(f"Failed to check Stack Overflow mentions for {package_name}: {e}") + return {"error": str(e), "status": "analysis_failed"} + + +async def _analyze_pypi_downloads_as_quality_indicator(package_name: str) -> Dict[str, Any]: + """Use download statistics as a quality and adoption indicator.""" + try: + # Use existing download stats functionality + from .download_stats import get_package_download_stats + + download_stats = await get_package_download_stats(package_name) + + # Convert download numbers to quality indicators + downloads = download_stats.get("downloads", {}) + last_month = downloads.get("last_month", 0) + + # Categorize adoption level + if last_month > 1000000: + adoption_level = "very_high" + elif last_month > 100000: + adoption_level = "high" + elif last_month > 10000: + adoption_level = "moderate" + elif last_month > 1000: + adoption_level = "growing" + else: + adoption_level = "emerging" + + return { + "download_stats": downloads, + "adoption_level": adoption_level, + "quality_indicator_score": min(100, (last_month / 10000) * 10), # Scale to 0-100 + "analysis_timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.warning(f"Failed to analyze download stats for {package_name}: {e}") + return {"error": str(e), "adoption_level": "unknown"} + + +async def _get_community_health_metrics(package_name: str) -> Dict[str, Any]: + """Get community health metrics from various sources.""" + try: + # Get GitHub repository for community metrics + github_info = await _find_github_repository(package_name) + + if github_info.get("repository_url"): + async with GitHubAPIClient() as github_client: + # Get community health data + community_profile = await github_client.get_community_profile( + github_info["owner"], + github_info["repo"] + ) + + return { + "github_community_health": community_profile, + "has_repository": True, + "repository_url": github_info["repository_url"], + } + else: + return { + "has_repository": False, + "note": "No GitHub repository found for community health analysis", + } + + except Exception as e: + logger.warning(f"Failed to get community health metrics for {package_name}: {e}") + return {"error": str(e), "has_repository": False} + + +async def _perform_sentiment_analysis(github_sentiment: Dict, stackoverflow_data: Dict) -> Dict[str, Any]: + """Perform sentiment analysis on community feedback.""" + try: + # Simple sentiment analysis based on available data + sentiment_scores = [] + + # Analyze GitHub sentiment + if github_sentiment.get("sentiment_analysis"): + github_score = github_sentiment["sentiment_analysis"].get("overall_sentiment_score", 50) + sentiment_scores.append(("github", github_score)) + + # Analyze Stack Overflow sentiment + if stackoverflow_data.get("sentiment_analysis"): + so_score = stackoverflow_data["sentiment_analysis"].get("overall_sentiment_score", 50) + sentiment_scores.append(("stackoverflow", so_score)) + + # Calculate overall sentiment + if sentiment_scores: + overall_score = sum(score for _, score in sentiment_scores) / len(sentiment_scores) + sentiment_level = "positive" if overall_score > 60 else "neutral" if overall_score > 40 else "negative" + else: + overall_score = 50 + sentiment_level = "neutral" + + return { + "overall_sentiment_score": round(overall_score, 1), + "sentiment_level": sentiment_level, + "source_scores": dict(sentiment_scores), + "confidence": "medium" if len(sentiment_scores) > 1 else "low", + "analysis_timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.warning(f"Failed to perform sentiment analysis: {e}") + return {"error": str(e), "sentiment_level": "unknown"} + + +def _calculate_community_score( + github_sentiment: Dict, + stackoverflow_data: Dict, + quality_indicators: Dict, + community_health: Dict +) -> Dict[str, Any]: + """Calculate overall community score based on multiple factors.""" + score_components = {} + total_weight = 0 + weighted_score = 0 + + # GitHub sentiment component (weight: 30%) + if github_sentiment.get("sentiment_analysis"): + github_score = github_sentiment["sentiment_analysis"].get("overall_sentiment_score", 50) + score_components["github_sentiment"] = github_score + weighted_score += github_score * 0.3 + total_weight += 0.3 + + # Stack Overflow mentions component (weight: 20%) + if stackoverflow_data.get("sentiment_analysis"): + so_score = stackoverflow_data["sentiment_analysis"].get("overall_sentiment_score", 50) + score_components["stackoverflow_sentiment"] = so_score + weighted_score += so_score * 0.2 + total_weight += 0.2 + + # Download/adoption component (weight: 25%) + if quality_indicators.get("quality_indicator_score"): + adoption_score = quality_indicators["quality_indicator_score"] + score_components["adoption_level"] = adoption_score + weighted_score += adoption_score * 0.25 + total_weight += 0.25 + + # Community health component (weight: 25%) + if community_health.get("github_community_health"): + # Simplified health score based on repository activity + health_score = 70 # Default moderate score + score_components["community_health"] = health_score + weighted_score += health_score * 0.25 + total_weight += 0.25 + + # Calculate final score + overall_score = weighted_score / total_weight if total_weight > 0 else 50 + + # Determine community status + if overall_score >= 80: + status = "excellent" + elif overall_score >= 65: + status = "good" + elif overall_score >= 50: + status = "moderate" + elif overall_score >= 35: + status = "limited" + else: + status = "poor" + + return { + "overall_score": round(overall_score, 1), + "community_status": status, + "score_components": score_components, + "data_reliability": total_weight, + "calculation_timestamp": datetime.now().isoformat(), + } + + +def _generate_community_insights( + github_sentiment: Dict, + stackoverflow_data: Dict, + community_score: Dict, + package_metadata: Dict +) -> Dict[str, Any]: + """Generate insights from community analysis.""" + insights = { + "key_insights": [], + "recommendations": [], + "community_strengths": [], + "areas_for_improvement": [], + } + + score = community_score.get("overall_score", 50) + + # Generate insights based on score + if score >= 70: + insights["key_insights"].append("Package has strong community support and positive sentiment") + insights["community_strengths"].append("Active and engaged community") + elif score >= 50: + insights["key_insights"].append("Package has moderate community engagement") + insights["recommendations"].append("Consider increasing community engagement initiatives") + else: + insights["key_insights"].append("Package has limited community feedback available") + insights["areas_for_improvement"].append("Community engagement and visibility") + + # Add specific insights from GitHub + if github_sentiment.get("repository_stats"): + stars = github_sentiment["repository_stats"].get("stargazers_count", 0) + if stars > 1000: + insights["community_strengths"].append("High GitHub stars indicating community appreciation") + elif stars > 100: + insights["community_strengths"].append("Moderate GitHub community following") + + # Add Stack Overflow insights + if stackoverflow_data.get("questions_found", 0) > 10: + insights["community_strengths"].append("Active discussion on Stack Overflow") + elif stackoverflow_data.get("questions_found", 0) > 0: + insights["key_insights"].append("Some Stack Overflow discussion available") + else: + insights["areas_for_improvement"].append("Limited presence in developer Q&A platforms") + + return insights + + +# Helper functions for discussion management + +async def _get_package_metadata_for_discussions(package_name: str) -> Dict[str, Any]: + """Get package metadata relevant for discussion management.""" + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + return package_data + except Exception as e: + logger.warning(f"Failed to get package metadata for discussions: {e}") + return {} + + +async def _get_current_discussion_status(package_name: str, package_data: Dict) -> Dict[str, Any]: + """Get current discussion status from various platforms.""" + try: + # Check GitHub Discussions + github_status = await _check_github_discussions_status(package_name, package_data) + + # Check other community platforms + community_platforms = await _check_community_platforms(package_name, package_data) + + return { + "github_discussions": github_status, + "community_platforms": community_platforms, + "native_pypi_discussions": { + "available": False, + "note": "PyPI does not currently support native discussions", + }, + } + + except Exception as e: + logger.warning(f"Failed to get discussion status for {package_name}: {e}") + return {"error": str(e)} + + +async def _get_discussion_status(package_name: str, current_status: Dict) -> Dict[str, Any]: + """Get comprehensive discussion status.""" + return { + "status": "retrieved", + "current_discussion_status": current_status, + "available_platforms": [ + { + "platform": "GitHub Discussions", + "status": current_status.get("github_discussions", {}).get("enabled", False), + "url": current_status.get("github_discussions", {}).get("url"), + } + ], + "management_options": [ + "Enable/disable GitHub Discussions", + "Configure community guidelines", + "Set up moderation policies", + "Monitor discussion metrics", + ], + } + + +async def _enable_discussions(package_name: str, settings: Optional[Dict], current_status: Dict) -> Dict[str, Any]: + """Enable discussions for the package.""" + # This is a placeholder for future implementation + # Would integrate with GitHub API to enable discussions + + return { + "status": "configured", + "action": "enable_discussions", + "message": "Discussion enabling configured (requires repository admin access)", + "settings_applied": settings or {}, + "next_steps": [ + "Verify repository admin access", + "Enable GitHub Discussions in repository settings", + "Configure community guidelines", + "Set up moderation policies", + ], + "note": "Actual enabling requires repository admin privileges and GitHub API integration", + } + + +async def _disable_discussions(package_name: str, current_status: Dict) -> Dict[str, Any]: + """Disable discussions for the package.""" + return { + "status": "configured", + "action": "disable_discussions", + "message": "Discussion disabling configured (requires repository admin access)", + "current_status": current_status, + "next_steps": [ + "Verify repository admin access", + "Disable GitHub Discussions in repository settings", + "Archive existing discussions if needed", + ], + "note": "Actual disabling requires repository admin privileges", + } + + +async def _configure_discussions(package_name: str, settings: Optional[Dict], current_status: Dict) -> Dict[str, Any]: + """Configure discussion settings.""" + return { + "status": "configured", + "action": "configure_discussions", + "settings": settings or {}, + "current_status": current_status, + "configuration_options": { + "categories": ["General", "Q&A", "Ideas", "Show and Tell"], + "moderation": ["Auto-moderation", "Manual review", "Community moderation"], + "notifications": ["Email notifications", "Web notifications", "Digest emails"], + }, + "note": "Configuration changes require repository admin access", + } + + +async def _moderate_discussions(package_name: str, moderator_controls: Optional[Dict], current_status: Dict) -> Dict[str, Any]: + """Apply moderation controls to discussions.""" + return { + "status": "moderation_configured", + "action": "moderate_discussions", + "moderator_controls": moderator_controls or {}, + "current_status": current_status, + "moderation_features": { + "content_filtering": "Automatic filtering of inappropriate content", + "user_management": "Block/unblock users, assign moderator roles", + "discussion_management": "Lock/unlock, pin/unpin discussions", + "reporting": "Community reporting and review systems", + }, + "note": "Moderation requires appropriate permissions and platform integration", + } + + +async def _get_discussion_metrics(package_name: str, current_status: Dict) -> Dict[str, Any]: + """Get discussion engagement metrics.""" + try: + github_metrics = {} + if current_status.get("github_discussions", {}).get("enabled"): + github_metrics = await _get_github_discussion_metrics(package_name) + + return { + "status": "metrics_retrieved", + "github_metrics": github_metrics, + "overall_engagement": _calculate_discussion_engagement(github_metrics), + "metrics_timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.warning(f"Failed to get discussion metrics: {e}") + return {"error": str(e), "status": "metrics_unavailable"} + + +# Helper functions for maintainer contacts + +async def _get_package_metadata_for_contacts(package_name: str) -> Dict[str, Any]: + """Get package metadata relevant for contact information.""" + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + return { + "name": info.get("name", package_name), + "author": info.get("author", ""), + "author_email": info.get("author_email", ""), + "maintainer": info.get("maintainer", ""), + "maintainer_email": info.get("maintainer_email", ""), + "home_page": info.get("home_page", ""), + "project_urls": info.get("project_urls", {}), + "download_url": info.get("download_url", ""), + } + + except Exception as e: + logger.warning(f"Failed to get package metadata for contacts: {e}") + return {"name": package_name} + + +async def _analyze_github_maintainer_info(package_name: str) -> Dict[str, Any]: + """Analyze GitHub repository for maintainer information.""" + try: + github_info = await _find_github_repository(package_name) + if not github_info.get("repository_url"): + return {"status": "no_github_repository"} + + async with GitHubAPIClient() as github_client: + # Get repository information + repo_data = await github_client.get_repository_info( + github_info["owner"], + github_info["repo"] + ) + + # Get contributors + contributors = await github_client.get_repository_contributors( + github_info["owner"], + github_info["repo"], + limit=10 + ) + + return { + "repository": github_info["repository_url"], + "owner": github_info["owner"], + "repository_data": repo_data, + "contributors": contributors, + "primary_maintainer": repo_data.get("owner", {}), + } + + except Exception as e: + logger.warning(f"Failed to analyze GitHub maintainer info: {e}") + return {"error": str(e), "status": "analysis_failed"} + + +async def _get_support_channels(package_name: str) -> Dict[str, Any]: + """Get available support channels for the package.""" + try: + # Check common support channels + support_channels = { + "issue_tracker": None, + "documentation": None, + "community_forum": None, + "chat_channels": [], + } + + # Get GitHub repository for issue tracker + github_info = await _find_github_repository(package_name) + if github_info.get("repository_url"): + support_channels["issue_tracker"] = f"{github_info['repository_url']}/issues" + + # Check for documentation links + async with GitHubAPIClient() as github_client: + repo_data = await github_client.get_repository_info( + github_info["owner"], + github_info["repo"] + ) + + if repo_data.get("has_pages"): + support_channels["documentation"] = f"https://{github_info['owner']}.github.io/{github_info['repo']}/" + + return support_channels + + except Exception as e: + logger.warning(f"Failed to get support channels: {e}") + return {} + + +async def _get_community_channels(package_name: str) -> Dict[str, Any]: + """Get available community channels for the package.""" + try: + community_channels = { + "github_discussions": None, + "stackoverflow_tag": None, + "reddit_community": None, + "discord_server": None, + } + + # Check for GitHub Discussions + github_info = await _find_github_repository(package_name) + if github_info.get("repository_url"): + # Check if discussions are enabled + discussions_enabled = await _check_github_discussions_enabled( + github_info["owner"], + github_info["repo"] + ) + if discussions_enabled: + community_channels["github_discussions"] = f"{github_info['repository_url']}/discussions" + + # Check for Stack Overflow tag + stackoverflow_tag = await _check_stackoverflow_tag(package_name) + if stackoverflow_tag: + community_channels["stackoverflow_tag"] = f"https://stackoverflow.com/questions/tagged/{stackoverflow_tag}" + + return community_channels + + except Exception as e: + logger.warning(f"Failed to get community channels: {e}") + return {} + + +async def _get_contribution_guidelines(package_name: str) -> Dict[str, Any]: + """Get contribution guidelines for the package.""" + try: + github_info = await _find_github_repository(package_name) + if not github_info.get("repository_url"): + return {"status": "no_repository"} + + async with GitHubAPIClient() as github_client: + # Check for common contribution files + contribution_files = await github_client.get_community_files( + github_info["owner"], + github_info["repo"] + ) + + return { + "repository": github_info["repository_url"], + "contribution_files": contribution_files, + "guidelines_available": bool(contribution_files), + } + + except Exception as e: + logger.warning(f"Failed to get contribution guidelines: {e}") + return {"error": str(e)} + + +def _extract_contact_info_from_metadata( + package_metadata: Dict, + contact_types: List[str], + respect_privacy: bool +) -> Dict[str, Any]: + """Extract contact information from package metadata.""" + contact_info = { + "available_contacts": {}, + "project_urls": {}, + "privacy_compliant": respect_privacy, + } + + # Extract email contacts if requested and available + if "email" in contact_types: + if package_metadata.get("author_email") and not respect_privacy: + contact_info["available_contacts"]["author_email"] = package_metadata["author_email"] + if package_metadata.get("maintainer_email") and not respect_privacy: + contact_info["available_contacts"]["maintainer_email"] = package_metadata["maintainer_email"] + if respect_privacy: + contact_info["available_contacts"]["email_note"] = "Email addresses hidden due to privacy settings" + + # Extract project URLs + project_urls = package_metadata.get("project_urls", {}) + for url_type, url in project_urls.items(): + if _is_relevant_project_url(url_type, contact_types): + contact_info["project_urls"][url_type] = url + + # Add homepage if available + if package_metadata.get("home_page"): + contact_info["project_urls"]["Homepage"] = package_metadata["home_page"] + + return contact_info + + +async def _get_social_profiles(package_name: str, github_info: Dict, respect_privacy: bool) -> Dict[str, Any]: + """Get social media profiles for maintainers.""" + if respect_privacy: + return { + "status": "privacy_protected", + "note": "Social profiles hidden due to privacy settings", + } + + # This would require additional API integrations + # For now, return a placeholder structure + return { + "status": "limited_data", + "note": "Social profile discovery requires additional API integrations", + "available_platforms": ["GitHub", "Twitter", "LinkedIn"], + } + + +def _generate_contact_recommendations( + contact_info: Dict, + github_info: Dict, + support_channels: Dict, + community_channels: Dict +) -> List[str]: + """Generate recommendations for contacting maintainers.""" + recommendations = [] + + # Recommend GitHub issues for bugs and features + if github_info.get("repository"): + recommendations.append("Use GitHub issues for bug reports and feature requests") + + # Recommend GitHub discussions for general questions + if community_channels.get("github_discussions"): + recommendations.append("Use GitHub Discussions for general questions and community interaction") + + # Recommend proper channels based on inquiry type + recommendations.extend([ + "Check existing issues before creating new ones", + "Follow contribution guidelines when proposing changes", + "Be respectful and patient when seeking support", + "Provide detailed information when reporting issues", + ]) + + return recommendations + + +def _assess_contact_accessibility( + contact_info: Dict, + github_info: Dict, + support_channels: Dict +) -> Dict[str, Any]: + """Assess how accessible maintainers are for contact.""" + accessibility_score = 0 + factors = [] + + # GitHub repository increases accessibility + if github_info.get("repository"): + accessibility_score += 40 + factors.append("GitHub repository available") + + # Issue tracker increases accessibility + if support_channels.get("issue_tracker"): + accessibility_score += 30 + factors.append("Issue tracker available") + + # Project URLs increase accessibility + if contact_info.get("project_urls"): + accessibility_score += 20 + factors.append("Project URLs provided") + + # Documentation increases accessibility + if support_channels.get("documentation"): + accessibility_score += 10 + factors.append("Documentation available") + + # Determine accessibility level + if accessibility_score >= 80: + level = "excellent" + elif accessibility_score >= 60: + level = "good" + elif accessibility_score >= 40: + level = "moderate" + else: + level = "limited" + + return { + "accessibility_score": accessibility_score, + "accessibility_level": level, + "contributing_factors": factors, + "assessment_timestamp": datetime.now().isoformat(), + } + + +def _generate_communication_guidelines( + contact_info: Dict, + github_info: Dict, + package_metadata: Dict +) -> Dict[str, Any]: + """Generate communication guidelines for contacting maintainers.""" + return { + "best_practices": [ + "Be clear and concise in your communication", + "Provide reproducible examples for bug reports", + "Search existing issues before creating new ones", + "Follow the project's code of conduct", + "Be patient and respectful in all interactions", + ], + "communication_channels": { + "bug_reports": "GitHub Issues (if available)", + "feature_requests": "GitHub Issues or Discussions", + "general_questions": "GitHub Discussions or community forums", + "security_issues": "Private communication via email or security contact", + }, + "response_expectations": { + "bug_reports": "Response within 1-7 days (varies by project)", + "feature_requests": "Response time varies significantly", + "general_questions": "Community may respond faster than maintainers", + "note": "Response times depend on maintainer availability and project activity", + }, + } + + +# Additional helper functions for GitHub and community analysis + +async def _find_github_repository(package_name: str) -> Dict[str, Any]: + """Find GitHub repository for a package.""" + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + + # Check project URLs for GitHub + project_urls = info.get("project_urls", {}) + for url_type, url in project_urls.items(): + if "github.com" in url.lower(): + return _parse_github_url(url) + + # Check homepage for GitHub + home_page = info.get("home_page", "") + if "github.com" in home_page.lower(): + return _parse_github_url(home_page) + + return {"status": "no_github_repository"} + + except Exception as e: + logger.warning(f"Failed to find GitHub repository: {e}") + return {"error": str(e)} + + +def _parse_github_url(url: str) -> Dict[str, str]: + """Parse GitHub URL to extract owner and repo.""" + try: + # Remove .git suffix and clean URL + clean_url = url.replace(".git", "").rstrip("/") + + # Parse URL + parsed = urlparse(clean_url) + if parsed.netloc != "github.com": + return {"status": "not_github"} + + # Extract owner and repo from path + path_parts = parsed.path.strip("/").split("/") + if len(path_parts) >= 2: + owner = path_parts[0] + repo = path_parts[1] + + return { + "repository_url": f"https://github.com/{owner}/{repo}", + "owner": owner, + "repo": repo, + } + else: + return {"status": "invalid_github_url"} + + except Exception as e: + logger.warning(f"Failed to parse GitHub URL {url}: {e}") + return {"error": str(e)} + + +def _analyze_issue_sentiment(issues_data: Dict) -> Dict[str, Any]: + """Analyze sentiment from GitHub issues.""" + issues = issues_data.get("issues", []) + if not issues: + return {"status": "no_issues", "overall_sentiment_score": 50} + + # Simple sentiment analysis based on issue characteristics + positive_indicators = 0 + negative_indicators = 0 + total_issues = len(issues) + + for issue in issues: + title = issue.get("title", "").lower() + labels = [label.get("name", "").lower() for label in issue.get("labels", [])] + state = issue.get("state", "") + + # Count positive indicators + if state == "closed": + positive_indicators += 1 + if any(label in ["enhancement", "feature", "good first issue"] for label in labels): + positive_indicators += 1 + + # Count negative indicators + if any(label in ["bug", "critical", "high priority"] for label in labels): + negative_indicators += 1 + if any(word in title for word in ["crash", "broken", "error", "fail"]): + negative_indicators += 1 + + # Calculate sentiment score (0-100) + if total_issues > 0: + positive_ratio = positive_indicators / (total_issues * 2) # Max 2 positive per issue + negative_ratio = negative_indicators / (total_issues * 2) # Max 2 negative per issue + sentiment_score = max(0, min(100, 50 + (positive_ratio - negative_ratio) * 100)) + else: + sentiment_score = 50 + + return { + "overall_sentiment_score": round(sentiment_score, 1), + "issues_analyzed": total_issues, + "positive_indicators": positive_indicators, + "negative_indicators": negative_indicators, + "sentiment_factors": { + "closed_issues": sum(1 for issue in issues if issue.get("state") == "closed"), + "open_issues": sum(1 for issue in issues if issue.get("state") == "open"), + "enhancement_requests": len([issue for issue in issues if any("enhancement" in label.get("name", "").lower() for label in issue.get("labels", []))]), + "bug_reports": len([issue for issue in issues if any("bug" in label.get("name", "").lower() for label in issue.get("labels", []))]), + }, + } + + +def _analyze_stackoverflow_sentiment(questions: List[Dict], package_name: str) -> Dict[str, Any]: + """Analyze sentiment from Stack Overflow questions.""" + if not questions: + return {"status": "no_questions", "overall_sentiment_score": 50} + + # Simple sentiment analysis based on question characteristics + positive_indicators = 0 + negative_indicators = 0 + total_questions = len(questions) + + for question in questions: + title = question.get("title", "").lower() + tags = question.get("tags", []) + score = question.get("score", 0) + is_answered = question.get("is_answered", False) + + # Count positive indicators + if is_answered: + positive_indicators += 1 + if score > 0: + positive_indicators += 1 + if any(word in title for word in ["how to", "tutorial", "example", "best practice"]): + positive_indicators += 1 + + # Count negative indicators + if any(word in title for word in ["error", "problem", "issue", "broken", "not working"]): + negative_indicators += 1 + if score < 0: + negative_indicators += 1 + + # Calculate sentiment score + if total_questions > 0: + positive_ratio = positive_indicators / (total_questions * 3) # Max 3 positive per question + negative_ratio = negative_indicators / (total_questions * 2) # Max 2 negative per question + sentiment_score = max(0, min(100, 50 + (positive_ratio - negative_ratio) * 100)) + else: + sentiment_score = 50 + + return { + "overall_sentiment_score": round(sentiment_score, 1), + "questions_analyzed": total_questions, + "positive_indicators": positive_indicators, + "negative_indicators": negative_indicators, + "question_characteristics": { + "answered_questions": sum(1 for q in questions if q.get("is_answered")), + "unanswered_questions": sum(1 for q in questions if not q.get("is_answered")), + "average_score": sum(q.get("score", 0) for q in questions) / total_questions if total_questions > 0 else 0, + }, + } + + +# Additional placeholder functions for future implementation + +async def _check_github_discussions_status(package_name: str, package_data: Dict) -> Dict[str, Any]: + """Check if GitHub Discussions are enabled for the repository.""" + github_info = await _find_github_repository(package_name) + if not github_info.get("repository_url"): + return {"enabled": False, "reason": "no_github_repository"} + + # This would require GitHub API integration to check discussions status + return { + "enabled": False, + "reason": "requires_github_api_integration", + "repository": github_info["repository_url"], + "note": "Checking discussions status requires GitHub API integration", + } + + +async def _check_community_platforms(package_name: str, package_data: Dict) -> Dict[str, Any]: + """Check other community platforms for discussions.""" + return { + "discord": {"available": False, "note": "Discord integration not implemented"}, + "reddit": {"available": False, "note": "Reddit integration not implemented"}, + "forums": {"available": False, "note": "Forum integration not implemented"}, + } + + +async def _get_github_discussion_metrics(package_name: str) -> Dict[str, Any]: + """Get GitHub discussion engagement metrics.""" + return { + "discussions_count": 0, + "participants": 0, + "note": "Requires GitHub API integration for actual metrics", + } + + +def _calculate_discussion_engagement(github_metrics: Dict) -> Dict[str, Any]: + """Calculate overall discussion engagement score.""" + return { + "engagement_score": 0, + "engagement_level": "unknown", + "note": "Engagement calculation requires actual discussion data", + } + + +async def _check_github_discussions_enabled(owner: str, repo: str) -> bool: + """Check if GitHub Discussions are enabled for a repository.""" + # This would require GitHub GraphQL API to check discussions + return False + + +async def _check_stackoverflow_tag(package_name: str) -> Optional[str]: + """Check if there's a Stack Overflow tag for the package.""" + # This would require Stack Exchange API integration + return None + + +def _is_relevant_project_url(url_type: str, contact_types: List[str]) -> bool: + """Check if a project URL is relevant for the requested contact types.""" + url_type_lower = url_type.lower() + + if "github" in contact_types and any(keyword in url_type_lower for keyword in ["repository", "source", "github"]): + return True + if "support" in contact_types and any(keyword in url_type_lower for keyword in ["support", "help", "issues", "bug"]): + return True + if "documentation" in contact_types and any(keyword in url_type_lower for keyword in ["documentation", "docs", "wiki"]): + return True + if "community" in contact_types and any(keyword in url_type_lower for keyword in ["community", "forum", "chat", "discussion"]): + return True + + return False \ No newline at end of file diff --git a/pypi_query_mcp/tools/compatibility_check.py b/pypi_query_mcp/tools/compatibility_check.py index b559035..3422e9a 100644 --- a/pypi_query_mcp/tools/compatibility_check.py +++ b/pypi_query_mcp/tools/compatibility_check.py @@ -39,7 +39,7 @@ async def check_python_compatibility( try: async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, use_cache) + package_data = await client.get_package_info(package_name, use_cache=use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") @@ -103,7 +103,7 @@ async def get_compatible_python_versions( try: async with PyPIClient() as client: - package_data = await client.get_package_info(package_name, use_cache) + package_data = await client.get_package_info(package_name, use_cache=use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") @@ -177,7 +177,7 @@ async def suggest_python_version_for_packages( async with PyPIClient() as client: for package_name in package_names: try: - package_data = await client.get_package_info(package_name, use_cache) + package_data = await client.get_package_info(package_name, use_cache=use_cache) info = package_data.get("info", {}) requires_python = info.get("requires_python") diff --git a/pypi_query_mcp/tools/discovery.py b/pypi_query_mcp/tools/discovery.py new file mode 100644 index 0000000..21a7c40 --- /dev/null +++ b/pypi_query_mcp/tools/discovery.py @@ -0,0 +1,1081 @@ +"""PyPI Discovery & Monitoring Tools for tracking new releases and trending packages.""" + +import asyncio +import json +import logging +import time +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional, Set +from urllib.parse import urlencode + +import httpx + +try: + from feedparser import parse as parse_feed + HAS_FEEDPARSER = True +except ImportError: + HAS_FEEDPARSER = False + def parse_feed(url_or_content): + """Fallback when feedparser is not available.""" + return {"entries": []} + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..core.pypi_client import PyPIClient + +logger = logging.getLogger(__name__) + + +class DiscoveryCache: + """Simple in-memory cache for discovery data with TTL.""" + + def __init__(self, default_ttl: int = 300): # 5 minutes default + self._cache: Dict[str, Dict[str, Any]] = {} + self._default_ttl = default_ttl + + def get(self, key: str) -> Optional[Any]: + """Get cached value if not expired.""" + if key in self._cache: + entry = self._cache[key] + if time.time() < entry["expires_at"]: + return entry["data"] + else: + del self._cache[key] + return None + + def set(self, key: str, data: Any, ttl: Optional[int] = None) -> None: + """Cache data with TTL.""" + expires_at = time.time() + (ttl or self._default_ttl) + self._cache[key] = {"data": data, "expires_at": expires_at} + + def clear(self) -> None: + """Clear all cached data.""" + self._cache.clear() + + +# Global cache instance +_discovery_cache = DiscoveryCache() + + +async def monitor_pypi_new_releases( + categories: Optional[List[str]] = None, + hours: int = 24, + min_downloads: Optional[int] = None, + maintainer_filter: Optional[str] = None, + enable_notifications: bool = False, + cache_ttl: int = 300, +) -> Dict[str, Any]: + """ + Track new releases in specified categories over a time period. + + Args: + categories: List of categories to monitor (e.g., ["web", "data-science", "ai", "cli"]) + hours: Number of hours to look back for new releases (default: 24) + min_downloads: Minimum monthly downloads to include (filters out very new packages) + maintainer_filter: Filter releases by specific maintainer names + enable_notifications: Whether to enable alert system for monitoring + cache_ttl: Cache time-to-live in seconds (default: 300) + + Returns: + Dictionary containing new releases with metadata and analysis + + Raises: + NetworkError: If unable to fetch release data + SearchError: If category filtering fails + """ + logger.info(f"Monitoring new PyPI releases for last {hours}h, categories: {categories}") + + # Generate cache key based on parameters + cache_key = f"new_releases_{categories}_{hours}_{min_downloads}_{maintainer_filter}" + cached_result = _discovery_cache.get(cache_key) + if cached_result: + logger.info("Returning cached new releases data") + return cached_result + + try: + # Use PyPI RSS feeds for recent releases + releases_data = await _fetch_recent_releases_from_rss(hours) + + # Enhance with package metadata + enhanced_releases = [] + async with PyPIClient() as client: + for release in releases_data: + try: + # Get full package info for filtering and categorization + package_info = await client.get_package_info(release["name"]) + info = package_info["info"] + + # Apply filters + if min_downloads: + # Skip packages that might not have download stats yet + try: + from .download_stats import get_package_download_stats + stats = await get_package_download_stats(release["name"], "month", use_cache=True) + if stats.get("recent_downloads", {}).get("last_month", 0) < min_downloads: + continue + except: + # If we can't get stats, assume it's a new package and include it + pass + + if maintainer_filter and maintainer_filter.lower() not in info.get("author", "").lower(): + continue + + # Categorize package + package_categories = await _categorize_package(info) + + # Apply category filter + if categories: + if not any(cat.lower() in [pc.lower() for pc in package_categories] for cat in categories): + continue + + enhanced_release = { + **release, + "summary": info.get("summary", ""), + "author": info.get("author", ""), + "license": info.get("license", ""), + "home_page": info.get("home_page", ""), + "keywords": info.get("keywords", ""), + "categories": package_categories, + "python_requires": info.get("requires_python", ""), + "project_urls": info.get("project_urls", {}), + "classifiers": info.get("classifiers", []), + } + + enhanced_releases.append(enhanced_release) + + except Exception as e: + logger.warning(f"Failed to enhance release data for {release['name']}: {e}") + # Include basic release info even if enhancement fails + enhanced_releases.append(release) + + # Sort by release time (most recent first) + enhanced_releases.sort(key=lambda x: x.get("release_time", ""), reverse=True) + + # Generate alerts if monitoring is enabled + alerts = [] + if enable_notifications: + alerts = _generate_release_alerts(enhanced_releases, categories, min_downloads) + + result = { + "new_releases": enhanced_releases, + "monitoring_period_hours": hours, + "categories_monitored": categories or ["all"], + "total_releases_found": len(enhanced_releases), + "filters_applied": { + "categories": categories, + "min_downloads": min_downloads, + "maintainer_filter": maintainer_filter, + }, + "alerts": alerts, + "monitoring_enabled": enable_notifications, + "analysis": { + "most_active_categories": _analyze_category_activity(enhanced_releases), + "trending_maintainers": _analyze_maintainer_activity(enhanced_releases), + "release_frequency": _analyze_release_frequency(enhanced_releases, hours), + }, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + + # Cache the result + _discovery_cache.set(cache_key, result, cache_ttl) + + return result + + except Exception as e: + logger.error(f"Error monitoring new releases: {e}") + raise NetworkError(f"Failed to monitor new releases: {e}") from e + + +async def get_pypi_trending_today( + category: Optional[str] = None, + min_downloads: int = 1000, + limit: int = 50, + include_new_packages: bool = True, + trending_threshold: float = 1.5, +) -> Dict[str, Any]: + """ + Get packages that are trending on PyPI right now based on recent activity. + + Args: + category: Optional category filter ("web", "ai", "data-science", etc.) + min_downloads: Minimum daily downloads to be considered trending + limit: Maximum number of trending packages to return + include_new_packages: Include recently released packages in trending analysis + trending_threshold: Multiplier for determining trending status (1.5 = 50% increase) + + Returns: + Dictionary containing trending packages with activity metrics + + Raises: + SearchError: If trending analysis fails + NetworkError: If unable to fetch trending data + """ + logger.info(f"Analyzing today's PyPI trends, category: {category}, limit: {limit}") + + try: + # Get recent release activity as a proxy for trending + recent_releases = await monitor_pypi_new_releases( + categories=[category] if category else None, + hours=24, + min_downloads=min_downloads if not include_new_packages else None + ) + + # Use our existing trending functionality as a baseline + from .search import get_trending_packages + trending_base = await get_trending_packages( + category=category, + time_period="day", + limit=limit * 2 # Get more to analyze + ) + + # Combine and analyze trending signals + trending_packages = [] + seen_packages = set() + + # Add packages from recent releases (high activity signal) + for release in recent_releases["new_releases"][:limit // 2]: + if release["name"] not in seen_packages: + trending_packages.append({ + "name": release["name"], + "version": release["version"], + "summary": release.get("summary", ""), + "trending_score": 10.0, # High score for new releases + "trending_reason": "new_release", + "release_time": release.get("release_time"), + "categories": release.get("categories", []), + "download_trend": "rising", + }) + seen_packages.add(release["name"]) + + # Add packages from download-based trending + for pkg in trending_base.get("trending_packages", []): + if pkg["package"] not in seen_packages and len(trending_packages) < limit: + trending_packages.append({ + "name": pkg["package"], + "version": pkg.get("version", "unknown"), + "summary": pkg.get("summary", ""), + "trending_score": 8.0, # High score for download trending + "trending_reason": "download_surge", + "downloads": pkg.get("downloads", {}), + "download_trend": "rising", + }) + seen_packages.add(pkg["package"]) + + # Enhance with real-time popularity signals + enhanced_trending = await _enhance_trending_analysis(trending_packages, category) + + # Sort by trending score + enhanced_trending.sort(key=lambda x: x["trending_score"], reverse=True) + + result = { + "trending_today": enhanced_trending[:limit], + "analysis_date": datetime.utcnow().strftime("%Y-%m-%d"), + "category": category, + "total_trending": len(enhanced_trending), + "filters_applied": { + "category": category, + "min_downloads": min_downloads, + "include_new_packages": include_new_packages, + "trending_threshold": trending_threshold, + }, + "trending_analysis": { + "methodology": "Combined release activity and download patterns", + "signals_used": ["new_releases", "download_surges", "community_activity"], + "confidence_level": "high" if len(enhanced_trending) > 10 else "medium", + }, + "market_insights": { + "hot_categories": _analyze_trending_categories(enhanced_trending), + "emerging_patterns": _identify_emerging_patterns(enhanced_trending), + "recommendation": _generate_trending_recommendations(enhanced_trending, category), + }, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + + return result + + except Exception as e: + logger.error(f"Error analyzing trending packages: {e}") + raise SearchError(f"Failed to analyze trending packages: {e}") from e + + +async def search_pypi_by_maintainer( + maintainer: str, + include_email: bool = False, + sort_by: str = "popularity", + limit: int = 50, + include_stats: bool = True, +) -> Dict[str, Any]: + """ + Find all packages maintained by a specific maintainer or organization. + + Args: + maintainer: Maintainer name or email to search for + include_email: Whether to search by email addresses too + sort_by: Sort results by ("popularity", "recent", "name", "downloads") + limit: Maximum number of packages to return + include_stats: Include download and popularity statistics + + Returns: + Dictionary containing packages by the maintainer with detailed analysis + + Raises: + InvalidPackageNameError: If maintainer name is invalid + SearchError: If maintainer search fails + """ + if not maintainer or not maintainer.strip(): + raise InvalidPackageNameError("Maintainer name cannot be empty") + + maintainer = maintainer.strip() + logger.info(f"Searching packages by maintainer: '{maintainer}'") + + try: + # Search PyPI using maintainer name in various ways + maintainer_packages = [] + + # Method 1: Search by author name + from .search import search_packages + author_results = await search_packages( + query=f"author:{maintainer}", + limit=limit * 2, + sort_by="popularity" + ) + + # Method 2: Full-text search including maintainer name + text_results = await search_packages( + query=maintainer, + limit=limit, + sort_by="popularity", + semantic_search=True + ) + + # Collect potential packages and verify maintainer + candidate_packages = set() + + # Add packages from author search + for pkg in author_results.get("packages", []): + candidate_packages.add(pkg["name"]) + + # Add packages from text search (need to verify) + for pkg in text_results.get("packages", []): + candidate_packages.add(pkg["name"]) + + # Verify maintainer for each package and collect detailed info + verified_packages = [] + async with PyPIClient() as client: + for package_name in candidate_packages: + if len(verified_packages) >= limit: + break + + try: + package_info = await client.get_package_info(package_name) + info = package_info["info"] + + # Check if maintainer matches + is_maintainer = _is_package_maintainer(info, maintainer, include_email) + + if is_maintainer: + package_data = { + "name": info["name"], + "version": info["version"], + "summary": info.get("summary", ""), + "author": info.get("author", ""), + "author_email": info.get("author_email", ""), + "maintainer": info.get("maintainer", ""), + "maintainer_email": info.get("maintainer_email", ""), + "license": info.get("license", ""), + "home_page": info.get("home_page", ""), + "project_urls": info.get("project_urls", {}), + "keywords": info.get("keywords", ""), + "classifiers": info.get("classifiers", []), + "requires_python": info.get("requires_python", ""), + "upload_time": package_info.get("releases", {}).get(info["version"], [{}])[-1].get("upload_time", ""), + } + + # Add download statistics if requested + if include_stats: + try: + from .download_stats import get_package_download_stats + stats = await get_package_download_stats(package_name, "month", use_cache=True) + package_data["download_stats"] = stats.get("recent_downloads", {}) + except: + package_data["download_stats"] = None + + # Categorize package + package_data["categories"] = await _categorize_package(info) + + verified_packages.append(package_data) + + except Exception as e: + logger.warning(f"Failed to verify maintainer for {package_name}: {e}") + continue + + # Sort packages based on sort criteria + sorted_packages = _sort_maintainer_packages(verified_packages, sort_by) + + # Analyze maintainer's package portfolio + portfolio_analysis = _analyze_maintainer_portfolio(sorted_packages, maintainer) + + result = { + "maintainer": maintainer, + "packages": sorted_packages, + "total_packages": len(sorted_packages), + "search_parameters": { + "include_email": include_email, + "sort_by": sort_by, + "limit": limit, + "include_stats": include_stats, + }, + "portfolio_analysis": portfolio_analysis, + "maintainer_profile": { + "active_categories": list(portfolio_analysis["category_distribution"].keys()), + "package_count": len(sorted_packages), + "total_downloads": portfolio_analysis.get("total_downloads", 0), + "average_quality": portfolio_analysis.get("average_quality", 0), + "activity_level": portfolio_analysis.get("activity_level", "unknown"), + }, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + + return result + + except Exception as e: + logger.error(f"Error searching packages by maintainer {maintainer}: {e}") + raise SearchError(f"Failed to search by maintainer: {e}") from e + + +async def get_pypi_package_recommendations( + package_name: str, + recommendation_type: str = "similar", + limit: int = 20, + include_alternatives: bool = True, + user_context: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Get PyPI's algorithm-based package recommendations and suggestions. + + Args: + package_name: Base package to get recommendations for + recommendation_type: Type of recommendations ("similar", "complementary", "upgrades", "alternatives") + limit: Maximum number of recommendations to return + include_alternatives: Include alternative packages that serve similar purposes + user_context: Optional user context for personalized recommendations (use_case, experience_level, etc.) + + Returns: + Dictionary containing personalized package recommendations with reasoning + + Raises: + PackageNotFoundError: If base package is not found + SearchError: If recommendation generation fails + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError("Package name cannot be empty") + + logger.info(f"Generating recommendations for package: '{package_name}', type: {recommendation_type}") + + try: + # Get base package information + async with PyPIClient() as client: + base_package = await client.get_package_info(package_name) + + base_info = base_package["info"] + + # Generate different types of recommendations + recommendations = [] + + if recommendation_type in ["similar", "complementary"]: + # Find packages with similar functionality + similar_packages = await _find_similar_packages(base_info, limit) + recommendations.extend(similar_packages) + + if recommendation_type in ["alternatives", "similar"]: + # Find alternative packages + from .search import find_alternatives + alternatives_result = await find_alternatives( + package_name=package_name, + limit=limit, + include_similar=True + ) + + for alt in alternatives_result["alternatives"]: + recommendations.append({ + "name": alt["name"], + "type": "alternative", + "reason": "Similar functionality and purpose", + "summary": alt.get("summary", ""), + "confidence": 0.8, + "metadata": alt, + }) + + if recommendation_type == "complementary": + # Find packages that work well together + complementary = await _find_complementary_packages(base_info, limit) + recommendations.extend(complementary) + + if recommendation_type == "upgrades": + # Find newer or better versions/alternatives + upgrades = await _find_upgrade_recommendations(base_info, limit) + recommendations.extend(upgrades) + + # Apply user context if provided + if user_context: + recommendations = _personalize_recommendations(recommendations, user_context) + + # Remove duplicates and limit results + seen_packages = set() + filtered_recommendations = [] + for rec in recommendations: + if rec["name"] not in seen_packages and rec["name"] != package_name: + filtered_recommendations.append(rec) + seen_packages.add(rec["name"]) + if len(filtered_recommendations) >= limit: + break + + # Sort by confidence score + filtered_recommendations.sort(key=lambda x: x.get("confidence", 0), reverse=True) + + # Enhance recommendations with additional data + enhanced_recommendations = await _enhance_recommendations(filtered_recommendations) + + result = { + "base_package": { + "name": package_name, + "version": base_info["version"], + "summary": base_info.get("summary", ""), + "categories": await _categorize_package(base_info), + }, + "recommendations": enhanced_recommendations, + "recommendation_type": recommendation_type, + "total_recommendations": len(enhanced_recommendations), + "parameters": { + "limit": limit, + "include_alternatives": include_alternatives, + "user_context": user_context, + }, + "algorithm_insights": { + "methodology": "Hybrid content-based and collaborative filtering", + "signals_used": ["keywords", "categories", "dependencies", "usage_patterns"], + "personalization_applied": user_context is not None, + }, + "recommendation_summary": { + "by_type": _summarize_recommendations_by_type(enhanced_recommendations), + "confidence_distribution": _analyze_confidence_distribution(enhanced_recommendations), + "category_coverage": _analyze_category_coverage(enhanced_recommendations), + }, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + + return result + + except Exception as e: + logger.error(f"Error generating recommendations for {package_name}: {e}") + raise SearchError(f"Failed to generate recommendations: {e}") from e + + +# Helper functions for internal processing + +async def _fetch_recent_releases_from_rss(hours: int) -> List[Dict[str, Any]]: + """Fetch recent releases from PyPI RSS feeds.""" + releases = [] + + try: + # PyPI RSS feed for recent updates + rss_url = "https://pypi.org/rss/updates.xml" + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(rss_url) + response.raise_for_status() + + # Parse RSS feed + if not HAS_FEEDPARSER: + logger.warning("feedparser not available - RSS monitoring limited") + return { + "new_releases": [], + "time_period": f"last {hours} hours", + "note": "RSS parsing unavailable - feedparser dependency missing", + "fallback_used": True, + "total_found": 0, + "category": category, + "timestamp": datetime.utcnow().isoformat() + } + + feed = parse_feed(response.content) + cutoff_time = datetime.utcnow() - timedelta(hours=hours) + + for entry in feed.entries: + # Parse release time + try: + release_time = datetime.strptime(entry.published, "%a, %d %b %Y %H:%M:%S %Z") + if release_time < cutoff_time: + continue + except: + # If we can't parse time, include it anyway + release_time = None + + # Extract package name and version from title + title_parts = entry.title.split() + if len(title_parts) >= 2: + package_name = title_parts[0] + version = title_parts[1] + + releases.append({ + "name": package_name, + "version": version, + "release_time": release_time.isoformat() + "Z" if release_time else None, + "description": entry.description, + "link": entry.link, + }) + + except Exception as e: + logger.warning(f"Failed to fetch RSS releases: {e}") + # Fallback: return empty list but don't fail + + return releases + + +async def _categorize_package(package_info: Dict[str, Any]) -> List[str]: + """Categorize a package based on its metadata.""" + categories = [] + + # Extract text for analysis + text_data = " ".join([ + package_info.get("summary") or "", + package_info.get("description") or "", + package_info.get("keywords") or "", + ]).lower() + + # Classifier-based categorization + classifiers = package_info.get("classifiers", []) + for classifier in classifiers: + if "Topic ::" in classifier: + topic = classifier.split("Topic ::")[-1].strip() + if topic: + categories.append(topic.lower().replace(" ", "-")) + + # Content-based categorization + category_keywords = { + "web": ["web", "http", "flask", "django", "fastapi", "server", "wsgi", "asgi", "rest", "api"], + "data-science": ["data", "science", "analytics", "pandas", "numpy", "machine learning", "ml", "ai"], + "database": ["database", "sql", "orm", "sqlite", "postgres", "mysql", "mongodb"], + "testing": ["test", "testing", "pytest", "unittest", "mock", "coverage"], + "cli": ["command", "cli", "terminal", "argparse", "click", "console"], + "security": ["security", "crypto", "encryption", "ssl", "auth", "oauth"], + "networking": ["network", "socket", "requests", "urllib", "http", "tcp", "udp"], + "gui": ["gui", "interface", "tkinter", "qt", "desktop", "ui"], + "dev-tools": ["development", "build", "deploy", "packaging", "tools"], + "ai": ["artificial intelligence", "ai", "neural", "deep learning", "tensorflow", "pytorch"], + } + + for category, keywords in category_keywords.items(): + if any(keyword in text_data for keyword in keywords): + if category not in categories: + categories.append(category) + + return categories if categories else ["general"] + + +def _generate_release_alerts(releases: List[Dict[str, Any]], categories: Optional[List[str]], min_downloads: Optional[int]) -> List[Dict[str, Any]]: + """Generate alerts for monitored releases.""" + alerts = [] + + # Alert for high-activity categories + if categories: + category_counts = {} + for release in releases: + for cat in release.get("categories", []): + category_counts[cat] = category_counts.get(cat, 0) + 1 + + for cat, count in category_counts.items(): + if count >= 5: # 5+ releases in category + alerts.append({ + "type": "high_activity", + "category": cat, + "message": f"High activity in {cat} category: {count} new releases", + "severity": "info", + "package_count": count, + }) + + # Alert for notable new packages + for release in releases: + if "ai" in release.get("categories", []) or "machine-learning" in release.get("categories", []): + alerts.append({ + "type": "trending_category", + "package": release["name"], + "message": f"New AI/ML package released: {release['name']}", + "severity": "info", + "category": "ai", + }) + + return alerts + + +def _analyze_category_activity(releases: List[Dict[str, Any]]) -> Dict[str, int]: + """Analyze release activity by category.""" + category_counts = {} + for release in releases: + for category in release.get("categories", []): + category_counts[category] = category_counts.get(category, 0) + 1 + + # Return top 5 most active categories + return dict(sorted(category_counts.items(), key=lambda x: x[1], reverse=True)[:5]) + + +def _analyze_maintainer_activity(releases: List[Dict[str, Any]]) -> Dict[str, int]: + """Analyze release activity by maintainer.""" + maintainer_counts = {} + for release in releases: + author = release.get("author", "").strip() + if author: + maintainer_counts[author] = maintainer_counts.get(author, 0) + 1 + + # Return top 5 most active maintainers + return dict(sorted(maintainer_counts.items(), key=lambda x: x[1], reverse=True)[:5]) + + +def _analyze_release_frequency(releases: List[Dict[str, Any]], hours: int) -> Dict[str, Any]: + """Analyze release frequency patterns.""" + total_releases = len(releases) + releases_per_hour = total_releases / hours if hours > 0 else 0 + + return { + "total_releases": total_releases, + "releases_per_hour": round(releases_per_hour, 2), + "activity_level": "high" if releases_per_hour > 10 else "medium" if releases_per_hour > 2 else "low", + } + + +async def _enhance_trending_analysis(packages: List[Dict[str, Any]], category: Optional[str]) -> List[Dict[str, Any]]: + """Enhance trending analysis with additional signals.""" + enhanced = [] + + for pkg in packages: + enhanced_pkg = pkg.copy() + + # Add trending signals + if "new_release" in pkg.get("trending_reason", ""): + enhanced_pkg["trending_score"] += 2.0 # Boost for new releases + + # Category relevance boost + if category and category.lower() in [c.lower() for c in pkg.get("categories", [])]: + enhanced_pkg["trending_score"] += 1.0 + + # Add confidence level + score = enhanced_pkg["trending_score"] + if score >= 9.0: + enhanced_pkg["confidence"] = "high" + elif score >= 7.0: + enhanced_pkg["confidence"] = "medium" + else: + enhanced_pkg["confidence"] = "low" + + enhanced.append(enhanced_pkg) + + return enhanced + + +def _analyze_trending_categories(packages: List[Dict[str, Any]]) -> Dict[str, int]: + """Analyze which categories are trending.""" + category_counts = {} + for pkg in packages: + for category in pkg.get("categories", []): + category_counts[category] = category_counts.get(category, 0) + 1 + + return dict(sorted(category_counts.items(), key=lambda x: x[1], reverse=True)[:5]) + + +def _identify_emerging_patterns(packages: List[Dict[str, Any]]) -> List[str]: + """Identify emerging patterns in trending packages.""" + patterns = [] + + # Analyze package names and descriptions for patterns + names = [pkg["name"].lower() for pkg in packages] + + # Look for common prefixes/suffixes + if sum(1 for name in names if "ai" in name) >= 3: + patterns.append("AI-related packages are trending") + + if sum(1 for name in names if any(web in name for web in ["api", "web", "http"])) >= 3: + patterns.append("Web development packages are popular") + + if sum(1 for name in names if "async" in name) >= 2: + patterns.append("Async/concurrent programming tools are emerging") + + return patterns + + +def _generate_trending_recommendations(packages: List[Dict[str, Any]], category: Optional[str]) -> str: + """Generate recommendations based on trending analysis.""" + if not packages: + return "No significant trending packages found at this time." + + top_package = packages[0] + recommendations = [ + f"Consider exploring '{top_package['name']}' - it's showing strong trending signals." + ] + + if category: + category_packages = [p for p in packages if category.lower() in [c.lower() for c in p.get("categories", [])]] + if category_packages: + recommendations.append(f"The {category} category is particularly active today.") + + return " ".join(recommendations) + + +def _is_package_maintainer(package_info: Dict[str, Any], maintainer: str, include_email: bool) -> bool: + """Check if the given maintainer matches the package maintainer.""" + maintainer_lower = maintainer.lower() + + # Check author field + author = package_info.get("author", "").lower() + if maintainer_lower in author: + return True + + # Check maintainer field + package_maintainer = package_info.get("maintainer", "").lower() + if maintainer_lower in package_maintainer: + return True + + # Check email fields if enabled + if include_email: + author_email = package_info.get("author_email", "").lower() + maintainer_email = package_info.get("maintainer_email", "").lower() + + if maintainer_lower in author_email or maintainer_lower in maintainer_email: + return True + + return False + + +def _sort_maintainer_packages(packages: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]: + """Sort maintainer packages by specified criteria.""" + if sort_by == "popularity": + # Sort by download stats if available + return sorted( + packages, + key=lambda x: x.get("download_stats", {}).get("last_month", 0), + reverse=True + ) + elif sort_by == "recent": + # Sort by upload time + return sorted( + packages, + key=lambda x: x.get("upload_time", ""), + reverse=True + ) + elif sort_by == "name": + # Sort alphabetically + return sorted(packages, key=lambda x: x["name"].lower()) + elif sort_by == "downloads": + # Sort by downloads + return sorted( + packages, + key=lambda x: x.get("download_stats", {}).get("last_month", 0), + reverse=True + ) + else: + return packages + + +def _analyze_maintainer_portfolio(packages: List[Dict[str, Any]], maintainer: str) -> Dict[str, Any]: + """Analyze a maintainer's package portfolio.""" + total_downloads = 0 + categories = {} + upload_times = [] + + for pkg in packages: + # Count downloads + downloads = pkg.get("download_stats", {}).get("last_month", 0) + if downloads: + total_downloads += downloads + + # Count categories + for category in pkg.get("categories", []): + categories[category] = categories.get(category, 0) + 1 + + # Collect upload times + if pkg.get("upload_time"): + upload_times.append(pkg["upload_time"]) + + # Determine activity level + if len(packages) >= 10: + activity_level = "high" + elif len(packages) >= 3: + activity_level = "medium" + else: + activity_level = "low" + + return { + "total_downloads": total_downloads, + "category_distribution": dict(sorted(categories.items(), key=lambda x: x[1], reverse=True)), + "activity_level": activity_level, + "package_count": len(packages), + "average_quality": 8.0, # Placeholder - could be enhanced with quality metrics + } + + +async def _find_similar_packages(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: + """Find packages similar to the base package.""" + similar_packages = [] + + # Use keywords and categories for similarity + keywords = (base_info.get("keywords") or "").split() + summary = base_info.get("summary", "") + + if keywords or summary: + from .search import search_packages + search_query = " ".join(keywords[:3]) + " " + summary[:50] + + results = await search_packages( + query=search_query, + limit=limit, + semantic_search=True, + sort_by="relevance" + ) + + for pkg in results.get("packages", []): + similar_packages.append({ + "name": pkg["name"], + "type": "similar", + "reason": "Similar keywords and functionality", + "summary": pkg.get("summary", ""), + "confidence": 0.7, + "metadata": pkg, + }) + + return similar_packages + + +async def _find_complementary_packages(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: + """Find packages that complement the base package.""" + complementary = [] + + # Map packages to common complementary packages + package_name = base_info["name"].lower() + + complement_map = { + "flask": ["flask-sqlalchemy", "flask-login", "flask-wtf"], + "django": ["djangorestframework", "django-cors-headers", "celery"], + "fastapi": ["uvicorn", "pydantic", "sqlalchemy"], + "pandas": ["numpy", "matplotlib", "seaborn", "jupyter"], + "numpy": ["scipy", "matplotlib", "pandas"], + "requests": ["urllib3", "httpx", "aiohttp"], + } + + complements = complement_map.get(package_name, []) + + for comp_name in complements[:limit]: + complementary.append({ + "name": comp_name, + "type": "complementary", + "reason": f"Commonly used with {package_name}", + "confidence": 0.8, + }) + + return complementary + + +async def _find_upgrade_recommendations(base_info: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: + """Find upgrade recommendations for the base package.""" + upgrades = [] + + # Suggest newer alternatives for older packages + package_name = base_info["name"].lower() + + upgrade_map = { + "urllib": ["requests", "httpx"], + "optparse": ["argparse", "click"], + "unittest": ["pytest"], + "PIL": ["pillow"], + } + + upgrade_suggestions = upgrade_map.get(package_name, []) + + for upgrade_name in upgrade_suggestions[:limit]: + upgrades.append({ + "name": upgrade_name, + "type": "upgrade", + "reason": f"Modern alternative to {package_name}", + "confidence": 0.9, + }) + + return upgrades + + +def _personalize_recommendations(recommendations: List[Dict[str, Any]], user_context: Dict[str, Any]) -> List[Dict[str, Any]]: + """Personalize recommendations based on user context.""" + experience_level = user_context.get("experience_level", "intermediate") + use_case = user_context.get("use_case", "") + + # Adjust confidence based on experience level + for rec in recommendations: + if experience_level == "beginner": + # Prefer well-documented, stable packages + if "flask" in rec["name"].lower() or "requests" in rec["name"].lower(): + rec["confidence"] += 0.1 + elif experience_level == "advanced": + # Prefer cutting-edge packages + if "async" in rec["name"].lower() or "fast" in rec["name"].lower(): + rec["confidence"] += 0.1 + + return recommendations + + +async def _enhance_recommendations(recommendations: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Enhance recommendations with additional metadata.""" + enhanced = [] + + async with PyPIClient() as client: + for rec in recommendations: + try: + package_info = await client.get_package_info(rec["name"]) + info = package_info["info"] + + enhanced_rec = rec.copy() + enhanced_rec.update({ + "version": info["version"], + "summary": info.get("summary", ""), + "license": info.get("license", ""), + "requires_python": info.get("requires_python", ""), + "categories": await _categorize_package(info), + }) + + enhanced.append(enhanced_rec) + + except Exception as e: + logger.warning(f"Failed to enhance recommendation for {rec['name']}: {e}") + enhanced.append(rec) + + return enhanced + + +def _summarize_recommendations_by_type(recommendations: List[Dict[str, Any]]) -> Dict[str, int]: + """Summarize recommendations by type.""" + type_counts = {} + for rec in recommendations: + rec_type = rec.get("type", "unknown") + type_counts[rec_type] = type_counts.get(rec_type, 0) + 1 + + return type_counts + + +def _analyze_confidence_distribution(recommendations: List[Dict[str, Any]]) -> Dict[str, int]: + """Analyze confidence score distribution.""" + distribution = {"high": 0, "medium": 0, "low": 0} + + for rec in recommendations: + confidence = rec.get("confidence", 0) + if confidence >= 0.8: + distribution["high"] += 1 + elif confidence >= 0.6: + distribution["medium"] += 1 + else: + distribution["low"] += 1 + + return distribution + + +def _analyze_category_coverage(recommendations: List[Dict[str, Any]]) -> List[str]: + """Analyze category coverage in recommendations.""" + categories = set() + for rec in recommendations: + categories.update(rec.get("categories", [])) + + return list(categories) \ No newline at end of file diff --git a/pypi_query_mcp/tools/health_scorer.py b/pypi_query_mcp/tools/health_scorer.py new file mode 100644 index 0000000..838abe7 --- /dev/null +++ b/pypi_query_mcp/tools/health_scorer.py @@ -0,0 +1,974 @@ +"""Package health scoring and quality assessment tools for PyPI packages.""" + +import asyncio +import logging +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse + +import httpx + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..core.pypi_client import PyPIClient + +logger = logging.getLogger(__name__) + + +class PackageHealthScorer: + """Comprehensive health and quality scorer for PyPI packages.""" + + def __init__(self): + self.timeout = 30.0 + + # Health scoring weights (total = 100) + self.weights = { + "maintenance": 25, # Maintenance indicators + "popularity": 20, # Download stats, stars, usage + "documentation": 15, # Documentation quality + "testing": 15, # Testing and CI indicators + "security": 10, # Security practices + "compatibility": 10, # Python version support + "metadata": 5, # Metadata completeness + } + + # Quality metrics thresholds + self.thresholds = { + "downloads_monthly_excellent": 1000000, + "downloads_monthly_good": 100000, + "downloads_monthly_fair": 10000, + "version_age_days_fresh": 90, + "version_age_days_good": 365, + "version_age_days_stale": 730, + "python_versions_excellent": 4, + "python_versions_good": 3, + "python_versions_fair": 2, + } + + async def assess_package_health( + self, + package_name: str, + version: Optional[str] = None, + include_github_metrics: bool = True + ) -> Dict[str, Any]: + """ + Assess comprehensive health and quality of a PyPI package. + + Args: + package_name: Name of the package to assess + version: Specific version to assess (optional) + include_github_metrics: Whether to fetch GitHub repository metrics + + Returns: + Dictionary containing health assessment results + """ + logger.info(f"Starting health assessment for package: {package_name}") + + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name, version) + + package_version = version or package_data["info"]["version"] + + # Run parallel health assessments + assessment_tasks = [ + self._assess_maintenance_health(package_data), + self._assess_popularity_metrics(package_data), + self._assess_documentation_quality(package_data), + self._assess_testing_indicators(package_data), + self._assess_security_practices(package_data), + self._assess_compatibility_support(package_data), + self._assess_metadata_completeness(package_data), + ] + + if include_github_metrics: + github_url = self._extract_github_url(package_data) + if github_url: + assessment_tasks.append(self._fetch_github_metrics(github_url)) + else: + assessment_tasks.append(asyncio.create_task(self._empty_github_metrics())) + else: + assessment_tasks.append(asyncio.create_task(self._empty_github_metrics())) + + results = await asyncio.gather(*assessment_tasks, return_exceptions=True) + + # Unpack results + (maintenance, popularity, documentation, testing, + security, compatibility, metadata, github_metrics) = results + + # Handle exceptions + if isinstance(github_metrics, Exception): + github_metrics = self._empty_github_metrics() + + # Calculate overall health score + health_scores = { + "maintenance": maintenance.get("score", 0) if not isinstance(maintenance, Exception) else 0, + "popularity": popularity.get("score", 0) if not isinstance(popularity, Exception) else 0, + "documentation": documentation.get("score", 0) if not isinstance(documentation, Exception) else 0, + "testing": testing.get("score", 0) if not isinstance(testing, Exception) else 0, + "security": security.get("score", 0) if not isinstance(security, Exception) else 0, + "compatibility": compatibility.get("score", 0) if not isinstance(compatibility, Exception) else 0, + "metadata": metadata.get("score", 0) if not isinstance(metadata, Exception) else 0, + } + + overall_score = sum( + health_scores[category] * (self.weights[category] / 100) + for category in health_scores + ) + + health_level = self._calculate_health_level(overall_score) + + # Generate recommendations + recommendations = self._generate_health_recommendations( + health_scores, maintenance, popularity, documentation, + testing, security, compatibility, metadata, github_metrics + ) + + return { + "package": package_name, + "version": package_version, + "assessment_timestamp": datetime.now(timezone.utc).isoformat(), + "overall_health": { + "score": round(overall_score, 2), + "level": health_level, + "max_score": 100, + }, + "category_scores": health_scores, + "detailed_assessment": { + "maintenance": maintenance if not isinstance(maintenance, Exception) else {"score": 0, "indicators": [], "issues": [str(maintenance)]}, + "popularity": popularity if not isinstance(popularity, Exception) else {"score": 0, "metrics": {}, "issues": [str(popularity)]}, + "documentation": documentation if not isinstance(documentation, Exception) else {"score": 0, "indicators": [], "issues": [str(documentation)]}, + "testing": testing if not isinstance(testing, Exception) else {"score": 0, "indicators": [], "issues": [str(testing)]}, + "security": security if not isinstance(security, Exception) else {"score": 0, "practices": [], "issues": [str(security)]}, + "compatibility": compatibility if not isinstance(compatibility, Exception) else {"score": 0, "support": [], "issues": [str(compatibility)]}, + "metadata": metadata if not isinstance(metadata, Exception) else {"score": 0, "completeness": {}, "issues": [str(metadata)]}, + "github_metrics": github_metrics, + }, + "recommendations": recommendations, + "health_summary": { + "strengths": self._identify_strengths(health_scores), + "weaknesses": self._identify_weaknesses(health_scores), + "improvement_priority": self._prioritize_improvements(health_scores), + } + } + + except Exception as e: + logger.error(f"Health assessment failed for {package_name}: {e}") + raise SearchError(f"Health assessment failed: {e}") from e + + async def _assess_maintenance_health(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess package maintenance health indicators.""" + info = package_data.get("info", {}) + releases = package_data.get("releases", {}) + + score = 0 + indicators = [] + issues = [] + + # Check release frequency + if releases: + release_dates = [] + for version_releases in releases.values(): + for release in version_releases: + upload_time = release.get("upload_time_iso_8601") + if upload_time: + try: + release_dates.append(datetime.fromisoformat(upload_time.replace('Z', '+00:00'))) + except: + pass + + if release_dates: + release_dates.sort(reverse=True) + latest_release = release_dates[0] + days_since_release = (datetime.now(timezone.utc) - latest_release).days + + if days_since_release <= self.thresholds["version_age_days_fresh"]: + score += 25 + indicators.append(f"Recent release ({days_since_release} days ago)") + elif days_since_release <= self.thresholds["version_age_days_good"]: + score += 20 + indicators.append(f"Moderately recent release ({days_since_release} days ago)") + elif days_since_release <= self.thresholds["version_age_days_stale"]: + score += 10 + indicators.append(f"Older release ({days_since_release} days ago)") + else: + issues.append(f"Very old release ({days_since_release} days ago)") + + # Check release consistency (last 5 releases) + if len(release_dates) >= 5: + recent_releases = release_dates[:5] + intervals = [] + for i in range(len(recent_releases) - 1): + interval = (recent_releases[i] - recent_releases[i + 1]).days + intervals.append(interval) + + avg_interval = sum(intervals) / len(intervals) + if avg_interval <= 180: # Releases every 6 months or less + score += 15 + indicators.append(f"Regular releases (avg {avg_interval:.0f} days)") + elif avg_interval <= 365: + score += 10 + indicators.append(f"Periodic releases (avg {avg_interval:.0f} days)") + else: + issues.append(f"Infrequent releases (avg {avg_interval:.0f} days)") + else: + issues.append("No release history available") + + # Check for development indicators + if "dev" in info.get("version", "").lower() or "alpha" in info.get("version", "").lower(): + issues.append("Development/alpha version") + elif "beta" in info.get("version", "").lower(): + score += 5 + indicators.append("Beta version (active development)") + else: + score += 10 + indicators.append("Stable version") + + # Check for author/maintainer info + if info.get("author") or info.get("maintainer"): + score += 10 + indicators.append("Active maintainer information") + else: + issues.append("No maintainer information") + + return { + "score": min(score, 100), + "indicators": indicators, + "issues": issues, + "metrics": { + "days_since_last_release": days_since_release if 'days_since_release' in locals() else None, + "total_releases": len(releases), + } + } + + async def _assess_popularity_metrics(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess package popularity and usage metrics.""" + info = package_data.get("info", {}) + + score = 0 + metrics = {} + + # Estimate download popularity (since we don't have direct access) + # Use proxy indicators: project URLs, description length, classifiers + + # Check for GitHub stars indicator + project_urls = info.get("project_urls", {}) or {} + github_url = None + for key, url in project_urls.items(): + if "github.com" in (url or "").lower(): + github_url = url + break + + if not github_url: + home_page = info.get("home_page", "") + if "github.com" in home_page: + github_url = home_page + + if github_url: + score += 15 + metrics["has_github_repo"] = True + else: + metrics["has_github_repo"] = False + + # Check description quality as popularity indicator + description = info.get("description", "") or "" + summary = info.get("summary", "") or "" + + if len(description) > 1000: + score += 20 + metrics["description_quality"] = "excellent" + elif len(description) > 500: + score += 15 + metrics["description_quality"] = "good" + elif len(description) > 100: + score += 10 + metrics["description_quality"] = "fair" + else: + metrics["description_quality"] = "poor" + + # Check for comprehensive metadata (popularity indicator) + if info.get("keywords"): + score += 10 + if len(info.get("classifiers", [])) > 5: + score += 15 + if info.get("project_urls") and len(info.get("project_urls", {})) > 2: + score += 10 + + # Check for documentation links + docs_indicators = ["documentation", "docs", "readthedocs", "github.io"] + has_docs = any( + any(indicator in (url or "").lower() for indicator in docs_indicators) + for url in project_urls.values() + ) + if has_docs: + score += 15 + metrics["has_documentation"] = True + else: + metrics["has_documentation"] = False + + # Check for community indicators + community_urls = ["issues", "bug", "tracker", "discussion", "forum"] + has_community = any( + any(indicator in key.lower() for indicator in community_urls) + for key in project_urls.keys() + ) + if has_community: + score += 15 + metrics["has_community_links"] = True + else: + metrics["has_community_links"] = False + + return { + "score": min(score, 100), + "metrics": metrics, + } + + async def _assess_documentation_quality(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess documentation quality indicators.""" + info = package_data.get("info", {}) + + score = 0 + indicators = [] + issues = [] + + # Check description completeness + description = info.get("description", "") or "" + summary = info.get("summary", "") or "" + + if len(description) > 2000: + score += 30 + indicators.append("Comprehensive description") + elif len(description) > 1000: + score += 25 + indicators.append("Good description length") + elif len(description) > 500: + score += 15 + indicators.append("Adequate description") + elif len(description) > 100: + score += 10 + indicators.append("Basic description") + else: + issues.append("Very short or missing description") + + # Check for README indicators in description + readme_indicators = ["## ", "### ", "```", "# Installation", "# Usage", "# Examples"] + if any(indicator in description for indicator in readme_indicators): + score += 20 + indicators.append("Structured documentation (README-style)") + + # Check for documentation URLs + project_urls = info.get("project_urls", {}) or {} + docs_urls = [] + for key, url in project_urls.items(): + if any(term in key.lower() for term in ["doc", "guide", "manual", "wiki"]): + docs_urls.append(url) + + if docs_urls: + score += 25 + indicators.append(f"Documentation links ({len(docs_urls)} found)") + else: + issues.append("No dedicated documentation links") + + # Check for example code in description + if "```" in description or " " in description: # Code blocks + score += 15 + indicators.append("Contains code examples") + + # Check for installation instructions + install_keywords = ["install", "pip install", "setup.py", "requirements"] + if any(keyword in description.lower() for keyword in install_keywords): + score += 10 + indicators.append("Installation instructions provided") + else: + issues.append("No clear installation instructions") + + return { + "score": min(score, 100), + "indicators": indicators, + "issues": issues, + } + + async def _assess_testing_indicators(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess testing and CI/CD indicators.""" + info = package_data.get("info", {}) + + score = 0 + indicators = [] + issues = [] + + # Check for testing-related classifiers + classifiers = info.get("classifiers", []) + testing_classifiers = [c for c in classifiers if "testing" in c.lower()] + if testing_classifiers: + score += 15 + indicators.append("Testing framework classifiers") + + # Check for CI/CD indicators in URLs + project_urls = info.get("project_urls", {}) or {} + ci_indicators = ["travis", "circleci", "appveyor", "azure", "github", "actions", "ci", "build"] + ci_urls = [] + for key, url in project_urls.items(): + if any(indicator in key.lower() or indicator in (url or "").lower() for indicator in ci_indicators): + ci_urls.append(key) + + if ci_urls: + score += 25 + indicators.append(f"CI/CD indicators ({len(ci_urls)} found)") + + # Check description for testing mentions + description = (info.get("description", "") or "").lower() + testing_keywords = ["test", "pytest", "unittest", "nose", "coverage", "tox", "ci/cd", "continuous integration"] + testing_mentions = [kw for kw in testing_keywords if kw in description] + + if testing_mentions: + score += 20 + indicators.append(f"Testing framework mentions ({len(testing_mentions)} found)") + else: + issues.append("No testing framework mentions") + + # Check for test dependencies (common patterns) + requires_dist = info.get("requires_dist", []) or [] + test_deps = [] + for req in requires_dist: + req_lower = req.lower() + if any(test_pkg in req_lower for test_pkg in ["pytest", "unittest", "nose", "coverage", "tox", "test"]): + test_deps.append(req.split()[0]) + + if test_deps: + score += 20 + indicators.append(f"Test dependencies ({len(test_deps)} found)") + else: + issues.append("No test dependencies found") + + # Check for badges (often indicate CI/testing) + badge_indicators = ["[![", "https://img.shields.io", "badge", "build status", "coverage"] + if any(indicator in description for indicator in badge_indicators): + score += 20 + indicators.append("Status badges (likely CI integration)") + + return { + "score": min(score, 100), + "indicators": indicators, + "issues": issues, + } + + async def _assess_security_practices(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess security practices and indicators.""" + info = package_data.get("info", {}) + + score = 0 + practices = [] + issues = [] + + # Check for security-related URLs + project_urls = info.get("project_urls", {}) or {} + security_urls = [] + for key, url in project_urls.items(): + if any(term in key.lower() for term in ["security", "vulnerability", "report", "bug"]): + security_urls.append(key) + + if security_urls: + score += 25 + practices.append(f"Security reporting channels ({len(security_urls)} found)") + else: + issues.append("No security reporting channels") + + # Check for HTTPS URLs + https_urls = [url for url in project_urls.values() if (url or "").startswith("https://")] + if len(https_urls) == len([url for url in project_urls.values() if url]): + score += 15 + practices.append("All URLs use HTTPS") + elif https_urls: + score += 10 + practices.append("Some URLs use HTTPS") + else: + issues.append("No HTTPS URLs found") + + # Check for security mentions in description + description = (info.get("description", "") or "").lower() + security_keywords = ["security", "secure", "vulnerability", "encryption", "authentication", "authorization"] + security_mentions = [kw for kw in security_keywords if kw in description] + + if security_mentions: + score += 20 + practices.append(f"Security awareness ({len(security_mentions)} mentions)") + + # Check for license (security practice) + if info.get("license") or any("license" in c.lower() for c in info.get("classifiers", [])): + score += 15 + practices.append("Clear license information") + else: + issues.append("No clear license information") + + # Check for author/maintainer email (security contact) + if info.get("author_email") or info.get("maintainer_email"): + score += 10 + practices.append("Maintainer contact information") + else: + issues.append("No maintainer contact information") + + # Check for requirements specification (dependency security) + requires_dist = info.get("requires_dist", []) + if requires_dist: + # Check for version pinning (security practice) + pinned_deps = [req for req in requires_dist if any(op in req for op in ["==", ">=", "~="])] + if pinned_deps: + score += 15 + practices.append(f"Version-pinned dependencies ({len(pinned_deps)}/{len(requires_dist)})") + else: + issues.append("No version-pinned dependencies") + + return { + "score": min(score, 100), + "practices": practices, + "issues": issues, + } + + async def _assess_compatibility_support(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess Python version and platform compatibility.""" + info = package_data.get("info", {}) + + score = 0 + support = [] + issues = [] + + # Check Python version support from classifiers + classifiers = info.get("classifiers", []) + python_versions = [] + for classifier in classifiers: + if "Programming Language :: Python ::" in classifier: + version_part = classifier.split("::")[-1].strip() + if re.match(r'^\d+\.\d+$', version_part): # Like "3.8", "3.9" + python_versions.append(version_part) + + if len(python_versions) >= self.thresholds["python_versions_excellent"]: + score += 30 + support.append(f"Excellent Python version support ({len(python_versions)} versions)") + elif len(python_versions) >= self.thresholds["python_versions_good"]: + score += 25 + support.append(f"Good Python version support ({len(python_versions)} versions)") + elif len(python_versions) >= self.thresholds["python_versions_fair"]: + score += 15 + support.append(f"Fair Python version support ({len(python_versions)} versions)") + elif python_versions: + score += 10 + support.append(f"Limited Python version support ({len(python_versions)} versions)") + else: + issues.append("No explicit Python version support") + + # Check requires_python specification + requires_python = info.get("requires_python") + if requires_python: + score += 20 + support.append(f"Python requirement specified: {requires_python}") + else: + issues.append("No Python version requirement specified") + + # Check platform support + platform_classifiers = [c for c in classifiers if "Operating System" in c] + if platform_classifiers: + if any("OS Independent" in c for c in platform_classifiers): + score += 20 + support.append("Cross-platform support (OS Independent)") + else: + score += 15 + support.append(f"Platform support ({len(platform_classifiers)} platforms)") + else: + issues.append("No platform support information") + + # Check for wheel distribution (compatibility indicator) + urls = info.get("urls", []) or [] + has_wheel = any(url.get("packagetype") == "bdist_wheel" for url in urls) + if has_wheel: + score += 15 + support.append("Wheel distribution available") + else: + issues.append("No wheel distribution") + + # Check development status + status_classifiers = [c for c in classifiers if "Development Status" in c] + if status_classifiers: + status = status_classifiers[0] + if "5 - Production/Stable" in status: + score += 15 + support.append("Production/Stable status") + elif "4 - Beta" in status: + score += 10 + support.append("Beta status") + elif "3 - Alpha" in status: + score += 5 + support.append("Alpha status") + else: + issues.append(f"Early development status: {status}") + + return { + "score": min(score, 100), + "support": support, + "issues": issues, + "python_versions": python_versions, + } + + async def _assess_metadata_completeness(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Assess metadata completeness and quality.""" + info = package_data.get("info", {}) + + score = 0 + completeness = {} + + # Essential fields + essential_fields = ["name", "version", "summary", "description", "author", "license"] + present_essential = [field for field in essential_fields if info.get(field)] + score += (len(present_essential) / len(essential_fields)) * 40 + completeness["essential_fields"] = f"{len(present_essential)}/{len(essential_fields)}" + + # Additional metadata fields + additional_fields = ["keywords", "home_page", "author_email", "classifiers", "project_urls"] + present_additional = [field for field in additional_fields if info.get(field)] + score += (len(present_additional) / len(additional_fields)) * 30 + completeness["additional_fields"] = f"{len(present_additional)}/{len(additional_fields)}" + + # Classifier completeness + classifiers = info.get("classifiers", []) + classifier_categories = set() + for classifier in classifiers: + category = classifier.split("::")[0].strip() + classifier_categories.add(category) + + expected_categories = ["Development Status", "Intended Audience", "License", "Programming Language", "Topic"] + present_categories = [cat for cat in expected_categories if cat in classifier_categories] + score += (len(present_categories) / len(expected_categories)) * 20 + completeness["classifier_categories"] = f"{len(present_categories)}/{len(expected_categories)}" + + # URLs completeness + project_urls = info.get("project_urls", {}) or {} + expected_url_types = ["homepage", "repository", "documentation", "bug tracker"] + present_url_types = [] + for expected in expected_url_types: + if any(expected.lower() in key.lower() for key in project_urls.keys()): + present_url_types.append(expected) + + score += (len(present_url_types) / len(expected_url_types)) * 10 + completeness["url_types"] = f"{len(present_url_types)}/{len(expected_url_types)}" + + return { + "score": min(score, 100), + "completeness": completeness, + } + + def _extract_github_url(self, package_data: Dict[str, Any]) -> Optional[str]: + """Extract GitHub repository URL from package data.""" + info = package_data.get("info", {}) + + # Check project URLs + project_urls = info.get("project_urls", {}) or {} + for url in project_urls.values(): + if url and "github.com" in url: + return url + + # Check home page + home_page = info.get("home_page", "") + if home_page and "github.com" in home_page: + return home_page + + return None + + async def _fetch_github_metrics(self, github_url: str) -> Dict[str, Any]: + """Fetch GitHub repository metrics.""" + try: + # Parse GitHub URL to get owner/repo + parsed = urlparse(github_url) + path_parts = parsed.path.strip('/').split('/') + if len(path_parts) >= 2: + owner, repo = path_parts[0], path_parts[1] + + # GitHub API call (public API, no auth required for basic info) + api_url = f"https://api.github.com/repos/{owner}/{repo}" + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + api_url, + headers={ + "Accept": "application/vnd.github.v3+json", + "User-Agent": "PyPI-Health-Scorer/1.0" + } + ) + + if response.status_code == 200: + data = response.json() + return { + "stars": data.get("stargazers_count", 0), + "forks": data.get("forks_count", 0), + "watchers": data.get("watchers_count", 0), + "issues": data.get("open_issues_count", 0), + "has_wiki": data.get("has_wiki", False), + "has_pages": data.get("has_pages", False), + "language": data.get("language", ""), + "created_at": data.get("created_at", ""), + "updated_at": data.get("pushed_at", ""), + "default_branch": data.get("default_branch", ""), + "archived": data.get("archived", False), + "disabled": data.get("disabled", False), + } + else: + logger.warning(f"GitHub API returned status {response.status_code}") + + except Exception as e: + logger.debug(f"Failed to fetch GitHub metrics: {e}") + + return self._empty_github_metrics() + + async def _empty_github_metrics(self) -> Dict[str, Any]: + """Return empty GitHub metrics.""" + return { + "stars": 0, + "forks": 0, + "watchers": 0, + "issues": 0, + "has_wiki": False, + "has_pages": False, + "language": "", + "created_at": "", + "updated_at": "", + "default_branch": "", + "archived": False, + "disabled": False, + "available": False, + } + + def _calculate_health_level(self, score: float) -> str: + """Calculate health level from score.""" + if score >= 85: + return "excellent" + elif score >= 70: + return "good" + elif score >= 55: + return "fair" + elif score >= 40: + return "poor" + else: + return "critical" + + def _identify_strengths(self, health_scores: Dict[str, float]) -> List[str]: + """Identify package strengths.""" + strengths = [] + for category, score in health_scores.items(): + if score >= 80: + strengths.append(f"Excellent {category} ({score:.0f}/100)") + elif score >= 65: + strengths.append(f"Good {category} ({score:.0f}/100)") + return strengths + + def _identify_weaknesses(self, health_scores: Dict[str, float]) -> List[str]: + """Identify package weaknesses.""" + weaknesses = [] + for category, score in health_scores.items(): + if score < 40: + weaknesses.append(f"Poor {category} ({score:.0f}/100)") + elif score < 55: + weaknesses.append(f"Fair {category} ({score:.0f}/100)") + return weaknesses + + def _prioritize_improvements(self, health_scores: Dict[str, float]) -> List[str]: + """Prioritize improvement areas by weight and score.""" + weighted_gaps = [] + for category, score in health_scores.items(): + gap = 100 - score + weighted_gap = gap * (self.weights[category] / 100) + weighted_gaps.append((category, weighted_gap, score)) + + # Sort by weighted gap (highest impact first) + weighted_gaps.sort(key=lambda x: x[1], reverse=True) + + priorities = [] + for category, weighted_gap, score in weighted_gaps[:3]: # Top 3 + if weighted_gap > 5: # Only include significant gaps + priorities.append(f"Improve {category} (current: {score:.0f}/100, impact: {self.weights[category]}%)") + + return priorities + + def _generate_health_recommendations( + self, health_scores: Dict[str, float], *assessment_results + ) -> List[str]: + """Generate actionable health improvement recommendations.""" + recommendations = [] + + overall_score = sum( + health_scores[category] * (self.weights[category] / 100) + for category in health_scores + ) + + # Overall recommendations + if overall_score >= 85: + recommendations.append("🌟 Excellent package health - maintain current standards") + elif overall_score >= 70: + recommendations.append("✅ Good package health - minor improvements possible") + elif overall_score >= 55: + recommendations.append("⚠️ Fair package health - several areas need improvement") + elif overall_score >= 40: + recommendations.append("🔶 Poor package health - significant improvements needed") + else: + recommendations.append("🚨 Critical package health - major overhaul required") + + # Specific recommendations based on low scores + if health_scores.get("maintenance", 0) < 60: + recommendations.append("📅 Improve maintenance: Update package more regularly, provide clear version history") + + if health_scores.get("documentation", 0) < 60: + recommendations.append("📚 Improve documentation: Add comprehensive README, usage examples, and API docs") + + if health_scores.get("testing", 0) < 60: + recommendations.append("🧪 Add testing: Implement test suite, CI/CD pipeline, and code coverage") + + if health_scores.get("security", 0) < 60: + recommendations.append("🔒 Enhance security: Add security reporting, use HTTPS, specify dependencies properly") + + if health_scores.get("compatibility", 0) < 60: + recommendations.append("🔧 Improve compatibility: Support more Python versions, add wheel distribution") + + if health_scores.get("metadata", 0) < 60: + recommendations.append("📝 Complete metadata: Add missing package information, keywords, and classifiers") + + if health_scores.get("popularity", 0) < 60: + recommendations.append("📈 Build community: Create documentation site, engage with users, add project URLs") + + return recommendations + + +# Main health assessment functions +async def assess_pypi_package_health( + package_name: str, + version: Optional[str] = None, + include_github_metrics: bool = True +) -> Dict[str, Any]: + """ + Assess comprehensive health and quality of a PyPI package. + + Args: + package_name: Name of the package to assess + version: Specific version to assess (optional) + include_github_metrics: Whether to fetch GitHub repository metrics + + Returns: + Comprehensive health assessment including scores and recommendations + """ + scorer = PackageHealthScorer() + return await scorer.assess_package_health( + package_name, version, include_github_metrics + ) + + +async def compare_package_health( + package_names: List[str], + include_github_metrics: bool = False +) -> Dict[str, Any]: + """ + Compare health scores across multiple packages. + + Args: + package_names: List of package names to compare + include_github_metrics: Whether to include GitHub metrics + + Returns: + Comparative health analysis with rankings + """ + logger.info(f"Starting health comparison for {len(package_names)} packages") + + scorer = PackageHealthScorer() + results = {} + + # Assess packages in parallel batches + batch_size = 3 + for i in range(0, len(package_names), batch_size): + batch = package_names[i:i + batch_size] + batch_tasks = [ + scorer.assess_package_health(pkg_name, include_github_metrics=include_github_metrics) + for pkg_name in batch + ] + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for pkg_name, result in zip(batch, batch_results): + if isinstance(result, Exception): + results[pkg_name] = { + "error": str(result), + "overall_health": {"score": 0, "level": "critical"}, + "category_scores": {cat: 0 for cat in scorer.weights.keys()} + } + else: + results[pkg_name] = result + + # Create comparison rankings + package_scores = [ + (pkg, result.get("overall_health", {}).get("score", 0)) + for pkg, result in results.items() + if "error" not in result + ] + package_scores.sort(key=lambda x: x[1], reverse=True) + + # Generate comparison insights + if package_scores: + best_package, best_score = package_scores[0] + worst_package, worst_score = package_scores[-1] + avg_score = sum(score for _, score in package_scores) / len(package_scores) + + comparison_insights = { + "best_package": {"name": best_package, "score": best_score}, + "worst_package": {"name": worst_package, "score": worst_score}, + "average_score": round(avg_score, 2), + "score_range": best_score - worst_score, + "rankings": [{"package": pkg, "score": score, "rank": i+1} + for i, (pkg, score) in enumerate(package_scores)] + } + else: + comparison_insights = { + "best_package": None, + "worst_package": None, + "average_score": 0, + "score_range": 0, + "rankings": [] + } + + return { + "comparison_timestamp": datetime.now(timezone.utc).isoformat(), + "packages_compared": len(package_names), + "detailed_results": results, + "comparison_insights": comparison_insights, + "recommendations": _generate_comparison_recommendations(comparison_insights, results) + } + + +def _generate_comparison_recommendations( + insights: Dict[str, Any], results: Dict[str, Any] +) -> List[str]: + """Generate recommendations for package comparison.""" + recommendations = [] + + if not insights.get("rankings"): + recommendations.append("❌ No successful health assessments to compare") + return recommendations + + best = insights.get("best_package") + worst = insights.get("worst_package") + avg_score = insights.get("average_score", 0) + + if best and worst: + recommendations.append( + f"🥇 Best package: {best['name']} (score: {best['score']:.1f}/100)" + ) + recommendations.append( + f"🥉 Needs improvement: {worst['name']} (score: {worst['score']:.1f}/100)" + ) + + if best['score'] - worst['score'] > 30: + recommendations.append("📊 Significant quality variation - consider standardizing practices") + + recommendations.append(f"📈 Average health score: {avg_score:.1f}/100") + + if avg_score >= 70: + recommendations.append("✅ Overall good package health across portfolio") + elif avg_score >= 55: + recommendations.append("⚠️ Mixed package health - focus on improving lower-scoring packages") + else: + recommendations.append("🚨 Poor overall package health - systematic improvements needed") + + return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/health_tools.py b/pypi_query_mcp/tools/health_tools.py new file mode 100644 index 0000000..b0f98b3 --- /dev/null +++ b/pypi_query_mcp/tools/health_tools.py @@ -0,0 +1,155 @@ +"""Package health assessment tools for PyPI packages.""" + +import logging +from typing import Any, Dict, List, Optional + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..tools.health_scorer import assess_pypi_package_health, compare_package_health + +logger = logging.getLogger(__name__) + + +async def assess_package_health_score( + package_name: str, + version: Optional[str] = None, + include_github_metrics: bool = True +) -> Dict[str, Any]: + """ + Assess comprehensive health and quality of a PyPI package. + + This tool evaluates package health across multiple dimensions including maintenance, + popularity, documentation, testing, security practices, compatibility, and metadata + completeness to provide an overall health score and actionable recommendations. + + Args: + package_name: Name of the package to assess for health and quality + version: Specific version to assess (optional, defaults to latest version) + include_github_metrics: Whether to fetch GitHub repository metrics for analysis + + Returns: + Dictionary containing comprehensive health assessment including: + - Overall health score (0-100) and level (excellent/good/fair/poor/critical) + - Category-specific scores (maintenance, popularity, documentation, testing, etc.) + - Detailed assessment breakdown with indicators and issues for each category + - GitHub repository metrics (stars, forks, activity) if available + - Actionable recommendations for health improvements + - Strengths, weaknesses, and improvement priorities analysis + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If health assessment fails + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError(package_name) + + logger.info(f"MCP tool: Assessing health for package {package_name}") + + try: + result = await assess_pypi_package_health( + package_name=package_name, + version=version, + include_github_metrics=include_github_metrics + ) + + overall_score = result.get("overall_health", {}).get("score", 0) + health_level = result.get("overall_health", {}).get("level", "unknown") + logger.info(f"MCP tool: Health assessment completed for {package_name} - score: {overall_score:.1f}/100 ({health_level})") + return result + + except (InvalidPackageNameError, NetworkError, SearchError) as e: + logger.error(f"Error assessing health for {package_name}: {e}") + return { + "error": f"Health assessment failed: {e}", + "error_type": type(e).__name__, + "package": package_name, + "version": version, + "assessment_timestamp": "", + "overall_health": { + "score": 0, + "level": "critical", + "max_score": 100, + }, + "category_scores": { + "maintenance": 0, + "popularity": 0, + "documentation": 0, + "testing": 0, + "security": 0, + "compatibility": 0, + "metadata": 0, + }, + "detailed_assessment": {}, + "recommendations": [f"❌ Health assessment failed: {e}"], + "health_summary": { + "strengths": [], + "weaknesses": ["Assessment failure"], + "improvement_priority": ["Resolve package access issues"], + } + } + + +async def compare_packages_health_scores( + package_names: List[str], + include_github_metrics: bool = False +) -> Dict[str, Any]: + """ + Compare health scores across multiple PyPI packages. + + This tool performs comparative health analysis across multiple packages, + providing rankings, insights, and recommendations to help evaluate + package ecosystem quality and identify the best options. + + Args: + package_names: List of package names to compare for health and quality + include_github_metrics: Whether to include GitHub metrics in the comparison + + Returns: + Dictionary containing comparative health analysis including: + - Detailed health results for each package + - Health score rankings with best/worst package identification + - Comparison insights (average scores, score ranges, rankings) + - Recommendations for package selection and improvements + - Statistical analysis of health across the package set + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during analysis + SearchError: If health comparison fails + """ + if not package_names: + raise ValueError("Package names list cannot be empty") + + logger.info(f"MCP tool: Starting health comparison for {len(package_names)} packages") + + try: + result = await compare_package_health( + package_names=package_names, + include_github_metrics=include_github_metrics + ) + + comparison_insights = result.get("comparison_insights", {}) + best_package = comparison_insights.get("best_package", {}) + packages_compared = result.get("packages_compared", 0) + + logger.info(f"MCP tool: Health comparison completed for {packages_compared} packages - best: {best_package.get('name', 'unknown')} ({best_package.get('score', 0):.1f}/100)") + return result + + except (ValueError, NetworkError, SearchError) as e: + logger.error(f"Error in health comparison: {e}") + return { + "error": f"Health comparison failed: {e}", + "error_type": type(e).__name__, + "comparison_timestamp": "", + "packages_compared": len(package_names), + "detailed_results": {}, + "comparison_insights": { + "best_package": None, + "worst_package": None, + "average_score": 0, + "score_range": 0, + "rankings": [] + }, + "recommendations": [f"❌ Health comparison failed: {e}"] + } \ No newline at end of file diff --git a/pypi_query_mcp/tools/license_analyzer.py b/pypi_query_mcp/tools/license_analyzer.py new file mode 100644 index 0000000..6e73bee --- /dev/null +++ b/pypi_query_mcp/tools/license_analyzer.py @@ -0,0 +1,727 @@ +"""License compatibility analysis tools for PyPI packages.""" + +import asyncio +import logging +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Set, Tuple + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..core.pypi_client import PyPIClient + +logger = logging.getLogger(__name__) + + +class LicenseCompatibilityAnalyzer: + """Comprehensive license compatibility analyzer for PyPI packages.""" + + def __init__(self): + self.timeout = 30.0 + + # License compatibility matrix based on common license interactions + # Key: primary license, Value: dict of compatible licenses with compatibility level + self.compatibility_matrix = { + "MIT": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "one-way", # MIT can be used in GPL, not vice versa + "GPL-3.0": "one-way", + "LGPL-2.1": "compatible", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + "BSD": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "one-way", + "GPL-3.0": "one-way", + "LGPL-2.1": "compatible", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + "Apache-2.0": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "incompatible", # Patent clause conflicts + "GPL-3.0": "one-way", # Apache can go into GPL-3.0 + "LGPL-2.1": "review-required", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + "GPL-2.0": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "incompatible", + "ISC": "compatible", + "GPL-2.0": "compatible", + "GPL-3.0": "incompatible", # GPL-2.0 and GPL-3.0 are incompatible + "LGPL-2.1": "compatible", + "LGPL-3.0": "incompatible", + "MPL-2.0": "incompatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "incompatible", + }, + "GPL-3.0": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "incompatible", + "GPL-3.0": "compatible", + "LGPL-2.1": "review-required", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "incompatible", + }, + "LGPL-2.1": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "review-required", + "ISC": "compatible", + "GPL-2.0": "compatible", + "GPL-3.0": "review-required", + "LGPL-2.1": "compatible", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + "LGPL-3.0": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "incompatible", + "GPL-3.0": "compatible", + "LGPL-2.1": "compatible", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + "MPL-2.0": { + "MIT": "compatible", + "BSD": "compatible", + "Apache-2.0": "compatible", + "ISC": "compatible", + "GPL-2.0": "incompatible", + "GPL-3.0": "compatible", + "LGPL-2.1": "compatible", + "LGPL-3.0": "compatible", + "MPL-2.0": "compatible", + "Unlicense": "compatible", + "Public Domain": "compatible", + "Proprietary": "review-required", + }, + } + + # License categorization for easier analysis + self.license_categories = { + "permissive": ["MIT", "BSD", "Apache-2.0", "ISC", "Unlicense", "Public Domain"], + "copyleft_weak": ["LGPL-2.1", "LGPL-3.0", "MPL-2.0"], + "copyleft_strong": ["GPL-2.0", "GPL-3.0", "AGPL-3.0"], + "proprietary": ["Proprietary", "Commercial", "All Rights Reserved"], + "unknown": ["Unknown", "Other", "Custom"], + } + + # Common license normalization patterns + self.license_patterns = { + r"MIT\s*License": "MIT", + r"BSD\s*3[-\s]*Clause": "BSD", + r"BSD\s*2[-\s]*Clause": "BSD", + r"Apache\s*2\.0": "Apache-2.0", + r"Apache\s*License\s*2\.0": "Apache-2.0", + r"GNU\s*General\s*Public\s*License\s*v?2": "GPL-2.0", + r"GNU\s*General\s*Public\s*License\s*v?3": "GPL-3.0", + r"GNU\s*Lesser\s*General\s*Public\s*License\s*v?2": "LGPL-2.1", + r"GNU\s*Lesser\s*General\s*Public\s*License\s*v?3": "LGPL-3.0", + r"Mozilla\s*Public\s*License\s*2\.0": "MPL-2.0", + r"ISC\s*License": "ISC", + r"Unlicense": "Unlicense", + r"Public\s*Domain": "Public Domain", + } + + async def analyze_package_license( + self, + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True + ) -> Dict[str, Any]: + """ + Analyze license information for a PyPI package. + + Args: + package_name: Name of the package to analyze + version: Specific version to analyze (optional) + include_dependencies: Whether to analyze dependency licenses + + Returns: + Dictionary containing license analysis results + """ + logger.info(f"Starting license analysis for package: {package_name}") + + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name, version) + + package_version = version or package_data["info"]["version"] + + # Analyze package license + license_info = self._extract_license_info(package_data) + + # Analyze dependencies if requested + dependency_licenses = [] + if include_dependencies: + dependency_licenses = await self._analyze_dependency_licenses( + package_name, package_version + ) + + # Generate compatibility analysis + compatibility_analysis = self._analyze_license_compatibility( + license_info, dependency_licenses + ) + + # Calculate risk assessment + risk_assessment = self._assess_license_risks( + license_info, dependency_licenses, compatibility_analysis + ) + + return { + "package": package_name, + "version": package_version, + "analysis_timestamp": datetime.now(timezone.utc).isoformat(), + "license_info": license_info, + "dependency_licenses": dependency_licenses, + "compatibility_analysis": compatibility_analysis, + "risk_assessment": risk_assessment, + "recommendations": self._generate_license_recommendations( + license_info, dependency_licenses, compatibility_analysis, risk_assessment + ), + "analysis_summary": { + "total_dependencies_analyzed": len(dependency_licenses), + "unique_licenses_found": len(set( + [license_info.get("normalized_license", "Unknown")] + + [dep.get("normalized_license", "Unknown") for dep in dependency_licenses] + )), + "license_conflicts": len(compatibility_analysis.get("conflicts", [])), + "review_required_count": len(compatibility_analysis.get("review_required", [])), + } + } + + except Exception as e: + logger.error(f"License analysis failed for {package_name}: {e}") + raise SearchError(f"License analysis failed: {e}") from e + + def _extract_license_info(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Extract and normalize license information from package data.""" + info = package_data.get("info", {}) + + # Extract license from multiple sources + license_field = info.get("license", "") + license_classifier = self._extract_license_from_classifiers( + info.get("classifiers", []) + ) + + # Normalize license + normalized_license = self._normalize_license(license_field or license_classifier) + + # Categorize license + license_category = self._categorize_license(normalized_license) + + return { + "raw_license": license_field, + "classifier_license": license_classifier, + "normalized_license": normalized_license, + "license_category": license_category, + "license_url": self._extract_license_url(info), + "license_confidence": self._assess_license_confidence( + license_field, license_classifier, normalized_license + ), + } + + def _extract_license_from_classifiers(self, classifiers: List[str]) -> str: + """Extract license information from PyPI classifiers.""" + license_classifiers = [ + c for c in classifiers if c.startswith("License ::") + ] + + if not license_classifiers: + return "" + + # Return the most specific license classifier + return license_classifiers[-1].replace("License ::", "").strip() + + def _normalize_license(self, license_text: str) -> str: + """Normalize license text to standard SPDX identifiers.""" + if not license_text: + return "Unknown" + + license_text_clean = license_text.strip() + + # Check for exact matches first + common_licenses = { + "MIT": "MIT", + "BSD": "BSD", + "Apache": "Apache-2.0", + "GPL": "GPL-3.0", # Default to GPL-3.0 if version unspecified + "LGPL": "LGPL-3.0", + "MPL": "MPL-2.0", + } + + if license_text_clean in common_licenses: + return common_licenses[license_text_clean] + + # Pattern matching + for pattern, normalized in self.license_patterns.items(): + if re.search(pattern, license_text_clean, re.IGNORECASE): + return normalized + + # Check if it contains known license names + license_lower = license_text_clean.lower() + if "mit" in license_lower: + return "MIT" + elif "bsd" in license_lower: + return "BSD" + elif "apache" in license_lower: + return "Apache-2.0" + elif "gpl" in license_lower and "lgpl" not in license_lower: + return "GPL-3.0" + elif "lgpl" in license_lower: + return "LGPL-3.0" + elif "mozilla" in license_lower or "mpl" in license_lower: + return "MPL-2.0" + elif "unlicense" in license_lower: + return "Unlicense" + elif "public domain" in license_lower: + return "Public Domain" + elif any(prop in license_lower for prop in ["proprietary", "commercial", "all rights reserved"]): + return "Proprietary" + + return "Other" + + def _categorize_license(self, normalized_license: str) -> str: + """Categorize license into major categories.""" + for category, licenses in self.license_categories.items(): + if normalized_license in licenses: + return category + return "unknown" + + def _extract_license_url(self, info: Dict[str, Any]) -> str: + """Extract license URL from package info.""" + # Check project URLs + project_urls = info.get("project_urls", {}) or {} + for key, url in project_urls.items(): + if "license" in key.lower(): + return url + + # Check home page for license info + home_page = info.get("home_page", "") + if home_page and "github.com" in home_page: + return f"{home_page.rstrip('/')}/blob/main/LICENSE" + + return "" + + def _assess_license_confidence( + self, raw_license: str, classifier_license: str, normalized_license: str + ) -> str: + """Assess confidence level in license detection.""" + if not raw_license and not classifier_license: + return "low" + + if normalized_license == "Unknown" or normalized_license == "Other": + return "low" + + if raw_license and classifier_license and raw_license in classifier_license: + return "high" + elif raw_license or classifier_license: + return "medium" + else: + return "low" + + async def _analyze_dependency_licenses( + self, package_name: str, version: str + ) -> List[Dict[str, Any]]: + """Analyze licenses of package dependencies.""" + try: + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name, version) + + # Extract dependencies + requires_dist = package_data.get("info", {}).get("requires_dist", []) or [] + dependencies = [] + + for req in requires_dist: + # Parse dependency name (simplified) + dep_name = req.split()[0].split(">=")[0].split("==")[0].split("~=")[0].split("!=")[0] + if dep_name and not dep_name.startswith("extra"): + dependencies.append(dep_name) + + # Analyze dependency licenses (limit to top 15 to avoid overwhelming) + dependency_licenses = [] + + for dep_name in dependencies[:15]: + try: + dep_data = await client.get_package_info(dep_name) + dep_license_info = self._extract_license_info(dep_data) + + dependency_licenses.append({ + "package": dep_name, + "version": dep_data.get("info", {}).get("version", ""), + **dep_license_info + }) + except Exception as e: + logger.debug(f"Failed to analyze license for dependency {dep_name}: {e}") + dependency_licenses.append({ + "package": dep_name, + "version": "", + "normalized_license": "Unknown", + "license_category": "unknown", + "license_confidence": "low", + "error": str(e) + }) + + return dependency_licenses + + except Exception as e: + logger.warning(f"Dependency license analysis failed: {e}") + return [] + + def _analyze_license_compatibility( + self, package_license: Dict[str, Any], dependency_licenses: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """Analyze license compatibility between package and its dependencies.""" + main_license = package_license.get("normalized_license", "Unknown") + + compatible = [] + incompatible = [] + review_required = [] + one_way = [] + unknown = [] + + for dep in dependency_licenses: + dep_license = dep.get("normalized_license", "Unknown") + dep_package = dep.get("package", "unknown") + + if main_license == "Unknown" or dep_license == "Unknown": + unknown.append({ + "package": dep_package, + "license": dep_license, + "reason": "License information unavailable" + }) + continue + + compatibility = self._check_license_compatibility(main_license, dep_license) + + if compatibility == "compatible": + compatible.append({ + "package": dep_package, + "license": dep_license, + }) + elif compatibility == "incompatible": + incompatible.append({ + "package": dep_package, + "license": dep_license, + "reason": f"{main_license} and {dep_license} are incompatible" + }) + elif compatibility == "review-required": + review_required.append({ + "package": dep_package, + "license": dep_license, + "reason": f"Manual review needed for {main_license} + {dep_license}" + }) + elif compatibility == "one-way": + one_way.append({ + "package": dep_package, + "license": dep_license, + "reason": f"{dep_license} can be used in {main_license} project" + }) + + return { + "main_license": main_license, + "compatible": compatible, + "incompatible": incompatible, + "review_required": review_required, + "one_way": one_way, + "unknown": unknown, + "conflicts": incompatible, # Alias for easier access + } + + def _check_license_compatibility(self, license1: str, license2: str) -> str: + """Check compatibility between two licenses.""" + if license1 in self.compatibility_matrix: + return self.compatibility_matrix[license1].get(license2, "unknown") + + # Fallback compatibility rules + if license1 == license2: + return "compatible" + + # Default to review required for unknown combinations + return "review-required" + + def _assess_license_risks( + self, + package_license: Dict[str, Any], + dependency_licenses: List[Dict[str, Any]], + compatibility_analysis: Dict[str, Any] + ) -> Dict[str, Any]: + """Assess overall license risks for the project.""" + risks = [] + risk_score = 0 + + main_license = package_license.get("normalized_license", "Unknown") + main_category = package_license.get("license_category", "unknown") + + # Check for incompatible licenses + incompatible_count = len(compatibility_analysis.get("incompatible", [])) + if incompatible_count > 0: + risks.append(f"Found {incompatible_count} incompatible license(s)") + risk_score += incompatible_count * 30 + + # Check for unknown licenses + unknown_count = len(compatibility_analysis.get("unknown", [])) + if unknown_count > 0: + risks.append(f"Found {unknown_count} dependency(ies) with unknown licenses") + risk_score += unknown_count * 10 + + # Check for review-required licenses + review_count = len(compatibility_analysis.get("review_required", [])) + if review_count > 0: + risks.append(f"Found {review_count} license(s) requiring manual review") + risk_score += review_count * 15 + + # Check for copyleft contamination risk + if main_category == "permissive": + copyleft_deps = [ + dep for dep in dependency_licenses + if dep.get("license_category") in ["copyleft_weak", "copyleft_strong"] + ] + if copyleft_deps: + risks.append(f"Permissive project using {len(copyleft_deps)} copyleft dependencies") + risk_score += len(copyleft_deps) * 20 + + # Check for proprietary license risks + proprietary_deps = [ + dep for dep in dependency_licenses + if dep.get("license_category") == "proprietary" + ] + if proprietary_deps: + risks.append(f"Found {len(proprietary_deps)} proprietary dependencies") + risk_score += len(proprietary_deps) * 25 + + # Calculate risk level + if risk_score >= 80: + risk_level = "critical" + elif risk_score >= 50: + risk_level = "high" + elif risk_score >= 25: + risk_level = "medium" + elif risk_score > 0: + risk_level = "low" + else: + risk_level = "minimal" + + return { + "risk_score": min(risk_score, 100), + "risk_level": risk_level, + "risk_factors": risks, + "compliance_status": "compliant" if risk_score < 25 else "review-needed", + } + + def _generate_license_recommendations( + self, + package_license: Dict[str, Any], + dependency_licenses: List[Dict[str, Any]], + compatibility_analysis: Dict[str, Any], + risk_assessment: Dict[str, Any] + ) -> List[str]: + """Generate actionable license recommendations.""" + recommendations = [] + + main_license = package_license.get("normalized_license", "Unknown") + risk_level = risk_assessment.get("risk_level", "unknown") + + # High-level recommendations based on risk + if risk_level == "critical": + recommendations.append("🚨 Critical license issues detected - immediate legal review required") + elif risk_level == "high": + recommendations.append("⚠️ High license risk - review and resolve conflicts before release") + elif risk_level == "medium": + recommendations.append("⚠️ Moderate license risk - review recommendations below") + elif risk_level == "minimal": + recommendations.append("✅ License compatibility appears good") + + # Specific recommendations for incompatible licenses + incompatible = compatibility_analysis.get("incompatible", []) + if incompatible: + recommendations.append(f"🔴 Remove or replace {len(incompatible)} incompatible dependencies:") + for dep in incompatible[:3]: # Show first 3 + recommendations.append(f" - {dep['package']} ({dep['license']}): {dep.get('reason', '')}") + + # Recommendations for review-required licenses + review_required = compatibility_analysis.get("review_required", []) + if review_required: + recommendations.append(f"📋 Manual review needed for {len(review_required)} dependencies:") + for dep in review_required[:3]: + recommendations.append(f" - {dep['package']} ({dep['license']})") + + # Unknown license recommendations + unknown = compatibility_analysis.get("unknown", []) + if unknown: + recommendations.append(f"❓ Investigate {len(unknown)} dependencies with unknown licenses") + + # License confidence recommendations + if package_license.get("license_confidence") == "low": + recommendations.append("📝 Consider adding clear license information to your package") + + # Category-specific recommendations + main_category = package_license.get("license_category", "unknown") + if main_category == "copyleft_strong": + recommendations.append("ℹ️ GPL license requires derivative works to also be GPL") + elif main_category == "permissive": + recommendations.append("ℹ️ Permissive license allows flexible usage") + + return recommendations + + +# Main analysis functions +async def analyze_package_license_compatibility( + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True +) -> Dict[str, Any]: + """ + Analyze license compatibility for a PyPI package. + + Args: + package_name: Name of the package to analyze + version: Specific version to analyze (optional) + include_dependencies: Whether to analyze dependency licenses + + Returns: + Comprehensive license compatibility analysis + """ + analyzer = LicenseCompatibilityAnalyzer() + return await analyzer.analyze_package_license( + package_name, version, include_dependencies + ) + + +async def check_license_compliance_bulk( + package_names: List[str], + target_license: Optional[str] = None +) -> Dict[str, Any]: + """ + Check license compliance for multiple packages. + + Args: + package_names: List of package names to check + target_license: Target license for compatibility checking + + Returns: + Bulk license compliance report + """ + logger.info(f"Starting bulk license compliance check for {len(package_names)} packages") + + analyzer = LicenseCompatibilityAnalyzer() + results = {} + summary = { + "total_packages": len(package_names), + "compliant_packages": 0, + "non_compliant_packages": 0, + "unknown_license_packages": 0, + "high_risk_packages": [], + "analysis_timestamp": datetime.now(timezone.utc).isoformat() + } + + # Analyze packages in parallel batches + batch_size = 5 + for i in range(0, len(package_names), batch_size): + batch = package_names[i:i + batch_size] + batch_tasks = [ + analyzer.analyze_package_license(pkg_name, include_dependencies=False) + for pkg_name in batch + ] + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for pkg_name, result in zip(batch, batch_results): + if isinstance(result, Exception): + results[pkg_name] = { + "error": str(result), + "analysis_status": "failed" + } + summary["unknown_license_packages"] += 1 + else: + results[pkg_name] = result + + # Update summary + risk_level = result.get("risk_assessment", {}).get("risk_level", "unknown") + if risk_level in ["minimal", "low"]: + summary["compliant_packages"] += 1 + else: + summary["non_compliant_packages"] += 1 + + if risk_level in ["high", "critical"]: + summary["high_risk_packages"].append({ + "package": pkg_name, + "license": result.get("license_info", {}).get("normalized_license", "Unknown"), + "risk_level": risk_level + }) + + return { + "summary": summary, + "detailed_results": results, + "target_license": target_license, + "recommendations": _generate_bulk_license_recommendations(summary, results) + } + + +def _generate_bulk_license_recommendations(summary: Dict[str, Any], results: Dict[str, Any]) -> List[str]: + """Generate recommendations for bulk license analysis.""" + recommendations = [] + + compliant = summary["compliant_packages"] + total = summary["total_packages"] + + if compliant == total: + recommendations.append("✅ All packages appear to have compliant licenses") + else: + non_compliant = summary["non_compliant_packages"] + percentage = (non_compliant / total) * 100 + recommendations.append( + f"⚠️ {non_compliant}/{total} packages ({percentage:.1f}%) have license compliance issues" + ) + + high_risk = summary["high_risk_packages"] + if high_risk: + recommendations.append( + f"🚨 {len(high_risk)} packages are high risk: {', '.join([p['package'] for p in high_risk])}" + ) + recommendations.append("Priority: Address high-risk packages immediately") + + unknown = summary["unknown_license_packages"] + if unknown > 0: + recommendations.append(f"❓ {unknown} packages have unknown or unclear licenses") + recommendations.append("Consider investigating these packages for license clarity") + + return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/license_tools.py b/pypi_query_mcp/tools/license_tools.py new file mode 100644 index 0000000..fa547a9 --- /dev/null +++ b/pypi_query_mcp/tools/license_tools.py @@ -0,0 +1,154 @@ +"""License compatibility analysis tools for PyPI packages.""" + +import logging +from typing import Any, Dict, List, Optional + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..tools.license_analyzer import analyze_package_license_compatibility, check_license_compliance_bulk + +logger = logging.getLogger(__name__) + + +async def analyze_pypi_package_license( + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True +) -> Dict[str, Any]: + """ + Analyze license compatibility for a PyPI package. + + This tool provides comprehensive license analysis including license identification, + dependency license scanning, compatibility checking, and risk assessment to help + ensure your project complies with open source license requirements. + + Args: + package_name: Name of the package to analyze for license compatibility + version: Specific version to analyze (optional, defaults to latest version) + include_dependencies: Whether to analyze dependency licenses for compatibility + + Returns: + Dictionary containing comprehensive license analysis including: + - License identification and normalization (SPDX format) + - License categorization (permissive, copyleft, proprietary, etc.) + - Dependency license analysis and compatibility matrix + - Risk assessment with score and risk level (minimal, low, medium, high, critical) + - Compatibility analysis highlighting conflicts and review-required combinations + - Actionable recommendations for license compliance + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If license analysis fails + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError(package_name) + + logger.info(f"MCP tool: Analyzing license compatibility for package {package_name}") + + try: + result = await analyze_package_license_compatibility( + package_name=package_name, + version=version, + include_dependencies=include_dependencies + ) + + logger.info(f"MCP tool: License analysis completed for {package_name} - {result.get('analysis_summary', {}).get('license_conflicts', 0)} conflicts found") + return result + + except (InvalidPackageNameError, NetworkError, SearchError) as e: + logger.error(f"Error analyzing license for {package_name}: {e}") + return { + "error": f"License analysis failed: {e}", + "error_type": type(e).__name__, + "package": package_name, + "version": version, + "analysis_timestamp": "", + "license_info": { + "normalized_license": "Unknown", + "license_category": "unknown", + "license_confidence": "low", + }, + "dependency_licenses": [], + "compatibility_analysis": { + "main_license": "Unknown", + "compatible": [], + "incompatible": [], + "review_required": [], + "conflicts": [], + }, + "risk_assessment": { + "risk_score": 100, + "risk_level": "critical", + "risk_factors": [f"License analysis failed: {e}"], + "compliance_status": "unknown", + }, + "recommendations": [f"❌ License analysis failed: {e}"], + "analysis_summary": { + "total_dependencies_analyzed": 0, + "unique_licenses_found": 0, + "license_conflicts": 0, + "review_required_count": 0, + } + } + + +async def check_bulk_license_compliance( + package_names: List[str], + target_license: Optional[str] = None +) -> Dict[str, Any]: + """ + Check license compliance for multiple PyPI packages. + + This tool performs bulk license compliance checking across multiple packages, + providing a consolidated report to help ensure your entire package ecosystem + complies with license requirements and identifying potential legal risks. + + Args: + package_names: List of package names to check for license compliance + target_license: Target license for compatibility checking (optional) + + Returns: + Dictionary containing bulk compliance analysis including: + - Summary statistics (total packages, compliant/non-compliant counts) + - Detailed license analysis for each package + - High-risk packages requiring immediate attention + - Unknown license packages needing investigation + - Prioritized recommendations for compliance remediation + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during analysis + SearchError: If bulk compliance checking fails + """ + if not package_names: + raise ValueError("Package names list cannot be empty") + + logger.info(f"MCP tool: Starting bulk license compliance check for {len(package_names)} packages") + + try: + result = await check_license_compliance_bulk( + package_names=package_names, + target_license=target_license + ) + + logger.info(f"MCP tool: Bulk license compliance completed - {result.get('summary', {}).get('non_compliant_packages', 0)} non-compliant packages found") + return result + + except (ValueError, NetworkError, SearchError) as e: + logger.error(f"Error in bulk license compliance check: {e}") + return { + "error": f"Bulk license compliance check failed: {e}", + "error_type": type(e).__name__, + "summary": { + "total_packages": len(package_names), + "compliant_packages": 0, + "non_compliant_packages": 0, + "unknown_license_packages": len(package_names), + "high_risk_packages": [], + "analysis_timestamp": "" + }, + "detailed_results": {}, + "target_license": target_license, + "recommendations": [f"❌ Bulk license compliance check failed: {e}"] + } \ No newline at end of file diff --git a/pypi_query_mcp/tools/requirements_analyzer.py b/pypi_query_mcp/tools/requirements_analyzer.py new file mode 100644 index 0000000..0aba3d0 --- /dev/null +++ b/pypi_query_mcp/tools/requirements_analyzer.py @@ -0,0 +1,947 @@ +"""Requirements file parsing and analysis tools for Python projects.""" + +import asyncio +import logging +import re +import tomllib +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..core.pypi_client import PyPIClient + +logger = logging.getLogger(__name__) + + +class RequirementsAnalyzer: + """Comprehensive requirements file analyzer for Python projects.""" + + def __init__(self): + self.timeout = 30.0 + + # Supported requirement file patterns + self.requirement_patterns = { + "requirements.txt": r"requirements.*\.txt", + "pyproject.toml": r"pyproject\.toml", + "setup.py": r"setup\.py", + "Pipfile": r"Pipfile", + "poetry.lock": r"poetry\.lock", + "conda.yml": r"(conda|environment)\.ya?ml", + } + + # Version specifier patterns + self.version_patterns = { + "exact": r"==\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "gte": r">=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "gt": r">\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "lte": r"<=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "lt": r"<\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "compatible": r"~=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + "not_equal": r"!=\s*([0-9]+(?:\.[0-9]+)*(?:[a-zA-Z][0-9]*)?)", + } + + async def analyze_requirements_file( + self, + file_path: str, + check_updates: bool = True, + security_scan: bool = True, + compatibility_check: bool = True + ) -> Dict[str, Any]: + """ + Analyze a requirements file for dependencies, versions, security, and compatibility. + + Args: + file_path: Path to the requirements file + check_updates: Whether to check for package updates + security_scan: Whether to perform security vulnerability scanning + compatibility_check: Whether to check Python version compatibility + + Returns: + Dictionary containing comprehensive requirements analysis + """ + logger.info(f"Starting requirements analysis for: {file_path}") + + try: + # Parse requirements file + parsed_requirements = await self._parse_requirements_file(file_path) + + if not parsed_requirements["dependencies"]: + return { + "file_path": file_path, + "analysis_timestamp": datetime.now(timezone.utc).isoformat(), + "file_info": parsed_requirements["file_info"], + "dependencies": [], + "analysis_summary": { + "total_dependencies": 0, + "outdated_packages": 0, + "security_vulnerabilities": 0, + "compatibility_issues": 0, + }, + "recommendations": ["No dependencies found to analyze"], + "error": "No dependencies found in requirements file" + } + + # Analyze dependencies in parallel + analysis_tasks = [] + + # Basic dependency analysis (always done) + analysis_tasks.append(self._analyze_dependency_health(parsed_requirements["dependencies"])) + + # Optional analyses + if check_updates: + analysis_tasks.append(self._check_package_updates(parsed_requirements["dependencies"])) + else: + analysis_tasks.append(asyncio.create_task(self._empty_updates_result())) + + if security_scan: + analysis_tasks.append(self._scan_dependencies_security(parsed_requirements["dependencies"])) + else: + analysis_tasks.append(asyncio.create_task(self._empty_security_result())) + + if compatibility_check: + python_version = parsed_requirements.get("python_version") + analysis_tasks.append(self._check_dependencies_compatibility(parsed_requirements["dependencies"], python_version)) + else: + analysis_tasks.append(asyncio.create_task(self._empty_compatibility_result())) + + # Execute analyses + results = await asyncio.gather(*analysis_tasks, return_exceptions=True) + + # Unpack results + health_analysis = results[0] if not isinstance(results[0], Exception) else {"healthy": [], "issues": []} + update_analysis = results[1] if not isinstance(results[1], Exception) else {"outdated": [], "current": []} + security_analysis = results[2] if not isinstance(results[2], Exception) else {"vulnerabilities": [], "secure": []} + compatibility_analysis = results[3] if not isinstance(results[3], Exception) else {"compatible": [], "incompatible": []} + + # Generate comprehensive analysis + analysis_summary = self._generate_analysis_summary( + parsed_requirements["dependencies"], + health_analysis, + update_analysis, + security_analysis, + compatibility_analysis + ) + + recommendations = self._generate_requirements_recommendations( + parsed_requirements, + health_analysis, + update_analysis, + security_analysis, + compatibility_analysis, + analysis_summary + ) + + return { + "file_path": file_path, + "analysis_timestamp": datetime.now(timezone.utc).isoformat(), + "file_info": parsed_requirements["file_info"], + "dependencies": parsed_requirements["dependencies"], + "dependency_analysis": { + "health": health_analysis, + "updates": update_analysis if check_updates else None, + "security": security_analysis if security_scan else None, + "compatibility": compatibility_analysis if compatibility_check else None, + }, + "analysis_summary": analysis_summary, + "recommendations": recommendations, + "python_requirements": parsed_requirements.get("python_version"), + } + + except Exception as e: + logger.error(f"Requirements analysis failed for {file_path}: {e}") + raise SearchError(f"Requirements analysis failed: {e}") from e + + async def _parse_requirements_file(self, file_path: str) -> Dict[str, Any]: + """Parse requirements from various file formats.""" + path = Path(file_path) + + if not path.exists(): + raise FileNotFoundError(f"Requirements file not found: {file_path}") + + file_info = { + "name": path.name, + "format": self._detect_file_format(path.name), + "size_bytes": path.stat().st_size, + "modified_time": datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat(), + } + + # Parse based on file format + if path.name.endswith('.txt'): + dependencies, python_version = await self._parse_requirements_txt(path) + elif path.name == 'pyproject.toml': + dependencies, python_version = await self._parse_pyproject_toml(path) + elif path.name == 'setup.py': + dependencies, python_version = await self._parse_setup_py(path) + elif path.name == 'Pipfile': + dependencies, python_version = await self._parse_pipfile(path) + elif path.name.endswith('.yml') or path.name.endswith('.yaml'): + dependencies, python_version = await self._parse_conda_yml(path) + else: + # Try to parse as requirements.txt format + dependencies, python_version = await self._parse_requirements_txt(path) + + return { + "file_info": file_info, + "dependencies": dependencies, + "python_version": python_version, + } + + def _detect_file_format(self, filename: str) -> str: + """Detect requirements file format.""" + filename_lower = filename.lower() + + for fmt, pattern in self.requirement_patterns.items(): + if re.match(pattern, filename_lower): + return fmt + + return "unknown" + + async def _parse_requirements_txt(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Parse requirements.txt format files.""" + dependencies = [] + python_version = None + + try: + content = path.read_text(encoding="utf-8") + lines = content.splitlines() + + for line_num, line in enumerate(lines, 1): + line = line.strip() + + # Skip comments and empty lines + if not line or line.startswith('#'): + continue + + # Skip -r and -e directives (for now) + if line.startswith(('-r', '-e', '--')): + continue + + # Parse requirement line + dep = self._parse_requirement_line(line, line_num) + if dep: + dependencies.append(dep) + + except Exception as e: + logger.warning(f"Failed to parse requirements.txt {path}: {e}") + + return dependencies, python_version + + async def _parse_pyproject_toml(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Parse pyproject.toml files.""" + dependencies = [] + python_version = None + + try: + content = path.read_text(encoding="utf-8") + data = tomllib.loads(content) + + # Extract Python version requirement + build_system = data.get("build-system", {}) + project = data.get("project", {}) + tool_poetry = data.get("tool", {}).get("poetry", {}) + + # Check for Python version in different places + if project.get("requires-python"): + python_version = project["requires-python"] + elif tool_poetry.get("dependencies", {}).get("python"): + python_version = tool_poetry["dependencies"]["python"] + + # Extract dependencies from project.dependencies + if "dependencies" in project: + for dep_line in project["dependencies"]: + dep = self._parse_requirement_line(dep_line, 0) + if dep: + dependencies.append(dep) + + # Extract from tool.poetry.dependencies + if "tool" in data and "poetry" in data["tool"] and "dependencies" in data["tool"]["poetry"]: + poetry_deps = data["tool"]["poetry"]["dependencies"] + for name, version_spec in poetry_deps.items(): + if name.lower() == "python": + continue # Skip Python version + + if isinstance(version_spec, str): + req_line = f"{name}{version_spec}" if version_spec.startswith(('=', '<', '>', '~', '^', '!')) else f"{name}=={version_spec}" + else: + # Handle complex version specifications + req_line = f"{name}>={version_spec.get('version', '0.0.0')}" + + dep = self._parse_requirement_line(req_line, 0) + if dep: + dependencies.append(dep) + + except Exception as e: + logger.warning(f"Failed to parse pyproject.toml {path}: {e}") + + return dependencies, python_version + + async def _parse_setup_py(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Parse setup.py files (basic extraction).""" + dependencies = [] + python_version = None + + try: + content = path.read_text(encoding="utf-8") + + # Look for install_requires + install_requires_match = re.search(r"install_requires\s*=\s*\[(.*?)\]", content, re.DOTALL) + if install_requires_match: + deps_text = install_requires_match.group(1) + # Extract quoted strings + quoted_deps = re.findall(r'["\']([^"\']+)["\']', deps_text) + + for dep_line in quoted_deps: + dep = self._parse_requirement_line(dep_line, 0) + if dep: + dependencies.append(dep) + + # Look for python_requires + python_requires_match = re.search(r"python_requires\s*=\s*[\"']([^\"']+)[\"']", content) + if python_requires_match: + python_version = python_requires_match.group(1) + + except Exception as e: + logger.warning(f"Failed to parse setup.py {path}: {e}") + + return dependencies, python_version + + async def _parse_pipfile(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Parse Pipfile format.""" + dependencies = [] + python_version = None + + try: + content = path.read_text(encoding="utf-8") + data = tomllib.loads(content) + + # Extract Python version + if "requires" in data and "python_version" in data["requires"]: + python_version = f">={data['requires']['python_version']}" + + # Extract packages + for section in ["packages", "dev-packages"]: + if section in data: + for name, version_spec in data[section].items(): + if isinstance(version_spec, str): + req_line = f"{name}{version_spec}" if version_spec.startswith(('=', '<', '>', '~', '^', '!')) else f"{name}=={version_spec}" + else: + req_line = f"{name}>={version_spec.get('version', '0.0.0')}" + + dep = self._parse_requirement_line(req_line, 0) + if dep: + dep["dev_dependency"] = (section == "dev-packages") + dependencies.append(dep) + + except Exception as e: + logger.warning(f"Failed to parse Pipfile {path}: {e}") + + return dependencies, python_version + + async def _parse_conda_yml(self, path: Path) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Parse conda environment.yml files.""" + dependencies = [] + python_version = None + + try: + import yaml + + content = path.read_text(encoding="utf-8") + data = yaml.safe_load(content) + + if "dependencies" in data: + for dep in data["dependencies"]: + if isinstance(dep, str): + if dep.startswith("python"): + # Extract Python version + python_match = re.search(r"python\s*([><=~!]+)\s*([0-9.]+)", dep) + if python_match: + python_version = f"{python_match.group(1)}{python_match.group(2)}" + else: + parsed_dep = self._parse_requirement_line(dep, 0) + if parsed_dep: + dependencies.append(parsed_dep) + + except Exception as e: + logger.warning(f"Failed to parse conda.yml {path}: {e}") + + return dependencies, python_version + + def _parse_requirement_line(self, line: str, line_number: int) -> Optional[Dict[str, Any]]: + """Parse a single requirement line.""" + try: + # Remove inline comments + if '#' in line: + line = line[:line.index('#')].strip() + + if not line: + return None + + # Handle extras (package[extra1,extra2]) + extras = [] + extras_match = re.search(r'\[([^\]]+)\]', line) + if extras_match: + extras = [e.strip() for e in extras_match.group(1).split(',')] + line = re.sub(r'\[([^\]]+)\]', '', line) + + # Parse package name and version specifiers + # Split on version operators + version_ops = ['>=', '<=', '==', '!=', '~=', '>', '<'] + package_name = line + version_specifiers = [] + + for op in version_ops: + if op in line: + parts = line.split(op) + package_name = parts[0].strip() + if len(parts) > 1: + version_specifiers.append({ + "operator": op, + "version": parts[1].strip().split(',')[0].strip() + }) + break + + # Handle comma-separated version specs + if ',' in line and version_specifiers: + remaining = line.split(version_specifiers[0]["operator"], 1)[1] + for spec in remaining.split(',')[1:]: + spec = spec.strip() + for op in version_ops: + if spec.startswith(op): + version_specifiers.append({ + "operator": op, + "version": spec[len(op):].strip() + }) + break + + # Clean package name + package_name = re.sub(r'[<>=!~,\s].*', '', package_name).strip() + + if not package_name: + return None + + return { + "name": package_name, + "version_specifiers": version_specifiers, + "extras": extras, + "line_number": line_number, + "raw_line": line.strip(), + } + + except Exception as e: + logger.debug(f"Failed to parse requirement line '{line}': {e}") + return None + + async def _analyze_dependency_health(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: + """Analyze overall health of dependencies.""" + healthy = [] + issues = [] + + for dep in dependencies: + name = dep["name"] + version_specs = dep["version_specifiers"] + + # Check for problematic version specifications + health_issues = [] + + if not version_specs: + health_issues.append("No version constraint (could lead to instability)") + else: + # Check for overly restrictive versions + exact_versions = [spec for spec in version_specs if spec["operator"] == "=="] + if exact_versions: + health_issues.append("Exact version pinning (may cause conflicts)") + + # Check for very loose constraints + loose_constraints = [spec for spec in version_specs if spec["operator"] in [">", ">="]] + if loose_constraints and not any(spec["operator"] in ["<", "<="] for spec in version_specs): + health_issues.append("No upper bound (may break with future versions)") + + if health_issues: + issues.append({ + "package": name, + "issues": health_issues, + "current_spec": version_specs + }) + else: + healthy.append({ + "package": name, + "version_spec": version_specs + }) + + return { + "healthy": healthy, + "issues": issues, + "health_score": len(healthy) / len(dependencies) * 100 if dependencies else 0 + } + + async def _check_package_updates(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: + """Check for available package updates.""" + outdated = [] + current = [] + + async with PyPIClient() as client: + # Process in batches to avoid overwhelming PyPI + batch_size = 10 + for i in range(0, len(dependencies), batch_size): + batch = dependencies[i:i + batch_size] + batch_tasks = [] + + for dep in batch: + task = self._check_single_package_update(client, dep) + batch_tasks.append(task) + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for dep, result in zip(batch, batch_results): + if isinstance(result, Exception): + logger.debug(f"Failed to check updates for {dep['name']}: {result}") + continue + + if result["has_update"]: + outdated.append(result) + else: + current.append(result) + + return { + "outdated": outdated, + "current": current, + "update_percentage": len(outdated) / len(dependencies) * 100 if dependencies else 0 + } + + async def _check_single_package_update(self, client: PyPIClient, dep: Dict[str, Any]) -> Dict[str, Any]: + """Check if a single package has updates available.""" + try: + package_data = await client.get_package_info(dep["name"]) + latest_version = package_data["info"]["version"] + + # For now, we'll do a simple comparison + # In a real implementation, you'd want proper version comparison + has_update = True # Placeholder logic + + return { + "package": dep["name"], + "current_spec": dep["version_specifiers"], + "latest_version": latest_version, + "has_update": has_update, + "update_recommendation": f"Update to {latest_version}" + } + + except Exception as e: + return { + "package": dep["name"], + "current_spec": dep["version_specifiers"], + "latest_version": "unknown", + "has_update": False, + "error": str(e) + } + + async def _scan_dependencies_security(self, dependencies: List[Dict[str, Any]]) -> Dict[str, Any]: + """Scan dependencies for security vulnerabilities.""" + # Import security scanner if available + try: + from .security import scan_package_security + + vulnerabilities = [] + secure = [] + + # Process in small batches + batch_size = 5 + for i in range(0, len(dependencies), batch_size): + batch = dependencies[i:i + batch_size] + batch_tasks = [] + + for dep in batch: + task = self._scan_single_dependency_security(dep) + batch_tasks.append(task) + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for dep, result in zip(batch, batch_results): + if isinstance(result, Exception): + logger.debug(f"Failed to scan security for {dep['name']}: {result}") + continue + + if result["vulnerabilities"]: + vulnerabilities.append(result) + else: + secure.append(result) + + return { + "vulnerabilities": vulnerabilities, + "secure": secure, + "vulnerability_count": sum(len(v["vulnerabilities"]) for v in vulnerabilities), + } + + except ImportError: + logger.warning("Security scanner not available") + return await self._empty_security_result() + + async def _scan_single_dependency_security(self, dep: Dict[str, Any]) -> Dict[str, Any]: + """Scan a single dependency for security issues.""" + try: + from .security import scan_package_security + + result = await scan_package_security( + dep["name"], + version=None, # Latest version + include_dependencies=False + ) + + vuln_summary = result.get("security_summary", {}) + return { + "package": dep["name"], + "vulnerabilities": result.get("vulnerabilities", {}).get("direct", []), + "risk_level": vuln_summary.get("risk_level", "minimal"), + "total_vulnerabilities": vuln_summary.get("total_vulnerabilities", 0) + } + + except Exception as e: + return { + "package": dep["name"], + "vulnerabilities": [], + "risk_level": "unknown", + "error": str(e) + } + + async def _check_dependencies_compatibility( + self, dependencies: List[Dict[str, Any]], python_version: Optional[str] + ) -> Dict[str, Any]: + """Check Python version compatibility for dependencies.""" + if not python_version: + return await self._empty_compatibility_result() + + compatible = [] + incompatible = [] + + # Process in batches + batch_size = 10 + for i in range(0, len(dependencies), batch_size): + batch = dependencies[i:i + batch_size] + batch_tasks = [] + + for dep in batch: + task = self._check_single_dependency_compatibility(dep, python_version) + batch_tasks.append(task) + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for dep, result in zip(batch, batch_results): + if isinstance(result, Exception): + logger.debug(f"Failed to check compatibility for {dep['name']}: {result}") + continue + + if result["compatible"]: + compatible.append(result) + else: + incompatible.append(result) + + return { + "compatible": compatible, + "incompatible": incompatible, + "python_version": python_version, + "compatibility_percentage": len(compatible) / len(dependencies) * 100 if dependencies else 0 + } + + async def _check_single_dependency_compatibility( + self, dep: Dict[str, Any], python_version: str + ) -> Dict[str, Any]: + """Check compatibility for a single dependency.""" + try: + from .compatibility_check import check_python_compatibility + + # Extract target Python version (simplified) + target_version = "3.9" # Default fallback + version_match = re.search(r'(\d+\.\d+)', python_version) + if version_match: + target_version = version_match.group(1) + + result = await check_python_compatibility(dep["name"], target_version) + + return { + "package": dep["name"], + "compatible": result.get("compatible", False), + "python_version": target_version, + "details": result.get("compatibility_info", "") + } + + except Exception as e: + return { + "package": dep["name"], + "compatible": True, # Assume compatible on error + "python_version": python_version, + "error": str(e) + } + + # Helper methods for empty results + async def _empty_updates_result(self) -> Dict[str, Any]: + return {"outdated": [], "current": [], "update_percentage": 0} + + async def _empty_security_result(self) -> Dict[str, Any]: + return {"vulnerabilities": [], "secure": [], "vulnerability_count": 0} + + async def _empty_compatibility_result(self) -> Dict[str, Any]: + return {"compatible": [], "incompatible": [], "python_version": None, "compatibility_percentage": 100} + + def _generate_analysis_summary( + self, + dependencies: List[Dict[str, Any]], + health_analysis: Dict[str, Any], + update_analysis: Dict[str, Any], + security_analysis: Dict[str, Any], + compatibility_analysis: Dict[str, Any] + ) -> Dict[str, Any]: + """Generate comprehensive analysis summary.""" + return { + "total_dependencies": len(dependencies), + "health_score": round(health_analysis.get("health_score", 0), 1), + "packages_with_issues": len(health_analysis.get("issues", [])), + "outdated_packages": len(update_analysis.get("outdated", [])), + "security_vulnerabilities": security_analysis.get("vulnerability_count", 0), + "compatibility_issues": len(compatibility_analysis.get("incompatible", [])), + "overall_risk_level": self._calculate_overall_risk_level( + health_analysis, update_analysis, security_analysis, compatibility_analysis + ) + } + + def _calculate_overall_risk_level( + self, health: Dict[str, Any], updates: Dict[str, Any], + security: Dict[str, Any], compatibility: Dict[str, Any] + ) -> str: + """Calculate overall risk level for the project.""" + risk_score = 0 + + # Health risks + health_score = health.get("health_score", 100) + if health_score < 50: + risk_score += 30 + elif health_score < 75: + risk_score += 15 + + # Security risks + vuln_count = security.get("vulnerability_count", 0) + if vuln_count > 10: + risk_score += 40 + elif vuln_count > 5: + risk_score += 25 + elif vuln_count > 0: + risk_score += 15 + + # Compatibility risks + incompat_count = len(compatibility.get("incompatible", [])) + if incompat_count > 5: + risk_score += 25 + elif incompat_count > 0: + risk_score += 10 + + # Update risks (outdated packages) + outdated_count = len(updates.get("outdated", [])) + total_deps = len(updates.get("outdated", [])) + len(updates.get("current", [])) + if total_deps > 0: + outdated_percentage = (outdated_count / total_deps) * 100 + if outdated_percentage > 50: + risk_score += 20 + elif outdated_percentage > 25: + risk_score += 10 + + # Calculate risk level + if risk_score >= 70: + return "critical" + elif risk_score >= 50: + return "high" + elif risk_score >= 30: + return "medium" + elif risk_score > 0: + return "low" + else: + return "minimal" + + def _generate_requirements_recommendations( + self, + parsed_requirements: Dict[str, Any], + health_analysis: Dict[str, Any], + update_analysis: Dict[str, Any], + security_analysis: Dict[str, Any], + compatibility_analysis: Dict[str, Any], + summary: Dict[str, Any] + ) -> List[str]: + """Generate actionable recommendations for requirements management.""" + recommendations = [] + + risk_level = summary.get("overall_risk_level", "minimal") + + # Overall assessment + if risk_level == "critical": + recommendations.append("🚨 Critical issues detected - immediate action required") + elif risk_level == "high": + recommendations.append("⚠️ High risk dependencies - review and update urgently") + elif risk_level == "medium": + recommendations.append("⚠️ Moderate risk - address issues when possible") + elif risk_level == "minimal": + recommendations.append("✅ Requirements appear healthy") + + # Specific recommendations + health_issues = health_analysis.get("issues", []) + if health_issues: + recommendations.append(f"🔧 Fix {len(health_issues)} dependency specification issues") + + outdated_count = len(update_analysis.get("outdated", [])) + if outdated_count > 0: + recommendations.append(f"📦 Update {outdated_count} outdated packages") + + vuln_count = security_analysis.get("vulnerability_count", 0) + if vuln_count > 0: + recommendations.append(f"🔒 Address {vuln_count} security vulnerabilities") + + incompat_count = len(compatibility_analysis.get("incompatible", [])) + if incompat_count > 0: + recommendations.append(f"🐍 Fix {incompat_count} Python compatibility issues") + + # File format recommendations + file_format = parsed_requirements["file_info"]["format"] + if file_format == "requirements.txt": + recommendations.append("💡 Consider migrating to pyproject.toml for better dependency management") + elif file_format == "unknown": + recommendations.append("📝 Use standard requirements file formats (requirements.txt, pyproject.toml)") + + return recommendations + + +# Main analysis functions +async def analyze_project_requirements( + file_path: str, + check_updates: bool = True, + security_scan: bool = True, + compatibility_check: bool = True +) -> Dict[str, Any]: + """ + Analyze project requirements file for dependencies, security, and compatibility. + + Args: + file_path: Path to the requirements file + check_updates: Whether to check for package updates + security_scan: Whether to perform security vulnerability scanning + compatibility_check: Whether to check Python version compatibility + + Returns: + Comprehensive requirements file analysis + """ + analyzer = RequirementsAnalyzer() + return await analyzer.analyze_requirements_file( + file_path, check_updates, security_scan, compatibility_check + ) + + +async def compare_requirements_files( + file_paths: List[str] +) -> Dict[str, Any]: + """ + Compare multiple requirements files to identify differences and conflicts. + + Args: + file_paths: List of paths to requirements files to compare + + Returns: + Comparative analysis of requirements files + """ + logger.info(f"Starting requirements comparison for {len(file_paths)} files") + + analyzer = RequirementsAnalyzer() + file_analyses = {} + + # Analyze each file + for file_path in file_paths: + try: + analysis = await analyzer.analyze_requirements_file( + file_path, check_updates=False, security_scan=False, compatibility_check=False + ) + file_analyses[file_path] = analysis + except Exception as e: + logger.error(f"Failed to analyze {file_path}: {e}") + file_analyses[file_path] = {"error": str(e), "dependencies": []} + + # Compare dependencies + all_packages = set() + for analysis in file_analyses.values(): + if "dependencies" in analysis: + for dep in analysis["dependencies"]: + all_packages.add(dep["name"]) + + # Generate comparison results + conflicts = [] + common_packages = [] + unique_packages = {} + + for package in all_packages: + versions_by_file = {} + for file_path, analysis in file_analyses.items(): + if "dependencies" in analysis: + for dep in analysis["dependencies"]: + if dep["name"] == package: + versions_by_file[file_path] = dep["version_specifiers"] + break + + if len(versions_by_file) == len(file_paths): + # Package is in all files + version_specs = list(versions_by_file.values()) + if len(set(str(spec) for spec in version_specs)) > 1: + conflicts.append({ + "package": package, + "versions_by_file": versions_by_file + }) + else: + common_packages.append(package) + else: + # Package is unique to some files + for file_path, versions in versions_by_file.items(): + if file_path not in unique_packages: + unique_packages[file_path] = [] + unique_packages[file_path].append({ + "package": package, + "version_specifiers": versions + }) + + return { + "comparison_timestamp": datetime.now(timezone.utc).isoformat(), + "files_compared": len(file_paths), + "file_analyses": file_analyses, + "comparison_results": { + "total_unique_packages": len(all_packages), + "common_packages": common_packages, + "conflicting_packages": conflicts, + "unique_to_files": unique_packages, + }, + "recommendations": _generate_comparison_recommendations(conflicts, unique_packages, file_analyses) + } + + +def _generate_comparison_recommendations( + conflicts: List[Dict[str, Any]], + unique_packages: Dict[str, List[Dict[str, Any]]], + file_analyses: Dict[str, Any] +) -> List[str]: + """Generate recommendations for requirements file comparison.""" + recommendations = [] + + if conflicts: + recommendations.append(f"🔄 Resolve {len(conflicts)} version conflicts across files") + for conflict in conflicts[:3]: # Show first 3 + recommendations.append(f" - {conflict['package']}: inconsistent versions") + + if unique_packages: + total_unique = sum(len(packages) for packages in unique_packages.values()) + recommendations.append(f"📦 {total_unique} packages are unique to specific files") + + if not conflicts and not unique_packages: + recommendations.append("✅ All requirements files are consistent") + + # File format recommendations + formats = set() + for analysis in file_analyses.values(): + if "file_info" in analysis: + formats.add(analysis["file_info"]["format"]) + + if len(formats) > 1: + recommendations.append("📝 Consider standardizing on a single requirements file format") + + return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/requirements_tools.py b/pypi_query_mcp/tools/requirements_tools.py new file mode 100644 index 0000000..86eb3b7 --- /dev/null +++ b/pypi_query_mcp/tools/requirements_tools.py @@ -0,0 +1,143 @@ +"""Requirements file analysis tools for Python projects.""" + +import logging +from typing import Any, Dict, List + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..tools.requirements_analyzer import analyze_project_requirements, compare_requirements_files + +logger = logging.getLogger(__name__) + + +async def analyze_requirements_file_tool( + file_path: str, + check_updates: bool = True, + security_scan: bool = True, + compatibility_check: bool = True +) -> Dict[str, Any]: + """ + Analyze project requirements file for dependencies, security, and compatibility. + + This tool provides comprehensive analysis of Python project requirements files + including dependency parsing, version checking, security vulnerability scanning, + Python compatibility assessment, and actionable recommendations for improvements. + + Args: + file_path: Path to the requirements file (requirements.txt, pyproject.toml, setup.py, etc.) + check_updates: Whether to check for available package updates + security_scan: Whether to perform security vulnerability scanning on dependencies + compatibility_check: Whether to check Python version compatibility for all dependencies + + Returns: + Dictionary containing comprehensive requirements analysis including: + - File information and detected format (requirements.txt, pyproject.toml, etc.) + - Parsed dependencies with version specifiers and extras + - Dependency health analysis with specification issues and recommendations + - Package update analysis showing outdated packages and latest versions + - Security vulnerability scan results for all dependencies + - Python version compatibility assessment + - Overall risk level and actionable improvement recommendations + + Raises: + FileNotFoundError: If the requirements file is not found + NetworkError: For network-related errors during analysis + SearchError: If requirements analysis fails + """ + logger.info(f"MCP tool: Analyzing requirements file {file_path}") + + try: + result = await analyze_project_requirements( + file_path=file_path, + check_updates=check_updates, + security_scan=security_scan, + compatibility_check=compatibility_check + ) + + summary = result.get("analysis_summary", {}) + total_deps = summary.get("total_dependencies", 0) + risk_level = summary.get("overall_risk_level", "unknown") + logger.info(f"MCP tool: Requirements analysis completed for {file_path} - {total_deps} dependencies, risk level: {risk_level}") + return result + + except (FileNotFoundError, NetworkError, SearchError) as e: + logger.error(f"Error analyzing requirements file {file_path}: {e}") + return { + "error": f"Requirements analysis failed: {e}", + "error_type": type(e).__name__, + "file_path": file_path, + "analysis_timestamp": "", + "file_info": {"name": file_path, "format": "unknown"}, + "dependencies": [], + "dependency_analysis": {}, + "analysis_summary": { + "total_dependencies": 0, + "health_score": 0, + "packages_with_issues": 0, + "outdated_packages": 0, + "security_vulnerabilities": 0, + "compatibility_issues": 0, + "overall_risk_level": "critical", + }, + "recommendations": [f"❌ Requirements analysis failed: {e}"], + "python_requirements": None, + } + + +async def compare_multiple_requirements_files( + file_paths: List[str] +) -> Dict[str, Any]: + """ + Compare multiple requirements files to identify differences and conflicts. + + This tool analyzes multiple requirements files simultaneously to identify + version conflicts, unique dependencies, and inconsistencies across different + project configurations or environments. + + Args: + file_paths: List of paths to requirements files to compare and analyze + + Returns: + Dictionary containing comparative requirements analysis including: + - Detailed analysis results for each individual file + - Common packages shared across all files + - Conflicting package versions between files with specific version details + - Packages unique to specific files + - Recommendations for resolving conflicts and standardizing requirements + - Statistics on package overlap and conflict rates + + Raises: + ValueError: If file_paths list is empty + NetworkError: For network-related errors during analysis + SearchError: If requirements comparison fails + """ + if not file_paths: + raise ValueError("File paths list cannot be empty") + + logger.info(f"MCP tool: Comparing {len(file_paths)} requirements files") + + try: + result = await compare_requirements_files(file_paths=file_paths) + + comparison_results = result.get("comparison_results", {}) + conflicts = len(comparison_results.get("conflicting_packages", [])) + total_packages = comparison_results.get("total_unique_packages", 0) + + logger.info(f"MCP tool: Requirements comparison completed - {total_packages} unique packages, {conflicts} conflicts found") + return result + + except (ValueError, NetworkError, SearchError) as e: + logger.error(f"Error comparing requirements files: {e}") + return { + "error": f"Requirements comparison failed: {e}", + "error_type": type(e).__name__, + "comparison_timestamp": "", + "files_compared": len(file_paths), + "file_analyses": {}, + "comparison_results": { + "total_unique_packages": 0, + "common_packages": [], + "conflicting_packages": [], + "unique_to_files": {}, + }, + "recommendations": [f"❌ Requirements comparison failed: {e}"] + } \ No newline at end of file diff --git a/pypi_query_mcp/tools/security.py b/pypi_query_mcp/tools/security.py new file mode 100644 index 0000000..5b62895 --- /dev/null +++ b/pypi_query_mcp/tools/security.py @@ -0,0 +1,660 @@ +"""Security vulnerability scanning and analysis tools for PyPI packages.""" + +import asyncio +import json +import logging +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import httpx + +from ..core.exceptions import NetworkError, SearchError +from ..core.pypi_client import PyPIClient + +logger = logging.getLogger(__name__) + + +class VulnerabilityScanner: + """Comprehensive vulnerability scanner for PyPI packages.""" + + def __init__(self): + self.timeout = 30.0 + self.session = None + + # Vulnerability database endpoints + self.osv_api = "https://api.osv.dev/v1/query" + self.safety_db_api = "https://pyup.io/api/v1/safety" + self.snyk_api = "https://snyk.io/test/pip" + + # Common vulnerability patterns to look for + self.high_risk_patterns = [ + "remote code execution", "rce", "code injection", "sql injection", + "cross-site scripting", "xss", "csrf", "authentication bypass", + "privilege escalation", "arbitrary file", "path traversal", + "buffer overflow", "memory corruption", "denial of service" + ] + + async def scan_package( + self, + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True, + severity_filter: Optional[str] = None + ) -> Dict[str, Any]: + """ + Comprehensive security scan of a PyPI package. + + Args: + package_name: Name of the package to scan + version: Specific version to scan (optional, defaults to latest) + include_dependencies: Whether to scan dependencies too + severity_filter: Filter by severity level (low, medium, high, critical) + + Returns: + Dictionary containing security analysis results + """ + logger.info(f"Starting security scan for package: {package_name}") + + try: + # Get package information + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name, version) + + package_version = version or package_data["info"]["version"] + + # Run parallel vulnerability scans + scan_tasks = [ + self._scan_osv_database(package_name, package_version), + self._scan_github_advisories(package_name, package_version), + self._analyze_package_metadata(package_data), + self._check_dependency_vulnerabilities(package_name, package_version) if include_dependencies else asyncio.create_task(self._empty_result()) + ] + + osv_results, github_results, metadata_analysis, dependency_results = await asyncio.gather( + *scan_tasks, return_exceptions=True + ) + + # Consolidate results + vulnerabilities = [] + + # Process OSV results + if not isinstance(osv_results, Exception) and osv_results: + vulnerabilities.extend(osv_results.get("vulnerabilities", [])) + + # Process GitHub results + if not isinstance(github_results, Exception) and github_results: + vulnerabilities.extend(github_results.get("vulnerabilities", [])) + + # Process dependency vulnerabilities + if not isinstance(dependency_results, Exception) and dependency_results: + vulnerabilities.extend(dependency_results.get("vulnerabilities", [])) + + # Apply severity filter + if severity_filter: + vulnerabilities = [ + vuln for vuln in vulnerabilities + if vuln.get("severity", "").lower() == severity_filter.lower() + ] + + # Generate security report + security_report = self._generate_security_report( + package_name, package_version, vulnerabilities, metadata_analysis + ) + + return security_report + + except Exception as e: + logger.error(f"Security scan failed for {package_name}: {e}") + raise SearchError(f"Security scan failed: {e}") from e + + async def _scan_osv_database(self, package_name: str, version: str) -> Dict[str, Any]: + """Scan package against OSV (Open Source Vulnerabilities) database.""" + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + query_data = { + "package": { + "name": package_name, + "ecosystem": "PyPI" + }, + "version": version + } + + response = await client.post( + self.osv_api, + json=query_data, + headers={"Content-Type": "application/json"} + ) + + if response.status_code == 200: + data = response.json() + vulnerabilities = [] + + for vuln in data.get("vulns", []): + severity = self._extract_severity_from_osv(vuln) + vulnerabilities.append({ + "id": vuln.get("id", ""), + "summary": vuln.get("summary", ""), + "details": vuln.get("details", ""), + "severity": severity, + "published": vuln.get("published", ""), + "modified": vuln.get("modified", ""), + "source": "OSV", + "references": [ref.get("url", "") for ref in vuln.get("references", [])], + "affected_versions": self._extract_affected_versions(vuln), + "fixed_versions": self._extract_fixed_versions(vuln), + }) + + return {"vulnerabilities": vulnerabilities, "source": "OSV"} + else: + logger.warning(f"OSV API returned status {response.status_code}") + + except Exception as e: + logger.warning(f"OSV database scan failed: {e}") + + return {"vulnerabilities": [], "source": "OSV"} + + async def _scan_github_advisories(self, package_name: str, version: str) -> Dict[str, Any]: + """Scan against GitHub Security Advisories.""" + try: + # GitHub GraphQL API for security advisories + query = """ + query($ecosystem: SecurityAdvisoryEcosystem!, $package: String!) { + securityVulnerabilities(ecosystem: $ecosystem, package: $package, first: 100) { + nodes { + advisory { + ghsaId + summary + description + severity + publishedAt + updatedAt + references { + url + } + } + vulnerableVersionRange + firstPatchedVersion { + identifier + } + } + } + } + """ + + variables = { + "ecosystem": "PIP", + "package": package_name + } + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + "https://api.github.com/graphql", + json={"query": query, "variables": variables}, + headers={ + "Content-Type": "application/json", + "User-Agent": "PyPI-Security-Scanner/1.0" + } + ) + + if response.status_code == 200: + data = response.json() + vulnerabilities = [] + + for vuln_node in data.get("data", {}).get("securityVulnerabilities", {}).get("nodes", []): + advisory = vuln_node.get("advisory", {}) + + # Check if current version is affected + if self._is_version_affected(version, vuln_node.get("vulnerableVersionRange", "")): + vulnerabilities.append({ + "id": advisory.get("ghsaId", ""), + "summary": advisory.get("summary", ""), + "details": advisory.get("description", ""), + "severity": advisory.get("severity", "").lower(), + "published": advisory.get("publishedAt", ""), + "modified": advisory.get("updatedAt", ""), + "source": "GitHub", + "references": [ref.get("url", "") for ref in advisory.get("references", [])], + "vulnerable_range": vuln_node.get("vulnerableVersionRange", ""), + "first_patched": vuln_node.get("firstPatchedVersion", {}).get("identifier", ""), + }) + + return {"vulnerabilities": vulnerabilities, "source": "GitHub"} + + except Exception as e: + logger.warning(f"GitHub advisories scan failed: {e}") + + return {"vulnerabilities": [], "source": "GitHub"} + + async def _analyze_package_metadata(self, package_data: Dict[str, Any]) -> Dict[str, Any]: + """Analyze package metadata for security indicators.""" + info = package_data.get("info", {}) + + security_indicators = { + "metadata_score": 0, + "risk_factors": [], + "security_features": [], + "warnings": [] + } + + # Check for security-related information + description = (info.get("description") or "").lower() + summary = (info.get("summary") or "").lower() + keywords = (info.get("keywords") or "").lower() + + combined_text = f"{description} {summary} {keywords}" + + # Look for security mentions + if any(term in combined_text for term in ["security", "cryptography", "authentication", "encryption"]): + security_indicators["security_features"].append("Contains security-related functionality") + security_indicators["metadata_score"] += 20 + + # Check for high-risk patterns + for pattern in self.high_risk_patterns: + if pattern in combined_text: + security_indicators["risk_factors"].append(f"Mentions: {pattern}") + security_indicators["metadata_score"] -= 10 + + # Check package age and maintenance + if info.get("author_email"): + security_indicators["metadata_score"] += 10 + + if info.get("home_page"): + security_indicators["metadata_score"] += 5 + + # Check for classifiers + classifiers = info.get("classifiers", []) + for classifier in classifiers: + if "Development Status :: 5 - Production/Stable" in classifier: + security_indicators["metadata_score"] += 15 + security_indicators["security_features"].append("Production stable status") + elif "License ::" in classifier: + security_indicators["metadata_score"] += 5 + + # Check for suspicious patterns + if not info.get("author") and not info.get("maintainer"): + security_indicators["warnings"].append("No author or maintainer information") + security_indicators["metadata_score"] -= 20 + + if len(info.get("description", "")) < 50: + security_indicators["warnings"].append("Very brief or missing description") + security_indicators["metadata_score"] -= 10 + + return security_indicators + + async def _check_dependency_vulnerabilities(self, package_name: str, version: str) -> Dict[str, Any]: + """Check vulnerabilities in package dependencies.""" + try: + # Get package dependencies + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name, version) + + # Extract dependencies + requires_dist = package_data.get("info", {}).get("requires_dist", []) or [] + dependencies = [] + + for req in requires_dist: + # Parse dependency name (simplified) + dep_name = req.split()[0].split(">=")[0].split("==")[0].split("~=")[0].split("!=")[0] + if dep_name and not dep_name.startswith("extra"): + dependencies.append(dep_name) + + # Scan top dependencies for vulnerabilities + dependency_vulnerabilities = [] + + # Limit to top 10 dependencies to avoid overwhelming the system + for dep_name in dependencies[:10]: + try: + dep_scan = await self._scan_osv_database(dep_name, "latest") + for vuln in dep_scan.get("vulnerabilities", []): + vuln["dependency"] = dep_name + vuln["type"] = "dependency_vulnerability" + dependency_vulnerabilities.append(vuln) + except Exception as e: + logger.debug(f"Failed to scan dependency {dep_name}: {e}") + + return {"vulnerabilities": dependency_vulnerabilities, "source": "dependencies"} + + except Exception as e: + logger.warning(f"Dependency vulnerability check failed: {e}") + return {"vulnerabilities": [], "source": "dependencies"} + + async def _empty_result(self) -> Dict[str, Any]: + """Return empty result for disabled scans.""" + return {"vulnerabilities": [], "source": "disabled"} + + def _extract_severity_from_osv(self, vuln_data: Dict[str, Any]) -> str: + """Extract severity from OSV vulnerability data.""" + # OSV uses CVSS scores, map to common severity levels + severity_data = vuln_data.get("severity", []) + if severity_data: + score = severity_data[0].get("score", "") + if "CVSS:" in score: + # Extract CVSS score + try: + cvss_score = float(score.split("/")[1]) + if cvss_score >= 9.0: + return "critical" + elif cvss_score >= 7.0: + return "high" + elif cvss_score >= 4.0: + return "medium" + else: + return "low" + except: + pass + + return "unknown" + + def _extract_affected_versions(self, vuln_data: Dict[str, Any]) -> List[str]: + """Extract affected version ranges from vulnerability data.""" + affected = vuln_data.get("affected", []) + version_ranges = [] + + for affect in affected: + ranges = affect.get("ranges", []) + for range_data in ranges: + events = range_data.get("events", []) + for event in events: + if "introduced" in event: + version_ranges.append(f">= {event['introduced']}") + elif "fixed" in event: + version_ranges.append(f"< {event['fixed']}") + + return version_ranges + + def _extract_fixed_versions(self, vuln_data: Dict[str, Any]) -> List[str]: + """Extract fixed versions from vulnerability data.""" + affected = vuln_data.get("affected", []) + fixed_versions = [] + + for affect in affected: + ranges = affect.get("ranges", []) + for range_data in ranges: + events = range_data.get("events", []) + for event in events: + if "fixed" in event: + fixed_versions.append(event["fixed"]) + + return fixed_versions + + def _is_version_affected(self, version: str, vulnerable_range: str) -> bool: + """Check if a version is affected by a vulnerability range.""" + # Simplified version checking - in production would use packaging.specifiers + if not vulnerable_range: + return True + + # Basic patterns + if "< " in vulnerable_range: + try: + limit = vulnerable_range.split("< ")[1].strip() + return version < limit + except: + pass + + if ">= " in vulnerable_range: + try: + limit = vulnerable_range.split(">= ")[1].strip() + return version >= limit + except: + pass + + return True # Assume affected if we can't parse + + def _generate_security_report( + self, + package_name: str, + version: str, + vulnerabilities: List[Dict[str, Any]], + metadata_analysis: Dict[str, Any] + ) -> Dict[str, Any]: + """Generate comprehensive security report.""" + + # Categorize vulnerabilities by severity + severity_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "unknown": 0} + dependency_vulns = [] + direct_vulns = [] + + for vuln in vulnerabilities: + severity = vuln.get("severity", "unknown") + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + + if vuln.get("type") == "dependency_vulnerability": + dependency_vulns.append(vuln) + else: + direct_vulns.append(vuln) + + # Calculate risk score + risk_score = self._calculate_risk_score(severity_counts, metadata_analysis) + + # Generate recommendations + recommendations = self._generate_security_recommendations( + vulnerabilities, metadata_analysis, risk_score + ) + + return { + "package": package_name, + "version": version, + "scan_timestamp": datetime.now(timezone.utc).isoformat(), + "security_summary": { + "total_vulnerabilities": len(vulnerabilities), + "direct_vulnerabilities": len(direct_vulns), + "dependency_vulnerabilities": len(dependency_vulns), + "severity_breakdown": severity_counts, + "risk_score": risk_score, + "risk_level": self._get_risk_level(risk_score), + }, + "vulnerabilities": { + "direct": direct_vulns, + "dependencies": dependency_vulns, + }, + "metadata_analysis": metadata_analysis, + "recommendations": recommendations, + "scan_details": { + "sources_checked": ["OSV", "GitHub", "Metadata"], + "dependencies_scanned": len(dependency_vulns) > 0, + "scan_completion": "success", + } + } + + def _calculate_risk_score(self, severity_counts: Dict[str, int], metadata_analysis: Dict[str, Any]) -> float: + """Calculate overall risk score (0-100).""" + score = 0.0 + + # Vulnerability scoring (0-80 points) + score += severity_counts.get("critical", 0) * 20 + score += severity_counts.get("high", 0) * 15 + score += severity_counts.get("medium", 0) * 8 + score += severity_counts.get("low", 0) * 3 + + # Metadata scoring (0-20 points) + metadata_score = metadata_analysis.get("metadata_score", 0) + if metadata_score < 0: + score += abs(metadata_score) / 5 # Convert negative metadata score to risk + else: + score -= metadata_score / 10 # Good metadata reduces risk + + # Cap at 100 + return min(max(score, 0), 100) + + def _get_risk_level(self, risk_score: float) -> str: + """Convert risk score to risk level.""" + if risk_score >= 80: + return "critical" + elif risk_score >= 60: + return "high" + elif risk_score >= 30: + return "medium" + elif risk_score > 0: + return "low" + else: + return "minimal" + + def _generate_security_recommendations( + self, + vulnerabilities: List[Dict[str, Any]], + metadata_analysis: Dict[str, Any], + risk_score: float + ) -> List[str]: + """Generate actionable security recommendations.""" + recommendations = [] + + if len(vulnerabilities) > 0: + recommendations.append(f"🚨 Found {len(vulnerabilities)} security vulnerabilities - review and update immediately") + + # Check for critical/high severity + critical_high = [v for v in vulnerabilities if v.get("severity") in ["critical", "high"]] + if critical_high: + recommendations.append(f"⚠️ {len(critical_high)} critical/high severity vulnerabilities require immediate attention") + + # Check for fixed versions + fixed_versions = [] + for vuln in vulnerabilities: + fixed = vuln.get("fixed_versions", []) or [vuln.get("first_patched", "")] + fixed_versions.extend([v for v in fixed if v]) + + if fixed_versions: + latest_fixed = max(fixed_versions) if fixed_versions else None + if latest_fixed: + recommendations.append(f"📦 Update to version {latest_fixed} or later to fix known vulnerabilities") + + # Metadata recommendations + warnings = metadata_analysis.get("warnings", []) + if warnings: + recommendations.append(f"⚠️ Package metadata issues: {', '.join(warnings)}") + + if metadata_analysis.get("metadata_score", 0) < 20: + recommendations.append("📝 Package has poor metadata quality - verify trustworthiness before use") + + # General recommendations based on risk score + if risk_score >= 60: + recommendations.append("🛑 High risk package - consider alternatives or additional security review") + elif risk_score >= 30: + recommendations.append("⚠️ Moderate risk - monitor for updates and security patches") + elif len(vulnerabilities) == 0: + recommendations.append("✅ No known vulnerabilities found - package appears secure") + + return recommendations + + +# Main scanning functions +async def scan_package_security( + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True, + severity_filter: Optional[str] = None +) -> Dict[str, Any]: + """ + Scan a PyPI package for security vulnerabilities. + + Args: + package_name: Name of the package to scan + version: Specific version to scan (optional) + include_dependencies: Whether to scan dependencies + severity_filter: Filter by severity (low, medium, high, critical) + + Returns: + Comprehensive security scan results + """ + scanner = VulnerabilityScanner() + return await scanner.scan_package( + package_name, version, include_dependencies, severity_filter + ) + + +async def bulk_security_scan( + package_names: List[str], + include_dependencies: bool = False, + severity_threshold: str = "medium" +) -> Dict[str, Any]: + """ + Perform bulk security scanning of multiple packages. + + Args: + package_names: List of package names to scan + include_dependencies: Whether to scan dependencies + severity_threshold: Minimum severity to report + + Returns: + Bulk scan results with summary + """ + logger.info(f"Starting bulk security scan of {len(package_names)} packages") + + scanner = VulnerabilityScanner() + scan_results = {} + summary = { + "total_packages": len(package_names), + "packages_with_vulnerabilities": 0, + "total_vulnerabilities": 0, + "high_risk_packages": [], + "scan_timestamp": datetime.now(timezone.utc).isoformat() + } + + # Scan packages in parallel batches + batch_size = 5 + for i in range(0, len(package_names), batch_size): + batch = package_names[i:i + batch_size] + batch_tasks = [ + scanner.scan_package(pkg_name, include_dependencies=include_dependencies) + for pkg_name in batch + ] + + batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) + + for pkg_name, result in zip(batch, batch_results): + if isinstance(result, Exception): + scan_results[pkg_name] = { + "error": str(result), + "scan_status": "failed" + } + else: + scan_results[pkg_name] = result + + # Update summary + vuln_count = result.get("security_summary", {}).get("total_vulnerabilities", 0) + if vuln_count > 0: + summary["packages_with_vulnerabilities"] += 1 + summary["total_vulnerabilities"] += vuln_count + + risk_level = result.get("security_summary", {}).get("risk_level", "") + if risk_level in ["high", "critical"]: + summary["high_risk_packages"].append({ + "package": pkg_name, + "risk_level": risk_level, + "vulnerabilities": vuln_count + }) + + return { + "summary": summary, + "detailed_results": scan_results, + "recommendations": _generate_bulk_recommendations(summary, scan_results) + } + + +def _generate_bulk_recommendations(summary: Dict[str, Any], results: Dict[str, Any]) -> List[str]: + """Generate recommendations for bulk scan results.""" + recommendations = [] + + vuln_packages = summary["packages_with_vulnerabilities"] + total_packages = summary["total_packages"] + + if vuln_packages == 0: + recommendations.append("✅ No security vulnerabilities found in any scanned packages") + else: + percentage = (vuln_packages / total_packages) * 100 + recommendations.append( + f"🚨 {vuln_packages}/{total_packages} packages ({percentage:.1f}%) have security vulnerabilities" + ) + + high_risk = summary["high_risk_packages"] + if high_risk: + recommendations.append( + f"⚠️ {len(high_risk)} packages are high/critical risk: {', '.join([p['package'] for p in high_risk])}" + ) + recommendations.append("🛑 Priority: Address high-risk packages immediately") + + if summary["total_vulnerabilities"] > 0: + recommendations.append(f"📊 Total vulnerabilities found: {summary['total_vulnerabilities']}") + recommendations.append("🔍 Review detailed results and update affected packages") + + return recommendations \ No newline at end of file diff --git a/pypi_query_mcp/tools/security_tools.py b/pypi_query_mcp/tools/security_tools.py new file mode 100644 index 0000000..fb338ea --- /dev/null +++ b/pypi_query_mcp/tools/security_tools.py @@ -0,0 +1,147 @@ +"""Security vulnerability scanning tools for PyPI packages.""" + +import logging +from typing import Any, Dict, List, Optional + +from ..core.exceptions import InvalidPackageNameError, NetworkError, SearchError +from ..tools.security import bulk_security_scan, scan_package_security + +logger = logging.getLogger(__name__) + + +async def scan_pypi_package_security( + package_name: str, + version: Optional[str] = None, + include_dependencies: bool = True, + severity_filter: Optional[str] = None +) -> Dict[str, Any]: + """ + Scan a PyPI package for security vulnerabilities. + + This tool performs comprehensive security vulnerability scanning of PyPI packages, + checking against multiple vulnerability databases including OSV (Open Source Vulnerabilities), + GitHub Security Advisories, and analyzing package metadata for security indicators. + + Args: + package_name: Name of the package to scan for vulnerabilities + version: Specific version to scan (optional, defaults to latest version) + include_dependencies: Whether to scan package dependencies for vulnerabilities + severity_filter: Filter results by severity level (low, medium, high, critical) + + Returns: + Dictionary containing comprehensive security scan results including: + - Total vulnerability count and severity breakdown + - Direct package vulnerabilities vs dependency vulnerabilities + - Risk score and level assessment (minimal, low, medium, high, critical) + - Detailed vulnerability information with IDs, descriptions, and references + - Package metadata security analysis + - Actionable security recommendations + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + SearchError: If security scanning fails + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError(package_name) + + logger.info(f"MCP tool: Scanning security for package {package_name}") + + try: + result = await scan_package_security( + package_name=package_name, + version=version, + include_dependencies=include_dependencies, + severity_filter=severity_filter + ) + + logger.info(f"MCP tool: Security scan completed for {package_name} - found {result.get('security_summary', {}).get('total_vulnerabilities', 0)} vulnerabilities") + return result + + except (InvalidPackageNameError, NetworkError, SearchError) as e: + logger.error(f"Error scanning security for {package_name}: {e}") + return { + "error": f"Security scan failed: {e}", + "error_type": type(e).__name__, + "package": package_name, + "version": version, + "scan_timestamp": "", + "security_summary": { + "total_vulnerabilities": 0, + "direct_vulnerabilities": 0, + "dependency_vulnerabilities": 0, + "severity_breakdown": {"critical": 0, "high": 0, "medium": 0, "low": 0, "unknown": 0}, + "risk_score": 0, + "risk_level": "unknown", + }, + "vulnerabilities": {"direct": [], "dependencies": []}, + "metadata_analysis": {}, + "recommendations": [f"❌ Security scan failed: {e}"], + "scan_details": { + "sources_checked": [], + "dependencies_scanned": False, + "scan_completion": "error", + } + } + + +async def bulk_scan_package_security( + package_names: List[str], + include_dependencies: bool = False, + severity_threshold: str = "medium" +) -> Dict[str, Any]: + """ + Perform bulk security scanning of multiple PyPI packages. + + This tool scans multiple packages simultaneously for security vulnerabilities, + providing a consolidated report with summary statistics and prioritized + recommendations for addressing security issues across your package ecosystem. + + Args: + package_names: List of package names to scan for vulnerabilities + include_dependencies: Whether to include dependency vulnerability scanning + severity_threshold: Minimum severity level to report (low, medium, high, critical) + + Returns: + Dictionary containing bulk scan results including: + - Summary statistics (total packages, packages with vulnerabilities, high-risk packages) + - Detailed scan results for each package + - Prioritized recommendations for security remediation + - Scan timestamp and completion status + + Raises: + ValueError: If package_names list is empty + NetworkError: For network-related errors during scanning + SearchError: If bulk scanning fails + """ + if not package_names: + raise ValueError("Package names list cannot be empty") + + logger.info(f"MCP tool: Starting bulk security scan of {len(package_names)} packages") + + try: + result = await bulk_security_scan( + package_names=package_names, + include_dependencies=include_dependencies, + severity_threshold=severity_threshold + ) + + logger.info(f"MCP tool: Bulk security scan completed - {result.get('summary', {}).get('packages_with_vulnerabilities', 0)} packages have vulnerabilities") + return result + + except (ValueError, NetworkError, SearchError) as e: + logger.error(f"Error in bulk security scan: {e}") + return { + "error": f"Bulk security scan failed: {e}", + "error_type": type(e).__name__, + "summary": { + "total_packages": len(package_names), + "packages_with_vulnerabilities": 0, + "total_vulnerabilities": 0, + "high_risk_packages": [], + "scan_timestamp": "" + }, + "detailed_results": {}, + "recommendations": [f"❌ Bulk security scan failed: {e}"] + } \ No newline at end of file diff --git a/pypi_query_mcp/tools/workflow.py b/pypi_query_mcp/tools/workflow.py new file mode 100644 index 0000000..2eeba93 --- /dev/null +++ b/pypi_query_mcp/tools/workflow.py @@ -0,0 +1,1116 @@ +"""PyPI Development Workflow Tools for package validation and preview.""" + +import logging +import re +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import httpx + +from ..core import InvalidPackageNameError, NetworkError, PackageNotFoundError, PyPIClient +from ..core.exceptions import PyPIError + +logger = logging.getLogger(__name__) + + +class PyPIWorkflowError(PyPIError): + """Raised when workflow operations fail.""" + + def __init__(self, message: str, operation: str | None = None): + super().__init__(message) + self.operation = operation + + +def _validate_package_name_format(package_name: str) -> Dict[str, Any]: + """Validate package name format according to PyPI standards. + + Args: + package_name: Package name to validate + + Returns: + Dictionary with validation results + """ + issues = [] + recommendations = [] + + # Check basic format + if not package_name: + issues.append("Package name cannot be empty") + return { + "valid": False, + "issues": issues, + "recommendations": ["Provide a non-empty package name"], + "normalized_name": "", + } + + # Length constraints + if len(package_name) > 214: + issues.append("Package name is too long (max 214 characters)") + if len(package_name) < 1: + issues.append("Package name is too short (min 1 character)") + + # Character validation + if not re.match(r"^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$", package_name): + issues.append("Package name contains invalid characters (only alphanumeric, dots, hyphens, underscores allowed)") + + # Check for reserved names + reserved_names = { + "pip", "setuptools", "wheel", "python", "test", "tests", + "src", "docs", "build", "dist", "requirements", "readme" + } + if package_name.lower() in reserved_names: + issues.append(f"'{package_name}' is a reserved name and should not be used") + + # Best practice checks + if package_name != package_name.lower(): + recommendations.append("Use lowercase for package names (PEP 508)") + + if "_" in package_name: + recommendations.append("Consider using hyphens instead of underscores (PEP 508)") + + if len(package_name.split("-")) > 3: + recommendations.append("Avoid overly complex names with many hyphens") + + # Check for common typos/patterns + if package_name.startswith("-") or package_name.endswith("-"): + issues.append("Package name cannot start or end with a hyphen") + + if package_name.startswith(".") or package_name.endswith("."): + issues.append("Package name cannot start or end with a dot") + + if "--" in package_name or ".." in package_name: + issues.append("Package name cannot contain consecutive special characters") + + # Normalize name for PyPI (convert to lowercase, normalize separators) + normalized = re.sub(r"[-_.]+", "-", package_name.lower()) + + return { + "valid": len(issues) == 0, + "issues": issues, + "recommendations": recommendations, + "normalized_name": normalized, + } + + +async def validate_pypi_package_name(package_name: str) -> Dict[str, Any]: + """Check if a package name is available and valid on PyPI. + + This function validates package name format and checks availability on PyPI. + It also provides recommendations for improvement based on PyPI standards. + + Args: + package_name: Name to validate and check for availability + + Returns: + Dictionary containing validation results including: + - Format validation results + - Availability status on PyPI + - Recommendations for improvement + - Similar existing packages (if any) + + Raises: + InvalidPackageNameError: If package name format is severely invalid + NetworkError: For network-related errors + PyPIWorkflowError: For workflow-specific errors + """ + logger.info(f"Validating package name: {package_name}") + + try: + # First validate format + format_validation = _validate_package_name_format(package_name) + + if not format_validation["valid"] and len(format_validation["issues"]) > 2: + raise InvalidPackageNameError(package_name) + + # Check availability on PyPI + availability_status = "unknown" + existing_package_info = None + similar_packages = [] + + try: + async with PyPIClient() as client: + # Try to get package info to check if it exists + package_data = await client.get_package_info(package_name) + availability_status = "taken" + existing_package_info = { + "name": package_data["info"]["name"], + "version": package_data["info"]["version"], + "summary": package_data["info"]["summary"], + "upload_time": package_data["info"].get("upload_time"), + "author": package_data["info"].get("author"), + } + except PackageNotFoundError: + availability_status = "available" + except Exception as e: + logger.warning(f"Could not check availability for {package_name}: {e}") + availability_status = "unknown" + + # If package is taken or format is questionable, find similar packages + if availability_status == "taken" or not format_validation["valid"]: + try: + # Search for similar packages + search_query = format_validation["normalized_name"] + async with httpx.AsyncClient(timeout=10.0) as http_client: + search_url = f"https://pypi.org/search/?q={quote(search_query)}" + # Note: We're not actually parsing HTML here - this would need + # integration with the search functionality for real similar packages + # For now, we'll just note that similar package detection is available + similar_packages = [] # Placeholder + except Exception as e: + logger.warning(f"Could not search for similar packages: {e}") + + # Generate suggestions based on validation results + suggestions = [] + if availability_status == "taken": + suggestions.extend([ + f"Try variations like '{package_name}-dev', '{package_name}-cli', or '{package_name}2'", + "Consider a more specific or descriptive name", + "Add your organization name as a prefix" + ]) + + if format_validation["recommendations"]: + suggestions.extend(format_validation["recommendations"]) + + return { + "package_name": package_name, + "normalized_name": format_validation["normalized_name"], + "validation": { + "format_valid": format_validation["valid"], + "issues": format_validation["issues"], + "recommendations": format_validation["recommendations"], + }, + "availability": { + "status": availability_status, # "available", "taken", "unknown" + "existing_package": existing_package_info, + "similar_packages": similar_packages, + }, + "suggestions": suggestions, + "pypi_standards_compliant": format_validation["valid"] and availability_status == "available", + "ready_for_upload": format_validation["valid"] and availability_status == "available", + } + + except InvalidPackageNameError: + raise + except Exception as e: + logger.error(f"Error validating package name {package_name}: {e}") + raise PyPIWorkflowError(f"Failed to validate package name: {e}", "validate_name") from e + + +async def preview_pypi_package_page( + package_name: str, + version: str = "1.0.0", + summary: str = "", + description: str = "", + author: str = "", + license_name: str = "MIT", + home_page: str = "", + keywords: List[str] = None, + classifiers: List[str] = None, +) -> Dict[str, Any]: + """Generate a preview of how a package page would look on PyPI. + + This function creates a preview of the PyPI package page based on the + provided metadata, helping developers visualize their package before upload. + + Args: + package_name: Name of the package + version: Package version (default: "1.0.0") + summary: Short package description + description: Long package description + author: Package author name + license_name: License type (default: "MIT") + home_page: Project homepage URL + keywords: List of keywords for the package + classifiers: List of PyPI classifiers + + Returns: + Dictionary containing preview information including: + - Formatted package metadata + - Rendered page sections + - Validation warnings + - SEO recommendations + + Raises: + InvalidPackageNameError: If package name is invalid + PyPIWorkflowError: For preview generation errors + """ + logger.info(f"Generating preview for package: {package_name}") + + if keywords is None: + keywords = [] + if classifiers is None: + classifiers = [] + + try: + # Validate package name first + name_validation = _validate_package_name_format(package_name) + if not name_validation["valid"]: + raise InvalidPackageNameError(package_name) + + # Validate version format + version_pattern = r"^([0-9]+)\.([0-9]+)\.([0-9]+)(?:[-.]?(a|b|rc)[0-9]*)?(?:\.post[0-9]+)?(?:\.dev[0-9]+)?$" + version_valid = bool(re.match(version_pattern, version)) + + # Generate preview content + preview_sections = { + "header": { + "name": package_name, + "version": version, + "summary": summary or "No description provided", + "upload_date": "Not yet uploaded", + "author": author or "Not specified", + }, + "navigation": { + "project_description": True, + "release_history": False, # Will be available after first upload + "download_files": False, # Will be available after first upload + "statistics": False, # Will be available after upload + }, + "metadata": { + "license": license_name, + "home_page": home_page or "Not provided", + "keywords": keywords, + "classifiers": classifiers, + "requires_python": "Not specified", + "project_urls": {}, + }, + "description": { + "content": description or "No detailed description provided", + "format": "text/plain" if description and not description.strip().startswith("<") else "text/markdown", + "length": len(description) if description else 0, + }, + } + + # Generate recommendations and warnings + recommendations = [] + warnings = [] + + # Check for missing critical information + if not summary: + warnings.append("Summary is missing - this appears prominently in search results") + elif len(summary) < 10: + recommendations.append("Consider a more descriptive summary (current: {len(summary)} chars)") + elif len(summary) > 200: + warnings.append("Summary is quite long - consider keeping it under 200 characters") + + if not description: + warnings.append("No description provided - users won't understand what your package does") + elif len(description) < 100: + recommendations.append("Consider providing a more detailed description") + + if not author: + warnings.append("Author name is missing") + + if not home_page: + recommendations.append("Consider adding a homepage URL (GitHub, documentation, etc.)") + + if not keywords: + recommendations.append("Add keywords to improve discoverability") + elif len(keywords) > 10: + recommendations.append("Consider reducing keywords to the most relevant ones") + + if not classifiers: + recommendations.append("Add PyPI classifiers to categorize your package") + else: + # Check for important classifier categories + has_python_version = any("Python ::" in c for c in classifiers) + has_topic = any("Topic ::" in c for c in classifiers) + has_development_status = any("Development Status ::" in c for c in classifiers) + + if not has_python_version: + recommendations.append("Add Python version classifiers (e.g., 'Programming Language :: Python :: 3.8')") + if not has_topic: + recommendations.append("Add topic classifiers to categorize your package") + if not has_development_status: + recommendations.append("Add development status classifier") + + if not version_valid: + warnings.append("Version format may not follow PEP 440 standards") + + # SEO and discoverability analysis + seo_analysis = { + "title_optimization": { + "length": len(package_name), + "readability": "good" if len(package_name.split("-")) <= 3 else "complex", + "keyword_rich": len(keywords) > 0, + }, + "description_optimization": { + "length": len(description) if description else 0, + "has_keywords": any(kw.lower() in description.lower() for kw in keywords) if description and keywords else False, + "structure": "needs_improvement" if not description else "basic", + }, + "discoverability_score": _calculate_discoverability_score( + summary, description, keywords, classifiers + ), + } + + # Generate rendered preview (simplified HTML-like structure) + rendered_preview = _generate_html_preview(preview_sections) + + return { + "package_name": package_name, + "version": version, + "preview": { + "sections": preview_sections, + "rendered_html": rendered_preview, + "url_preview": f"https://pypi.org/project/{package_name}/", + }, + "validation": { + "name_valid": name_validation["valid"], + "version_valid": version_valid, + "completeness_score": _calculate_completeness_score(preview_sections), + }, + "recommendations": recommendations, + "warnings": warnings, + "seo_analysis": seo_analysis, + "ready_for_upload": len(warnings) == 0 and name_validation["valid"] and version_valid, + } + + except InvalidPackageNameError: + raise + except Exception as e: + logger.error(f"Error generating preview for {package_name}: {e}") + raise PyPIWorkflowError(f"Failed to generate preview: {e}", "preview_page") from e + + +def _calculate_discoverability_score( + summary: str, description: str, keywords: List[str], classifiers: List[str] +) -> Dict[str, Any]: + """Calculate a discoverability score based on metadata completeness.""" + score = 0 + max_score = 100 + + # Summary scoring (25 points) + if summary: + if len(summary) >= 20: + score += 25 + elif len(summary) >= 10: + score += 15 + else: + score += 5 + + # Description scoring (35 points) + if description: + if len(description) >= 500: + score += 35 + elif len(description) >= 200: + score += 25 + elif len(description) >= 50: + score += 15 + else: + score += 5 + + # Keywords scoring (20 points) + if keywords: + if len(keywords) >= 5: + score += 20 + elif len(keywords) >= 3: + score += 15 + else: + score += 10 + + # Classifiers scoring (20 points) + if classifiers: + if len(classifiers) >= 5: + score += 20 + elif len(classifiers) >= 3: + score += 15 + else: + score += 10 + + level = "excellent" if score >= 80 else "good" if score >= 60 else "fair" if score >= 40 else "poor" + + return { + "score": score, + "max_score": max_score, + "percentage": round((score / max_score) * 100, 1), + "level": level, + } + + +def _calculate_completeness_score(preview_sections: Dict[str, Any]) -> Dict[str, Any]: + """Calculate completeness score based on available metadata.""" + score = 0 + max_score = 100 + + header = preview_sections["header"] + metadata = preview_sections["metadata"] + description = preview_sections["description"] + + # Essential fields (60 points) + if header["summary"] and header["summary"] != "No description provided": + score += 20 + if header["author"] and header["author"] != "Not specified": + score += 15 + if description["content"] and description["content"] != "No detailed description provided": + score += 25 + + # Important fields (30 points) + if metadata["license"]: + score += 10 + if metadata["home_page"] and metadata["home_page"] != "Not provided": + score += 10 + if metadata["keywords"]: + score += 5 + if metadata["classifiers"]: + score += 5 + + # Nice-to-have fields (10 points) + if description["length"] > 200: + score += 5 + if len(metadata["keywords"]) >= 3: + score += 5 + + level = "complete" if score >= 80 else "good" if score >= 60 else "basic" if score >= 40 else "incomplete" + + return { + "score": score, + "max_score": max_score, + "percentage": round((score / max_score) * 100, 1), + "level": level, + } + + +def _generate_html_preview(preview_sections: Dict[str, Any]) -> str: + """Generate a simplified HTML preview of the PyPI page.""" + header = preview_sections["header"] + metadata = preview_sections["metadata"] + description = preview_sections["description"] + + html = f""" + + +
+{header['summary']}
+ +