MCP/pypi-query-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

chore: upgrade all Python packages and fix linting issues

Browse source 
- Update all dependencies to latest versions (fastmcp, httpx, packaging, etc.)
- Downgrade click from yanked 8.2.2 to stable 8.1.7
- Fix code formatting and linting issues with ruff
- Most tests passing (2 test failures in dependency resolver need investigation)
This commit is contained in:
Ryan Malloy 2025-08-15 20:23:14 -06:00
parent 503ea589f1
commit 8b43927493
34 changed files with 2276 additions and 1593 deletions
Download patch file Download diff file Expand all files Collapse all files

27
pypi_query_mcp/core/dependency_parser.py
View file

@ -117,12 +117,29 @@ class DependencyParser:
Dictionary with categorized dependencies
"""
categories = {"runtime": [], "development": [], "optional": {}, "extras": {}}
# Define development-related extra names
dev_extra_names = {
'dev', 'development', 'test', 'testing', 'tests', 'lint', 'linting',
'doc', 'docs', 'documentation', 'build', 'check', 'cover', 'coverage',
'type', 'typing', 'mypy', 'style', 'format', 'quality'
"dev",
"development",
"test",
"testing",
"tests",
"lint",
"linting",
"doc",
"docs",
"documentation",
"build",
"check",
"cover",
"coverage",
"type",
"typing",
"mypy",
"style",
"format",
"quality",
}
for req in requirements:
@ -141,7 +158,7 @@ class DependencyParser:
if extra_name not in categories["extras"]:
categories["extras"][extra_name] = []
categories["extras"][extra_name].append(req)
# Check if this extra is development-related
if extra_name.lower() in dev_extra_names:
categories["development"].append(req)

65
pypi_query_mcp/core/github_client.py
View file

@ -2,7 +2,7 @@
import asyncio
import logging
from typing import Any, Dict, Optional
from typing import Any
import httpx
@ -17,7 +17,7 @@ class GitHubAPIClient:
timeout: float = 10.0,
max_retries: int = 2,
retry_delay: float = 1.0,
github_token: Optional[str] = None,
github_token: str | None = None,
):
"""Initialize GitHub API client.
@ -33,7 +33,7 @@ class GitHubAPIClient:
self.retry_delay = retry_delay
# Simple in-memory cache for repository data
self._cache: Dict[str, Dict[str, Any]] = {}
self._cache: dict[str, dict[str, Any]] = {}
self._cache_ttl = 3600 # 1 hour cache
# HTTP client configuration
@ -41,7 +41,7 @@ class GitHubAPIClient:
"Accept": "application/vnd.github.v3+json",
"User-Agent": "pypi-query-mcp-server/0.1.0",
}
if github_token:
headers["Authorization"] = f"token {github_token}"
@ -67,12 +67,13 @@ class GitHubAPIClient:
"""Generate cache key for repository data."""
return f"repo:{repo}"
def _is_cache_valid(self, cache_entry: Dict[str, Any]) -> bool:
def _is_cache_valid(self, cache_entry: dict[str, Any]) -> bool:
"""Check if cache entry is still valid."""
import time
return time.time() - cache_entry.get("timestamp", 0) < self._cache_ttl
async def _make_request(self, url: str) -> Optional[Dict[str, Any]]:
async def _make_request(self, url: str) -> dict[str, Any] | None:
"""Make HTTP request with retry logic and error handling.
Args:
@ -85,7 +86,9 @@ class GitHubAPIClient:
for attempt in range(self.max_retries + 1):
try:
logger.debug(f"Making GitHub API request to {url} (attempt {attempt + 1})")
logger.debug(
f"Making GitHub API request to {url} (attempt {attempt + 1})"
)
response = await self._client.get(url)
@ -100,12 +103,16 @@ class GitHubAPIClient:
logger.warning(f"GitHub API rate limit or permission denied: {url}")
return None
elif response.status_code >= 500:
logger.warning(f"GitHub API server error {response.status_code}: {url}")
logger.warning(
f"GitHub API server error {response.status_code}: {url}"
)
if attempt < self.max_retries:
continue
return None
else:
logger.warning(f"Unexpected GitHub API status {response.status_code}: {url}")
logger.warning(
f"Unexpected GitHub API status {response.status_code}: {url}"
)
return None
except httpx.TimeoutException:
@ -120,13 +127,17 @@ class GitHubAPIClient:
# Wait before retry (except on last attempt)
if attempt < self.max_retries:
await asyncio.sleep(self.retry_delay * (2 ** attempt))
await asyncio.sleep(self.retry_delay * (2**attempt))
# If we get here, all retries failed
logger.error(f"Failed to fetch GitHub data after {self.max_retries + 1} attempts: {last_exception}")
logger.error(
f"Failed to fetch GitHub data after {self.max_retries + 1} attempts: {last_exception}"
)
return None
async def get_repository_stats(self, repo_path: str, use_cache: bool = True) -> Optional[Dict[str, Any]]:
async def get_repository_stats(
self, repo_path: str, use_cache: bool = True
) -> dict[str, Any] | None:
"""Get repository statistics from GitHub API.
Args:
@ -147,10 +158,10 @@ class GitHubAPIClient:
# Make API request
url = f"{self.base_url}/repos/{repo_path}"
try:
data = await self._make_request(url)
if data:
# Extract relevant statistics
stats = {
@ -171,14 +182,19 @@ class GitHubAPIClient:
"has_wiki": data.get("has_wiki", False),
"archived": data.get("archived", False),
"disabled": data.get("disabled", False),
"license": data.get("license", {}).get("name") if data.get("license") else None,
"license": data.get("license", {}).get("name")
if data.get("license")
else None,
}
# Cache the result
import time
self._cache[cache_key] = {"data": stats, "timestamp": time.time()}
logger.debug(f"Fetched GitHub stats for {repo_path}: {stats['stars']} stars")
logger.debug(
f"Fetched GitHub stats for {repo_path}: {stats['stars']} stars"
)
return stats
else:
return None
@ -188,11 +204,8 @@ class GitHubAPIClient:
return None
async def get_multiple_repo_stats(
self,
repo_paths: list[str],
use_cache: bool = True,
max_concurrent: int = 5
) -> Dict[str, Optional[Dict[str, Any]]]:
self, repo_paths: list[str], use_cache: bool = True, max_concurrent: int = 5
) -> dict[str, dict[str, Any] | None]:
"""Get statistics for multiple repositories concurrently.
Args:
@ -205,7 +218,7 @@ class GitHubAPIClient:
"""
semaphore = asyncio.Semaphore(max_concurrent)
async def fetch_repo_stats(repo_path: str) -> tuple[str, Optional[Dict[str, Any]]]:
async def fetch_repo_stats(repo_path: str) -> tuple[str, dict[str, Any] | None]:
async with semaphore:
stats = await self.get_repository_stats(repo_path, use_cache)
return repo_path, stats
@ -220,7 +233,7 @@ class GitHubAPIClient:
if isinstance(result, Exception):
logger.error(f"Error in concurrent GitHub fetch: {result}")
continue
repo_path, stats = result
repo_stats[repo_path] = stats
@ -231,14 +244,14 @@ class GitHubAPIClient:
self._cache.clear()
logger.debug("GitHub cache cleared")
async def get_rate_limit(self) -> Optional[Dict[str, Any]]:
async def get_rate_limit(self) -> dict[str, Any] | None:
"""Get current GitHub API rate limit status.
Returns:
Dictionary containing rate limit information
"""
url = f"{self.base_url}/rate_limit"
try:
data = await self._make_request(url)
if data:
@ -246,4 +259,4 @@ class GitHubAPIClient:
return None
except Exception as e:
logger.error(f"Error fetching GitHub rate limit: {e}")
return None
return None

30
pypi_query_mcp/core/pypi_client.py
View file

@ -182,7 +182,7 @@ class PyPIClient:
NetworkError: For network-related errors
"""
normalized_name = self._validate_package_name(package_name)
# Create cache key that includes version info
cache_suffix = f"v{version}" if version else "latest"
cache_key = self._get_cache_key(normalized_name, f"info_{cache_suffix}")
@ -191,13 +191,17 @@ class PyPIClient:
if use_cache and cache_key in self._cache:
cache_entry = self._cache[cache_key]
if self._is_cache_valid(cache_entry):
logger.debug(f"Using cached data for package: {normalized_name} version: {version or 'latest'}")
logger.debug(
f"Using cached data for package: {normalized_name} version: {version or 'latest'}"
)
return cache_entry["data"]
# Build URL - include version if specified
if version:
url = f"{self.base_url}/{quote(normalized_name)}/{quote(version)}/json"
logger.info(f"Fetching package info for: {normalized_name} version {version}")
logger.info(
f"Fetching package info for: {normalized_name} version {version}"
)
else:
url = f"{self.base_url}/{quote(normalized_name)}/json"
logger.info(f"Fetching package info for: {normalized_name} (latest)")
@ -215,13 +219,19 @@ class PyPIClient:
except PackageNotFoundError as e:
if version:
# More specific error message for version not found
logger.error(f"Version {version} not found for package {normalized_name}")
raise PackageNotFoundError(f"Version {version} not found for package {normalized_name}")
logger.error(
f"Version {version} not found for package {normalized_name}"
)
raise PackageNotFoundError(
f"Version {version} not found for package {normalized_name}"
)
else:
logger.error(f"Failed to fetch package info for {normalized_name}: {e}")
raise
except Exception as e:
logger.error(f"Failed to fetch package info for {normalized_name} version {version or 'latest'}: {e}")
logger.error(
f"Failed to fetch package info for {normalized_name} version {version or 'latest'}: {e}"
)
raise
async def get_package_versions(
@ -236,7 +246,9 @@ class PyPIClient:
Returns:
List of version strings
"""
package_info = await self.get_package_info(package_name, version=None, use_cache=use_cache)
package_info = await self.get_package_info(
package_name, version=None, use_cache=use_cache
)
releases = package_info.get("releases", {})
return list(releases.keys())
@ -252,7 +264,9 @@ class PyPIClient:
Returns:
Latest version string
"""
package_info = await self.get_package_info(package_name, version=None, use_cache=use_cache)
package_info = await self.get_package_info(
package_name, version=None, use_cache=use_cache
)
return package_info.get("info", {}).get("version", "")
def clear_cache(self):

224
pypi_query_mcp/core/stats_client.py
View file

@ -5,7 +5,7 @@ import logging
import random
import time
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from typing import Any
import httpx
@ -50,7 +50,7 @@ class PyPIStatsClient:
self._cache: dict[str, dict[str, Any]] = {}
self._cache_ttl = 86400 # 24 hours (increased for resilience)
self._fallback_cache_ttl = 604800 # 7 days for fallback data
# Track API health for smart fallback decisions
self._api_health = {
"last_success": None,
@ -106,31 +106,33 @@ class PyPIStatsClient:
)
return f"{endpoint}:{package_name}:{param_str}"
def _is_cache_valid(self, cache_entry: dict[str, Any], fallback: bool = False) -> bool:
def _is_cache_valid(
self, cache_entry: dict[str, Any], fallback: bool = False
) -> bool:
"""Check if cache entry is still valid.
Args:
cache_entry: Cache entry to validate
fallback: Whether to use fallback cache TTL (longer for resilience)
"""
ttl = self._fallback_cache_ttl if fallback else self._cache_ttl
return time.time() - cache_entry.get("timestamp", 0) < ttl
def _should_use_fallback(self) -> bool:
"""Determine if fallback mechanisms should be used based on API health."""
if not self.fallback_enabled:
return False
# Use fallback if we've had multiple consecutive failures
if self._api_health["consecutive_failures"] >= 3:
return True
# Use fallback if last success was more than 1 hour ago
if self._api_health["last_success"]:
time_since_success = time.time() - self._api_health["last_success"]
if time_since_success > 3600: # 1 hour
return True
return False
async def _make_request(self, url: str) -> dict[str, Any]:
@ -152,7 +154,9 @@ class PyPIStatsClient:
for attempt in range(self.max_retries + 1):
try:
logger.debug(f"Making request to {url} (attempt {attempt + 1}/{self.max_retries + 1})")
logger.debug(
f"Making request to {url} (attempt {attempt + 1}/{self.max_retries + 1})"
)
response = await self._client.get(url)
@ -171,16 +175,25 @@ class PyPIStatsClient:
elif response.status_code == 429:
retry_after = response.headers.get("Retry-After")
retry_after_int = int(retry_after) if retry_after else None
self._update_api_failure(f"Rate limit exceeded (retry after {retry_after_int}s)")
self._update_api_failure(
f"Rate limit exceeded (retry after {retry_after_int}s)"
)
raise RateLimitError(retry_after_int)
elif response.status_code >= 500:
error_msg = f"Server error: HTTP {response.status_code}"
self._update_api_failure(error_msg)
# For 502/503/504 errors, continue retrying
if response.status_code in [502, 503, 504] and attempt < self.max_retries:
last_exception = PyPIServerError(response.status_code, error_msg)
logger.warning(f"Retryable server error {response.status_code}, attempt {attempt + 1}")
if (
response.status_code in [502, 503, 504]
and attempt < self.max_retries
):
last_exception = PyPIServerError(
response.status_code, error_msg
)
logger.warning(
f"Retryable server error {response.status_code}, attempt {attempt + 1}"
)
else:
raise PyPIServerError(response.status_code, error_msg)
else:
@ -205,7 +218,9 @@ class PyPIStatsClient:
# Only retry certain server errors
if e.status_code in [502, 503, 504] and attempt < self.max_retries:
last_exception = e
logger.warning(f"Retrying server error {e.status_code}, attempt {attempt + 1}")
logger.warning(
f"Retrying server error {e.status_code}, attempt {attempt + 1}"
)
else:
raise
except Exception as e:
@ -216,7 +231,7 @@ class PyPIStatsClient:
# Calculate exponential backoff with jitter
if attempt < self.max_retries:
base_delay = self.retry_delay * (2 ** attempt)
base_delay = self.retry_delay * (2**attempt)
jitter = random.uniform(0.1, 0.3) * base_delay # Add 10-30% jitter
delay = base_delay + jitter
logger.debug(f"Waiting {delay:.2f}s before retry...")
@ -227,21 +242,25 @@ class PyPIStatsClient:
raise last_exception
else:
raise NetworkError("All retry attempts failed with unknown error")
def _update_api_failure(self, error_msg: str) -> None:
"""Update API health tracking on failure."""
self._api_health["consecutive_failures"] += 1
self._api_health["last_error"] = error_msg
logger.debug(f"API failure count: {self._api_health['consecutive_failures']}, error: {error_msg}")
def _generate_fallback_recent_downloads(self, package_name: str, period: str = "month") -> dict[str, Any]:
logger.debug(
f"API failure count: {self._api_health['consecutive_failures']}, error: {error_msg}"
)
def _generate_fallback_recent_downloads(
self, package_name: str, period: str = "month"
) -> dict[str, Any]:
"""Generate fallback download statistics when API is unavailable.
This provides estimated download counts based on package popularity patterns
to ensure the system remains functional during API outages.
"""
logger.warning(f"Generating fallback download data for {package_name}")
# Base estimates for popular packages (these are conservative estimates)
popular_packages = {
"requests": {"day": 1500000, "week": 10500000, "month": 45000000},
@ -270,39 +289,50 @@ class PyPIStatsClient:
"pandas": {"day": 200000, "week": 1400000, "month": 6000000},
"sqlalchemy": {"day": 90000, "week": 630000, "month": 2700000},
}
# Get estimates for known packages or generate based on package name characteristics
if package_name.lower() in popular_packages:
estimates = popular_packages[package_name.lower()]
else:
# Generate estimates based on common package patterns
if any(keyword in package_name.lower() for keyword in ["test", "dev", "debug"]):
if any(
keyword in package_name.lower() for keyword in ["test", "dev", "debug"]
):
# Development/testing packages - lower usage
base_daily = random.randint(100, 1000)
elif any(keyword in package_name.lower() for keyword in ["aws", "google", "microsoft", "azure"]):
elif any(
keyword in package_name.lower()
for keyword in ["aws", "google", "microsoft", "azure"]
):
# Cloud provider packages - higher usage
base_daily = random.randint(10000, 50000)
elif any(keyword in package_name.lower() for keyword in ["http", "request", "client", "api"]):
elif any(
keyword in package_name.lower()
for keyword in ["http", "request", "client", "api"]
):
# HTTP/API packages - moderate to high usage
base_daily = random.randint(5000, 25000)
elif any(keyword in package_name.lower() for keyword in ["data", "pandas", "numpy", "scipy"]):
elif any(
keyword in package_name.lower()
for keyword in ["data", "pandas", "numpy", "scipy"]
):
# Data science packages - high usage
base_daily = random.randint(15000, 75000)
else:
# Generic packages - moderate usage
base_daily = random.randint(1000, 10000)
estimates = {
"day": base_daily,
"week": base_daily * 7,
"month": base_daily * 30,
}
# Add some realistic variation (±20%)
variation = random.uniform(0.8, 1.2)
for key in estimates:
estimates[key] = int(estimates[key] * variation)
return {
"data": {
"last_day": estimates["day"],
@ -314,42 +344,48 @@ class PyPIStatsClient:
"source": "fallback_estimates",
"note": "Estimated data due to API unavailability. Actual values may differ.",
}
def _generate_fallback_overall_downloads(self, package_name: str, mirrors: bool = False) -> dict[str, Any]:
def _generate_fallback_overall_downloads(
self, package_name: str, mirrors: bool = False
) -> dict[str, Any]:
"""Generate fallback time series data when API is unavailable."""
logger.warning(f"Generating fallback time series data for {package_name}")
# Generate 180 days of synthetic time series data
time_series = []
base_date = datetime.now() - timedelta(days=180)
# Get base daily estimate from recent downloads fallback
recent_fallback = self._generate_fallback_recent_downloads(package_name)
base_daily = recent_fallback["data"]["last_day"]
for i in range(180):
current_date = base_date + timedelta(days=i)
# Add weekly and seasonal patterns
day_of_week = current_date.weekday()
# Lower downloads on weekends
week_factor = 0.7 if day_of_week >= 5 else 1.0
# Add some growth trend (packages generally grow over time)
growth_factor = 1.0 + (i / 180) * 0.3 # 30% growth over 180 days
# Add random daily variation
daily_variation = random.uniform(0.7, 1.3)
daily_downloads = int(base_daily * week_factor * growth_factor * daily_variation)
daily_downloads = int(
base_daily * week_factor * growth_factor * daily_variation
)
category = "with_mirrors" if mirrors else "without_mirrors"
time_series.append({
"category": category,
"date": current_date.strftime("%Y-%m-%d"),
"downloads": daily_downloads,
})
time_series.append(
{
"category": category,
"date": current_date.strftime("%Y-%m-%d"),
"downloads": daily_downloads,
}
)
return {
"data": time_series,
"package": package_name,
@ -385,16 +421,24 @@ class PyPIStatsClient:
if self._is_cache_valid(cache_entry):
logger.debug(f"Using cached recent downloads for: {normalized_name}")
return cache_entry["data"]
elif self._should_use_fallback() and self._is_cache_valid(cache_entry, fallback=True):
logger.info(f"Using extended cache (fallback mode) for: {normalized_name}")
elif self._should_use_fallback() and self._is_cache_valid(
cache_entry, fallback=True
):
logger.info(
f"Using extended cache (fallback mode) for: {normalized_name}"
)
cache_entry["data"]["note"] = "Extended cache data due to API issues"
return cache_entry["data"]
# Check if we should use fallback immediately
if self._should_use_fallback():
logger.warning(f"API health poor, using fallback data for: {normalized_name}")
fallback_data = self._generate_fallback_recent_downloads(normalized_name, period)
logger.warning(
f"API health poor, using fallback data for: {normalized_name}"
)
fallback_data = self._generate_fallback_recent_downloads(
normalized_name, period
)
# Cache fallback data with extended TTL
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
return fallback_data
@ -418,28 +462,39 @@ class PyPIStatsClient:
except (PyPIServerError, NetworkError) as e:
logger.error(f"API request failed for {normalized_name}: {e}")
# Try to use stale cache data if available
if use_cache and cache_key in self._cache:
cache_entry = self._cache[cache_key]
logger.warning(f"Using stale cache data for {normalized_name} due to API failure")
logger.warning(
f"Using stale cache data for {normalized_name} due to API failure"
)
cache_entry["data"]["note"] = f"Stale cache data due to API error: {e}"
return cache_entry["data"]
# Last resort: generate fallback data
if self.fallback_enabled:
logger.warning(f"Generating fallback data for {normalized_name} due to API failure")
fallback_data = self._generate_fallback_recent_downloads(normalized_name, period)
logger.warning(
f"Generating fallback data for {normalized_name} due to API failure"
)
fallback_data = self._generate_fallback_recent_downloads(
normalized_name, period
)
# Cache fallback data
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
self._cache[cache_key] = {
"data": fallback_data,
"timestamp": time.time(),
}
return fallback_data
# If fallback is disabled, re-raise the original exception
raise
except Exception as e:
logger.error(f"Unexpected error fetching recent downloads for {normalized_name}: {e}")
logger.error(
f"Unexpected error fetching recent downloads for {normalized_name}: {e}"
)
raise
async def get_overall_downloads(
@ -469,16 +524,24 @@ class PyPIStatsClient:
if self._is_cache_valid(cache_entry):
logger.debug(f"Using cached overall downloads for: {normalized_name}")
return cache_entry["data"]
elif self._should_use_fallback() and self._is_cache_valid(cache_entry, fallback=True):
logger.info(f"Using extended cache (fallback mode) for: {normalized_name}")
elif self._should_use_fallback() and self._is_cache_valid(
cache_entry, fallback=True
):
logger.info(
f"Using extended cache (fallback mode) for: {normalized_name}"
)
cache_entry["data"]["note"] = "Extended cache data due to API issues"
return cache_entry["data"]
# Check if we should use fallback immediately
if self._should_use_fallback():
logger.warning(f"API health poor, using fallback data for: {normalized_name}")
fallback_data = self._generate_fallback_overall_downloads(normalized_name, mirrors)
logger.warning(
f"API health poor, using fallback data for: {normalized_name}"
)
fallback_data = self._generate_fallback_overall_downloads(
normalized_name, mirrors
)
# Cache fallback data with extended TTL
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
return fallback_data
@ -502,28 +565,39 @@ class PyPIStatsClient:
except (PyPIServerError, NetworkError) as e:
logger.error(f"API request failed for {normalized_name}: {e}")
# Try to use stale cache data if available
if use_cache and cache_key in self._cache:
cache_entry = self._cache[cache_key]
logger.warning(f"Using stale cache data for {normalized_name} due to API failure")
logger.warning(
f"Using stale cache data for {normalized_name} due to API failure"
)
cache_entry["data"]["note"] = f"Stale cache data due to API error: {e}"
return cache_entry["data"]
# Last resort: generate fallback data
if self.fallback_enabled:
logger.warning(f"Generating fallback data for {normalized_name} due to API failure")
fallback_data = self._generate_fallback_overall_downloads(normalized_name, mirrors)
logger.warning(
f"Generating fallback data for {normalized_name} due to API failure"
)
fallback_data = self._generate_fallback_overall_downloads(
normalized_name, mirrors
)
# Cache fallback data
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
self._cache[cache_key] = {
"data": fallback_data,
"timestamp": time.time(),
}
return fallback_data
# If fallback is disabled, re-raise the original exception
raise
except Exception as e:
logger.error(f"Unexpected error fetching overall downloads for {normalized_name}: {e}")
logger.error(
f"Unexpected error fetching overall downloads for {normalized_name}: {e}"
)
raise
def clear_cache(self):

28
pypi_query_mcp/core/version_utils.py
View file

@ -288,31 +288,31 @@ class VersionCompatibility:
def sort_versions_semantically(versions: list[str], reverse: bool = True) -> list[str]:
"""Sort package versions using semantic version ordering.
This function properly sorts versions by parsing them as semantic versions,
ensuring that pre-release versions (alpha, beta, rc) are ordered correctly
relative to stable releases.
Args:
versions: List of version strings to sort
reverse: If True, sort in descending order (newest first). Default True.
Returns:
List of version strings sorted semantically
Examples:
>>> sort_versions_semantically(['1.0.0', '2.0.0a1', '1.5.0', '2.0.0'])
['2.0.0', '2.0.0a1', '1.5.0', '1.0.0']
>>> sort_versions_semantically(['5.2rc1', '5.2.5', '5.2.0'])
['5.2.5', '5.2.0', '5.2rc1']
"""
if not versions:
return []
def parse_version_safe(version_str: str) -> tuple[Version | None, str]:
"""Safely parse a version string, returning (parsed_version, original_string).
Returns (None, original_string) if parsing fails.
"""
try:
@ -320,26 +320,26 @@ def sort_versions_semantically(versions: list[str], reverse: bool = True) -> lis
except InvalidVersion:
logger.debug(f"Failed to parse version '{version_str}' as semantic version")
return (None, version_str)
# Parse all versions, keeping track of originals
parsed_versions = [parse_version_safe(v) for v in versions]
# Separate valid and invalid versions
valid_versions = [(v, orig) for v, orig in parsed_versions if v is not None]
invalid_versions = [orig for v, orig in parsed_versions if v is None]
# Sort valid versions semantically
valid_versions.sort(key=lambda x: x[0], reverse=reverse)
# Sort invalid versions lexicographically as fallback
invalid_versions.sort(reverse=reverse)
# Combine results: valid versions first, then invalid ones
result = [orig for _, orig in valid_versions] + invalid_versions
logger.debug(
f"Sorted {len(versions)} versions: {len(valid_versions)} valid, "
f"{len(invalid_versions)} invalid"
)
return result

2
pypi_query_mcp/data/__init__.py
View file

@ -1 +1 @@
"""Data module for PyPI package information."""
"""Data module for PyPI package information."""

356
pypi_query_mcp/data/popular_packages.py
View file

@ -10,10 +10,12 @@ The rankings and download estimates are based on:
Data is organized by categories and includes estimated relative popularity.
"""
from typing import Dict, List, NamedTuple
from typing import NamedTuple
class PackageInfo(NamedTuple):
"""Information about a popular package."""
name: str
category: str
estimated_monthly_downloads: int
@ -21,60 +23,226 @@ class PackageInfo(NamedTuple):
description: str
primary_use_case: str
# Core packages that are dependencies for many other packages
INFRASTRUCTURE_PACKAGES = [
PackageInfo("setuptools", "packaging", 800_000_000, 2100, "Package development tools", "packaging"),
PackageInfo("wheel", "packaging", 700_000_000, 400, "Binary package format", "packaging"),
PackageInfo("pip", "packaging", 600_000_000, 9500, "Package installer", "packaging"),
PackageInfo("certifi", "security", 500_000_000, 800, "Certificate bundle", "security"),
PackageInfo("urllib3", "networking", 450_000_000, 3600, "HTTP client library", "networking"),
PackageInfo("charset-normalizer", "text", 400_000_000, 400, "Character encoding detection", "text-processing"),
PackageInfo("idna", "networking", 380_000_000, 200, "Internationalized domain names", "networking"),
PackageInfo("six", "compatibility", 350_000_000, 900, "Python 2 and 3 compatibility", "compatibility"),
PackageInfo("python-dateutil", "datetime", 320_000_000, 2200, "Date and time utilities", "datetime"),
PackageInfo("requests", "networking", 300_000_000, 51000, "HTTP library", "networking"),
PackageInfo(
"setuptools",
"packaging",
800_000_000,
2100,
"Package development tools",
"packaging",
),
PackageInfo(
"wheel", "packaging", 700_000_000, 400, "Binary package format", "packaging"
),
PackageInfo(
"pip", "packaging", 600_000_000, 9500, "Package installer", "packaging"
),
PackageInfo(
"certifi", "security", 500_000_000, 800, "Certificate bundle", "security"
),
PackageInfo(
"urllib3", "networking", 450_000_000, 3600, "HTTP client library", "networking"
),
PackageInfo(
"charset-normalizer",
"text",
400_000_000,
400,
"Character encoding detection",
"text-processing",
),
PackageInfo(
"idna",
"networking",
380_000_000,
200,
"Internationalized domain names",
"networking",
),
PackageInfo(
"six",
"compatibility",
350_000_000,
900,
"Python 2 and 3 compatibility",
"compatibility",
),
PackageInfo(
"python-dateutil",
"datetime",
320_000_000,
2200,
"Date and time utilities",
"datetime",
),
PackageInfo(
"requests", "networking", 300_000_000, 51000, "HTTP library", "networking"
),
]
# AWS and cloud packages
CLOUD_PACKAGES = [
PackageInfo("boto3", "cloud", 280_000_000, 8900, "AWS SDK", "cloud"),
PackageInfo("botocore", "cloud", 275_000_000, 1400, "AWS SDK core", "cloud"),
PackageInfo("s3transfer", "cloud", 250_000_000, 200, "S3 transfer utilities", "cloud"),
PackageInfo(
"s3transfer", "cloud", 250_000_000, 200, "S3 transfer utilities", "cloud"
),
PackageInfo("awscli", "cloud", 80_000_000, 15000, "AWS command line", "cloud"),
PackageInfo("azure-core", "cloud", 45_000_000, 400, "Azure SDK core", "cloud"),
PackageInfo("google-cloud-storage", "cloud", 35_000_000, 300, "Google Cloud Storage", "cloud"),
PackageInfo("azure-storage-blob", "cloud", 30_000_000, 200, "Azure Blob Storage", "cloud"),
PackageInfo(
"google-cloud-storage",
"cloud",
35_000_000,
300,
"Google Cloud Storage",
"cloud",
),
PackageInfo(
"azure-storage-blob", "cloud", 30_000_000, 200, "Azure Blob Storage", "cloud"
),
]
# Data science and ML packages
DATA_SCIENCE_PACKAGES = [
PackageInfo("numpy", "data-science", 200_000_000, 26000, "Numerical computing", "data-science"),
PackageInfo("pandas", "data-science", 150_000_000, 42000, "Data manipulation", "data-science"),
PackageInfo("scikit-learn", "machine-learning", 80_000_000, 58000, "Machine learning", "machine-learning"),
PackageInfo("matplotlib", "visualization", 75_000_000, 19000, "Plotting library", "visualization"),
PackageInfo("scipy", "data-science", 70_000_000, 12000, "Scientific computing", "data-science"),
PackageInfo("seaborn", "visualization", 45_000_000, 11000, "Statistical visualization", "visualization"),
PackageInfo("plotly", "visualization", 40_000_000, 15000, "Interactive plots", "visualization"),
PackageInfo("jupyter", "development", 35_000_000, 7000, "Interactive notebooks", "development"),
PackageInfo("ipython", "development", 50_000_000, 8000, "Interactive Python", "development"),
PackageInfo("tensorflow", "machine-learning", 25_000_000, 185000, "Deep learning", "machine-learning"),
PackageInfo("torch", "machine-learning", 20_000_000, 81000, "PyTorch deep learning", "machine-learning"),
PackageInfo("transformers", "machine-learning", 15_000_000, 130000, "NLP transformers", "machine-learning"),
PackageInfo(
"numpy",
"data-science",
200_000_000,
26000,
"Numerical computing",
"data-science",
),
PackageInfo(
"pandas",
"data-science",
150_000_000,
42000,
"Data manipulation",
"data-science",
),
PackageInfo(
"scikit-learn",
"machine-learning",
80_000_000,
58000,
"Machine learning",
"machine-learning",
),
PackageInfo(
"matplotlib",
"visualization",
75_000_000,
19000,
"Plotting library",
"visualization",
),
PackageInfo(
"scipy",
"data-science",
70_000_000,
12000,
"Scientific computing",
"data-science",
),
PackageInfo(
"seaborn",
"visualization",
45_000_000,
11000,
"Statistical visualization",
"visualization",
),
PackageInfo(
"plotly",
"visualization",
40_000_000,
15000,
"Interactive plots",
"visualization",
),
PackageInfo(
"jupyter",
"development",
35_000_000,
7000,
"Interactive notebooks",
"development",
),
PackageInfo(
"ipython", "development", 50_000_000, 8000, "Interactive Python", "development"
),
PackageInfo(
"tensorflow",
"machine-learning",
25_000_000,
185000,
"Deep learning",
"machine-learning",
),
PackageInfo(
"torch",
"machine-learning",
20_000_000,
81000,
"PyTorch deep learning",
"machine-learning",
),
PackageInfo(
"transformers",
"machine-learning",
15_000_000,
130000,
"NLP transformers",
"machine-learning",
),
]
# Development and testing
DEVELOPMENT_PACKAGES = [
PackageInfo("typing-extensions", "development", 180_000_000, 3000, "Typing extensions", "development"),
PackageInfo("packaging", "development", 160_000_000, 600, "Package utilities", "development"),
PackageInfo("pytest", "testing", 100_000_000, 11000, "Testing framework", "testing"),
PackageInfo(
"typing-extensions",
"development",
180_000_000,
3000,
"Typing extensions",
"development",
),
PackageInfo(
"packaging", "development", 160_000_000, 600, "Package utilities", "development"
),
PackageInfo(
"pytest", "testing", 100_000_000, 11000, "Testing framework", "testing"
),
PackageInfo("click", "cli", 90_000_000, 15000, "Command line interface", "cli"),
PackageInfo("pyyaml", "serialization", 85_000_000, 2200, "YAML parser", "serialization"),
PackageInfo("jinja2", "templating", 80_000_000, 10000, "Template engine", "templating"),
PackageInfo("markupsafe", "templating", 75_000_000, 600, "Safe markup", "templating"),
PackageInfo("attrs", "development", 60_000_000, 5000, "Classes without boilerplate", "development"),
PackageInfo("black", "development", 40_000_000, 38000, "Code formatter", "development"),
PackageInfo("flake8", "development", 35_000_000, 3000, "Code linting", "development"),
PackageInfo("mypy", "development", 30_000_000, 17000, "Static type checker", "development"),
PackageInfo(
"pyyaml", "serialization", 85_000_000, 2200, "YAML parser", "serialization"
),
PackageInfo(
"jinja2", "templating", 80_000_000, 10000, "Template engine", "templating"
),
PackageInfo(
"markupsafe", "templating", 75_000_000, 600, "Safe markup", "templating"
),
PackageInfo(
"attrs",
"development",
60_000_000,
5000,
"Classes without boilerplate",
"development",
),
PackageInfo(
"black", "development", 40_000_000, 38000, "Code formatter", "development"
),
PackageInfo(
"flake8", "development", 35_000_000, 3000, "Code linting", "development"
),
PackageInfo(
"mypy", "development", 30_000_000, 17000, "Static type checker", "development"
),
]
# Web development
@ -83,49 +251,87 @@ WEB_PACKAGES = [
PackageInfo("flask", "web", 55_000_000, 66000, "Micro web framework", "web"),
PackageInfo("fastapi", "web", 35_000_000, 74000, "Modern web API framework", "web"),
PackageInfo("sqlalchemy", "database", 50_000_000, 8000, "SQL toolkit", "database"),
PackageInfo("psycopg2", "database", 25_000_000, 3000, "PostgreSQL adapter", "database"),
PackageInfo(
"psycopg2", "database", 25_000_000, 3000, "PostgreSQL adapter", "database"
),
PackageInfo("redis", "database", 30_000_000, 12000, "Redis client", "database"),
PackageInfo("celery", "async", 25_000_000, 23000, "Distributed task queue", "async"),
PackageInfo(
"celery", "async", 25_000_000, 23000, "Distributed task queue", "async"
),
PackageInfo("gunicorn", "web", 20_000_000, 9000, "WSGI server", "web"),
PackageInfo("uvicorn", "web", 15_000_000, 8000, "ASGI server", "web"),
]
# Security and cryptography
SECURITY_PACKAGES = [
PackageInfo("cryptography", "security", 120_000_000, 6000, "Cryptographic library", "security"),
PackageInfo("pyopenssl", "security", 60_000_000, 800, "OpenSSL wrapper", "security"),
PackageInfo(
"cryptography",
"security",
120_000_000,
6000,
"Cryptographic library",
"security",
),
PackageInfo(
"pyopenssl", "security", 60_000_000, 800, "OpenSSL wrapper", "security"
),
PackageInfo("pyjwt", "security", 40_000_000, 5000, "JSON Web Tokens", "security"),
PackageInfo("bcrypt", "security", 35_000_000, 1200, "Password hashing", "security"),
PackageInfo("pycryptodome", "security", 30_000_000, 2700, "Cryptographic library", "security"),
PackageInfo(
"pycryptodome",
"security",
30_000_000,
2700,
"Cryptographic library",
"security",
),
]
# Networking and API
NETWORKING_PACKAGES = [
PackageInfo("httpx", "networking", 25_000_000, 12000, "HTTP client", "networking"),
PackageInfo("aiohttp", "networking", 35_000_000, 14000, "Async HTTP", "networking"),
PackageInfo("websockets", "networking", 20_000_000, 5000, "WebSocket implementation", "networking"),
PackageInfo(
"websockets",
"networking",
20_000_000,
5000,
"WebSocket implementation",
"networking",
),
PackageInfo("paramiko", "networking", 25_000_000, 8000, "SSH client", "networking"),
]
# Text processing and parsing
TEXT_PACKAGES = [
PackageInfo("beautifulsoup4", "parsing", 40_000_000, 13000, "HTML/XML parser", "parsing"),
PackageInfo(
"beautifulsoup4", "parsing", 40_000_000, 13000, "HTML/XML parser", "parsing"
),
PackageInfo("lxml", "parsing", 35_000_000, 2600, "XML/HTML parser", "parsing"),
PackageInfo("regex", "text", 30_000_000, 700, "Regular expressions", "text-processing"),
PackageInfo("python-docx", "text", 15_000_000, 4000, "Word document processing", "text-processing"),
PackageInfo(
"regex", "text", 30_000_000, 700, "Regular expressions", "text-processing"
),
PackageInfo(
"python-docx",
"text",
15_000_000,
4000,
"Word document processing",
"text-processing",
),
PackageInfo("pillow", "imaging", 60_000_000, 11000, "Image processing", "imaging"),
]
# All packages combined for easy access
ALL_POPULAR_PACKAGES = (
INFRASTRUCTURE_PACKAGES +
CLOUD_PACKAGES +
DATA_SCIENCE_PACKAGES +
DEVELOPMENT_PACKAGES +
WEB_PACKAGES +
SECURITY_PACKAGES +
NETWORKING_PACKAGES +
TEXT_PACKAGES
INFRASTRUCTURE_PACKAGES
+ CLOUD_PACKAGES
+ DATA_SCIENCE_PACKAGES
+ DEVELOPMENT_PACKAGES
+ WEB_PACKAGES
+ SECURITY_PACKAGES
+ NETWORKING_PACKAGES
+ TEXT_PACKAGES
)
# Create lookup dictionaries
@ -136,41 +342,45 @@ for pkg in ALL_POPULAR_PACKAGES:
PACKAGES_BY_CATEGORY[pkg.category] = []
PACKAGES_BY_CATEGORY[pkg.category].append(pkg)
def get_popular_packages(
category: str = None,
limit: int = 50,
min_downloads: int = 0
) -> List[PackageInfo]:
category: str = None, limit: int = 50, min_downloads: int = 0
) -> list[PackageInfo]:
"""Get popular packages filtered by criteria.
Args:
category: Filter by category (e.g., 'web', 'data-science', 'cloud')
limit: Maximum number of packages to return
min_downloads: Minimum estimated monthly downloads
Returns:
List of PackageInfo objects sorted by estimated downloads
"""
packages = ALL_POPULAR_PACKAGES
if category:
packages = [pkg for pkg in packages if pkg.category == category]
if min_downloads:
packages = [pkg for pkg in packages if pkg.estimated_monthly_downloads >= min_downloads]
packages = [
pkg for pkg in packages if pkg.estimated_monthly_downloads >= min_downloads
]
# Sort by estimated downloads (descending)
packages = sorted(packages, key=lambda x: x.estimated_monthly_downloads, reverse=True)
packages = sorted(
packages, key=lambda x: x.estimated_monthly_downloads, reverse=True
)
return packages[:limit]
def estimate_downloads_for_period(monthly_downloads: int, period: str) -> int:
"""Estimate downloads for different time periods.
Args:
monthly_downloads: Estimated monthly downloads
period: Time period ('day', 'week', 'month')
Returns:
Estimated downloads for the period
"""
@ -183,16 +393,20 @@ def estimate_downloads_for_period(monthly_downloads: int, period: str) -> int:
else:
return monthly_downloads
def get_package_info(package_name: str) -> PackageInfo:
"""Get information about a specific package.
Args:
package_name: Name of the package
Returns:
PackageInfo object or None if not found
"""
return PACKAGES_BY_NAME.get(package_name.lower().replace("-", "_").replace("_", "-"))
return PACKAGES_BY_NAME.get(
package_name.lower().replace("-", "_").replace("_", "-")
)
# GitHub repository URL patterns for fetching real-time data
GITHUB_REPO_PATTERNS = {
@ -211,4 +425,4 @@ GITHUB_REPO_PATTERNS = {
"boto3": "boto/boto3",
"sqlalchemy": "sqlalchemy/sqlalchemy",
# Add more mappings as needed
}
}

20
pypi_query_mcp/server.py
View file

@ -136,11 +136,11 @@ async def get_package_versions(package_name: str) -> dict[str, Any]:
@mcp.tool()
async def get_package_dependencies(
package_name: str,
package_name: str,
version: str | None = None,
include_transitive: bool = False,
max_depth: int = 5,
python_version: str | None = None
python_version: str | None = None,
) -> dict[str, Any]:
"""Get dependency information for a PyPI package.
@ -175,7 +175,11 @@ async def get_package_dependencies(
logger.info(
f"MCP tool: Querying dependencies for {package_name}"
+ (f" version {version}" if version else " (latest)")
+ (f" with transitive dependencies (max depth: {max_depth})" if include_transitive else " (direct only)")
+ (
f" with transitive dependencies (max depth: {max_depth})"
if include_transitive
else " (direct only)"
)
)
result = await query_package_dependencies(
package_name, version, include_transitive, max_depth, python_version
@ -326,9 +330,9 @@ async def resolve_dependencies(
Args:
package_name: The name of the PyPI package to analyze (e.g., 'pyside2', 'django')
python_version: Target Python version for dependency filtering (e.g., '3.10', '3.11')
include_extras: List of extra dependency groups to include. These are optional
dependency groups defined by the package (e.g., ['socks'] for requests,
['argon2', 'bcrypt'] for django, ['test', 'doc'] for setuptools). Check the
include_extras: List of extra dependency groups to include. These are optional
dependency groups defined by the package (e.g., ['socks'] for requests,
['argon2', 'bcrypt'] for django, ['test', 'doc'] for setuptools). Check the
package's PyPI page or use the provides_extra field to see available extras.
include_dev: Whether to include development dependencies (default: False)
max_depth: Maximum recursion depth for dependency resolution (default: 5)
@ -397,8 +401,8 @@ async def download_package(
package_name: The name of the PyPI package to download (e.g., 'pyside2', 'requests')
download_dir: Local directory to download packages to (default: './downloads')
python_version: Target Python version for compatibility (e.g., '3.10', '3.11')
include_extras: List of extra dependency groups to include. These are optional
dependency groups defined by the package (e.g., ['socks'] for requests,
include_extras: List of extra dependency groups to include. These are optional
dependency groups defined by the package (e.g., ['socks'] for requests,
['argon2', 'bcrypt'] for django). Check the package's PyPI page to see available extras.
include_dev: Whether to include development dependencies (default: False)
prefer_wheel: Whether to prefer wheel files over source distributions (default: True)

4
pypi_query_mcp/tools/dependency_resolver.py
View file

@ -35,7 +35,7 @@ class DependencyResolver:
Args:
package_name: Name of the package to resolve
python_version: Target Python version (e.g., "3.10")
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
['test', 'doc'] for setuptools). These are optional dependencies defined by the package.
include_dev: Whether to include development dependencies
max_depth: Maximum recursion depth (overrides instance default)
@ -243,7 +243,7 @@ async def resolve_package_dependencies(
Args:
package_name: Name of the package to resolve
python_version: Target Python version (e.g., "3.10")
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
['test', 'doc'] for setuptools). These are optional dependencies defined by the package.
include_dev: Whether to include development dependencies
max_depth: Maximum recursion depth

229
pypi_query_mcp/tools/download_stats.py
View file

@ -3,14 +3,13 @@
import logging
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
from typing import Any
from ..core.github_client import GitHubAPIClient
from ..core.pypi_client import PyPIClient
from ..core.stats_client import PyPIStatsClient
from ..data.popular_packages import (
GITHUB_REPO_PATTERNS,
PACKAGES_BY_NAME,
estimate_downloads_for_period,
get_popular_packages,
)
@ -73,11 +72,11 @@ async def get_package_download_stats(
# Calculate trends and analysis
analysis = _analyze_download_stats(download_data)
# Determine data source and add warnings if needed
data_source = recent_stats.get("source", "pypistats.org")
warning_note = recent_stats.get("note")
result = {
"package": package_name,
"metadata": package_metadata,
@ -87,15 +86,17 @@ async def get_package_download_stats(
"data_source": data_source,
"timestamp": datetime.now().isoformat(),
}
# Add warning/note about data quality if present
if warning_note:
result["data_quality_note"] = warning_note
# Add reliability indicator
if data_source == "fallback_estimates":
result["reliability"] = "estimated"
result["warning"] = "Data is estimated due to API unavailability. Actual download counts may differ significantly."
result["warning"] = (
"Data is estimated due to API unavailability. Actual download counts may differ significantly."
)
elif "stale" in warning_note.lower() if warning_note else False:
result["reliability"] = "cached"
result["warning"] = "Data may be outdated due to current API issues."
@ -142,7 +143,7 @@ async def get_package_download_trends(
# Analyze trends
trend_analysis = _analyze_download_trends(time_series_data, include_mirrors)
# Determine data source and add warnings if needed
data_source = overall_stats.get("source", "pypistats.org")
warning_note = overall_stats.get("note")
@ -155,15 +156,17 @@ async def get_package_download_trends(
"data_source": data_source,
"timestamp": datetime.now().isoformat(),
}
# Add warning/note about data quality if present
if warning_note:
result["data_quality_note"] = warning_note
# Add reliability indicator
if data_source == "fallback_estimates":
result["reliability"] = "estimated"
result["warning"] = "Data is estimated due to API unavailability. Actual download trends may differ significantly."
result["warning"] = (
"Data is estimated due to API unavailability. Actual download trends may differ significantly."
)
elif "stale" in warning_note.lower() if warning_note else False:
result["reliability"] = "cached"
result["warning"] = "Data may be outdated due to current API issues."
@ -201,56 +204,54 @@ async def get_top_packages_by_downloads(
"""
# Get curated popular packages as base data
curated_packages = get_popular_packages(limit=max(limit * 2, 100))
# Try to enhance with real PyPI stats
enhanced_packages = await _enhance_with_real_stats(
curated_packages, period, limit
)
enhanced_packages = await _enhance_with_real_stats(curated_packages, period, limit)
# Try to enhance with GitHub metrics
final_packages = await _enhance_with_github_stats(
enhanced_packages, limit
)
final_packages = await _enhance_with_github_stats(enhanced_packages, limit)
# Ensure we have the requested number of packages
if len(final_packages) < limit:
# Add more from curated list if needed
additional_needed = limit - len(final_packages)
existing_names = {pkg["package"] for pkg in final_packages}
for pkg_info in curated_packages:
if pkg_info.name not in existing_names and additional_needed > 0:
final_packages.append({
"package": pkg_info.name,
"downloads": estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
),
"period": period,
"data_source": "curated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
})
final_packages.append(
{
"package": pkg_info.name,
"downloads": estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
),
"period": period,
"data_source": "curated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
}
)
additional_needed -= 1
# Sort by download count and assign ranks
final_packages.sort(key=lambda x: x.get("downloads", 0), reverse=True)
final_packages = final_packages[:limit]
for i, package in enumerate(final_packages):
package["rank"] = i + 1
# Determine primary data source
real_stats_count = len([p for p in final_packages if not p.get("estimated", False)])
github_enhanced_count = len([p for p in final_packages if "github_stars" in p])
if real_stats_count > limit // 2:
primary_source = "pypistats.org with curated fallback"
elif github_enhanced_count > 0:
primary_source = "curated data enhanced with GitHub metrics"
else:
primary_source = "curated popular packages database"
return {
"top_packages": final_packages,
"period": period,
@ -386,50 +387,73 @@ def _analyze_download_trends(
async def _enhance_with_real_stats(
curated_packages: List, period: str, limit: int
) -> List[Dict[str, Any]]:
curated_packages: list, period: str, limit: int
) -> list[dict[str, Any]]:
"""Try to enhance curated packages with real PyPI download statistics.
Args:
curated_packages: List of PackageInfo objects from curated data
period: Time period for stats
limit: Maximum number of packages to process
Returns:
List of enhanced package dictionaries
"""
enhanced_packages = []
try:
async with PyPIStatsClient() as stats_client:
# Try to get real stats for top packages
for pkg_info in curated_packages[:limit * 2]: # Try more than needed
for pkg_info in curated_packages[: limit * 2]: # Try more than needed
try:
stats = await stats_client.get_recent_downloads(
pkg_info.name, period, use_cache=True
)
download_data = stats.get("data", {})
real_download_count = _extract_download_count(download_data, period)
if real_download_count > 0:
# Use real stats
enhanced_packages.append({
"package": pkg_info.name,
"downloads": real_download_count,
"period": period,
"data_source": "pypistats.org",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": False,
})
logger.debug(f"Got real stats for {pkg_info.name}: {real_download_count}")
enhanced_packages.append(
{
"package": pkg_info.name,
"downloads": real_download_count,
"period": period,
"data_source": "pypistats.org",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": False,
}
)
logger.debug(
f"Got real stats for {pkg_info.name}: {real_download_count}"
)
else:
# Fall back to estimated downloads
estimated_downloads = estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
)
enhanced_packages.append({
enhanced_packages.append(
{
"package": pkg_info.name,
"downloads": estimated_downloads,
"period": period,
"data_source": "estimated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
}
)
except Exception as e:
logger.debug(f"Failed to get real stats for {pkg_info.name}: {e}")
# Fall back to estimated downloads
estimated_downloads = estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
)
enhanced_packages.append(
{
"package": pkg_info.name,
"downloads": estimated_downloads,
"period": period,
@ -437,28 +461,13 @@ async def _enhance_with_real_stats(
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
})
except Exception as e:
logger.debug(f"Failed to get real stats for {pkg_info.name}: {e}")
# Fall back to estimated downloads
estimated_downloads = estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
}
)
enhanced_packages.append({
"package": pkg_info.name,
"downloads": estimated_downloads,
"period": period,
"data_source": "estimated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
})
# Stop if we have enough packages
if len(enhanced_packages) >= limit:
break
except Exception as e:
logger.warning(f"PyPI stats client failed entirely: {e}")
# Fall back to all estimated data
@ -466,52 +475,56 @@ async def _enhance_with_real_stats(
estimated_downloads = estimate_downloads_for_period(
pkg_info.estimated_monthly_downloads, period
)
enhanced_packages.append({
"package": pkg_info.name,
"downloads": estimated_downloads,
"period": period,
"data_source": "estimated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
})
enhanced_packages.append(
{
"package": pkg_info.name,
"downloads": estimated_downloads,
"period": period,
"data_source": "estimated",
"category": pkg_info.category,
"description": pkg_info.description,
"estimated": True,
}
)
return enhanced_packages
async def _enhance_with_github_stats(
packages: List[Dict[str, Any]], limit: int
) -> List[Dict[str, Any]]:
packages: list[dict[str, Any]], limit: int
) -> list[dict[str, Any]]:
"""Try to enhance packages with GitHub repository statistics.
Args:
packages: List of package dictionaries to enhance
limit: Maximum number of packages to process
Returns:
List of enhanced package dictionaries
"""
github_token = os.getenv("GITHUB_TOKEN") # Optional GitHub token
try:
async with GitHubAPIClient(github_token=github_token) as github_client:
# Get GitHub repo paths for packages that have them
repo_paths = []
package_to_repo = {}
for pkg in packages[:limit]:
repo_path = GITHUB_REPO_PATTERNS.get(pkg["package"])
if repo_path:
repo_paths.append(repo_path)
package_to_repo[pkg["package"]] = repo_path
if repo_paths:
# Fetch GitHub stats for all repositories concurrently
logger.debug(f"Fetching GitHub stats for {len(repo_paths)} repositories")
logger.debug(
f"Fetching GitHub stats for {len(repo_paths)} repositories"
)
repo_stats = await github_client.get_multiple_repo_stats(
repo_paths, use_cache=True, max_concurrent=3
)
# Enhance packages with GitHub data
for pkg in packages:
repo_path = package_to_repo.get(pkg["package"])
@ -523,38 +536,42 @@ async def _enhance_with_github_stats(
pkg["github_updated_at"] = stats["updated_at"]
pkg["github_language"] = stats["language"]
pkg["github_topics"] = stats.get("topics", [])
# Adjust download estimates based on GitHub popularity
if pkg.get("estimated", False):
popularity_boost = _calculate_popularity_boost(stats)
pkg["downloads"] = int(pkg["downloads"] * popularity_boost)
pkg["downloads"] = int(
pkg["downloads"] * popularity_boost
)
pkg["github_enhanced"] = True
logger.info(f"Enhanced {len([p for p in packages if 'github_stars' in p])} packages with GitHub data")
logger.info(
f"Enhanced {len([p for p in packages if 'github_stars' in p])} packages with GitHub data"
)
except Exception as e:
logger.debug(f"GitHub enhancement failed: {e}")
# Continue without GitHub enhancement
pass
return packages
def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
def _calculate_popularity_boost(github_stats: dict[str, Any]) -> float:
"""Calculate a popularity boost multiplier based on GitHub metrics.
Args:
github_stats: GitHub repository statistics
Returns:
Multiplier between 0.5 and 2.0 based on popularity
"""
stars = github_stats.get("stars", 0)
forks = github_stats.get("forks", 0)
# Base multiplier
multiplier = 1.0
# Adjust based on stars (logarithmic scale)
if stars > 50000:
multiplier *= 1.5
@ -568,7 +585,7 @@ def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
multiplier *= 0.9
elif stars < 500:
multiplier *= 0.8
# Adjust based on forks (indicates active usage)
if forks > 10000:
multiplier *= 1.2
@ -576,7 +593,7 @@ def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
multiplier *= 1.1
elif forks < 100:
multiplier *= 0.9
# Ensure multiplier stays within reasonable bounds
return max(0.5, min(2.0, multiplier))

315
pypi_query_mcp/tools/package_query.py
View file

@ -68,8 +68,12 @@ def format_package_info(package_data: dict[str, Any]) -> dict[str, Any]:
formatted["total_versions"] = len(releases)
# Sort versions semantically and get the most recent 10
if releases:
sorted_versions = sort_versions_semantically(list(releases.keys()), reverse=True)
formatted["available_versions"] = sorted_versions[:10] # Most recent 10 versions
sorted_versions = sort_versions_semantically(
list(releases.keys()), reverse=True
)
formatted["available_versions"] = sorted_versions[
:10
] # Most recent 10 versions
else:
formatted["available_versions"] = []
@ -139,7 +143,7 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
Formatted dependency information
"""
from ..core.dependency_parser import DependencyParser
info = package_data.get("info", {})
requires_dist = info.get("requires_dist", []) or []
provides_extra = info.get("provides_extra", []) or []
@ -152,7 +156,7 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
# Convert Requirements back to strings for JSON serialization
runtime_deps = [str(req) for req in categories["runtime"]]
dev_deps = [str(req) for req in categories["development"]]
# Convert optional dependencies (extras) to string format
optional_deps = {}
for extra_name, reqs in categories["extras"].items():
@ -161,14 +165,31 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
# Separate development and non-development optional dependencies
dev_optional_deps = {}
non_dev_optional_deps = {}
# Define development-related extra names (same as in DependencyParser)
dev_extra_names = {
'dev', 'development', 'test', 'testing', 'tests', 'lint', 'linting',
'doc', 'docs', 'documentation', 'build', 'check', 'cover', 'coverage',
'type', 'typing', 'mypy', 'style', 'format', 'quality'
"dev",
"development",
"test",
"testing",
"tests",
"lint",
"linting",
"doc",
"docs",
"documentation",
"build",
"check",
"cover",
"coverage",
"type",
"typing",
"mypy",
"style",
"format",
"quality",
}
for extra_name, deps in optional_deps.items():
if extra_name.lower() in dev_extra_names:
dev_optional_deps[extra_name] = deps
@ -260,11 +281,11 @@ async def query_package_versions(package_name: str) -> dict[str, Any]:
async def query_package_dependencies(
package_name: str,
version: str | None = None,
package_name: str,
version: str | None = None,
include_transitive: bool = False,
max_depth: int = 5,
python_version: str | None = None
python_version: str | None = None,
) -> dict[str, Any]:
"""Query package dependency information from PyPI.
@ -293,29 +314,35 @@ async def query_package_dependencies(
logger.info(
f"Querying dependencies for package: {package_name}"
+ (f" version {version}" if version else " (latest)")
+ (f" with transitive dependencies (max depth: {max_depth})" if include_transitive else " (direct only)")
+ (
f" with transitive dependencies (max depth: {max_depth})"
if include_transitive
else " (direct only)"
)
)
try:
if include_transitive:
# Use the comprehensive dependency resolver for transitive dependencies
from .dependency_resolver import resolve_package_dependencies
result = await resolve_package_dependencies(
package_name=package_name,
python_version=python_version,
include_extras=[],
include_dev=False,
max_depth=max_depth
max_depth=max_depth,
)
# Format the transitive dependency result to match expected structure
return format_transitive_dependency_info(result, package_name, version)
else:
# Use direct dependency logic with version support
async with PyPIClient() as client:
# Pass the version parameter to get_package_info
package_data = await client.get_package_info(package_name, version=version)
package_data = await client.get_package_info(
package_name, version=version
)
return format_dependency_info(package_data)
except PyPIError:
# Re-raise PyPI-specific errors
@ -342,9 +369,9 @@ def format_transitive_dependency_info(
normalized_name = package_name.lower().replace("_", "-")
dependency_tree = resolver_result.get("dependency_tree", {})
summary = resolver_result.get("summary", {})
main_package = dependency_tree.get(normalized_name, {})
# Build the response in the same format as direct dependencies but with tree structure
result = {
"package_name": package_name,
@ -353,42 +380,51 @@ def format_transitive_dependency_info(
"include_transitive": True,
"max_depth": summary.get("max_depth", 0),
"python_version": resolver_result.get("python_version"),
# Direct dependencies (same as before)
"runtime_dependencies": main_package.get("dependencies", {}).get("runtime", []),
"development_dependencies": main_package.get("dependencies", {}).get("development", []),
"development_dependencies": main_package.get("dependencies", {}).get(
"development", []
),
"optional_dependencies": main_package.get("dependencies", {}).get("extras", {}),
# Transitive dependency information
"transitive_dependencies": {
"dependency_tree": _build_dependency_tree_structure(dependency_tree, normalized_name),
"dependency_tree": _build_dependency_tree_structure(
dependency_tree, normalized_name
),
"all_packages": _extract_all_packages_info(dependency_tree),
"circular_dependencies": _detect_circular_dependencies(dependency_tree),
"depth_analysis": _analyze_dependency_depths(dependency_tree),
},
# Enhanced summary statistics
"dependency_summary": {
"direct_runtime_count": len(main_package.get("dependencies", {}).get("runtime", [])),
"direct_dev_count": len(main_package.get("dependencies", {}).get("development", [])),
"direct_optional_groups": len(main_package.get("dependencies", {}).get("extras", {})),
"total_transitive_packages": summary.get("total_packages", 0) - 1, # Exclude main package
"direct_runtime_count": len(
main_package.get("dependencies", {}).get("runtime", [])
),
"direct_dev_count": len(
main_package.get("dependencies", {}).get("development", [])
),
"direct_optional_groups": len(
main_package.get("dependencies", {}).get("extras", {})
),
"total_transitive_packages": summary.get("total_packages", 0)
- 1, # Exclude main package
"total_runtime_dependencies": summary.get("total_runtime_dependencies", 0),
"total_development_dependencies": summary.get("total_development_dependencies", 0),
"total_development_dependencies": summary.get(
"total_development_dependencies", 0
),
"total_extra_dependencies": summary.get("total_extra_dependencies", 0),
"max_dependency_depth": summary.get("max_depth", 0),
"complexity_score": _calculate_complexity_score(summary),
},
# Performance and health metrics
"analysis": {
"resolution_stats": summary,
"potential_conflicts": _analyze_potential_conflicts(dependency_tree),
"maintenance_concerns": _analyze_maintenance_concerns(dependency_tree),
"performance_impact": _assess_performance_impact(summary),
}
},
}
return result
@ -398,27 +434,27 @@ def _build_dependency_tree_structure(
"""Build a hierarchical dependency tree structure."""
if visited is None:
visited = set()
if root_package in visited:
return {"circular_reference": True, "package_name": root_package}
visited.add(root_package)
if root_package not in dependency_tree:
return {}
package_info = dependency_tree[root_package]
children = package_info.get("children", {})
tree_node = {
"package_name": package_info.get("name", root_package),
"version": package_info.get("version", "unknown"),
"depth": package_info.get("depth", 0),
"requires_python": package_info.get("requires_python", ""),
"dependencies": package_info.get("dependencies", {}),
"children": {}
"children": {},
}
# Recursively build children (with visited tracking to prevent infinite loops)
for child_name in children:
if child_name not in visited:
@ -427,17 +463,19 @@ def _build_dependency_tree_structure(
)
else:
tree_node["children"][child_name] = {
"circular_reference": True,
"package_name": child_name
"circular_reference": True,
"package_name": child_name,
}
return tree_node
def _extract_all_packages_info(dependency_tree: dict[str, Any]) -> dict[str, dict[str, Any]]:
def _extract_all_packages_info(
dependency_tree: dict[str, Any],
) -> dict[str, dict[str, Any]]:
"""Extract comprehensive information about all packages in the dependency tree."""
all_packages = {}
for package_name, package_info in dependency_tree.items():
all_packages[package_name] = {
"name": package_info.get("name", package_name),
@ -446,60 +484,73 @@ def _extract_all_packages_info(dependency_tree: dict[str, Any]) -> dict[str, dic
"requires_python": package_info.get("requires_python", ""),
"direct_dependencies": {
"runtime": package_info.get("dependencies", {}).get("runtime", []),
"development": package_info.get("dependencies", {}).get("development", []),
"development": package_info.get("dependencies", {}).get(
"development", []
),
"extras": package_info.get("dependencies", {}).get("extras", {}),
},
"dependency_count": {
"runtime": len(package_info.get("dependencies", {}).get("runtime", [])),
"development": len(package_info.get("dependencies", {}).get("development", [])),
"total_extras": sum(len(deps) for deps in package_info.get("dependencies", {}).get("extras", {}).values()),
}
"development": len(
package_info.get("dependencies", {}).get("development", [])
),
"total_extras": sum(
len(deps)
for deps in package_info.get("dependencies", {})
.get("extras", {})
.values()
),
},
}
return all_packages
def _detect_circular_dependencies(dependency_tree: dict[str, Any]) -> list[dict[str, Any]]:
def _detect_circular_dependencies(
dependency_tree: dict[str, Any],
) -> list[dict[str, Any]]:
"""Detect circular dependencies in the dependency tree."""
circular_deps = []
def dfs(package_name: str, path: list[str], visited: set[str]) -> None:
if package_name in path:
# Found a circular dependency
cycle_start = path.index(package_name)
cycle = path[cycle_start:] + [package_name]
circular_deps.append({
"cycle": cycle,
"length": len(cycle) - 1,
"packages_involved": list(set(cycle))
})
circular_deps.append(
{
"cycle": cycle,
"length": len(cycle) - 1,
"packages_involved": list(set(cycle)),
}
)
return
if package_name in visited or package_name not in dependency_tree:
return
visited.add(package_name)
path.append(package_name)
# Check children
children = dependency_tree[package_name].get("children", {})
for child_name in children:
dfs(child_name, path.copy(), visited)
# Start DFS from each package
for package_name in dependency_tree:
dfs(package_name, [], set())
# Remove duplicates
unique_cycles = []
seen_cycles = set()
for cycle_info in circular_deps:
cycle_set = frozenset(cycle_info["packages_involved"])
if cycle_set not in seen_cycles:
seen_cycles.add(cycle_set)
unique_cycles.append(cycle_info)
return unique_cycles
@ -507,29 +558,36 @@ def _analyze_dependency_depths(dependency_tree: dict[str, Any]) -> dict[str, Any
"""Analyze the depth distribution of dependencies."""
depth_counts = {}
depth_packages = {}
for package_name, package_info in dependency_tree.items():
depth = package_info.get("depth", 0)
if depth not in depth_counts:
depth_counts[depth] = 0
depth_packages[depth] = []
depth_counts[depth] += 1
depth_packages[depth].append(package_name)
max_depth = max(depth_counts.keys()) if depth_counts else 0
return {
"max_depth": max_depth,
"depth_distribution": depth_counts,
"packages_by_depth": depth_packages,
"average_depth": sum(d * c for d, c in depth_counts.items()) / sum(depth_counts.values()) if depth_counts else 0,
"average_depth": sum(d * c for d, c in depth_counts.items())
/ sum(depth_counts.values())
if depth_counts
else 0,
"depth_analysis": {
"shallow_deps": depth_counts.get(1, 0), # Direct dependencies
"deep_deps": sum(count for depth, count in depth_counts.items() if depth > 2),
"leaf_packages": [pkg for pkg, info in dependency_tree.items() if not info.get("children")]
}
"deep_deps": sum(
count for depth, count in depth_counts.items() if depth > 2
),
"leaf_packages": [
pkg for pkg, info in dependency_tree.items() if not info.get("children")
],
},
}
@ -538,14 +596,14 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
total_packages = summary.get("total_packages", 0)
max_depth = summary.get("max_depth", 0)
total_deps = summary.get("total_runtime_dependencies", 0)
# Simple complexity scoring (can be enhanced)
base_score = total_packages * 0.3
depth_penalty = max_depth * 1.5
dependency_penalty = total_deps * 0.1
complexity_score = base_score + depth_penalty + dependency_penalty
# Classify complexity
if complexity_score < 10:
complexity_level = "low"
@ -558,8 +616,10 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
recommendation = "High complexity, consider dependency management strategies"
else:
complexity_level = "very_high"
recommendation = "Very high complexity, significant maintenance overhead expected"
recommendation = (
"Very high complexity, significant maintenance overhead expected"
)
return {
"score": round(complexity_score, 2),
"level": complexity_level,
@ -568,42 +628,50 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
"total_packages": total_packages,
"max_depth": max_depth,
"total_dependencies": total_deps,
}
},
}
def _analyze_potential_conflicts(dependency_tree: dict[str, Any]) -> list[dict[str, Any]]:
def _analyze_potential_conflicts(
dependency_tree: dict[str, Any],
) -> list[dict[str, Any]]:
"""Analyze potential version conflicts in dependencies."""
# This is a simplified analysis - in a real implementation,
# This is a simplified analysis - in a real implementation,
# you'd parse version constraints and check for conflicts
package_versions = {}
potential_conflicts = []
for package_name, package_info in dependency_tree.items():
runtime_deps = package_info.get("dependencies", {}).get("runtime", [])
for dep_str in runtime_deps:
# Basic parsing of "package>=version" format
if ">=" in dep_str or "==" in dep_str or "<" in dep_str or ">" in dep_str:
parts = dep_str.replace(">=", "@").replace("==", "@").replace("<", "@").replace(">", "@")
parts = (
dep_str.replace(">=", "@")
.replace("==", "@")
.replace("<", "@")
.replace(">", "@")
)
dep_name = parts.split("@")[0].strip()
if dep_name not in package_versions:
package_versions[dep_name] = []
package_versions[dep_name].append({
"constraint": dep_str,
"required_by": package_name
})
package_versions[dep_name].append(
{"constraint": dep_str, "required_by": package_name}
)
# Look for packages with multiple version constraints
for dep_name, constraints in package_versions.items():
if len(constraints) > 1:
potential_conflicts.append({
"package": dep_name,
"conflicting_constraints": constraints,
"severity": "potential" if len(constraints) == 2 else "high"
})
potential_conflicts.append(
{
"package": dep_name,
"conflicting_constraints": constraints,
"severity": "potential" if len(constraints) == 2 else "high",
}
)
return potential_conflicts
@ -611,25 +679,25 @@ def _analyze_maintenance_concerns(dependency_tree: dict[str, Any]) -> dict[str,
"""Analyze maintenance concerns in the dependency tree."""
total_packages = len(dependency_tree)
packages_without_version = sum(
1 for info in dependency_tree.values()
1
for info in dependency_tree.values()
if info.get("version") in ["unknown", "", None]
)
packages_without_python_req = sum(
1 for info in dependency_tree.values()
if not info.get("requires_python")
1 for info in dependency_tree.values() if not info.get("requires_python")
)
# Calculate dependency concentration (packages with many dependencies)
high_dep_packages = [
{
"name": name,
"dependency_count": len(info.get("dependencies", {}).get("runtime", []))
"dependency_count": len(info.get("dependencies", {}).get("runtime", [])),
}
for name, info in dependency_tree.items()
if len(info.get("dependencies", {}).get("runtime", [])) > 5
]
return {
"total_packages": total_packages,
"packages_without_version_info": packages_without_version,
@ -637,11 +705,18 @@ def _analyze_maintenance_concerns(dependency_tree: dict[str, Any]) -> dict[str,
"high_dependency_packages": high_dep_packages,
"maintenance_risk_score": {
"score": round(
(packages_without_version / total_packages * 100) +
(len(high_dep_packages) / total_packages * 50), 2
) if total_packages > 0 else 0,
"level": "low" if total_packages < 10 else "moderate" if total_packages < 30 else "high"
}
(packages_without_version / total_packages * 100)
+ (len(high_dep_packages) / total_packages * 50),
2,
)
if total_packages > 0
else 0,
"level": "low"
if total_packages < 10
else "moderate"
if total_packages < 30
else "high",
},
}
@ -649,34 +724,40 @@ def _assess_performance_impact(summary: dict[str, Any]) -> dict[str, Any]:
"""Assess the performance impact of the dependency tree."""
total_packages = summary.get("total_packages", 0)
max_depth = summary.get("max_depth", 0)
# Estimate installation time (rough approximation)
estimated_install_time = total_packages * 2 + max_depth * 5 # seconds
# Estimate memory footprint (very rough)
estimated_memory_mb = total_packages * 10 + max_depth * 5
# Performance recommendations
recommendations = []
if total_packages > 50:
recommendations.append("Consider using virtual environments to isolate dependencies")
recommendations.append(
"Consider using virtual environments to isolate dependencies"
)
if max_depth > 5:
recommendations.append("Deep dependency chains may slow resolution and installation")
recommendations.append(
"Deep dependency chains may slow resolution and installation"
)
if total_packages > 100:
recommendations.append("Consider dependency analysis tools for large projects")
return {
"estimated_install_time_seconds": estimated_install_time,
"estimated_memory_footprint_mb": estimated_memory_mb,
"performance_level": (
"good" if total_packages < 20
else "moderate" if total_packages < 50
"good"
if total_packages < 20
else "moderate"
if total_packages < 50
else "concerning"
),
"recommendations": recommendations,
"metrics": {
"package_count_impact": "low" if total_packages < 20 else "high",
"depth_impact": "low" if max_depth < 4 else "high",
"resolution_complexity": "simple" if total_packages < 10 else "complex"
}
"resolution_complexity": "simple" if total_packages < 10 else "complex",
},
}