chore: upgrade all Python packages and fix linting issues
- Update all dependencies to latest versions (fastmcp, httpx, packaging, etc.) - Downgrade click from yanked 8.2.2 to stable 8.1.7 - Fix code formatting and linting issues with ruff - Most tests passing (2 test failures in dependency resolver need investigation)
This commit is contained in:
parent
503ea589f1
commit
8b43927493
34 changed files with 2276 additions and 1593 deletions
|
|
@ -117,12 +117,29 @@ class DependencyParser:
|
|||
Dictionary with categorized dependencies
|
||||
"""
|
||||
categories = {"runtime": [], "development": [], "optional": {}, "extras": {}}
|
||||
|
||||
|
||||
# Define development-related extra names
|
||||
dev_extra_names = {
|
||||
'dev', 'development', 'test', 'testing', 'tests', 'lint', 'linting',
|
||||
'doc', 'docs', 'documentation', 'build', 'check', 'cover', 'coverage',
|
||||
'type', 'typing', 'mypy', 'style', 'format', 'quality'
|
||||
"dev",
|
||||
"development",
|
||||
"test",
|
||||
"testing",
|
||||
"tests",
|
||||
"lint",
|
||||
"linting",
|
||||
"doc",
|
||||
"docs",
|
||||
"documentation",
|
||||
"build",
|
||||
"check",
|
||||
"cover",
|
||||
"coverage",
|
||||
"type",
|
||||
"typing",
|
||||
"mypy",
|
||||
"style",
|
||||
"format",
|
||||
"quality",
|
||||
}
|
||||
|
||||
for req in requirements:
|
||||
|
|
@ -141,7 +158,7 @@ class DependencyParser:
|
|||
if extra_name not in categories["extras"]:
|
||||
categories["extras"][extra_name] = []
|
||||
categories["extras"][extra_name].append(req)
|
||||
|
||||
|
||||
# Check if this extra is development-related
|
||||
if extra_name.lower() in dev_extra_names:
|
||||
categories["development"].append(req)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ class GitHubAPIClient:
|
|||
timeout: float = 10.0,
|
||||
max_retries: int = 2,
|
||||
retry_delay: float = 1.0,
|
||||
github_token: Optional[str] = None,
|
||||
github_token: str | None = None,
|
||||
):
|
||||
"""Initialize GitHub API client.
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ class GitHubAPIClient:
|
|||
self.retry_delay = retry_delay
|
||||
|
||||
# Simple in-memory cache for repository data
|
||||
self._cache: Dict[str, Dict[str, Any]] = {}
|
||||
self._cache: dict[str, dict[str, Any]] = {}
|
||||
self._cache_ttl = 3600 # 1 hour cache
|
||||
|
||||
# HTTP client configuration
|
||||
|
|
@ -41,7 +41,7 @@ class GitHubAPIClient:
|
|||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "pypi-query-mcp-server/0.1.0",
|
||||
}
|
||||
|
||||
|
||||
if github_token:
|
||||
headers["Authorization"] = f"token {github_token}"
|
||||
|
||||
|
|
@ -67,12 +67,13 @@ class GitHubAPIClient:
|
|||
"""Generate cache key for repository data."""
|
||||
return f"repo:{repo}"
|
||||
|
||||
def _is_cache_valid(self, cache_entry: Dict[str, Any]) -> bool:
|
||||
def _is_cache_valid(self, cache_entry: dict[str, Any]) -> bool:
|
||||
"""Check if cache entry is still valid."""
|
||||
import time
|
||||
|
||||
return time.time() - cache_entry.get("timestamp", 0) < self._cache_ttl
|
||||
|
||||
async def _make_request(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
async def _make_request(self, url: str) -> dict[str, Any] | None:
|
||||
"""Make HTTP request with retry logic and error handling.
|
||||
|
||||
Args:
|
||||
|
|
@ -85,7 +86,9 @@ class GitHubAPIClient:
|
|||
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
logger.debug(f"Making GitHub API request to {url} (attempt {attempt + 1})")
|
||||
logger.debug(
|
||||
f"Making GitHub API request to {url} (attempt {attempt + 1})"
|
||||
)
|
||||
|
||||
response = await self._client.get(url)
|
||||
|
||||
|
|
@ -100,12 +103,16 @@ class GitHubAPIClient:
|
|||
logger.warning(f"GitHub API rate limit or permission denied: {url}")
|
||||
return None
|
||||
elif response.status_code >= 500:
|
||||
logger.warning(f"GitHub API server error {response.status_code}: {url}")
|
||||
logger.warning(
|
||||
f"GitHub API server error {response.status_code}: {url}"
|
||||
)
|
||||
if attempt < self.max_retries:
|
||||
continue
|
||||
return None
|
||||
else:
|
||||
logger.warning(f"Unexpected GitHub API status {response.status_code}: {url}")
|
||||
logger.warning(
|
||||
f"Unexpected GitHub API status {response.status_code}: {url}"
|
||||
)
|
||||
return None
|
||||
|
||||
except httpx.TimeoutException:
|
||||
|
|
@ -120,13 +127,17 @@ class GitHubAPIClient:
|
|||
|
||||
# Wait before retry (except on last attempt)
|
||||
if attempt < self.max_retries:
|
||||
await asyncio.sleep(self.retry_delay * (2 ** attempt))
|
||||
await asyncio.sleep(self.retry_delay * (2**attempt))
|
||||
|
||||
# If we get here, all retries failed
|
||||
logger.error(f"Failed to fetch GitHub data after {self.max_retries + 1} attempts: {last_exception}")
|
||||
logger.error(
|
||||
f"Failed to fetch GitHub data after {self.max_retries + 1} attempts: {last_exception}"
|
||||
)
|
||||
return None
|
||||
|
||||
async def get_repository_stats(self, repo_path: str, use_cache: bool = True) -> Optional[Dict[str, Any]]:
|
||||
async def get_repository_stats(
|
||||
self, repo_path: str, use_cache: bool = True
|
||||
) -> dict[str, Any] | None:
|
||||
"""Get repository statistics from GitHub API.
|
||||
|
||||
Args:
|
||||
|
|
@ -147,10 +158,10 @@ class GitHubAPIClient:
|
|||
|
||||
# Make API request
|
||||
url = f"{self.base_url}/repos/{repo_path}"
|
||||
|
||||
|
||||
try:
|
||||
data = await self._make_request(url)
|
||||
|
||||
|
||||
if data:
|
||||
# Extract relevant statistics
|
||||
stats = {
|
||||
|
|
@ -171,14 +182,19 @@ class GitHubAPIClient:
|
|||
"has_wiki": data.get("has_wiki", False),
|
||||
"archived": data.get("archived", False),
|
||||
"disabled": data.get("disabled", False),
|
||||
"license": data.get("license", {}).get("name") if data.get("license") else None,
|
||||
"license": data.get("license", {}).get("name")
|
||||
if data.get("license")
|
||||
else None,
|
||||
}
|
||||
|
||||
# Cache the result
|
||||
import time
|
||||
|
||||
self._cache[cache_key] = {"data": stats, "timestamp": time.time()}
|
||||
|
||||
logger.debug(f"Fetched GitHub stats for {repo_path}: {stats['stars']} stars")
|
||||
logger.debug(
|
||||
f"Fetched GitHub stats for {repo_path}: {stats['stars']} stars"
|
||||
)
|
||||
return stats
|
||||
else:
|
||||
return None
|
||||
|
|
@ -188,11 +204,8 @@ class GitHubAPIClient:
|
|||
return None
|
||||
|
||||
async def get_multiple_repo_stats(
|
||||
self,
|
||||
repo_paths: list[str],
|
||||
use_cache: bool = True,
|
||||
max_concurrent: int = 5
|
||||
) -> Dict[str, Optional[Dict[str, Any]]]:
|
||||
self, repo_paths: list[str], use_cache: bool = True, max_concurrent: int = 5
|
||||
) -> dict[str, dict[str, Any] | None]:
|
||||
"""Get statistics for multiple repositories concurrently.
|
||||
|
||||
Args:
|
||||
|
|
@ -205,7 +218,7 @@ class GitHubAPIClient:
|
|||
"""
|
||||
semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def fetch_repo_stats(repo_path: str) -> tuple[str, Optional[Dict[str, Any]]]:
|
||||
async def fetch_repo_stats(repo_path: str) -> tuple[str, dict[str, Any] | None]:
|
||||
async with semaphore:
|
||||
stats = await self.get_repository_stats(repo_path, use_cache)
|
||||
return repo_path, stats
|
||||
|
|
@ -220,7 +233,7 @@ class GitHubAPIClient:
|
|||
if isinstance(result, Exception):
|
||||
logger.error(f"Error in concurrent GitHub fetch: {result}")
|
||||
continue
|
||||
|
||||
|
||||
repo_path, stats = result
|
||||
repo_stats[repo_path] = stats
|
||||
|
||||
|
|
@ -231,14 +244,14 @@ class GitHubAPIClient:
|
|||
self._cache.clear()
|
||||
logger.debug("GitHub cache cleared")
|
||||
|
||||
async def get_rate_limit(self) -> Optional[Dict[str, Any]]:
|
||||
async def get_rate_limit(self) -> dict[str, Any] | None:
|
||||
"""Get current GitHub API rate limit status.
|
||||
|
||||
Returns:
|
||||
Dictionary containing rate limit information
|
||||
"""
|
||||
url = f"{self.base_url}/rate_limit"
|
||||
|
||||
|
||||
try:
|
||||
data = await self._make_request(url)
|
||||
if data:
|
||||
|
|
@ -246,4 +259,4 @@ class GitHubAPIClient:
|
|||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching GitHub rate limit: {e}")
|
||||
return None
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ class PyPIClient:
|
|||
NetworkError: For network-related errors
|
||||
"""
|
||||
normalized_name = self._validate_package_name(package_name)
|
||||
|
||||
|
||||
# Create cache key that includes version info
|
||||
cache_suffix = f"v{version}" if version else "latest"
|
||||
cache_key = self._get_cache_key(normalized_name, f"info_{cache_suffix}")
|
||||
|
|
@ -191,13 +191,17 @@ class PyPIClient:
|
|||
if use_cache and cache_key in self._cache:
|
||||
cache_entry = self._cache[cache_key]
|
||||
if self._is_cache_valid(cache_entry):
|
||||
logger.debug(f"Using cached data for package: {normalized_name} version: {version or 'latest'}")
|
||||
logger.debug(
|
||||
f"Using cached data for package: {normalized_name} version: {version or 'latest'}"
|
||||
)
|
||||
return cache_entry["data"]
|
||||
|
||||
# Build URL - include version if specified
|
||||
if version:
|
||||
url = f"{self.base_url}/{quote(normalized_name)}/{quote(version)}/json"
|
||||
logger.info(f"Fetching package info for: {normalized_name} version {version}")
|
||||
logger.info(
|
||||
f"Fetching package info for: {normalized_name} version {version}"
|
||||
)
|
||||
else:
|
||||
url = f"{self.base_url}/{quote(normalized_name)}/json"
|
||||
logger.info(f"Fetching package info for: {normalized_name} (latest)")
|
||||
|
|
@ -215,13 +219,19 @@ class PyPIClient:
|
|||
except PackageNotFoundError as e:
|
||||
if version:
|
||||
# More specific error message for version not found
|
||||
logger.error(f"Version {version} not found for package {normalized_name}")
|
||||
raise PackageNotFoundError(f"Version {version} not found for package {normalized_name}")
|
||||
logger.error(
|
||||
f"Version {version} not found for package {normalized_name}"
|
||||
)
|
||||
raise PackageNotFoundError(
|
||||
f"Version {version} not found for package {normalized_name}"
|
||||
)
|
||||
else:
|
||||
logger.error(f"Failed to fetch package info for {normalized_name}: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch package info for {normalized_name} version {version or 'latest'}: {e}")
|
||||
logger.error(
|
||||
f"Failed to fetch package info for {normalized_name} version {version or 'latest'}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
async def get_package_versions(
|
||||
|
|
@ -236,7 +246,9 @@ class PyPIClient:
|
|||
Returns:
|
||||
List of version strings
|
||||
"""
|
||||
package_info = await self.get_package_info(package_name, version=None, use_cache=use_cache)
|
||||
package_info = await self.get_package_info(
|
||||
package_name, version=None, use_cache=use_cache
|
||||
)
|
||||
releases = package_info.get("releases", {})
|
||||
return list(releases.keys())
|
||||
|
||||
|
|
@ -252,7 +264,9 @@ class PyPIClient:
|
|||
Returns:
|
||||
Latest version string
|
||||
"""
|
||||
package_info = await self.get_package_info(package_name, version=None, use_cache=use_cache)
|
||||
package_info = await self.get_package_info(
|
||||
package_name, version=None, use_cache=use_cache
|
||||
)
|
||||
return package_info.get("info", {}).get("version", "")
|
||||
|
||||
def clear_cache(self):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import logging
|
|||
import random
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ class PyPIStatsClient:
|
|||
self._cache: dict[str, dict[str, Any]] = {}
|
||||
self._cache_ttl = 86400 # 24 hours (increased for resilience)
|
||||
self._fallback_cache_ttl = 604800 # 7 days for fallback data
|
||||
|
||||
|
||||
# Track API health for smart fallback decisions
|
||||
self._api_health = {
|
||||
"last_success": None,
|
||||
|
|
@ -106,31 +106,33 @@ class PyPIStatsClient:
|
|||
)
|
||||
return f"{endpoint}:{package_name}:{param_str}"
|
||||
|
||||
def _is_cache_valid(self, cache_entry: dict[str, Any], fallback: bool = False) -> bool:
|
||||
def _is_cache_valid(
|
||||
self, cache_entry: dict[str, Any], fallback: bool = False
|
||||
) -> bool:
|
||||
"""Check if cache entry is still valid.
|
||||
|
||||
|
||||
Args:
|
||||
cache_entry: Cache entry to validate
|
||||
fallback: Whether to use fallback cache TTL (longer for resilience)
|
||||
"""
|
||||
ttl = self._fallback_cache_ttl if fallback else self._cache_ttl
|
||||
return time.time() - cache_entry.get("timestamp", 0) < ttl
|
||||
|
||||
|
||||
def _should_use_fallback(self) -> bool:
|
||||
"""Determine if fallback mechanisms should be used based on API health."""
|
||||
if not self.fallback_enabled:
|
||||
return False
|
||||
|
||||
|
||||
# Use fallback if we've had multiple consecutive failures
|
||||
if self._api_health["consecutive_failures"] >= 3:
|
||||
return True
|
||||
|
||||
|
||||
# Use fallback if last success was more than 1 hour ago
|
||||
if self._api_health["last_success"]:
|
||||
time_since_success = time.time() - self._api_health["last_success"]
|
||||
if time_since_success > 3600: # 1 hour
|
||||
return True
|
||||
|
||||
|
||||
return False
|
||||
|
||||
async def _make_request(self, url: str) -> dict[str, Any]:
|
||||
|
|
@ -152,7 +154,9 @@ class PyPIStatsClient:
|
|||
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
logger.debug(f"Making request to {url} (attempt {attempt + 1}/{self.max_retries + 1})")
|
||||
logger.debug(
|
||||
f"Making request to {url} (attempt {attempt + 1}/{self.max_retries + 1})"
|
||||
)
|
||||
|
||||
response = await self._client.get(url)
|
||||
|
||||
|
|
@ -171,16 +175,25 @@ class PyPIStatsClient:
|
|||
elif response.status_code == 429:
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
retry_after_int = int(retry_after) if retry_after else None
|
||||
self._update_api_failure(f"Rate limit exceeded (retry after {retry_after_int}s)")
|
||||
self._update_api_failure(
|
||||
f"Rate limit exceeded (retry after {retry_after_int}s)"
|
||||
)
|
||||
raise RateLimitError(retry_after_int)
|
||||
elif response.status_code >= 500:
|
||||
error_msg = f"Server error: HTTP {response.status_code}"
|
||||
self._update_api_failure(error_msg)
|
||||
|
||||
|
||||
# For 502/503/504 errors, continue retrying
|
||||
if response.status_code in [502, 503, 504] and attempt < self.max_retries:
|
||||
last_exception = PyPIServerError(response.status_code, error_msg)
|
||||
logger.warning(f"Retryable server error {response.status_code}, attempt {attempt + 1}")
|
||||
if (
|
||||
response.status_code in [502, 503, 504]
|
||||
and attempt < self.max_retries
|
||||
):
|
||||
last_exception = PyPIServerError(
|
||||
response.status_code, error_msg
|
||||
)
|
||||
logger.warning(
|
||||
f"Retryable server error {response.status_code}, attempt {attempt + 1}"
|
||||
)
|
||||
else:
|
||||
raise PyPIServerError(response.status_code, error_msg)
|
||||
else:
|
||||
|
|
@ -205,7 +218,9 @@ class PyPIStatsClient:
|
|||
# Only retry certain server errors
|
||||
if e.status_code in [502, 503, 504] and attempt < self.max_retries:
|
||||
last_exception = e
|
||||
logger.warning(f"Retrying server error {e.status_code}, attempt {attempt + 1}")
|
||||
logger.warning(
|
||||
f"Retrying server error {e.status_code}, attempt {attempt + 1}"
|
||||
)
|
||||
else:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
|
@ -216,7 +231,7 @@ class PyPIStatsClient:
|
|||
|
||||
# Calculate exponential backoff with jitter
|
||||
if attempt < self.max_retries:
|
||||
base_delay = self.retry_delay * (2 ** attempt)
|
||||
base_delay = self.retry_delay * (2**attempt)
|
||||
jitter = random.uniform(0.1, 0.3) * base_delay # Add 10-30% jitter
|
||||
delay = base_delay + jitter
|
||||
logger.debug(f"Waiting {delay:.2f}s before retry...")
|
||||
|
|
@ -227,21 +242,25 @@ class PyPIStatsClient:
|
|||
raise last_exception
|
||||
else:
|
||||
raise NetworkError("All retry attempts failed with unknown error")
|
||||
|
||||
|
||||
def _update_api_failure(self, error_msg: str) -> None:
|
||||
"""Update API health tracking on failure."""
|
||||
self._api_health["consecutive_failures"] += 1
|
||||
self._api_health["last_error"] = error_msg
|
||||
logger.debug(f"API failure count: {self._api_health['consecutive_failures']}, error: {error_msg}")
|
||||
|
||||
def _generate_fallback_recent_downloads(self, package_name: str, period: str = "month") -> dict[str, Any]:
|
||||
logger.debug(
|
||||
f"API failure count: {self._api_health['consecutive_failures']}, error: {error_msg}"
|
||||
)
|
||||
|
||||
def _generate_fallback_recent_downloads(
|
||||
self, package_name: str, period: str = "month"
|
||||
) -> dict[str, Any]:
|
||||
"""Generate fallback download statistics when API is unavailable.
|
||||
|
||||
|
||||
This provides estimated download counts based on package popularity patterns
|
||||
to ensure the system remains functional during API outages.
|
||||
"""
|
||||
logger.warning(f"Generating fallback download data for {package_name}")
|
||||
|
||||
|
||||
# Base estimates for popular packages (these are conservative estimates)
|
||||
popular_packages = {
|
||||
"requests": {"day": 1500000, "week": 10500000, "month": 45000000},
|
||||
|
|
@ -270,39 +289,50 @@ class PyPIStatsClient:
|
|||
"pandas": {"day": 200000, "week": 1400000, "month": 6000000},
|
||||
"sqlalchemy": {"day": 90000, "week": 630000, "month": 2700000},
|
||||
}
|
||||
|
||||
|
||||
# Get estimates for known packages or generate based on package name characteristics
|
||||
if package_name.lower() in popular_packages:
|
||||
estimates = popular_packages[package_name.lower()]
|
||||
else:
|
||||
# Generate estimates based on common package patterns
|
||||
if any(keyword in package_name.lower() for keyword in ["test", "dev", "debug"]):
|
||||
if any(
|
||||
keyword in package_name.lower() for keyword in ["test", "dev", "debug"]
|
||||
):
|
||||
# Development/testing packages - lower usage
|
||||
base_daily = random.randint(100, 1000)
|
||||
elif any(keyword in package_name.lower() for keyword in ["aws", "google", "microsoft", "azure"]):
|
||||
elif any(
|
||||
keyword in package_name.lower()
|
||||
for keyword in ["aws", "google", "microsoft", "azure"]
|
||||
):
|
||||
# Cloud provider packages - higher usage
|
||||
base_daily = random.randint(10000, 50000)
|
||||
elif any(keyword in package_name.lower() for keyword in ["http", "request", "client", "api"]):
|
||||
elif any(
|
||||
keyword in package_name.lower()
|
||||
for keyword in ["http", "request", "client", "api"]
|
||||
):
|
||||
# HTTP/API packages - moderate to high usage
|
||||
base_daily = random.randint(5000, 25000)
|
||||
elif any(keyword in package_name.lower() for keyword in ["data", "pandas", "numpy", "scipy"]):
|
||||
elif any(
|
||||
keyword in package_name.lower()
|
||||
for keyword in ["data", "pandas", "numpy", "scipy"]
|
||||
):
|
||||
# Data science packages - high usage
|
||||
base_daily = random.randint(15000, 75000)
|
||||
else:
|
||||
# Generic packages - moderate usage
|
||||
base_daily = random.randint(1000, 10000)
|
||||
|
||||
|
||||
estimates = {
|
||||
"day": base_daily,
|
||||
"week": base_daily * 7,
|
||||
"month": base_daily * 30,
|
||||
}
|
||||
|
||||
|
||||
# Add some realistic variation (±20%)
|
||||
variation = random.uniform(0.8, 1.2)
|
||||
for key in estimates:
|
||||
estimates[key] = int(estimates[key] * variation)
|
||||
|
||||
|
||||
return {
|
||||
"data": {
|
||||
"last_day": estimates["day"],
|
||||
|
|
@ -314,42 +344,48 @@ class PyPIStatsClient:
|
|||
"source": "fallback_estimates",
|
||||
"note": "Estimated data due to API unavailability. Actual values may differ.",
|
||||
}
|
||||
|
||||
def _generate_fallback_overall_downloads(self, package_name: str, mirrors: bool = False) -> dict[str, Any]:
|
||||
|
||||
def _generate_fallback_overall_downloads(
|
||||
self, package_name: str, mirrors: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""Generate fallback time series data when API is unavailable."""
|
||||
logger.warning(f"Generating fallback time series data for {package_name}")
|
||||
|
||||
|
||||
# Generate 180 days of synthetic time series data
|
||||
time_series = []
|
||||
base_date = datetime.now() - timedelta(days=180)
|
||||
|
||||
|
||||
# Get base daily estimate from recent downloads fallback
|
||||
recent_fallback = self._generate_fallback_recent_downloads(package_name)
|
||||
base_daily = recent_fallback["data"]["last_day"]
|
||||
|
||||
|
||||
for i in range(180):
|
||||
current_date = base_date + timedelta(days=i)
|
||||
|
||||
|
||||
# Add weekly and seasonal patterns
|
||||
day_of_week = current_date.weekday()
|
||||
# Lower downloads on weekends
|
||||
week_factor = 0.7 if day_of_week >= 5 else 1.0
|
||||
|
||||
|
||||
# Add some growth trend (packages generally grow over time)
|
||||
growth_factor = 1.0 + (i / 180) * 0.3 # 30% growth over 180 days
|
||||
|
||||
|
||||
# Add random daily variation
|
||||
daily_variation = random.uniform(0.7, 1.3)
|
||||
|
||||
daily_downloads = int(base_daily * week_factor * growth_factor * daily_variation)
|
||||
|
||||
|
||||
daily_downloads = int(
|
||||
base_daily * week_factor * growth_factor * daily_variation
|
||||
)
|
||||
|
||||
category = "with_mirrors" if mirrors else "without_mirrors"
|
||||
time_series.append({
|
||||
"category": category,
|
||||
"date": current_date.strftime("%Y-%m-%d"),
|
||||
"downloads": daily_downloads,
|
||||
})
|
||||
|
||||
time_series.append(
|
||||
{
|
||||
"category": category,
|
||||
"date": current_date.strftime("%Y-%m-%d"),
|
||||
"downloads": daily_downloads,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"data": time_series,
|
||||
"package": package_name,
|
||||
|
|
@ -385,16 +421,24 @@ class PyPIStatsClient:
|
|||
if self._is_cache_valid(cache_entry):
|
||||
logger.debug(f"Using cached recent downloads for: {normalized_name}")
|
||||
return cache_entry["data"]
|
||||
elif self._should_use_fallback() and self._is_cache_valid(cache_entry, fallback=True):
|
||||
logger.info(f"Using extended cache (fallback mode) for: {normalized_name}")
|
||||
elif self._should_use_fallback() and self._is_cache_valid(
|
||||
cache_entry, fallback=True
|
||||
):
|
||||
logger.info(
|
||||
f"Using extended cache (fallback mode) for: {normalized_name}"
|
||||
)
|
||||
cache_entry["data"]["note"] = "Extended cache data due to API issues"
|
||||
return cache_entry["data"]
|
||||
|
||||
# Check if we should use fallback immediately
|
||||
if self._should_use_fallback():
|
||||
logger.warning(f"API health poor, using fallback data for: {normalized_name}")
|
||||
fallback_data = self._generate_fallback_recent_downloads(normalized_name, period)
|
||||
|
||||
logger.warning(
|
||||
f"API health poor, using fallback data for: {normalized_name}"
|
||||
)
|
||||
fallback_data = self._generate_fallback_recent_downloads(
|
||||
normalized_name, period
|
||||
)
|
||||
|
||||
# Cache fallback data with extended TTL
|
||||
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
|
||||
return fallback_data
|
||||
|
|
@ -418,28 +462,39 @@ class PyPIStatsClient:
|
|||
|
||||
except (PyPIServerError, NetworkError) as e:
|
||||
logger.error(f"API request failed for {normalized_name}: {e}")
|
||||
|
||||
|
||||
# Try to use stale cache data if available
|
||||
if use_cache and cache_key in self._cache:
|
||||
cache_entry = self._cache[cache_key]
|
||||
logger.warning(f"Using stale cache data for {normalized_name} due to API failure")
|
||||
logger.warning(
|
||||
f"Using stale cache data for {normalized_name} due to API failure"
|
||||
)
|
||||
cache_entry["data"]["note"] = f"Stale cache data due to API error: {e}"
|
||||
return cache_entry["data"]
|
||||
|
||||
|
||||
# Last resort: generate fallback data
|
||||
if self.fallback_enabled:
|
||||
logger.warning(f"Generating fallback data for {normalized_name} due to API failure")
|
||||
fallback_data = self._generate_fallback_recent_downloads(normalized_name, period)
|
||||
|
||||
logger.warning(
|
||||
f"Generating fallback data for {normalized_name} due to API failure"
|
||||
)
|
||||
fallback_data = self._generate_fallback_recent_downloads(
|
||||
normalized_name, period
|
||||
)
|
||||
|
||||
# Cache fallback data
|
||||
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
|
||||
self._cache[cache_key] = {
|
||||
"data": fallback_data,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
return fallback_data
|
||||
|
||||
|
||||
# If fallback is disabled, re-raise the original exception
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching recent downloads for {normalized_name}: {e}")
|
||||
logger.error(
|
||||
f"Unexpected error fetching recent downloads for {normalized_name}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
async def get_overall_downloads(
|
||||
|
|
@ -469,16 +524,24 @@ class PyPIStatsClient:
|
|||
if self._is_cache_valid(cache_entry):
|
||||
logger.debug(f"Using cached overall downloads for: {normalized_name}")
|
||||
return cache_entry["data"]
|
||||
elif self._should_use_fallback() and self._is_cache_valid(cache_entry, fallback=True):
|
||||
logger.info(f"Using extended cache (fallback mode) for: {normalized_name}")
|
||||
elif self._should_use_fallback() and self._is_cache_valid(
|
||||
cache_entry, fallback=True
|
||||
):
|
||||
logger.info(
|
||||
f"Using extended cache (fallback mode) for: {normalized_name}"
|
||||
)
|
||||
cache_entry["data"]["note"] = "Extended cache data due to API issues"
|
||||
return cache_entry["data"]
|
||||
|
||||
# Check if we should use fallback immediately
|
||||
if self._should_use_fallback():
|
||||
logger.warning(f"API health poor, using fallback data for: {normalized_name}")
|
||||
fallback_data = self._generate_fallback_overall_downloads(normalized_name, mirrors)
|
||||
|
||||
logger.warning(
|
||||
f"API health poor, using fallback data for: {normalized_name}"
|
||||
)
|
||||
fallback_data = self._generate_fallback_overall_downloads(
|
||||
normalized_name, mirrors
|
||||
)
|
||||
|
||||
# Cache fallback data with extended TTL
|
||||
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
|
||||
return fallback_data
|
||||
|
|
@ -502,28 +565,39 @@ class PyPIStatsClient:
|
|||
|
||||
except (PyPIServerError, NetworkError) as e:
|
||||
logger.error(f"API request failed for {normalized_name}: {e}")
|
||||
|
||||
|
||||
# Try to use stale cache data if available
|
||||
if use_cache and cache_key in self._cache:
|
||||
cache_entry = self._cache[cache_key]
|
||||
logger.warning(f"Using stale cache data for {normalized_name} due to API failure")
|
||||
logger.warning(
|
||||
f"Using stale cache data for {normalized_name} due to API failure"
|
||||
)
|
||||
cache_entry["data"]["note"] = f"Stale cache data due to API error: {e}"
|
||||
return cache_entry["data"]
|
||||
|
||||
|
||||
# Last resort: generate fallback data
|
||||
if self.fallback_enabled:
|
||||
logger.warning(f"Generating fallback data for {normalized_name} due to API failure")
|
||||
fallback_data = self._generate_fallback_overall_downloads(normalized_name, mirrors)
|
||||
|
||||
logger.warning(
|
||||
f"Generating fallback data for {normalized_name} due to API failure"
|
||||
)
|
||||
fallback_data = self._generate_fallback_overall_downloads(
|
||||
normalized_name, mirrors
|
||||
)
|
||||
|
||||
# Cache fallback data
|
||||
self._cache[cache_key] = {"data": fallback_data, "timestamp": time.time()}
|
||||
self._cache[cache_key] = {
|
||||
"data": fallback_data,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
return fallback_data
|
||||
|
||||
|
||||
# If fallback is disabled, re-raise the original exception
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching overall downloads for {normalized_name}: {e}")
|
||||
logger.error(
|
||||
f"Unexpected error fetching overall downloads for {normalized_name}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
def clear_cache(self):
|
||||
|
|
|
|||
|
|
@ -288,31 +288,31 @@ class VersionCompatibility:
|
|||
|
||||
def sort_versions_semantically(versions: list[str], reverse: bool = True) -> list[str]:
|
||||
"""Sort package versions using semantic version ordering.
|
||||
|
||||
|
||||
This function properly sorts versions by parsing them as semantic versions,
|
||||
ensuring that pre-release versions (alpha, beta, rc) are ordered correctly
|
||||
relative to stable releases.
|
||||
|
||||
|
||||
Args:
|
||||
versions: List of version strings to sort
|
||||
reverse: If True, sort in descending order (newest first). Default True.
|
||||
|
||||
|
||||
Returns:
|
||||
List of version strings sorted semantically
|
||||
|
||||
|
||||
Examples:
|
||||
>>> sort_versions_semantically(['1.0.0', '2.0.0a1', '1.5.0', '2.0.0'])
|
||||
['2.0.0', '2.0.0a1', '1.5.0', '1.0.0']
|
||||
|
||||
|
||||
>>> sort_versions_semantically(['5.2rc1', '5.2.5', '5.2.0'])
|
||||
['5.2.5', '5.2.0', '5.2rc1']
|
||||
"""
|
||||
if not versions:
|
||||
return []
|
||||
|
||||
|
||||
def parse_version_safe(version_str: str) -> tuple[Version | None, str]:
|
||||
"""Safely parse a version string, returning (parsed_version, original_string).
|
||||
|
||||
|
||||
Returns (None, original_string) if parsing fails.
|
||||
"""
|
||||
try:
|
||||
|
|
@ -320,26 +320,26 @@ def sort_versions_semantically(versions: list[str], reverse: bool = True) -> lis
|
|||
except InvalidVersion:
|
||||
logger.debug(f"Failed to parse version '{version_str}' as semantic version")
|
||||
return (None, version_str)
|
||||
|
||||
|
||||
# Parse all versions, keeping track of originals
|
||||
parsed_versions = [parse_version_safe(v) for v in versions]
|
||||
|
||||
|
||||
# Separate valid and invalid versions
|
||||
valid_versions = [(v, orig) for v, orig in parsed_versions if v is not None]
|
||||
invalid_versions = [orig for v, orig in parsed_versions if v is None]
|
||||
|
||||
|
||||
# Sort valid versions semantically
|
||||
valid_versions.sort(key=lambda x: x[0], reverse=reverse)
|
||||
|
||||
|
||||
# Sort invalid versions lexicographically as fallback
|
||||
invalid_versions.sort(reverse=reverse)
|
||||
|
||||
|
||||
# Combine results: valid versions first, then invalid ones
|
||||
result = [orig for _, orig in valid_versions] + invalid_versions
|
||||
|
||||
|
||||
logger.debug(
|
||||
f"Sorted {len(versions)} versions: {len(valid_versions)} valid, "
|
||||
f"{len(invalid_versions)} invalid"
|
||||
)
|
||||
|
||||
|
||||
return result
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue