feat: add PyPI package download statistics and popularity analysis tools

- Add PyPIStatsClient for pypistats.org API integration
- Implement get_package_download_stats for recent download statistics
- Implement get_package_download_trends for time series analysis
- Implement get_top_packages_by_downloads for popularity rankings
- Add comprehensive MCP tools for download statistics
- Include download trends analysis with growth indicators
- Add repository information and metadata integration
- Provide comprehensive test coverage
- Add demo script and usage examples
- Update README with new features and examples

Signed-off-by: longhao <hal.long@outlook.com>

pypi_query_mcp/core/stats_client.py

@@ -0,0 +1,257 @@
+"""PyPI download statistics client using pypistats.org API."""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Any
+
+import httpx
+
+from .exceptions import (
+    InvalidPackageNameError,
+    NetworkError,
+    PackageNotFoundError,
+    PyPIServerError,
+    RateLimitError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class PyPIStatsClient:
+    """Async client for PyPI download statistics API."""
+
+    def __init__(
+        self,
+        base_url: str = "https://pypistats.org/api",
+        timeout: float = 30.0,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+    ):
+        """Initialize PyPI stats client.
+
+        Args:
+            base_url: Base URL for pypistats API
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts
+            retry_delay: Delay between retries in seconds
+        """
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+
+        # Simple in-memory cache
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._cache_ttl = 3600  # 1 hour (data updates daily)
+
+        # HTTP client configuration
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(timeout),
+            headers={
+                "User-Agent": "pypi-query-mcp-server/0.1.0",
+                "Accept": "application/json",
+            },
+            follow_redirects=True,
+        )
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()
+
+    async def close(self):
+        """Close the HTTP client."""
+        await self._client.aclose()
+
+    def _validate_package_name(self, package_name: str) -> str:
+        """Validate and normalize package name.
+
+        Args:
+            package_name: Package name to validate
+
+        Returns:
+            Normalized package name
+
+        Raises:
+            InvalidPackageNameError: If package name is invalid
+        """
+        if not package_name or not package_name.strip():
+            raise InvalidPackageNameError(package_name)
+
+        # Basic validation
+        normalized = package_name.strip().lower()
+        return normalized
+
+    def _get_cache_key(self, endpoint: str, package_name: str = "", **params) -> str:
+        """Generate cache key for API data."""
+        param_str = "&".join(f"{k}={v}" for k, v in sorted(params.items()) if v is not None)
+        return f"{endpoint}:{package_name}:{param_str}"
+
+    def _is_cache_valid(self, cache_entry: dict[str, Any]) -> bool:
+        """Check if cache entry is still valid."""
+        import time
+        return time.time() - cache_entry.get("timestamp", 0) < self._cache_ttl
+
+    async def _make_request(self, url: str) -> dict[str, Any]:
+        """Make HTTP request with retry logic.
+
+        Args:
+            url: URL to request
+
+        Returns:
+            JSON response data
+
+        Raises:
+            NetworkError: For network-related errors
+            PackageNotFoundError: When package is not found
+            RateLimitError: When rate limit is exceeded
+            PyPIServerError: For server errors
+        """
+        last_exception = None
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                logger.debug(f"Making request to {url} (attempt {attempt + 1})")
+
+                response = await self._client.get(url)
+
+                # Handle different HTTP status codes
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 404:
+                    # Extract package name from URL for better error message
+                    package_name = url.split("/")[-2] if "/" in url else "unknown"
+                    raise PackageNotFoundError(package_name)
+                elif response.status_code == 429:
+                    retry_after = response.headers.get("Retry-After")
+                    retry_after_int = int(retry_after) if retry_after else None
+                    raise RateLimitError(retry_after_int)
+                elif response.status_code >= 500:
+                    raise PyPIServerError(response.status_code)
+                else:
+                    raise PyPIServerError(
+                        response.status_code,
+                        f"Unexpected status code: {response.status_code}",
+                    )
+
+            except httpx.TimeoutException as e:
+                last_exception = NetworkError(f"Request timeout: {e}", e)
+            except httpx.NetworkError as e:
+                last_exception = NetworkError(f"Network error: {e}", e)
+            except (PackageNotFoundError, RateLimitError, PyPIServerError):
+                # Don't retry these errors
+                raise
+            except Exception as e:
+                last_exception = NetworkError(f"Unexpected error: {e}", e)
+
+            # Wait before retry (except on last attempt)
+            if attempt < self.max_retries:
+                await asyncio.sleep(self.retry_delay * (2**attempt))
+
+        # If we get here, all retries failed
+        raise last_exception
+
+    async def get_recent_downloads(
+        self, package_name: str, period: str = "month", use_cache: bool = True
+    ) -> dict[str, Any]:
+        """Get recent download statistics for a package.
+
+        Args:
+            package_name: Name of the package to query
+            period: Time period ('day', 'week', 'month')
+            use_cache: Whether to use cached data if available
+
+        Returns:
+            Dictionary containing recent download statistics
+
+        Raises:
+            InvalidPackageNameError: If package name is invalid
+            PackageNotFoundError: If package is not found
+            NetworkError: For network-related errors
+        """
+        normalized_name = self._validate_package_name(package_name)
+        cache_key = self._get_cache_key("recent", normalized_name, period=period)
+
+        # Check cache first
+        if use_cache and cache_key in self._cache:
+            cache_entry = self._cache[cache_key]
+            if self._is_cache_valid(cache_entry):
+                logger.debug(f"Using cached recent downloads for: {normalized_name}")
+                return cache_entry["data"]
+
+        # Make API request
+        url = f"{self.base_url}/packages/{normalized_name}/recent"
+        if period and period != "all":
+            url += f"?period={period}"
+
+        logger.info(f"Fetching recent downloads for: {normalized_name} (period: {period})")
+
+        try:
+            data = await self._make_request(url)
+
+            # Cache the result
+            import time
+            self._cache[cache_key] = {"data": data, "timestamp": time.time()}
+
+            return data
+
+        except Exception as e:
+            logger.error(f"Failed to fetch recent downloads for {normalized_name}: {e}")
+            raise
+
+    async def get_overall_downloads(
+        self, package_name: str, mirrors: bool = False, use_cache: bool = True
+    ) -> dict[str, Any]:
+        """Get overall download time series for a package.
+
+        Args:
+            package_name: Name of the package to query
+            mirrors: Whether to include mirror downloads
+            use_cache: Whether to use cached data if available
+
+        Returns:
+            Dictionary containing overall download time series
+
+        Raises:
+            InvalidPackageNameError: If package name is invalid
+            PackageNotFoundError: If package is not found
+            NetworkError: For network-related errors
+        """
+        normalized_name = self._validate_package_name(package_name)
+        cache_key = self._get_cache_key("overall", normalized_name, mirrors=mirrors)
+
+        # Check cache first
+        if use_cache and cache_key in self._cache:
+            cache_entry = self._cache[cache_key]
+            if self._is_cache_valid(cache_entry):
+                logger.debug(f"Using cached overall downloads for: {normalized_name}")
+                return cache_entry["data"]
+
+        # Make API request
+        url = f"{self.base_url}/packages/{normalized_name}/overall"
+        if mirrors is not None:
+            url += f"?mirrors={'true' if mirrors else 'false'}"
+
+        logger.info(f"Fetching overall downloads for: {normalized_name} (mirrors: {mirrors})")
+
+        try:
+            data = await self._make_request(url)
+
+            # Cache the result
+            import time
+            self._cache[cache_key] = {"data": data, "timestamp": time.time()}
+
+            return data
+
+        except Exception as e:
+            logger.error(f"Failed to fetch overall downloads for {normalized_name}: {e}")
+            raise
+
+    def clear_cache(self):
+        """Clear the internal cache."""
+        self._cache.clear()
+        logger.debug("Stats cache cleared")

pypi_query_mcp/server.py

@@ -11,6 +11,9 @@ from .tools import (
     check_python_compatibility,
     download_package_with_dependencies,
     get_compatible_python_versions,
+    get_package_download_stats,
+    get_package_download_trends,
+    get_top_packages_by_downloads,
     query_package_dependencies,
     query_package_info,
     query_package_versions,
@@ -407,6 +410,149 @@ async def download_package(
         }
 
 
+@mcp.tool()
+async def get_download_statistics(
+    package_name: str, period: str = "month", use_cache: bool = True
+) -> dict[str, Any]:
+    """Get download statistics for a PyPI package.
+
+    This tool retrieves comprehensive download statistics for a Python package,
+    including recent download counts, trends, and analysis.
+
+    Args:
+        package_name: The name of the PyPI package to analyze (e.g., 'requests', 'numpy')
+        period: Time period for recent downloads ('day', 'week', 'month', default: 'month')
+        use_cache: Whether to use cached data for faster responses (default: True)
+
+    Returns:
+        Dictionary containing download statistics including:
+        - Recent download counts (last day/week/month)
+        - Package metadata and repository information
+        - Download trends and growth analysis
+        - Data source and timestamp information
+
+    Raises:
+        InvalidPackageNameError: If package name is empty or invalid
+        PackageNotFoundError: If package is not found on PyPI
+        NetworkError: For network-related errors
+    """
+    try:
+        logger.info(f"MCP tool: Getting download statistics for {package_name} (period: {period})")
+        result = await get_package_download_stats(package_name, period, use_cache)
+        logger.info(f"Successfully retrieved download statistics for package: {package_name}")
+        return result
+    except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e:
+        logger.error(f"Error getting download statistics for {package_name}: {e}")
+        return {
+            "error": str(e),
+            "error_type": type(e).__name__,
+            "package_name": package_name,
+            "period": period,
+        }
+    except Exception as e:
+        logger.error(f"Unexpected error getting download statistics for {package_name}: {e}")
+        return {
+            "error": f"Unexpected error: {e}",
+            "error_type": "UnexpectedError",
+            "package_name": package_name,
+            "period": period,
+        }
+
+
+@mcp.tool()
+async def get_download_trends(
+    package_name: str, include_mirrors: bool = False, use_cache: bool = True
+) -> dict[str, Any]:
+    """Get download trends and time series for a PyPI package.
+
+    This tool retrieves detailed download trends and time series data for a Python package,
+    providing insights into download patterns over the last 180 days.
+
+    Args:
+        package_name: The name of the PyPI package to analyze (e.g., 'django', 'flask')
+        include_mirrors: Whether to include mirror downloads in analysis (default: False)
+        use_cache: Whether to use cached data for faster responses (default: True)
+
+    Returns:
+        Dictionary containing download trends including:
+        - Time series data for the last 180 days
+        - Trend analysis (increasing/decreasing/stable)
+        - Peak download periods and statistics
+        - Average daily downloads and growth indicators
+
+    Raises:
+        InvalidPackageNameError: If package name is empty or invalid
+        PackageNotFoundError: If package is not found on PyPI
+        NetworkError: For network-related errors
+    """
+    try:
+        logger.info(
+            f"MCP tool: Getting download trends for {package_name} "
+            f"(include_mirrors: {include_mirrors})"
+        )
+        result = await get_package_download_trends(package_name, include_mirrors, use_cache)
+        logger.info(f"Successfully retrieved download trends for package: {package_name}")
+        return result
+    except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e:
+        logger.error(f"Error getting download trends for {package_name}: {e}")
+        return {
+            "error": str(e),
+            "error_type": type(e).__name__,
+            "package_name": package_name,
+            "include_mirrors": include_mirrors,
+        }
+    except Exception as e:
+        logger.error(f"Unexpected error getting download trends for {package_name}: {e}")
+        return {
+            "error": f"Unexpected error: {e}",
+            "error_type": "UnexpectedError",
+            "package_name": package_name,
+            "include_mirrors": include_mirrors,
+        }
+
+
+@mcp.tool()
+async def get_top_downloaded_packages(
+    period: str = "month", limit: int = 20
+) -> dict[str, Any]:
+    """Get the most downloaded PyPI packages.
+
+    This tool retrieves a list of the most popular Python packages by download count,
+    helping you discover trending and widely-used packages in the Python ecosystem.
+
+    Args:
+        period: Time period for download ranking ('day', 'week', 'month', default: 'month')
+        limit: Maximum number of packages to return (default: 20, max: 50)
+
+    Returns:
+        Dictionary containing top packages information including:
+        - Ranked list of packages with download counts
+        - Package metadata and repository links
+        - Period and ranking information
+        - Data source and limitations
+
+    Note:
+        Due to API limitations, this tool provides results based on known popular packages.
+        For comprehensive data analysis, consider using Google BigQuery with PyPI datasets.
+    """
+    try:
+        # Limit the maximum number of packages to prevent excessive API calls
+        actual_limit = min(limit, 50)
+
+        logger.info(f"MCP tool: Getting top {actual_limit} packages for period: {period}")
+        result = await get_top_packages_by_downloads(period, actual_limit)
+        logger.info(f"Successfully retrieved top packages list")
+        return result
+    except Exception as e:
+        logger.error(f"Error getting top packages: {e}")
+        return {
+            "error": f"Unexpected error: {e}",
+            "error_type": "UnexpectedError",
+            "period": period,
+            "limit": limit,
+        }
+
+
 @click.command()
 @click.option(
     "--log-level",

pypi_query_mcp/tools/__init__.py

@@ -10,6 +10,11 @@ from .compatibility_check import (
     suggest_python_version_for_packages,
 )
 from .dependency_resolver import resolve_package_dependencies
+from .download_stats import (
+    get_package_download_stats,
+    get_package_download_trends,
+    get_top_packages_by_downloads,
+)
 from .package_downloader import download_package_with_dependencies
 from .package_query import (
     query_package_dependencies,
@@ -26,4 +31,7 @@ __all__ = [
     "suggest_python_version_for_packages",
     "resolve_package_dependencies",
     "download_package_with_dependencies",
+    "get_package_download_stats",
+    "get_package_download_trends",
+    "get_top_packages_by_downloads",
 ]

pypi_query_mcp/tools/download_stats.py

@@ -0,0 +1,322 @@
+"""PyPI package download statistics tools."""
+
+import logging
+from datetime import datetime, timedelta
+from typing import Any
+
+from ..core.pypi_client import PyPIClient
+from ..core.stats_client import PyPIStatsClient
+from ..core.exceptions import InvalidPackageNameError, NetworkError, PackageNotFoundError
+
+logger = logging.getLogger(__name__)
+
+
+async def get_package_download_stats(
+    package_name: str, period: str = "month", use_cache: bool = True
+) -> dict[str, Any]:
+    """Get download statistics for a PyPI package.
+
+    Args:
+        package_name: Name of the package to query
+        period: Time period for recent downloads ('day', 'week', 'month')
+        use_cache: Whether to use cached data
+
+    Returns:
+        Dictionary containing download statistics including:
+        - Recent download counts (last day/week/month)
+        - Package metadata
+        - Download trends and analysis
+
+    Raises:
+        InvalidPackageNameError: If package name is invalid
+        PackageNotFoundError: If package is not found
+        NetworkError: For network-related errors
+    """
+    async with PyPIStatsClient() as stats_client, PyPIClient() as pypi_client:
+        try:
+            # Get recent download statistics
+            recent_stats = await stats_client.get_recent_downloads(
+                package_name, period, use_cache
+            )
+
+            # Get basic package info for metadata
+            try:
+                package_info = await pypi_client.get_package_info(package_name, use_cache)
+                package_metadata = {
+                    "name": package_info.get("info", {}).get("name", package_name),
+                    "version": package_info.get("info", {}).get("version", "unknown"),
+                    "summary": package_info.get("info", {}).get("summary", ""),
+                    "author": package_info.get("info", {}).get("author", ""),
+                    "home_page": package_info.get("info", {}).get("home_page", ""),
+                    "project_url": package_info.get("info", {}).get("project_url", ""),
+                    "project_urls": package_info.get("info", {}).get("project_urls", {}),
+                }
+            except Exception as e:
+                logger.warning(f"Could not fetch package metadata for {package_name}: {e}")
+                package_metadata = {"name": package_name}
+
+            # Extract download data
+            download_data = recent_stats.get("data", {})
+            
+            # Calculate trends and analysis
+            analysis = _analyze_download_stats(download_data)
+
+            return {
+                "package": package_name,
+                "metadata": package_metadata,
+                "downloads": download_data,
+                "analysis": analysis,
+                "period": period,
+                "data_source": "pypistats.org",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+        except Exception as e:
+            logger.error(f"Error getting download stats for {package_name}: {e}")
+            raise
+
+
+async def get_package_download_trends(
+    package_name: str, include_mirrors: bool = False, use_cache: bool = True
+) -> dict[str, Any]:
+    """Get download trends and time series for a PyPI package.
+
+    Args:
+        package_name: Name of the package to query
+        include_mirrors: Whether to include mirror downloads
+        use_cache: Whether to use cached data
+
+    Returns:
+        Dictionary containing download trends including:
+        - Time series data for the last 180 days
+        - Trend analysis and statistics
+        - Peak download periods
+
+    Raises:
+        InvalidPackageNameError: If package name is invalid
+        PackageNotFoundError: If package is not found
+        NetworkError: For network-related errors
+    """
+    async with PyPIStatsClient() as stats_client:
+        try:
+            # Get overall download time series
+            overall_stats = await stats_client.get_overall_downloads(
+                package_name, include_mirrors, use_cache
+            )
+
+            # Process time series data
+            time_series_data = overall_stats.get("data", [])
+            
+            # Analyze trends
+            trend_analysis = _analyze_download_trends(time_series_data, include_mirrors)
+
+            return {
+                "package": package_name,
+                "time_series": time_series_data,
+                "trend_analysis": trend_analysis,
+                "include_mirrors": include_mirrors,
+                "data_source": "pypistats.org",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+        except Exception as e:
+            logger.error(f"Error getting download trends for {package_name}: {e}")
+            raise
+
+
+async def get_top_packages_by_downloads(
+    period: str = "month", limit: int = 20
+) -> dict[str, Any]:
+    """Get top PyPI packages by download count.
+
+    Note: This function provides a simulated response based on known popular packages
+    since pypistats.org doesn't provide a direct API for top packages.
+
+    Args:
+        period: Time period ('day', 'week', 'month')
+        limit: Maximum number of packages to return
+
+    Returns:
+        Dictionary containing top packages information including:
+        - List of top packages with download counts
+        - Period and ranking information
+        - Data source and timestamp
+    """
+    # Known popular packages (this would ideally come from an API)
+    popular_packages = [
+        "boto3", "urllib3", "requests", "certifi", "charset-normalizer",
+        "idna", "setuptools", "python-dateutil", "six", "botocore",
+        "typing-extensions", "packaging", "numpy", "pip", "pyyaml",
+        "cryptography", "click", "jinja2", "markupsafe", "wheel"
+    ]
+
+    async with PyPIStatsClient() as stats_client:
+        try:
+            top_packages = []
+            
+            # Get download stats for popular packages
+            for i, package_name in enumerate(popular_packages[:limit]):
+                try:
+                    stats = await stats_client.get_recent_downloads(
+                        package_name, period, use_cache=True
+                    )
+                    
+                    download_data = stats.get("data", {})
+                    download_count = _extract_download_count(download_data, period)
+                    
+                    top_packages.append({
+                        "rank": i + 1,
+                        "package": package_name,
+                        "downloads": download_count,
+                        "period": period,
+                    })
+                    
+                except Exception as e:
+                    logger.warning(f"Could not get stats for {package_name}: {e}")
+                    continue
+
+            # Sort by download count (descending)
+            top_packages.sort(key=lambda x: x.get("downloads", 0), reverse=True)
+            
+            # Update ranks after sorting
+            for i, package in enumerate(top_packages):
+                package["rank"] = i + 1
+
+            return {
+                "top_packages": top_packages,
+                "period": period,
+                "limit": limit,
+                "total_found": len(top_packages),
+                "data_source": "pypistats.org",
+                "note": "Based on known popular packages due to API limitations",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+        except Exception as e:
+            logger.error(f"Error getting top packages: {e}")
+            raise
+
+
+def _analyze_download_stats(download_data: dict[str, Any]) -> dict[str, Any]:
+    """Analyze download statistics data.
+
+    Args:
+        download_data: Raw download data from API
+
+    Returns:
+        Dictionary containing analysis results
+    """
+    analysis = {
+        "total_downloads": 0,
+        "periods_available": [],
+        "highest_period": None,
+        "growth_indicators": {},
+    }
+
+    if not download_data:
+        return analysis
+
+    # Extract available periods and counts
+    for period, count in download_data.items():
+        if period.startswith("last_") and isinstance(count, int):
+            analysis["periods_available"].append(period)
+            analysis["total_downloads"] += count
+            
+            if analysis["highest_period"] is None or count > download_data.get(analysis["highest_period"], 0):
+                analysis["highest_period"] = period
+
+    # Calculate growth indicators
+    last_day = download_data.get("last_day", 0)
+    last_week = download_data.get("last_week", 0)
+    last_month = download_data.get("last_month", 0)
+
+    if last_day and last_week:
+        analysis["growth_indicators"]["daily_vs_weekly"] = round(last_day * 7 / last_week, 2)
+    
+    if last_week and last_month:
+        analysis["growth_indicators"]["weekly_vs_monthly"] = round(last_week * 4 / last_month, 2)
+
+    return analysis
+
+
+def _analyze_download_trends(time_series_data: list[dict], include_mirrors: bool) -> dict[str, Any]:
+    """Analyze download trends from time series data.
+
+    Args:
+        time_series_data: Time series download data
+        include_mirrors: Whether mirrors are included
+
+    Returns:
+        Dictionary containing trend analysis
+    """
+    analysis = {
+        "total_downloads": 0,
+        "data_points": len(time_series_data),
+        "date_range": {},
+        "peak_day": None,
+        "average_daily": 0,
+        "trend_direction": "stable",
+    }
+
+    if not time_series_data:
+        return analysis
+
+    # Filter data based on mirror preference
+    category_filter = "with_mirrors" if include_mirrors else "without_mirrors"
+    filtered_data = [
+        item for item in time_series_data 
+        if item.get("category") == category_filter
+    ]
+
+    if not filtered_data:
+        return analysis
+
+    # Calculate statistics
+    total_downloads = sum(item.get("downloads", 0) for item in filtered_data)
+    analysis["total_downloads"] = total_downloads
+    analysis["data_points"] = len(filtered_data)
+
+    if filtered_data:
+        dates = [item.get("date") for item in filtered_data if item.get("date")]
+        if dates:
+            analysis["date_range"] = {
+                "start": min(dates),
+                "end": max(dates),
+            }
+
+        # Find peak day
+        peak_item = max(filtered_data, key=lambda x: x.get("downloads", 0))
+        analysis["peak_day"] = {
+            "date": peak_item.get("date"),
+            "downloads": peak_item.get("downloads", 0),
+        }
+
+        # Calculate average
+        if len(filtered_data) > 0:
+            analysis["average_daily"] = round(total_downloads / len(filtered_data), 2)
+
+        # Simple trend analysis (compare first and last week)
+        if len(filtered_data) >= 14:
+            first_week = sum(item.get("downloads", 0) for item in filtered_data[:7])
+            last_week = sum(item.get("downloads", 0) for item in filtered_data[-7:])
+            
+            if last_week > first_week * 1.1:
+                analysis["trend_direction"] = "increasing"
+            elif last_week < first_week * 0.9:
+                analysis["trend_direction"] = "decreasing"
+
+    return analysis
+
+
+def _extract_download_count(download_data: dict[str, Any], period: str) -> int:
+    """Extract download count for a specific period.
+
+    Args:
+        download_data: Download data from API
+        period: Period to extract ('day', 'week', 'month')
+
+    Returns:
+        Download count for the specified period
+    """
+    period_key = f"last_{period}"
+    return download_data.get(period_key, 0)