Resolve merge conflict: use enhanced top packages implementation

pypi_query_mcp/core/github_client.py

@@ -0,0 +1,249 @@
+"""GitHub API client for fetching repository statistics and popularity metrics."""
+
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class GitHubAPIClient:
+    """Async client for GitHub API to fetch repository statistics."""
+
+    def __init__(
+        self,
+        timeout: float = 10.0,
+        max_retries: int = 2,
+        retry_delay: float = 1.0,
+        github_token: Optional[str] = None,
+    ):
+        """Initialize GitHub API client.
+
+        Args:
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts
+            retry_delay: Delay between retries in seconds
+            github_token: Optional GitHub API token for higher rate limits
+        """
+        self.base_url = "https://api.github.com"
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+
+        # Simple in-memory cache for repository data
+        self._cache: Dict[str, Dict[str, Any]] = {}
+        self._cache_ttl = 3600  # 1 hour cache
+
+        # HTTP client configuration
+        headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "pypi-query-mcp-server/0.1.0",
+        }
+        
+        if github_token:
+            headers["Authorization"] = f"token {github_token}"
+
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(timeout),
+            headers=headers,
+            follow_redirects=True,
+        )
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()
+
+    async def close(self):
+        """Close the HTTP client."""
+        await self._client.aclose()
+
+    def _get_cache_key(self, repo: str) -> str:
+        """Generate cache key for repository data."""
+        return f"repo:{repo}"
+
+    def _is_cache_valid(self, cache_entry: Dict[str, Any]) -> bool:
+        """Check if cache entry is still valid."""
+        import time
+        return time.time() - cache_entry.get("timestamp", 0) < self._cache_ttl
+
+    async def _make_request(self, url: str) -> Optional[Dict[str, Any]]:
+        """Make HTTP request with retry logic and error handling.
+
+        Args:
+            url: URL to request
+
+        Returns:
+            JSON response data or None if failed
+        """
+        last_exception = None
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                logger.debug(f"Making GitHub API request to {url} (attempt {attempt + 1})")
+
+                response = await self._client.get(url)
+
+                # Handle different HTTP status codes
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 404:
+                    logger.warning(f"GitHub repository not found: {url}")
+                    return None
+                elif response.status_code == 403:
+                    # Rate limit or permission issue
+                    logger.warning(f"GitHub API rate limit or permission denied: {url}")
+                    return None
+                elif response.status_code >= 500:
+                    logger.warning(f"GitHub API server error {response.status_code}: {url}")
+                    if attempt < self.max_retries:
+                        continue
+                    return None
+                else:
+                    logger.warning(f"Unexpected GitHub API status {response.status_code}: {url}")
+                    return None
+
+            except httpx.TimeoutException:
+                last_exception = f"Request timeout for {url}"
+                logger.warning(last_exception)
+            except httpx.NetworkError as e:
+                last_exception = f"Network error for {url}: {e}"
+                logger.warning(last_exception)
+            except Exception as e:
+                last_exception = f"Unexpected error for {url}: {e}"
+                logger.warning(last_exception)
+
+            # Wait before retry (except on last attempt)
+            if attempt < self.max_retries:
+                await asyncio.sleep(self.retry_delay * (2 ** attempt))
+
+        # If we get here, all retries failed
+        logger.error(f"Failed to fetch GitHub data after {self.max_retries + 1} attempts: {last_exception}")
+        return None
+
+    async def get_repository_stats(self, repo_path: str, use_cache: bool = True) -> Optional[Dict[str, Any]]:
+        """Get repository statistics from GitHub API.
+
+        Args:
+            repo_path: Repository path in format "owner/repo"
+            use_cache: Whether to use cached data if available
+
+        Returns:
+            Dictionary containing repository statistics or None if failed
+        """
+        cache_key = self._get_cache_key(repo_path)
+
+        # Check cache first
+        if use_cache and cache_key in self._cache:
+            cache_entry = self._cache[cache_key]
+            if self._is_cache_valid(cache_entry):
+                logger.debug(f"Using cached GitHub data for: {repo_path}")
+                return cache_entry["data"]
+
+        # Make API request
+        url = f"{self.base_url}/repos/{repo_path}"
+        
+        try:
+            data = await self._make_request(url)
+            
+            if data:
+                # Extract relevant statistics
+                stats = {
+                    "stars": data.get("stargazers_count", 0),
+                    "forks": data.get("forks_count", 0),
+                    "watchers": data.get("watchers_count", 0),
+                    "open_issues": data.get("open_issues_count", 0),
+                    "size": data.get("size", 0),
+                    "language": data.get("language"),
+                    "created_at": data.get("created_at"),
+                    "updated_at": data.get("updated_at"),
+                    "pushed_at": data.get("pushed_at"),
+                    "description": data.get("description"),
+                    "topics": data.get("topics", []),
+                    "homepage": data.get("homepage"),
+                    "has_issues": data.get("has_issues", False),
+                    "has_projects": data.get("has_projects", False),
+                    "has_wiki": data.get("has_wiki", False),
+                    "archived": data.get("archived", False),
+                    "disabled": data.get("disabled", False),
+                    "license": data.get("license", {}).get("name") if data.get("license") else None,
+                }
+
+                # Cache the result
+                import time
+                self._cache[cache_key] = {"data": stats, "timestamp": time.time()}
+
+                logger.debug(f"Fetched GitHub stats for {repo_path}: {stats['stars']} stars")
+                return stats
+            else:
+                return None
+
+        except Exception as e:
+            logger.error(f"Error fetching GitHub stats for {repo_path}: {e}")
+            return None
+
+    async def get_multiple_repo_stats(
+        self, 
+        repo_paths: list[str], 
+        use_cache: bool = True,
+        max_concurrent: int = 5
+    ) -> Dict[str, Optional[Dict[str, Any]]]:
+        """Get statistics for multiple repositories concurrently.
+
+        Args:
+            repo_paths: List of repository paths in format "owner/repo"
+            use_cache: Whether to use cached data if available
+            max_concurrent: Maximum number of concurrent requests
+
+        Returns:
+            Dictionary mapping repo paths to their statistics
+        """
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def fetch_repo_stats(repo_path: str) -> tuple[str, Optional[Dict[str, Any]]]:
+            async with semaphore:
+                stats = await self.get_repository_stats(repo_path, use_cache)
+                return repo_path, stats
+
+        # Fetch all repositories concurrently
+        tasks = [fetch_repo_stats(repo) for repo in repo_paths]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Process results
+        repo_stats = {}
+        for result in results:
+            if isinstance(result, Exception):
+                logger.error(f"Error in concurrent GitHub fetch: {result}")
+                continue
+            
+            repo_path, stats = result
+            repo_stats[repo_path] = stats
+
+        return repo_stats
+
+    def clear_cache(self):
+        """Clear the internal cache."""
+        self._cache.clear()
+        logger.debug("GitHub cache cleared")
+
+    async def get_rate_limit(self) -> Optional[Dict[str, Any]]:
+        """Get current GitHub API rate limit status.
+
+        Returns:
+            Dictionary containing rate limit information
+        """
+        url = f"{self.base_url}/rate_limit"
+        
+        try:
+            data = await self._make_request(url)
+            if data:
+                return data.get("rate", {})
+            return None
+        except Exception as e:
+            logger.error(f"Error fetching GitHub rate limit: {e}")
+            return None

pypi_query_mcp/data/__init__.py

@@ -0,0 +1 @@
+"""Data module for PyPI package information."""

pypi_query_mcp/data/popular_packages.py

@@ -0,0 +1,214 @@
+"""Curated lists of popular PyPI packages organized by category and estimated download rankings.
+
+This data provides fallback information when PyPI statistics APIs are unavailable.
+The rankings and download estimates are based on:
+- Historical PyPI download statistics
+- GitHub star counts and activity
+- Community surveys and package popularity
+- Industry usage patterns
+
+Data is organized by categories and includes estimated relative popularity.
+"""
+
+from typing import Dict, List, NamedTuple
+
+class PackageInfo(NamedTuple):
+    """Information about a popular package."""
+    name: str
+    category: str
+    estimated_monthly_downloads: int
+    github_stars: int  # Approximate, for popularity estimation
+    description: str
+    primary_use_case: str
+
+# Core packages that are dependencies for many other packages
+INFRASTRUCTURE_PACKAGES = [
+    PackageInfo("setuptools", "packaging", 800_000_000, 2100, "Package development tools", "packaging"),
+    PackageInfo("wheel", "packaging", 700_000_000, 400, "Binary package format", "packaging"),
+    PackageInfo("pip", "packaging", 600_000_000, 9500, "Package installer", "packaging"),
+    PackageInfo("certifi", "security", 500_000_000, 800, "Certificate bundle", "security"),
+    PackageInfo("urllib3", "networking", 450_000_000, 3600, "HTTP client library", "networking"),
+    PackageInfo("charset-normalizer", "text", 400_000_000, 400, "Character encoding detection", "text-processing"),
+    PackageInfo("idna", "networking", 380_000_000, 200, "Internationalized domain names", "networking"),
+    PackageInfo("six", "compatibility", 350_000_000, 900, "Python 2 and 3 compatibility", "compatibility"),
+    PackageInfo("python-dateutil", "datetime", 320_000_000, 2200, "Date and time utilities", "datetime"),
+    PackageInfo("requests", "networking", 300_000_000, 51000, "HTTP library", "networking"),
+]
+
+# AWS and cloud packages
+CLOUD_PACKAGES = [
+    PackageInfo("boto3", "cloud", 280_000_000, 8900, "AWS SDK", "cloud"),
+    PackageInfo("botocore", "cloud", 275_000_000, 1400, "AWS SDK core", "cloud"),
+    PackageInfo("s3transfer", "cloud", 250_000_000, 200, "S3 transfer utilities", "cloud"),
+    PackageInfo("awscli", "cloud", 80_000_000, 15000, "AWS command line", "cloud"),
+    PackageInfo("azure-core", "cloud", 45_000_000, 400, "Azure SDK core", "cloud"),
+    PackageInfo("google-cloud-storage", "cloud", 35_000_000, 300, "Google Cloud Storage", "cloud"),
+    PackageInfo("azure-storage-blob", "cloud", 30_000_000, 200, "Azure Blob Storage", "cloud"),
+]
+
+# Data science and ML packages
+DATA_SCIENCE_PACKAGES = [
+    PackageInfo("numpy", "data-science", 200_000_000, 26000, "Numerical computing", "data-science"),
+    PackageInfo("pandas", "data-science", 150_000_000, 42000, "Data manipulation", "data-science"),
+    PackageInfo("scikit-learn", "machine-learning", 80_000_000, 58000, "Machine learning", "machine-learning"),
+    PackageInfo("matplotlib", "visualization", 75_000_000, 19000, "Plotting library", "visualization"),
+    PackageInfo("scipy", "data-science", 70_000_000, 12000, "Scientific computing", "data-science"),
+    PackageInfo("seaborn", "visualization", 45_000_000, 11000, "Statistical visualization", "visualization"),
+    PackageInfo("plotly", "visualization", 40_000_000, 15000, "Interactive plots", "visualization"),
+    PackageInfo("jupyter", "development", 35_000_000, 7000, "Interactive notebooks", "development"),
+    PackageInfo("ipython", "development", 50_000_000, 8000, "Interactive Python", "development"),
+    PackageInfo("tensorflow", "machine-learning", 25_000_000, 185000, "Deep learning", "machine-learning"),
+    PackageInfo("torch", "machine-learning", 20_000_000, 81000, "PyTorch deep learning", "machine-learning"),
+    PackageInfo("transformers", "machine-learning", 15_000_000, 130000, "NLP transformers", "machine-learning"),
+]
+
+# Development and testing
+DEVELOPMENT_PACKAGES = [
+    PackageInfo("typing-extensions", "development", 180_000_000, 3000, "Typing extensions", "development"),
+    PackageInfo("packaging", "development", 160_000_000, 600, "Package utilities", "development"),
+    PackageInfo("pytest", "testing", 100_000_000, 11000, "Testing framework", "testing"),
+    PackageInfo("click", "cli", 90_000_000, 15000, "Command line interface", "cli"),
+    PackageInfo("pyyaml", "serialization", 85_000_000, 2200, "YAML parser", "serialization"),
+    PackageInfo("jinja2", "templating", 80_000_000, 10000, "Template engine", "templating"),
+    PackageInfo("markupsafe", "templating", 75_000_000, 600, "Safe markup", "templating"),
+    PackageInfo("attrs", "development", 60_000_000, 5000, "Classes without boilerplate", "development"),
+    PackageInfo("black", "development", 40_000_000, 38000, "Code formatter", "development"),
+    PackageInfo("flake8", "development", 35_000_000, 3000, "Code linting", "development"),
+    PackageInfo("mypy", "development", 30_000_000, 17000, "Static type checker", "development"),
+]
+
+# Web development
+WEB_PACKAGES = [
+    PackageInfo("django", "web", 60_000_000, 77000, "Web framework", "web"),
+    PackageInfo("flask", "web", 55_000_000, 66000, "Micro web framework", "web"),
+    PackageInfo("fastapi", "web", 35_000_000, 74000, "Modern web API framework", "web"),
+    PackageInfo("sqlalchemy", "database", 50_000_000, 8000, "SQL toolkit", "database"),
+    PackageInfo("psycopg2", "database", 25_000_000, 3000, "PostgreSQL adapter", "database"),
+    PackageInfo("redis", "database", 30_000_000, 12000, "Redis client", "database"),
+    PackageInfo("celery", "async", 25_000_000, 23000, "Distributed task queue", "async"),
+    PackageInfo("gunicorn", "web", 20_000_000, 9000, "WSGI server", "web"),
+    PackageInfo("uvicorn", "web", 15_000_000, 8000, "ASGI server", "web"),
+]
+
+# Security and cryptography
+SECURITY_PACKAGES = [
+    PackageInfo("cryptography", "security", 120_000_000, 6000, "Cryptographic library", "security"),
+    PackageInfo("pyopenssl", "security", 60_000_000, 800, "OpenSSL wrapper", "security"),
+    PackageInfo("pyjwt", "security", 40_000_000, 5000, "JSON Web Tokens", "security"),
+    PackageInfo("bcrypt", "security", 35_000_000, 1200, "Password hashing", "security"),
+    PackageInfo("pycryptodome", "security", 30_000_000, 2700, "Cryptographic library", "security"),
+]
+
+# Networking and API
+NETWORKING_PACKAGES = [
+    PackageInfo("httpx", "networking", 25_000_000, 12000, "HTTP client", "networking"),
+    PackageInfo("aiohttp", "networking", 35_000_000, 14000, "Async HTTP", "networking"),
+    PackageInfo("websockets", "networking", 20_000_000, 5000, "WebSocket implementation", "networking"),
+    PackageInfo("paramiko", "networking", 25_000_000, 8000, "SSH client", "networking"),
+]
+
+# Text processing and parsing
+TEXT_PACKAGES = [
+    PackageInfo("beautifulsoup4", "parsing", 40_000_000, 13000, "HTML/XML parser", "parsing"),
+    PackageInfo("lxml", "parsing", 35_000_000, 2600, "XML/HTML parser", "parsing"),
+    PackageInfo("regex", "text", 30_000_000, 700, "Regular expressions", "text-processing"),
+    PackageInfo("python-docx", "text", 15_000_000, 4000, "Word document processing", "text-processing"),
+    PackageInfo("pillow", "imaging", 60_000_000, 11000, "Image processing", "imaging"),
+]
+
+# All packages combined for easy access
+ALL_POPULAR_PACKAGES = (
+    INFRASTRUCTURE_PACKAGES +
+    CLOUD_PACKAGES +
+    DATA_SCIENCE_PACKAGES +
+    DEVELOPMENT_PACKAGES +
+    WEB_PACKAGES +
+    SECURITY_PACKAGES +
+    NETWORKING_PACKAGES +
+    TEXT_PACKAGES
+)
+
+# Create lookup dictionaries
+PACKAGES_BY_NAME = {pkg.name: pkg for pkg in ALL_POPULAR_PACKAGES}
+PACKAGES_BY_CATEGORY = {}
+for pkg in ALL_POPULAR_PACKAGES:
+    if pkg.category not in PACKAGES_BY_CATEGORY:
+        PACKAGES_BY_CATEGORY[pkg.category] = []
+    PACKAGES_BY_CATEGORY[pkg.category].append(pkg)
+
+def get_popular_packages(
+    category: str = None,
+    limit: int = 50,
+    min_downloads: int = 0
+) -> List[PackageInfo]:
+    """Get popular packages filtered by criteria.
+    
+    Args:
+        category: Filter by category (e.g., 'web', 'data-science', 'cloud')
+        limit: Maximum number of packages to return
+        min_downloads: Minimum estimated monthly downloads
+        
+    Returns:
+        List of PackageInfo objects sorted by estimated downloads
+    """
+    packages = ALL_POPULAR_PACKAGES
+    
+    if category:
+        packages = [pkg for pkg in packages if pkg.category == category]
+    
+    if min_downloads:
+        packages = [pkg for pkg in packages if pkg.estimated_monthly_downloads >= min_downloads]
+    
+    # Sort by estimated downloads (descending)
+    packages = sorted(packages, key=lambda x: x.estimated_monthly_downloads, reverse=True)
+    
+    return packages[:limit]
+
+def estimate_downloads_for_period(monthly_downloads: int, period: str) -> int:
+    """Estimate downloads for different time periods.
+    
+    Args:
+        monthly_downloads: Estimated monthly downloads
+        period: Time period ('day', 'week', 'month')
+        
+    Returns:
+        Estimated downloads for the period
+    """
+    if period == "day":
+        return int(monthly_downloads / 30)
+    elif period == "week":
+        return int(monthly_downloads / 4.3)  # ~4.3 weeks per month
+    elif period == "month":
+        return monthly_downloads
+    else:
+        return monthly_downloads
+
+def get_package_info(package_name: str) -> PackageInfo:
+    """Get information about a specific package.
+    
+    Args:
+        package_name: Name of the package
+        
+    Returns:
+        PackageInfo object or None if not found
+    """
+    return PACKAGES_BY_NAME.get(package_name.lower().replace("-", "_").replace("_", "-"))
+
+# GitHub repository URL patterns for fetching real-time data
+GITHUB_REPO_PATTERNS = {
+    "requests": "psf/requests",
+    "django": "django/django",
+    "flask": "pallets/flask",
+    "fastapi": "tiangolo/fastapi",
+    "numpy": "numpy/numpy",
+    "pandas": "pandas-dev/pandas",
+    "scikit-learn": "scikit-learn/scikit-learn",
+    "tensorflow": "tensorflow/tensorflow",
+    "torch": "pytorch/pytorch",
+    "transformers": "huggingface/transformers",
+    "click": "pallets/click",
+    "black": "psf/black",
+    "boto3": "boto/boto3",
+    "sqlalchemy": "sqlalchemy/sqlalchemy",
+    # Add more mappings as needed
+}

pypi_query_mcp/tools/download_stats.py

@@ -1,11 +1,19 @@
-"""PyPI package download statistics tools."""
+"""PyPI package download statistics tools with robust fallback mechanisms."""
 
 import logging
+import os
 from datetime import datetime
-from typing import Any
+from typing import Any, Dict, List, Optional
 
+from ..core.github_client import GitHubAPIClient
 from ..core.pypi_client import PyPIClient
 from ..core.stats_client import PyPIStatsClient
+from ..data.popular_packages import (
+    GITHUB_REPO_PATTERNS,
+    PACKAGES_BY_NAME,
+    estimate_downloads_for_period,
+    get_popular_packages,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -172,10 +180,13 @@ async def get_package_download_trends(
 async def get_top_packages_by_downloads(
     period: str = "month", limit: int = 20
 ) -> dict[str, Any]:
-    """Get top PyPI packages by download count.
+    """Get top PyPI packages by download count with robust fallback mechanisms.
 
-    Note: This function provides a simulated response based on known popular packages
-    since pypistats.org doesn't provide a direct API for top packages.
+    This function implements a multi-tier fallback strategy:
+    1. Try to get real download stats from pypistats.org API
+    2. If API fails, use curated popular packages with estimated downloads
+    3. Enhance estimates with real-time GitHub popularity metrics
+    4. Always return meaningful results even when all external APIs fail
 
     Args:
         period: Time period ('day', 'week', 'month')
@@ -185,128 +196,75 @@ async def get_top_packages_by_downloads(
         Dictionary containing top packages information including:
         - List of top packages with download counts
         - Period and ranking information
-        - Data source and timestamp
+        - Data source and methodology
+        - Enhanced metadata from multiple sources
     """
-    # Known popular packages (this would ideally come from an API)
-    popular_packages = [
-        "boto3",
-        "urllib3",
-        "requests",
-        "certifi",
-        "charset-normalizer",
-        "idna",
-        "setuptools",
-        "python-dateutil",
-        "six",
-        "botocore",
-        "typing-extensions",
-        "packaging",
-        "numpy",
-        "pip",
-        "pyyaml",
-        "cryptography",
-        "click",
-        "jinja2",
-        "markupsafe",
-        "wheel",
-    ]
-
-    async with PyPIStatsClient() as stats_client:
-        try:
-            top_packages = []
-            data_sources_used = set()
-            has_estimated_data = False
-            has_stale_data = False
-            successful_requests = 0
-
-            # Get download stats for popular packages
-            for i, package_name in enumerate(popular_packages[:limit]):
-                try:
-                    stats = await stats_client.get_recent_downloads(
-                        package_name, period, use_cache=True
-                    )
-
-                    download_data = stats.get("data", {})
-                    download_count = _extract_download_count(download_data, period)
-                    
-                    # Track data sources and quality
-                    source = stats.get("source", "pypistats.org")
-                    data_sources_used.add(source)
-                    
-                    if source == "fallback_estimates":
-                        has_estimated_data = True
-                    elif stats.get("note") and "stale" in stats.get("note", "").lower():
-                        has_stale_data = True
-                    
-                    successful_requests += 1
-
-                    package_entry = {
-                        "rank": i + 1,
-                        "package": package_name,
-                        "downloads": download_count,
-                        "period": period,
-                        "data_source": source,
-                    }
-                    
-                    # Add warning note if data is estimated or stale
-                    if source == "fallback_estimates":
-                        package_entry["reliability"] = "estimated"
-                    elif stats.get("note") and "stale" in stats.get("note", "").lower():
-                        package_entry["reliability"] = "cached"
-                    else:
-                        package_entry["reliability"] = "live"
-                        
-                    top_packages.append(package_entry)
-
-                except Exception as e:
-                    logger.warning(f"Could not get stats for {package_name}: {e}")
-                    continue
-
-            # Sort by download count (descending)
-            top_packages.sort(key=lambda x: x.get("downloads", 0), reverse=True)
-
-            # Update ranks after sorting
-            for i, package in enumerate(top_packages):
-                package["rank"] = i + 1
-
-            # Determine overall data quality
-            primary_source = "pypistats.org" if "pypistats.org" in data_sources_used else list(data_sources_used)[0] if data_sources_used else "unknown"
-            
-            result = {
-                "top_packages": top_packages,
-                "period": period,
-                "limit": limit,
-                "total_found": len(top_packages),
-                "successful_requests": successful_requests,
-                "data_source": primary_source,
-                "data_sources_used": list(data_sources_used),
-                "note": "Based on known popular packages due to API limitations",
-                "timestamp": datetime.now().isoformat(),
-            }
-            
-            # Add data quality warnings
-            if has_estimated_data:
-                result["warning"] = "Some data is estimated due to API unavailability. Rankings may not reflect actual current downloads."
-                result["reliability"] = "mixed_estimated"
-            elif has_stale_data:
-                result["warning"] = "Some data may be outdated due to current API issues."
-                result["reliability"] = "mixed_cached"
-            else:
-                result["reliability"] = "live"
-                
-            # Add information about data collection success rate
-            expected_requests = min(limit, len(popular_packages))
-            success_rate = (successful_requests / expected_requests) * 100 if expected_requests > 0 else 0
-            result["data_collection_success_rate"] = f"{success_rate:.1f}%"
-            
-            if success_rate < 50:
-                result["data_quality_warning"] = "Low data collection success rate. Results may be incomplete."
-
-            return result
-
-        except Exception as e:
-            logger.error(f"Error getting top packages: {e}")
-            raise
+    # Get curated popular packages as base data
+    curated_packages = get_popular_packages(limit=max(limit * 2, 100))
+    
+    # Try to enhance with real PyPI stats
+    enhanced_packages = await _enhance_with_real_stats(
+        curated_packages, period, limit
+    )
+    
+    # Try to enhance with GitHub metrics
+    final_packages = await _enhance_with_github_stats(
+        enhanced_packages, limit
+    )
+    
+    # Ensure we have the requested number of packages
+    if len(final_packages) < limit:
+        # Add more from curated list if needed
+        additional_needed = limit - len(final_packages)
+        existing_names = {pkg["package"] for pkg in final_packages}
+        
+        for pkg_info in curated_packages:
+            if pkg_info.name not in existing_names and additional_needed > 0:
+                final_packages.append({
+                    "package": pkg_info.name,
+                    "downloads": estimate_downloads_for_period(
+                        pkg_info.estimated_monthly_downloads, period
+                    ),
+                    "period": period,
+                    "data_source": "curated",
+                    "category": pkg_info.category,
+                    "description": pkg_info.description,
+                    "estimated": True,
+                })
+                additional_needed -= 1
+    
+    # Sort by download count and assign ranks
+    final_packages.sort(key=lambda x: x.get("downloads", 0), reverse=True)
+    final_packages = final_packages[:limit]
+    
+    for i, package in enumerate(final_packages):
+        package["rank"] = i + 1
+    
+    # Determine primary data source
+    real_stats_count = len([p for p in final_packages if not p.get("estimated", False)])
+    github_enhanced_count = len([p for p in final_packages if "github_stars" in p])
+    
+    if real_stats_count > limit // 2:
+        primary_source = "pypistats.org with curated fallback"
+    elif github_enhanced_count > 0:
+        primary_source = "curated data enhanced with GitHub metrics"
+    else:
+        primary_source = "curated popular packages database"
+    
+    return {
+        "top_packages": final_packages,
+        "period": period,
+        "limit": limit,
+        "total_found": len(final_packages),
+        "data_source": primary_source,
+        "methodology": {
+            "real_stats": real_stats_count,
+            "github_enhanced": github_enhanced_count,
+            "estimated": len(final_packages) - real_stats_count,
+        },
+        "note": "Multi-source data with intelligent fallbacks for reliability",
+        "timestamp": datetime.now().isoformat(),
+    }
 
 
 def _analyze_download_stats(download_data: dict[str, Any]) -> dict[str, Any]:
@@ -427,6 +385,202 @@ def _analyze_download_trends(
     return analysis
 
 
+async def _enhance_with_real_stats(
+    curated_packages: List, period: str, limit: int
+) -> List[Dict[str, Any]]:
+    """Try to enhance curated packages with real PyPI download statistics.
+    
+    Args:
+        curated_packages: List of PackageInfo objects from curated data
+        period: Time period for stats
+        limit: Maximum number of packages to process
+        
+    Returns:
+        List of enhanced package dictionaries
+    """
+    enhanced_packages = []
+    
+    try:
+        async with PyPIStatsClient() as stats_client:
+            # Try to get real stats for top packages
+            for pkg_info in curated_packages[:limit * 2]:  # Try more than needed
+                try:
+                    stats = await stats_client.get_recent_downloads(
+                        pkg_info.name, period, use_cache=True
+                    )
+                    
+                    download_data = stats.get("data", {})
+                    real_download_count = _extract_download_count(download_data, period)
+                    
+                    if real_download_count > 0:
+                        # Use real stats
+                        enhanced_packages.append({
+                            "package": pkg_info.name,
+                            "downloads": real_download_count,
+                            "period": period,
+                            "data_source": "pypistats.org",
+                            "category": pkg_info.category,
+                            "description": pkg_info.description,
+                            "estimated": False,
+                        })
+                        logger.debug(f"Got real stats for {pkg_info.name}: {real_download_count}")
+                    else:
+                        # Fall back to estimated downloads
+                        estimated_downloads = estimate_downloads_for_period(
+                            pkg_info.estimated_monthly_downloads, period
+                        )
+                        enhanced_packages.append({
+                            "package": pkg_info.name,
+                            "downloads": estimated_downloads,
+                            "period": period,
+                            "data_source": "estimated",
+                            "category": pkg_info.category,
+                            "description": pkg_info.description,
+                            "estimated": True,
+                        })
+                        
+                except Exception as e:
+                    logger.debug(f"Failed to get real stats for {pkg_info.name}: {e}")
+                    # Fall back to estimated downloads
+                    estimated_downloads = estimate_downloads_for_period(
+                        pkg_info.estimated_monthly_downloads, period
+                    )
+                    enhanced_packages.append({
+                        "package": pkg_info.name,
+                        "downloads": estimated_downloads,
+                        "period": period,
+                        "data_source": "estimated",
+                        "category": pkg_info.category,
+                        "description": pkg_info.description,
+                        "estimated": True,
+                    })
+                    
+                # Stop if we have enough packages
+                if len(enhanced_packages) >= limit:
+                    break
+                    
+    except Exception as e:
+        logger.warning(f"PyPI stats client failed entirely: {e}")
+        # Fall back to all estimated data
+        for pkg_info in curated_packages[:limit]:
+            estimated_downloads = estimate_downloads_for_period(
+                pkg_info.estimated_monthly_downloads, period
+            )
+            enhanced_packages.append({
+                "package": pkg_info.name,
+                "downloads": estimated_downloads,
+                "period": period,
+                "data_source": "estimated",
+                "category": pkg_info.category,
+                "description": pkg_info.description,
+                "estimated": True,
+            })
+    
+    return enhanced_packages
+
+
+async def _enhance_with_github_stats(
+    packages: List[Dict[str, Any]], limit: int
+) -> List[Dict[str, Any]]:
+    """Try to enhance packages with GitHub repository statistics.
+    
+    Args:
+        packages: List of package dictionaries to enhance
+        limit: Maximum number of packages to process
+        
+    Returns:
+        List of enhanced package dictionaries
+    """
+    github_token = os.getenv("GITHUB_TOKEN")  # Optional GitHub token
+    
+    try:
+        async with GitHubAPIClient(github_token=github_token) as github_client:
+            # Get GitHub repo paths for packages that have them
+            repo_paths = []
+            package_to_repo = {}
+            
+            for pkg in packages[:limit]:
+                repo_path = GITHUB_REPO_PATTERNS.get(pkg["package"])
+                if repo_path:
+                    repo_paths.append(repo_path)
+                    package_to_repo[pkg["package"]] = repo_path
+            
+            if repo_paths:
+                # Fetch GitHub stats for all repositories concurrently
+                logger.debug(f"Fetching GitHub stats for {len(repo_paths)} repositories")
+                repo_stats = await github_client.get_multiple_repo_stats(
+                    repo_paths, use_cache=True, max_concurrent=3
+                )
+                
+                # Enhance packages with GitHub data
+                for pkg in packages:
+                    repo_path = package_to_repo.get(pkg["package"])
+                    if repo_path and repo_path in repo_stats:
+                        stats = repo_stats[repo_path]
+                        if stats:
+                            pkg["github_stars"] = stats["stars"]
+                            pkg["github_forks"] = stats["forks"]
+                            pkg["github_updated_at"] = stats["updated_at"]
+                            pkg["github_language"] = stats["language"]
+                            pkg["github_topics"] = stats.get("topics", [])
+                            
+                            # Adjust download estimates based on GitHub popularity
+                            if pkg.get("estimated", False):
+                                popularity_boost = _calculate_popularity_boost(stats)
+                                pkg["downloads"] = int(pkg["downloads"] * popularity_boost)
+                                pkg["github_enhanced"] = True
+                                
+                logger.info(f"Enhanced {len([p for p in packages if 'github_stars' in p])} packages with GitHub data")
+                
+    except Exception as e:
+        logger.debug(f"GitHub enhancement failed: {e}")
+        # Continue without GitHub enhancement
+        pass
+    
+    return packages
+
+
+def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
+    """Calculate a popularity boost multiplier based on GitHub metrics.
+    
+    Args:
+        github_stats: GitHub repository statistics
+        
+    Returns:
+        Multiplier between 0.5 and 2.0 based on popularity
+    """
+    stars = github_stats.get("stars", 0)
+    forks = github_stats.get("forks", 0)
+    
+    # Base multiplier
+    multiplier = 1.0
+    
+    # Adjust based on stars (logarithmic scale)
+    if stars > 50000:
+        multiplier *= 1.5
+    elif stars > 20000:
+        multiplier *= 1.3
+    elif stars > 10000:
+        multiplier *= 1.2
+    elif stars > 5000:
+        multiplier *= 1.1
+    elif stars < 1000:
+        multiplier *= 0.9
+    elif stars < 500:
+        multiplier *= 0.8
+    
+    # Adjust based on forks (indicates active usage)
+    if forks > 10000:
+        multiplier *= 1.2
+    elif forks > 5000:
+        multiplier *= 1.1
+    elif forks < 100:
+        multiplier *= 0.9
+    
+    # Ensure multiplier stays within reasonable bounds
+    return max(0.5, min(2.0, multiplier))
+
+
 def _extract_download_count(download_data: dict[str, Any], period: str) -> int:
     """Extract download count for a specific period.