chore: upgrade all Python packages and fix linting issues
- Update all dependencies to latest versions (fastmcp, httpx, packaging, etc.) - Downgrade click from yanked 8.2.2 to stable 8.1.7 - Fix code formatting and linting issues with ruff - Most tests passing (2 test failures in dependency resolver need investigation)
This commit is contained in:
parent
503ea589f1
commit
8b43927493
34 changed files with 2276 additions and 1593 deletions
|
|
@ -35,7 +35,7 @@ class DependencyResolver:
|
|||
Args:
|
||||
package_name: Name of the package to resolve
|
||||
python_version: Target Python version (e.g., "3.10")
|
||||
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
|
||||
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
|
||||
['test', 'doc'] for setuptools). These are optional dependencies defined by the package.
|
||||
include_dev: Whether to include development dependencies
|
||||
max_depth: Maximum recursion depth (overrides instance default)
|
||||
|
|
@ -243,7 +243,7 @@ async def resolve_package_dependencies(
|
|||
Args:
|
||||
package_name: Name of the package to resolve
|
||||
python_version: Target Python version (e.g., "3.10")
|
||||
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
|
||||
include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests,
|
||||
['test', 'doc'] for setuptools). These are optional dependencies defined by the package.
|
||||
include_dev: Whether to include development dependencies
|
||||
max_depth: Maximum recursion depth
|
||||
|
|
|
|||
|
|
@ -3,14 +3,13 @@
|
|||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
from ..core.github_client import GitHubAPIClient
|
||||
from ..core.pypi_client import PyPIClient
|
||||
from ..core.stats_client import PyPIStatsClient
|
||||
from ..data.popular_packages import (
|
||||
GITHUB_REPO_PATTERNS,
|
||||
PACKAGES_BY_NAME,
|
||||
estimate_downloads_for_period,
|
||||
get_popular_packages,
|
||||
)
|
||||
|
|
@ -73,11 +72,11 @@ async def get_package_download_stats(
|
|||
|
||||
# Calculate trends and analysis
|
||||
analysis = _analyze_download_stats(download_data)
|
||||
|
||||
|
||||
# Determine data source and add warnings if needed
|
||||
data_source = recent_stats.get("source", "pypistats.org")
|
||||
warning_note = recent_stats.get("note")
|
||||
|
||||
|
||||
result = {
|
||||
"package": package_name,
|
||||
"metadata": package_metadata,
|
||||
|
|
@ -87,15 +86,17 @@ async def get_package_download_stats(
|
|||
"data_source": data_source,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
# Add warning/note about data quality if present
|
||||
if warning_note:
|
||||
result["data_quality_note"] = warning_note
|
||||
|
||||
|
||||
# Add reliability indicator
|
||||
if data_source == "fallback_estimates":
|
||||
result["reliability"] = "estimated"
|
||||
result["warning"] = "Data is estimated due to API unavailability. Actual download counts may differ significantly."
|
||||
result["warning"] = (
|
||||
"Data is estimated due to API unavailability. Actual download counts may differ significantly."
|
||||
)
|
||||
elif "stale" in warning_note.lower() if warning_note else False:
|
||||
result["reliability"] = "cached"
|
||||
result["warning"] = "Data may be outdated due to current API issues."
|
||||
|
|
@ -142,7 +143,7 @@ async def get_package_download_trends(
|
|||
|
||||
# Analyze trends
|
||||
trend_analysis = _analyze_download_trends(time_series_data, include_mirrors)
|
||||
|
||||
|
||||
# Determine data source and add warnings if needed
|
||||
data_source = overall_stats.get("source", "pypistats.org")
|
||||
warning_note = overall_stats.get("note")
|
||||
|
|
@ -155,15 +156,17 @@ async def get_package_download_trends(
|
|||
"data_source": data_source,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
# Add warning/note about data quality if present
|
||||
if warning_note:
|
||||
result["data_quality_note"] = warning_note
|
||||
|
||||
|
||||
# Add reliability indicator
|
||||
if data_source == "fallback_estimates":
|
||||
result["reliability"] = "estimated"
|
||||
result["warning"] = "Data is estimated due to API unavailability. Actual download trends may differ significantly."
|
||||
result["warning"] = (
|
||||
"Data is estimated due to API unavailability. Actual download trends may differ significantly."
|
||||
)
|
||||
elif "stale" in warning_note.lower() if warning_note else False:
|
||||
result["reliability"] = "cached"
|
||||
result["warning"] = "Data may be outdated due to current API issues."
|
||||
|
|
@ -201,56 +204,54 @@ async def get_top_packages_by_downloads(
|
|||
"""
|
||||
# Get curated popular packages as base data
|
||||
curated_packages = get_popular_packages(limit=max(limit * 2, 100))
|
||||
|
||||
|
||||
# Try to enhance with real PyPI stats
|
||||
enhanced_packages = await _enhance_with_real_stats(
|
||||
curated_packages, period, limit
|
||||
)
|
||||
|
||||
enhanced_packages = await _enhance_with_real_stats(curated_packages, period, limit)
|
||||
|
||||
# Try to enhance with GitHub metrics
|
||||
final_packages = await _enhance_with_github_stats(
|
||||
enhanced_packages, limit
|
||||
)
|
||||
|
||||
final_packages = await _enhance_with_github_stats(enhanced_packages, limit)
|
||||
|
||||
# Ensure we have the requested number of packages
|
||||
if len(final_packages) < limit:
|
||||
# Add more from curated list if needed
|
||||
additional_needed = limit - len(final_packages)
|
||||
existing_names = {pkg["package"] for pkg in final_packages}
|
||||
|
||||
|
||||
for pkg_info in curated_packages:
|
||||
if pkg_info.name not in existing_names and additional_needed > 0:
|
||||
final_packages.append({
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
),
|
||||
"period": period,
|
||||
"data_source": "curated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
})
|
||||
final_packages.append(
|
||||
{
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
),
|
||||
"period": period,
|
||||
"data_source": "curated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
}
|
||||
)
|
||||
additional_needed -= 1
|
||||
|
||||
|
||||
# Sort by download count and assign ranks
|
||||
final_packages.sort(key=lambda x: x.get("downloads", 0), reverse=True)
|
||||
final_packages = final_packages[:limit]
|
||||
|
||||
|
||||
for i, package in enumerate(final_packages):
|
||||
package["rank"] = i + 1
|
||||
|
||||
|
||||
# Determine primary data source
|
||||
real_stats_count = len([p for p in final_packages if not p.get("estimated", False)])
|
||||
github_enhanced_count = len([p for p in final_packages if "github_stars" in p])
|
||||
|
||||
|
||||
if real_stats_count > limit // 2:
|
||||
primary_source = "pypistats.org with curated fallback"
|
||||
elif github_enhanced_count > 0:
|
||||
primary_source = "curated data enhanced with GitHub metrics"
|
||||
else:
|
||||
primary_source = "curated popular packages database"
|
||||
|
||||
|
||||
return {
|
||||
"top_packages": final_packages,
|
||||
"period": period,
|
||||
|
|
@ -386,50 +387,73 @@ def _analyze_download_trends(
|
|||
|
||||
|
||||
async def _enhance_with_real_stats(
|
||||
curated_packages: List, period: str, limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
curated_packages: list, period: str, limit: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Try to enhance curated packages with real PyPI download statistics.
|
||||
|
||||
|
||||
Args:
|
||||
curated_packages: List of PackageInfo objects from curated data
|
||||
period: Time period for stats
|
||||
limit: Maximum number of packages to process
|
||||
|
||||
|
||||
Returns:
|
||||
List of enhanced package dictionaries
|
||||
"""
|
||||
enhanced_packages = []
|
||||
|
||||
|
||||
try:
|
||||
async with PyPIStatsClient() as stats_client:
|
||||
# Try to get real stats for top packages
|
||||
for pkg_info in curated_packages[:limit * 2]: # Try more than needed
|
||||
for pkg_info in curated_packages[: limit * 2]: # Try more than needed
|
||||
try:
|
||||
stats = await stats_client.get_recent_downloads(
|
||||
pkg_info.name, period, use_cache=True
|
||||
)
|
||||
|
||||
|
||||
download_data = stats.get("data", {})
|
||||
real_download_count = _extract_download_count(download_data, period)
|
||||
|
||||
|
||||
if real_download_count > 0:
|
||||
# Use real stats
|
||||
enhanced_packages.append({
|
||||
"package": pkg_info.name,
|
||||
"downloads": real_download_count,
|
||||
"period": period,
|
||||
"data_source": "pypistats.org",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": False,
|
||||
})
|
||||
logger.debug(f"Got real stats for {pkg_info.name}: {real_download_count}")
|
||||
enhanced_packages.append(
|
||||
{
|
||||
"package": pkg_info.name,
|
||||
"downloads": real_download_count,
|
||||
"period": period,
|
||||
"data_source": "pypistats.org",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": False,
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
f"Got real stats for {pkg_info.name}: {real_download_count}"
|
||||
)
|
||||
else:
|
||||
# Fall back to estimated downloads
|
||||
estimated_downloads = estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
)
|
||||
enhanced_packages.append({
|
||||
enhanced_packages.append(
|
||||
{
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimated_downloads,
|
||||
"period": period,
|
||||
"data_source": "estimated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get real stats for {pkg_info.name}: {e}")
|
||||
# Fall back to estimated downloads
|
||||
estimated_downloads = estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
)
|
||||
enhanced_packages.append(
|
||||
{
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimated_downloads,
|
||||
"period": period,
|
||||
|
|
@ -437,28 +461,13 @@ async def _enhance_with_real_stats(
|
|||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get real stats for {pkg_info.name}: {e}")
|
||||
# Fall back to estimated downloads
|
||||
estimated_downloads = estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
}
|
||||
)
|
||||
enhanced_packages.append({
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimated_downloads,
|
||||
"period": period,
|
||||
"data_source": "estimated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
})
|
||||
|
||||
|
||||
# Stop if we have enough packages
|
||||
if len(enhanced_packages) >= limit:
|
||||
break
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"PyPI stats client failed entirely: {e}")
|
||||
# Fall back to all estimated data
|
||||
|
|
@ -466,52 +475,56 @@ async def _enhance_with_real_stats(
|
|||
estimated_downloads = estimate_downloads_for_period(
|
||||
pkg_info.estimated_monthly_downloads, period
|
||||
)
|
||||
enhanced_packages.append({
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimated_downloads,
|
||||
"period": period,
|
||||
"data_source": "estimated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
})
|
||||
|
||||
enhanced_packages.append(
|
||||
{
|
||||
"package": pkg_info.name,
|
||||
"downloads": estimated_downloads,
|
||||
"period": period,
|
||||
"data_source": "estimated",
|
||||
"category": pkg_info.category,
|
||||
"description": pkg_info.description,
|
||||
"estimated": True,
|
||||
}
|
||||
)
|
||||
|
||||
return enhanced_packages
|
||||
|
||||
|
||||
async def _enhance_with_github_stats(
|
||||
packages: List[Dict[str, Any]], limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
packages: list[dict[str, Any]], limit: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Try to enhance packages with GitHub repository statistics.
|
||||
|
||||
|
||||
Args:
|
||||
packages: List of package dictionaries to enhance
|
||||
limit: Maximum number of packages to process
|
||||
|
||||
|
||||
Returns:
|
||||
List of enhanced package dictionaries
|
||||
"""
|
||||
github_token = os.getenv("GITHUB_TOKEN") # Optional GitHub token
|
||||
|
||||
|
||||
try:
|
||||
async with GitHubAPIClient(github_token=github_token) as github_client:
|
||||
# Get GitHub repo paths for packages that have them
|
||||
repo_paths = []
|
||||
package_to_repo = {}
|
||||
|
||||
|
||||
for pkg in packages[:limit]:
|
||||
repo_path = GITHUB_REPO_PATTERNS.get(pkg["package"])
|
||||
if repo_path:
|
||||
repo_paths.append(repo_path)
|
||||
package_to_repo[pkg["package"]] = repo_path
|
||||
|
||||
|
||||
if repo_paths:
|
||||
# Fetch GitHub stats for all repositories concurrently
|
||||
logger.debug(f"Fetching GitHub stats for {len(repo_paths)} repositories")
|
||||
logger.debug(
|
||||
f"Fetching GitHub stats for {len(repo_paths)} repositories"
|
||||
)
|
||||
repo_stats = await github_client.get_multiple_repo_stats(
|
||||
repo_paths, use_cache=True, max_concurrent=3
|
||||
)
|
||||
|
||||
|
||||
# Enhance packages with GitHub data
|
||||
for pkg in packages:
|
||||
repo_path = package_to_repo.get(pkg["package"])
|
||||
|
|
@ -523,38 +536,42 @@ async def _enhance_with_github_stats(
|
|||
pkg["github_updated_at"] = stats["updated_at"]
|
||||
pkg["github_language"] = stats["language"]
|
||||
pkg["github_topics"] = stats.get("topics", [])
|
||||
|
||||
|
||||
# Adjust download estimates based on GitHub popularity
|
||||
if pkg.get("estimated", False):
|
||||
popularity_boost = _calculate_popularity_boost(stats)
|
||||
pkg["downloads"] = int(pkg["downloads"] * popularity_boost)
|
||||
pkg["downloads"] = int(
|
||||
pkg["downloads"] * popularity_boost
|
||||
)
|
||||
pkg["github_enhanced"] = True
|
||||
|
||||
logger.info(f"Enhanced {len([p for p in packages if 'github_stars' in p])} packages with GitHub data")
|
||||
|
||||
|
||||
logger.info(
|
||||
f"Enhanced {len([p for p in packages if 'github_stars' in p])} packages with GitHub data"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"GitHub enhancement failed: {e}")
|
||||
# Continue without GitHub enhancement
|
||||
pass
|
||||
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
|
||||
def _calculate_popularity_boost(github_stats: dict[str, Any]) -> float:
|
||||
"""Calculate a popularity boost multiplier based on GitHub metrics.
|
||||
|
||||
|
||||
Args:
|
||||
github_stats: GitHub repository statistics
|
||||
|
||||
|
||||
Returns:
|
||||
Multiplier between 0.5 and 2.0 based on popularity
|
||||
"""
|
||||
stars = github_stats.get("stars", 0)
|
||||
forks = github_stats.get("forks", 0)
|
||||
|
||||
|
||||
# Base multiplier
|
||||
multiplier = 1.0
|
||||
|
||||
|
||||
# Adjust based on stars (logarithmic scale)
|
||||
if stars > 50000:
|
||||
multiplier *= 1.5
|
||||
|
|
@ -568,7 +585,7 @@ def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
|
|||
multiplier *= 0.9
|
||||
elif stars < 500:
|
||||
multiplier *= 0.8
|
||||
|
||||
|
||||
# Adjust based on forks (indicates active usage)
|
||||
if forks > 10000:
|
||||
multiplier *= 1.2
|
||||
|
|
@ -576,7 +593,7 @@ def _calculate_popularity_boost(github_stats: Dict[str, Any]) -> float:
|
|||
multiplier *= 1.1
|
||||
elif forks < 100:
|
||||
multiplier *= 0.9
|
||||
|
||||
|
||||
# Ensure multiplier stays within reasonable bounds
|
||||
return max(0.5, min(2.0, multiplier))
|
||||
|
||||
|
|
|
|||
|
|
@ -68,8 +68,12 @@ def format_package_info(package_data: dict[str, Any]) -> dict[str, Any]:
|
|||
formatted["total_versions"] = len(releases)
|
||||
# Sort versions semantically and get the most recent 10
|
||||
if releases:
|
||||
sorted_versions = sort_versions_semantically(list(releases.keys()), reverse=True)
|
||||
formatted["available_versions"] = sorted_versions[:10] # Most recent 10 versions
|
||||
sorted_versions = sort_versions_semantically(
|
||||
list(releases.keys()), reverse=True
|
||||
)
|
||||
formatted["available_versions"] = sorted_versions[
|
||||
:10
|
||||
] # Most recent 10 versions
|
||||
else:
|
||||
formatted["available_versions"] = []
|
||||
|
||||
|
|
@ -139,7 +143,7 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
|
|||
Formatted dependency information
|
||||
"""
|
||||
from ..core.dependency_parser import DependencyParser
|
||||
|
||||
|
||||
info = package_data.get("info", {})
|
||||
requires_dist = info.get("requires_dist", []) or []
|
||||
provides_extra = info.get("provides_extra", []) or []
|
||||
|
|
@ -152,7 +156,7 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
|
|||
# Convert Requirements back to strings for JSON serialization
|
||||
runtime_deps = [str(req) for req in categories["runtime"]]
|
||||
dev_deps = [str(req) for req in categories["development"]]
|
||||
|
||||
|
||||
# Convert optional dependencies (extras) to string format
|
||||
optional_deps = {}
|
||||
for extra_name, reqs in categories["extras"].items():
|
||||
|
|
@ -161,14 +165,31 @@ def format_dependency_info(package_data: dict[str, Any]) -> dict[str, Any]:
|
|||
# Separate development and non-development optional dependencies
|
||||
dev_optional_deps = {}
|
||||
non_dev_optional_deps = {}
|
||||
|
||||
|
||||
# Define development-related extra names (same as in DependencyParser)
|
||||
dev_extra_names = {
|
||||
'dev', 'development', 'test', 'testing', 'tests', 'lint', 'linting',
|
||||
'doc', 'docs', 'documentation', 'build', 'check', 'cover', 'coverage',
|
||||
'type', 'typing', 'mypy', 'style', 'format', 'quality'
|
||||
"dev",
|
||||
"development",
|
||||
"test",
|
||||
"testing",
|
||||
"tests",
|
||||
"lint",
|
||||
"linting",
|
||||
"doc",
|
||||
"docs",
|
||||
"documentation",
|
||||
"build",
|
||||
"check",
|
||||
"cover",
|
||||
"coverage",
|
||||
"type",
|
||||
"typing",
|
||||
"mypy",
|
||||
"style",
|
||||
"format",
|
||||
"quality",
|
||||
}
|
||||
|
||||
|
||||
for extra_name, deps in optional_deps.items():
|
||||
if extra_name.lower() in dev_extra_names:
|
||||
dev_optional_deps[extra_name] = deps
|
||||
|
|
@ -260,11 +281,11 @@ async def query_package_versions(package_name: str) -> dict[str, Any]:
|
|||
|
||||
|
||||
async def query_package_dependencies(
|
||||
package_name: str,
|
||||
version: str | None = None,
|
||||
package_name: str,
|
||||
version: str | None = None,
|
||||
include_transitive: bool = False,
|
||||
max_depth: int = 5,
|
||||
python_version: str | None = None
|
||||
python_version: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Query package dependency information from PyPI.
|
||||
|
||||
|
|
@ -293,29 +314,35 @@ async def query_package_dependencies(
|
|||
logger.info(
|
||||
f"Querying dependencies for package: {package_name}"
|
||||
+ (f" version {version}" if version else " (latest)")
|
||||
+ (f" with transitive dependencies (max depth: {max_depth})" if include_transitive else " (direct only)")
|
||||
+ (
|
||||
f" with transitive dependencies (max depth: {max_depth})"
|
||||
if include_transitive
|
||||
else " (direct only)"
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
if include_transitive:
|
||||
# Use the comprehensive dependency resolver for transitive dependencies
|
||||
from .dependency_resolver import resolve_package_dependencies
|
||||
|
||||
|
||||
result = await resolve_package_dependencies(
|
||||
package_name=package_name,
|
||||
python_version=python_version,
|
||||
include_extras=[],
|
||||
include_dev=False,
|
||||
max_depth=max_depth
|
||||
max_depth=max_depth,
|
||||
)
|
||||
|
||||
|
||||
# Format the transitive dependency result to match expected structure
|
||||
return format_transitive_dependency_info(result, package_name, version)
|
||||
else:
|
||||
# Use direct dependency logic with version support
|
||||
async with PyPIClient() as client:
|
||||
# Pass the version parameter to get_package_info
|
||||
package_data = await client.get_package_info(package_name, version=version)
|
||||
package_data = await client.get_package_info(
|
||||
package_name, version=version
|
||||
)
|
||||
return format_dependency_info(package_data)
|
||||
except PyPIError:
|
||||
# Re-raise PyPI-specific errors
|
||||
|
|
@ -342,9 +369,9 @@ def format_transitive_dependency_info(
|
|||
normalized_name = package_name.lower().replace("_", "-")
|
||||
dependency_tree = resolver_result.get("dependency_tree", {})
|
||||
summary = resolver_result.get("summary", {})
|
||||
|
||||
|
||||
main_package = dependency_tree.get(normalized_name, {})
|
||||
|
||||
|
||||
# Build the response in the same format as direct dependencies but with tree structure
|
||||
result = {
|
||||
"package_name": package_name,
|
||||
|
|
@ -353,42 +380,51 @@ def format_transitive_dependency_info(
|
|||
"include_transitive": True,
|
||||
"max_depth": summary.get("max_depth", 0),
|
||||
"python_version": resolver_result.get("python_version"),
|
||||
|
||||
# Direct dependencies (same as before)
|
||||
"runtime_dependencies": main_package.get("dependencies", {}).get("runtime", []),
|
||||
"development_dependencies": main_package.get("dependencies", {}).get("development", []),
|
||||
"development_dependencies": main_package.get("dependencies", {}).get(
|
||||
"development", []
|
||||
),
|
||||
"optional_dependencies": main_package.get("dependencies", {}).get("extras", {}),
|
||||
|
||||
# Transitive dependency information
|
||||
"transitive_dependencies": {
|
||||
"dependency_tree": _build_dependency_tree_structure(dependency_tree, normalized_name),
|
||||
"dependency_tree": _build_dependency_tree_structure(
|
||||
dependency_tree, normalized_name
|
||||
),
|
||||
"all_packages": _extract_all_packages_info(dependency_tree),
|
||||
"circular_dependencies": _detect_circular_dependencies(dependency_tree),
|
||||
"depth_analysis": _analyze_dependency_depths(dependency_tree),
|
||||
},
|
||||
|
||||
# Enhanced summary statistics
|
||||
"dependency_summary": {
|
||||
"direct_runtime_count": len(main_package.get("dependencies", {}).get("runtime", [])),
|
||||
"direct_dev_count": len(main_package.get("dependencies", {}).get("development", [])),
|
||||
"direct_optional_groups": len(main_package.get("dependencies", {}).get("extras", {})),
|
||||
"total_transitive_packages": summary.get("total_packages", 0) - 1, # Exclude main package
|
||||
"direct_runtime_count": len(
|
||||
main_package.get("dependencies", {}).get("runtime", [])
|
||||
),
|
||||
"direct_dev_count": len(
|
||||
main_package.get("dependencies", {}).get("development", [])
|
||||
),
|
||||
"direct_optional_groups": len(
|
||||
main_package.get("dependencies", {}).get("extras", {})
|
||||
),
|
||||
"total_transitive_packages": summary.get("total_packages", 0)
|
||||
- 1, # Exclude main package
|
||||
"total_runtime_dependencies": summary.get("total_runtime_dependencies", 0),
|
||||
"total_development_dependencies": summary.get("total_development_dependencies", 0),
|
||||
"total_development_dependencies": summary.get(
|
||||
"total_development_dependencies", 0
|
||||
),
|
||||
"total_extra_dependencies": summary.get("total_extra_dependencies", 0),
|
||||
"max_dependency_depth": summary.get("max_depth", 0),
|
||||
"complexity_score": _calculate_complexity_score(summary),
|
||||
},
|
||||
|
||||
# Performance and health metrics
|
||||
"analysis": {
|
||||
"resolution_stats": summary,
|
||||
"potential_conflicts": _analyze_potential_conflicts(dependency_tree),
|
||||
"maintenance_concerns": _analyze_maintenance_concerns(dependency_tree),
|
||||
"performance_impact": _assess_performance_impact(summary),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
|
@ -398,27 +434,27 @@ def _build_dependency_tree_structure(
|
|||
"""Build a hierarchical dependency tree structure."""
|
||||
if visited is None:
|
||||
visited = set()
|
||||
|
||||
|
||||
if root_package in visited:
|
||||
return {"circular_reference": True, "package_name": root_package}
|
||||
|
||||
|
||||
visited.add(root_package)
|
||||
|
||||
|
||||
if root_package not in dependency_tree:
|
||||
return {}
|
||||
|
||||
|
||||
package_info = dependency_tree[root_package]
|
||||
children = package_info.get("children", {})
|
||||
|
||||
|
||||
tree_node = {
|
||||
"package_name": package_info.get("name", root_package),
|
||||
"version": package_info.get("version", "unknown"),
|
||||
"depth": package_info.get("depth", 0),
|
||||
"requires_python": package_info.get("requires_python", ""),
|
||||
"dependencies": package_info.get("dependencies", {}),
|
||||
"children": {}
|
||||
"children": {},
|
||||
}
|
||||
|
||||
|
||||
# Recursively build children (with visited tracking to prevent infinite loops)
|
||||
for child_name in children:
|
||||
if child_name not in visited:
|
||||
|
|
@ -427,17 +463,19 @@ def _build_dependency_tree_structure(
|
|||
)
|
||||
else:
|
||||
tree_node["children"][child_name] = {
|
||||
"circular_reference": True,
|
||||
"package_name": child_name
|
||||
"circular_reference": True,
|
||||
"package_name": child_name,
|
||||
}
|
||||
|
||||
|
||||
return tree_node
|
||||
|
||||
|
||||
def _extract_all_packages_info(dependency_tree: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
def _extract_all_packages_info(
|
||||
dependency_tree: dict[str, Any],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Extract comprehensive information about all packages in the dependency tree."""
|
||||
all_packages = {}
|
||||
|
||||
|
||||
for package_name, package_info in dependency_tree.items():
|
||||
all_packages[package_name] = {
|
||||
"name": package_info.get("name", package_name),
|
||||
|
|
@ -446,60 +484,73 @@ def _extract_all_packages_info(dependency_tree: dict[str, Any]) -> dict[str, dic
|
|||
"requires_python": package_info.get("requires_python", ""),
|
||||
"direct_dependencies": {
|
||||
"runtime": package_info.get("dependencies", {}).get("runtime", []),
|
||||
"development": package_info.get("dependencies", {}).get("development", []),
|
||||
"development": package_info.get("dependencies", {}).get(
|
||||
"development", []
|
||||
),
|
||||
"extras": package_info.get("dependencies", {}).get("extras", {}),
|
||||
},
|
||||
"dependency_count": {
|
||||
"runtime": len(package_info.get("dependencies", {}).get("runtime", [])),
|
||||
"development": len(package_info.get("dependencies", {}).get("development", [])),
|
||||
"total_extras": sum(len(deps) for deps in package_info.get("dependencies", {}).get("extras", {}).values()),
|
||||
}
|
||||
"development": len(
|
||||
package_info.get("dependencies", {}).get("development", [])
|
||||
),
|
||||
"total_extras": sum(
|
||||
len(deps)
|
||||
for deps in package_info.get("dependencies", {})
|
||||
.get("extras", {})
|
||||
.values()
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
return all_packages
|
||||
|
||||
|
||||
def _detect_circular_dependencies(dependency_tree: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _detect_circular_dependencies(
|
||||
dependency_tree: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Detect circular dependencies in the dependency tree."""
|
||||
circular_deps = []
|
||||
|
||||
|
||||
def dfs(package_name: str, path: list[str], visited: set[str]) -> None:
|
||||
if package_name in path:
|
||||
# Found a circular dependency
|
||||
cycle_start = path.index(package_name)
|
||||
cycle = path[cycle_start:] + [package_name]
|
||||
circular_deps.append({
|
||||
"cycle": cycle,
|
||||
"length": len(cycle) - 1,
|
||||
"packages_involved": list(set(cycle))
|
||||
})
|
||||
circular_deps.append(
|
||||
{
|
||||
"cycle": cycle,
|
||||
"length": len(cycle) - 1,
|
||||
"packages_involved": list(set(cycle)),
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
if package_name in visited or package_name not in dependency_tree:
|
||||
return
|
||||
|
||||
|
||||
visited.add(package_name)
|
||||
path.append(package_name)
|
||||
|
||||
|
||||
# Check children
|
||||
children = dependency_tree[package_name].get("children", {})
|
||||
for child_name in children:
|
||||
dfs(child_name, path.copy(), visited)
|
||||
|
||||
|
||||
# Start DFS from each package
|
||||
for package_name in dependency_tree:
|
||||
dfs(package_name, [], set())
|
||||
|
||||
|
||||
# Remove duplicates
|
||||
unique_cycles = []
|
||||
seen_cycles = set()
|
||||
|
||||
|
||||
for cycle_info in circular_deps:
|
||||
cycle_set = frozenset(cycle_info["packages_involved"])
|
||||
if cycle_set not in seen_cycles:
|
||||
seen_cycles.add(cycle_set)
|
||||
unique_cycles.append(cycle_info)
|
||||
|
||||
|
||||
return unique_cycles
|
||||
|
||||
|
||||
|
|
@ -507,29 +558,36 @@ def _analyze_dependency_depths(dependency_tree: dict[str, Any]) -> dict[str, Any
|
|||
"""Analyze the depth distribution of dependencies."""
|
||||
depth_counts = {}
|
||||
depth_packages = {}
|
||||
|
||||
|
||||
for package_name, package_info in dependency_tree.items():
|
||||
depth = package_info.get("depth", 0)
|
||||
|
||||
|
||||
if depth not in depth_counts:
|
||||
depth_counts[depth] = 0
|
||||
depth_packages[depth] = []
|
||||
|
||||
|
||||
depth_counts[depth] += 1
|
||||
depth_packages[depth].append(package_name)
|
||||
|
||||
|
||||
max_depth = max(depth_counts.keys()) if depth_counts else 0
|
||||
|
||||
|
||||
return {
|
||||
"max_depth": max_depth,
|
||||
"depth_distribution": depth_counts,
|
||||
"packages_by_depth": depth_packages,
|
||||
"average_depth": sum(d * c for d, c in depth_counts.items()) / sum(depth_counts.values()) if depth_counts else 0,
|
||||
"average_depth": sum(d * c for d, c in depth_counts.items())
|
||||
/ sum(depth_counts.values())
|
||||
if depth_counts
|
||||
else 0,
|
||||
"depth_analysis": {
|
||||
"shallow_deps": depth_counts.get(1, 0), # Direct dependencies
|
||||
"deep_deps": sum(count for depth, count in depth_counts.items() if depth > 2),
|
||||
"leaf_packages": [pkg for pkg, info in dependency_tree.items() if not info.get("children")]
|
||||
}
|
||||
"deep_deps": sum(
|
||||
count for depth, count in depth_counts.items() if depth > 2
|
||||
),
|
||||
"leaf_packages": [
|
||||
pkg for pkg, info in dependency_tree.items() if not info.get("children")
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -538,14 +596,14 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
|
|||
total_packages = summary.get("total_packages", 0)
|
||||
max_depth = summary.get("max_depth", 0)
|
||||
total_deps = summary.get("total_runtime_dependencies", 0)
|
||||
|
||||
|
||||
# Simple complexity scoring (can be enhanced)
|
||||
base_score = total_packages * 0.3
|
||||
depth_penalty = max_depth * 1.5
|
||||
dependency_penalty = total_deps * 0.1
|
||||
|
||||
|
||||
complexity_score = base_score + depth_penalty + dependency_penalty
|
||||
|
||||
|
||||
# Classify complexity
|
||||
if complexity_score < 10:
|
||||
complexity_level = "low"
|
||||
|
|
@ -558,8 +616,10 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
|
|||
recommendation = "High complexity, consider dependency management strategies"
|
||||
else:
|
||||
complexity_level = "very_high"
|
||||
recommendation = "Very high complexity, significant maintenance overhead expected"
|
||||
|
||||
recommendation = (
|
||||
"Very high complexity, significant maintenance overhead expected"
|
||||
)
|
||||
|
||||
return {
|
||||
"score": round(complexity_score, 2),
|
||||
"level": complexity_level,
|
||||
|
|
@ -568,42 +628,50 @@ def _calculate_complexity_score(summary: dict[str, Any]) -> dict[str, Any]:
|
|||
"total_packages": total_packages,
|
||||
"max_depth": max_depth,
|
||||
"total_dependencies": total_deps,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _analyze_potential_conflicts(dependency_tree: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _analyze_potential_conflicts(
|
||||
dependency_tree: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Analyze potential version conflicts in dependencies."""
|
||||
# This is a simplified analysis - in a real implementation,
|
||||
# This is a simplified analysis - in a real implementation,
|
||||
# you'd parse version constraints and check for conflicts
|
||||
package_versions = {}
|
||||
potential_conflicts = []
|
||||
|
||||
|
||||
for package_name, package_info in dependency_tree.items():
|
||||
runtime_deps = package_info.get("dependencies", {}).get("runtime", [])
|
||||
|
||||
|
||||
for dep_str in runtime_deps:
|
||||
# Basic parsing of "package>=version" format
|
||||
if ">=" in dep_str or "==" in dep_str or "<" in dep_str or ">" in dep_str:
|
||||
parts = dep_str.replace(">=", "@").replace("==", "@").replace("<", "@").replace(">", "@")
|
||||
parts = (
|
||||
dep_str.replace(">=", "@")
|
||||
.replace("==", "@")
|
||||
.replace("<", "@")
|
||||
.replace(">", "@")
|
||||
)
|
||||
dep_name = parts.split("@")[0].strip()
|
||||
|
||||
|
||||
if dep_name not in package_versions:
|
||||
package_versions[dep_name] = []
|
||||
package_versions[dep_name].append({
|
||||
"constraint": dep_str,
|
||||
"required_by": package_name
|
||||
})
|
||||
|
||||
package_versions[dep_name].append(
|
||||
{"constraint": dep_str, "required_by": package_name}
|
||||
)
|
||||
|
||||
# Look for packages with multiple version constraints
|
||||
for dep_name, constraints in package_versions.items():
|
||||
if len(constraints) > 1:
|
||||
potential_conflicts.append({
|
||||
"package": dep_name,
|
||||
"conflicting_constraints": constraints,
|
||||
"severity": "potential" if len(constraints) == 2 else "high"
|
||||
})
|
||||
|
||||
potential_conflicts.append(
|
||||
{
|
||||
"package": dep_name,
|
||||
"conflicting_constraints": constraints,
|
||||
"severity": "potential" if len(constraints) == 2 else "high",
|
||||
}
|
||||
)
|
||||
|
||||
return potential_conflicts
|
||||
|
||||
|
||||
|
|
@ -611,25 +679,25 @@ def _analyze_maintenance_concerns(dependency_tree: dict[str, Any]) -> dict[str,
|
|||
"""Analyze maintenance concerns in the dependency tree."""
|
||||
total_packages = len(dependency_tree)
|
||||
packages_without_version = sum(
|
||||
1 for info in dependency_tree.values()
|
||||
1
|
||||
for info in dependency_tree.values()
|
||||
if info.get("version") in ["unknown", "", None]
|
||||
)
|
||||
|
||||
|
||||
packages_without_python_req = sum(
|
||||
1 for info in dependency_tree.values()
|
||||
if not info.get("requires_python")
|
||||
1 for info in dependency_tree.values() if not info.get("requires_python")
|
||||
)
|
||||
|
||||
|
||||
# Calculate dependency concentration (packages with many dependencies)
|
||||
high_dep_packages = [
|
||||
{
|
||||
"name": name,
|
||||
"dependency_count": len(info.get("dependencies", {}).get("runtime", []))
|
||||
"dependency_count": len(info.get("dependencies", {}).get("runtime", [])),
|
||||
}
|
||||
for name, info in dependency_tree.items()
|
||||
if len(info.get("dependencies", {}).get("runtime", [])) > 5
|
||||
]
|
||||
|
||||
|
||||
return {
|
||||
"total_packages": total_packages,
|
||||
"packages_without_version_info": packages_without_version,
|
||||
|
|
@ -637,11 +705,18 @@ def _analyze_maintenance_concerns(dependency_tree: dict[str, Any]) -> dict[str,
|
|||
"high_dependency_packages": high_dep_packages,
|
||||
"maintenance_risk_score": {
|
||||
"score": round(
|
||||
(packages_without_version / total_packages * 100) +
|
||||
(len(high_dep_packages) / total_packages * 50), 2
|
||||
) if total_packages > 0 else 0,
|
||||
"level": "low" if total_packages < 10 else "moderate" if total_packages < 30 else "high"
|
||||
}
|
||||
(packages_without_version / total_packages * 100)
|
||||
+ (len(high_dep_packages) / total_packages * 50),
|
||||
2,
|
||||
)
|
||||
if total_packages > 0
|
||||
else 0,
|
||||
"level": "low"
|
||||
if total_packages < 10
|
||||
else "moderate"
|
||||
if total_packages < 30
|
||||
else "high",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -649,34 +724,40 @@ def _assess_performance_impact(summary: dict[str, Any]) -> dict[str, Any]:
|
|||
"""Assess the performance impact of the dependency tree."""
|
||||
total_packages = summary.get("total_packages", 0)
|
||||
max_depth = summary.get("max_depth", 0)
|
||||
|
||||
|
||||
# Estimate installation time (rough approximation)
|
||||
estimated_install_time = total_packages * 2 + max_depth * 5 # seconds
|
||||
|
||||
|
||||
# Estimate memory footprint (very rough)
|
||||
estimated_memory_mb = total_packages * 10 + max_depth * 5
|
||||
|
||||
|
||||
# Performance recommendations
|
||||
recommendations = []
|
||||
if total_packages > 50:
|
||||
recommendations.append("Consider using virtual environments to isolate dependencies")
|
||||
recommendations.append(
|
||||
"Consider using virtual environments to isolate dependencies"
|
||||
)
|
||||
if max_depth > 5:
|
||||
recommendations.append("Deep dependency chains may slow resolution and installation")
|
||||
recommendations.append(
|
||||
"Deep dependency chains may slow resolution and installation"
|
||||
)
|
||||
if total_packages > 100:
|
||||
recommendations.append("Consider dependency analysis tools for large projects")
|
||||
|
||||
|
||||
return {
|
||||
"estimated_install_time_seconds": estimated_install_time,
|
||||
"estimated_memory_footprint_mb": estimated_memory_mb,
|
||||
"performance_level": (
|
||||
"good" if total_packages < 20
|
||||
else "moderate" if total_packages < 50
|
||||
"good"
|
||||
if total_packages < 20
|
||||
else "moderate"
|
||||
if total_packages < 50
|
||||
else "concerning"
|
||||
),
|
||||
"recommendations": recommendations,
|
||||
"metrics": {
|
||||
"package_count_impact": "low" if total_packages < 20 else "high",
|
||||
"depth_impact": "low" if max_depth < 4 else "high",
|
||||
"resolution_complexity": "simple" if total_packages < 10 else "complex"
|
||||
}
|
||||
"resolution_complexity": "simple" if total_packages < 10 else "complex",
|
||||
},
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue