Implement pg-orrery-catalog: TLE catalog builder for pg_orrery

Core modules:
- tle.py: NORAD decoding (Alpha-5 + Super-5, matching get_el.c),
  3LE/2LE parsing, TLERecord dataclass with epoch-based dedup
- config.py: TOML config + env var overlay (XDG-compliant paths)
- cache.py: File-based cache with staleness checking
- catalog.py: Multi-source merge with MergeStats tracking
- regime.py: LEO/MEO/GEO/HEO classification by mean motion

Source downloaders (httpx):
- celestrak.py: Active catalog + supplemental GP groups
- satnogs.py: JSON API with 3LE conversion
- spacetrack.py: POST auth flow, bulk GP download

Output formatters:
- sql.py: pg_orrery-compatible INSERT generation (E'' strings)
- tle_file.py: Standard 3LE text output
- json_out.py: JSON with orbital metadata and regime

CLI (Click + Rich):
- download: Cache TLEs from all sources
- build: Merge + output SQL/3LE/JSON (pipes to psql)
- load: Direct DB load via psycopg (optional [pg] extra)
- info: Cache stats and configuration display

58 tests covering NORAD decoding (all 4 encoding cases),
parsing, merge/dedup, SQL escaping, regime classification.
This commit is contained in:
Ryan Malloy 2026-02-18 00:31:46 -07:00
parent 23eb205fe8
commit 1d36729bed
18 changed files with 2079 additions and 0 deletions

78
tests/test_catalog.py Normal file
View file

@ -0,0 +1,78 @@
"""Tests for catalog merge and deduplication logic."""
import textwrap
from pg_orrery_catalog.catalog import merge_records, merge_sources
from pg_orrery_catalog.tle import TLERecord
def _make_record(norad_id: int, epoch: float, source: str = "") -> TLERecord:
"""Helper to create a minimal TLERecord for testing."""
line1 = f"1 {norad_id:05d}U 98067A {epoch:014.8f} .00000000 00000-0 00000-0 0 9990"
line2 = f"2 {norad_id:05d} 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990"
return TLERecord(
line1=line1, line2=line2, name=f"SAT-{norad_id}",
norad_id=norad_id, epoch=epoch, source=source,
)
class TestMergeRecords:
def test_merge_disjoint(self):
a = {1: _make_record(1, 24001.0, "A")}
b = {2: _make_record(2, 24002.0, "B")}
merged = merge_records(a, b)
assert len(merged) == 2
assert 1 in merged
assert 2 in merged
def test_merge_newer_wins(self):
old = {1: _make_record(1, 24001.0, "old")}
new = {1: _make_record(1, 24005.0, "new")}
merged = merge_records(old, new)
assert len(merged) == 1
assert merged[1].source == "new"
assert merged[1].epoch == 24005.0
def test_merge_older_loses(self):
new = {1: _make_record(1, 24005.0, "new")}
old = {1: _make_record(1, 24001.0, "old")}
merged = merge_records(new, old)
assert merged[1].source == "new"
def test_merge_three_sources(self):
a = {1: _make_record(1, 24001.0, "A"), 2: _make_record(2, 24001.0, "A")}
b = {1: _make_record(1, 24003.0, "B"), 3: _make_record(3, 24002.0, "B")}
c = {1: _make_record(1, 24002.0, "C")} # older than B
merged = merge_records(a, b, c)
assert len(merged) == 3
assert merged[1].source == "B" # newest epoch
class TestMergeSources:
def test_merge_files(self, tmp_path):
tle1 = textwrap.dedent("""\
SAT-A
1 00001U 98001A 24001.50000000 .00000000 00000-0 00000-0 0 9990
2 00001 30.0000 100.0000 0010000 45.0000 315.0000 15.00000000999990
""")
tle2 = textwrap.dedent("""\
SAT-A-NEWER
1 00001U 98001A 24005.50000000 .00000000 00000-0 00000-0 0 9990
2 00001 30.0000 100.0000 0010000 45.0000 315.0000 15.00000000999990
SAT-B
1 00002U 98002A 24003.00000000 .00000000 00000-0 00000-0 0 9990
2 00002 45.0000 200.0000 0020000 90.0000 270.0000 14.50000000999990
""")
f1 = tmp_path / "source1.tle"
f2 = tmp_path / "source2.tle"
f1.write_text(tle1)
f2.write_text(tle2)
merged, stats = merge_sources([f1, f2])
assert stats.total_unique == 2
assert len(stats.sources) == 2
# Source 2 should have 1 new (SAT-B) and 1 updated (SAT-A-NEWER)
assert stats.sources[1].new == 1
assert stats.sources[1].updated == 1
# SAT-A should have the newer name
assert merged[1].name == "SAT-A-NEWER"

124
tests/test_output.py Normal file
View file

@ -0,0 +1,124 @@
"""Tests for output formatters (SQL, 3LE, JSON)."""
import json
from pg_orrery_catalog.output.json_out import generate_json
from pg_orrery_catalog.output.sql import escape_sql_string, generate_sql
from pg_orrery_catalog.output.tle_file import generate_3le
from pg_orrery_catalog.tle import TLERecord
def _make_record(norad_id: int, name: str = "", epoch: float = 24001.0) -> TLERecord:
line1 = f"1 {norad_id:05d}U 98067A {epoch:014.8f} .00000000 00000-0 00000-0 0 9990"
line2 = f"2 {norad_id:05d} 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990"
return TLERecord(
line1=line1, line2=line2, name=name or f"SAT-{norad_id}",
norad_id=norad_id, epoch=epoch,
)
class TestSQLEscaping:
def test_single_quote(self):
assert escape_sql_string("it's") == "it''s"
def test_backslash(self):
assert escape_sql_string("a\\b") == "a\\\\b"
def test_both(self):
assert escape_sql_string("it's a\\b") == "it''s a\\\\b"
def test_clean(self):
assert escape_sql_string("HELLO") == "HELLO"
class TestSQLGeneration:
def test_basic_output(self):
records = {25544: _make_record(25544, "ISS (ZARYA)")}
sql = generate_sql(records, table="test_table")
assert "DROP TABLE IF EXISTS test_table;" in sql
assert "CREATE TABLE IF NOT EXISTS test_table" in sql
assert "INSERT INTO test_table" in sql
assert "ISS (ZARYA)" in sql
assert "E'" in sql # escape string syntax
assert "\\n" in sql # newline between TLE lines
def test_no_drop(self):
records = {1: _make_record(1)}
sql = generate_sql(records, table="t", drop_existing=False)
assert "DROP TABLE" not in sql
def test_sorted_by_norad(self):
records = {
99999: _make_record(99999),
1: _make_record(1),
25544: _make_record(25544),
}
sql = generate_sql(records)
lines = [line for line in sql.split("\n") if line.startswith("INSERT")]
assert len(lines) == 3
# First INSERT should be NORAD 1 (lowest)
assert "SAT-1" in lines[0]
def test_name_escaping(self):
records = {1: _make_record(1, "O'BRIEN SAT")}
sql = generate_sql(records)
assert "O''BRIEN SAT" in sql
def test_default_name(self):
"""When TLERecord has no name, SQL output uses 'NORAD {id}'."""
rec = _make_record(1)
rec.name = "" # clear the name to test fallback
records = {1: rec}
sql = generate_sql(records)
assert "NORAD 1" in sql
class TestTLEFileGeneration:
def test_3le_format(self):
records = {25544: _make_record(25544, "ISS (ZARYA)")}
text = generate_3le(records)
lines = text.strip().split("\n")
assert len(lines) == 3
assert lines[0].startswith("0 ")
assert lines[1].startswith("1 ")
assert lines[2].startswith("2 ")
def test_sorted_output(self):
records = {
2: _make_record(2, "B"),
1: _make_record(1, "A"),
}
text = generate_3le(records)
lines = text.strip().split("\n")
assert "A" in lines[0]
def test_name_prefix(self):
records = {1: _make_record(1, "0 ALREADY PREFIXED")}
text = generate_3le(records)
# Should not double-prefix
assert "0 0 ALREADY PREFIXED" not in text
assert "0 ALREADY PREFIXED" in text
class TestJSONGeneration:
def test_valid_json(self):
records = {25544: _make_record(25544, "ISS")}
text = generate_json(records)
data = json.loads(text)
assert isinstance(data, list)
assert len(data) == 1
assert data[0]["norad_id"] == 25544
assert data[0]["name"] == "ISS"
def test_fields_present(self):
records = {1: _make_record(1)}
data = json.loads(generate_json(records))
entry = data[0]
for field in ("norad_id", "name", "line1", "line2", "epoch", "mean_motion",
"inclination", "eccentricity", "regime", "source"):
assert field in entry
def test_regime_classification(self):
records = {1: _make_record(1)} # mean_motion ~15.49 → LEO
data = json.loads(generate_json(records))
assert data[0]["regime"] == "LEO"

53
tests/test_regime.py Normal file
View file

@ -0,0 +1,53 @@
"""Tests for orbital regime classification."""
from pg_orrery_catalog.regime import classify_regime, regime_summary
from pg_orrery_catalog.tle import TLERecord
def _make_record_with_mm(norad_id: int, mean_motion: float) -> TLERecord:
"""Create a TLERecord with a specific mean motion in line2."""
line1 = f"1 {norad_id:05d}U 98067A 24001.50000000 .00000000 00000-0 00000-0 0 9990"
mm_str = f"{mean_motion:011.8f}"
line2 = f"2 {norad_id:05d} 51.6400 100.0000 0007417 30.0000 330.1234 {mm_str}999990"
return TLERecord(
line1=line1, line2=line2, name=f"SAT-{norad_id}",
norad_id=norad_id, epoch=24001.0,
)
class TestClassifyRegime:
def test_leo(self):
assert classify_regime(15.5) == "LEO" # ISS-like
assert classify_regime(11.26) == "LEO" # boundary
def test_meo(self):
assert classify_regime(2.0) == "MEO" # GPS-like
assert classify_regime(11.25) == "MEO" # just below LEO
def test_geo(self):
assert classify_regime(1.0) == "GEO" # near-synchronous
assert classify_regime(0.91) == "GEO"
def test_heo(self):
assert classify_regime(0.5) == "HEO" # Molniya-like
assert classify_regime(0.9) == "HEO" # boundary
assert classify_regime(0.1) == "HEO" # deep space
class TestRegimeSummary:
def test_mixed(self):
records = {
1: _make_record_with_mm(1, 15.5), # LEO
2: _make_record_with_mm(2, 2.0), # MEO
3: _make_record_with_mm(3, 1.0), # GEO
4: _make_record_with_mm(4, 0.5), # HEO
}
summary = regime_summary(records)
assert summary["LEO"] == 1
assert summary["MEO"] == 1
assert summary["GEO"] == 1
assert summary["HEO"] == 1
def test_empty(self):
summary = regime_summary({})
assert summary == {"LEO": 0, "MEO": 0, "GEO": 0, "HEO": 0}

234
tests/test_tle.py Normal file
View file

@ -0,0 +1,234 @@
"""Tests for TLE parsing and NORAD ID decoding.
decode_norad() must match get_norad_number() in pg_orrery's src/sgp4/get_el.c.
Test vectors verified against the C implementation.
"""
import textwrap
import pytest
from pg_orrery_catalog.tle import (
_base64_to_int,
decode_norad,
parse_3le_file,
parse_3le_text,
)
# ── base64_to_int ────────────────────────────────────────────
class TestBase64ToInt:
def test_digits(self):
for i in range(10):
assert _base64_to_int(str(i)) == i
def test_uppercase(self):
assert _base64_to_int("A") == 10
assert _base64_to_int("Z") == 35
def test_lowercase(self):
assert _base64_to_int("a") == 36
assert _base64_to_int("z") == 61
def test_special(self):
assert _base64_to_int(" ") == 0
assert _base64_to_int("+") == 62
assert _base64_to_int("-") == 63
def test_invalid(self):
assert _base64_to_int("!") == -1
assert _base64_to_int("@") == -1
# ── decode_norad: Case (1) traditional 5-digit ──────────────
class TestDecodeNoradTraditional:
def test_simple(self):
assert decode_norad("00001") == 1
assert decode_norad("25544") == 25544
assert decode_norad("99999") == 99999
def test_with_spaces(self):
assert decode_norad(" 5") == 5
assert decode_norad(" 405") == 405
def test_zero(self):
assert decode_norad("00000") == 0
# ── decode_norad: Case (2) Alpha-5 ──────────────────────────
class TestDecodeNoradAlpha5:
"""Alpha-5 encoding: letter + 4 digits, I and O skipped.
A=10, B=11, ..., H=17, J=18 (I skipped), ..., N=22, P=23 (O skipped), ..., Z=33
value = (letter_value - 10) * 10000 + digits + 100000
But the skip logic means: A0, B1, ..., H7, J8, K9, ..., N12, P13, ..., Z23
So: A0000 = 100000, Z9999 = 339999
"""
def test_a0001(self):
# A → val=0, result = 0*10000 + 1 + 100000 = 100001
assert decode_norad("A0001") == 100001
def test_a0000(self):
assert decode_norad("A0000") == 100000
def test_t0002(self):
# T → ord('T')-ord('A') = 19, minus 1 for I, minus 1 for O = 17
# 17 * 10000 + 2 + 100000 = 270002
assert decode_norad("T0002") == 270002
def test_z9999(self):
# Z → ord('Z')-ord('A') = 25, minus 1 for I, minus 1 for O = 23
# 23 * 10000 + 9999 + 100000 = 339999
assert decode_norad("Z9999") == 339999
def test_b0000(self):
assert decode_norad("B0000") == 110000
def test_h9999(self):
# H → 7, skip nothing before I
assert decode_norad("H9999") == 179999
def test_j0000(self):
# J → ord('J')-ord('A')=9, minus 1 for I = 8
assert decode_norad("J0000") == 180000
def test_p0000(self):
# P → ord('P')-ord('A')=15, minus 1 for I, minus 1 for O = 13
assert decode_norad("P0000") == 230000
# ── decode_norad: Case (3) Super-5 (last char uppercase) ────
class TestDecodeNoradSuper5Case3:
"""Case (3): xxxxX — last character is non-digit base64.
rval = 340000 + (digits[4] - 10) + 54 * (d3 + d2*64 + d1*64^2 + d0*64^3)
"""
def test_0000A(self):
# digits = [0, 0, 0, 0, 10]
# rval = 340000 + (10-10) + 54*(0 + 0 + 0 + 0) = 340000
assert decode_norad("0000A") == 340000
def test_0000B(self):
# digits = [0, 0, 0, 0, 11]
# rval = 340000 + 1 = 340001
assert decode_norad("0000B") == 340001
def test_0001A(self):
# digits = [0, 0, 0, 1, 10]
# rval = 340000 + 0 + 54*(1) = 340054
assert decode_norad("0001A") == 340054
# ── decode_norad: Case (4) Super-5 (4th char non-digit) ─────
class TestDecodeNoradSuper5Case4:
"""Case (4): xxxXd — 4th character is non-digit.
rval = 340000 + 905969664 + d4 + (d3-10)*10 + 540*(d2 + d1*64 + d0*64^2)
"""
def test_000A0(self):
# digits = [0, 0, 0, 10, 0]
# rval = 340000 + 905969664 + 0 + 0 + 0 = 906309664
assert decode_norad("000A0") == 906309664
# ── decode_norad: edge cases ────────────────────────────────
class TestDecodeNoradEdgeCases:
def test_empty(self):
assert decode_norad("") is None
assert decode_norad(" ") is None
def test_short(self):
assert decode_norad("123") is None
def test_invalid_chars(self):
assert decode_norad("!@#$%") is None
# ── parse_3le_text ──────────────────────────────────────────
class TestParse3LE:
SAMPLE_3LE = textwrap.dedent("""\
ISS (ZARYA)
1 25544U 98067A 24001.50000000 .00016717 00000-0 10270-3 0 9018
2 25544 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990
STARLINK-1234
1 T0002U 20001A 24002.50000000 .00001234 00000-0 12345-4 0 9990
2 T0002 53.0000 200.0000 0001234 45.0000 315.0000 15.12345678123456
""")
def test_parses_standard_norad(self):
records = parse_3le_text(self.SAMPLE_3LE)
assert 25544 in records
assert records[25544].name == "ISS (ZARYA)"
def test_parses_alpha5_norad(self):
records = parse_3le_text(self.SAMPLE_3LE)
# T0002 → 270002
assert 270002 in records
assert records[270002].name == "STARLINK-1234"
def test_epoch_extraction(self):
records = parse_3le_text(self.SAMPLE_3LE)
assert records[25544].epoch == pytest.approx(24001.5, rel=1e-6)
def test_record_properties(self):
records = parse_3le_text(self.SAMPLE_3LE)
iss = records[25544]
assert iss.mean_motion == pytest.approx(15.49456789, rel=1e-6)
assert iss.inclination == pytest.approx(51.64, rel=1e-2)
def test_2le_format(self):
"""2LE format (no name lines) should still parse."""
tle_2le = textwrap.dedent("""\
1 25544U 98067A 24001.50000000 .00016717 00000-0 10270-3 0 9018
2 25544 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990
""")
records = parse_3le_text(tle_2le)
assert 25544 in records
assert records[25544].name == ""
def test_epoch_dedup(self):
"""When same NORAD ID appears twice, newest epoch wins."""
tle = textwrap.dedent("""\
OLD
1 25544U 98067A 24001.50000000 .00016717 00000-0 10270-3 0 9018
2 25544 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990
NEW
1 25544U 98067A 24005.50000000 .00016717 00000-0 10270-3 0 9018
2 25544 51.6400 100.0000 0007417 30.0000 330.1234 15.49456789999990
""")
records = parse_3le_text(tle)
assert records[25544].name == "NEW"
assert records[25544].epoch == pytest.approx(24005.5, rel=1e-6)
def test_source_propagation(self):
records = parse_3le_text(self.SAMPLE_3LE, source="test-source")
assert records[25544].source == "test-source"
# ── parse_3le_file ──────────────────────────────────────────
class TestParse3LEFile:
def test_missing_file(self, capsys):
records = parse_3le_file("/nonexistent/file.tle")
assert records == {}
def test_file_roundtrip(self, tmp_path):
tle_content = textwrap.dedent("""\
TEST SAT
1 00001U 58001A 24001.50000000 .00000000 00000-0 00000-0 0 9990
2 00001 30.0000 100.0000 0010000 45.0000 315.0000 15.00000000999990
""")
p = tmp_path / "test.tle"
p.write_text(tle_content)
records = parse_3le_file(p)
assert 1 in records
assert records[1].name == "TEST SAT"