bandsaunter/tests/test_signals_named.py

"""The formats that say what they are: ADS-B, utility meters, weather sensors.

What these three have in common is that none of them needs to be believed.
Every ADS-B frame carries a 24-bit checksum, every meter message a 16-bit BCH,
every AcuRite message a checksum and four parity bits -- so the only question
a test can usefully ask is whether the fields come back holding what was put
in, and whether anything that is not one of these is ever mistaken for one.

Each generator is written from the published format rather than from the
decoder beside it, so a round trip is evidence about the format.
"""
import numpy as np
import pytest

import adsb_gen as gen
from bandsaunter.adsb import (AircraftRegistry, SAMPLE_RATE, crc24,
                              decode_adsb, decode_frames, global_position)
from bandsaunter.ism import (SCM_PREAMBLE, acurite_frame, decode_acurite,
                             decode_ism, decode_scm, scm_frame)
from bandsaunter.payload import (MIN_TEXT, hexdump, interpret, read_text,
                                 readable)

RATE = 2_000_000.0


# ---------------------------------------------------------------------------
# ADS-B
# ---------------------------------------------------------------------------

def test_the_parity_is_the_one_mode_s_uses():
    """A good frame leaves nothing behind; a corrupted one does."""
    frame = gen.identification(0x4CA1FA, "RYR1234")
    assert crc24(frame) == 0
    broken = bytearray(frame)
    broken[5] ^= 0x01
    assert crc24(bytes(broken)) != 0


def test_an_aircraft_saying_its_callsign_is_read_back():
    frames, registry = decode_adsb(
        gen.modulate([gen.identification(0x4CA1FA, "RYR1234")]), RATE)
    assert len(frames) == 1
    assert registry.aircraft["4CA1FA"].callsign == "RYR1234"


@pytest.mark.parametrize("lat,lon", [(51.5, -0.12), (40.7, -74.0),
                                     (-33.9, 151.2), (0.5, 0.5),
                                     (60.2, 24.9)])
def test_a_position_needs_two_frames_and_comes_back_exactly(lat, lon):
    """One frame is ambiguous by hundreds of miles; the pair is not."""
    frames = [gen.airborne_position(0xA0B1C2, lat, lon, 30000, odd=False),
              gen.airborne_position(0xA0B1C2, lat, lon, 30000, odd=True)]
    _, registry = decode_adsb(gen.modulate(frames), RATE)
    craft = registry.aircraft["A0B1C2"]
    assert craft.latitude == pytest.approx(lat, abs=0.01)
    assert craft.longitude == pytest.approx(lon, abs=0.01)


def test_one_position_frame_alone_places_nothing():
    _, registry = decode_adsb(gen.modulate(
        [gen.airborne_position(0xA0B1C2, 51.5, -0.12, 30000, odd=False)]), RATE)
    assert not registry.aircraft["A0B1C2"].located


@pytest.mark.parametrize("feet", [0, 1000, 12000, 35000, 43000])
def test_altitude_comes_back_in_feet(feet):
    _, registry = decode_adsb(gen.modulate(
        [gen.airborne_position(0xABCDEF, 51.5, -0.12, feet, odd=False)]), RATE)
    assert registry.aircraft["ABCDEF"].altitude_ft == pytest.approx(feet,
                                                                    abs=25)


def test_speed_and_climb_rate_come_back():
    _, registry = decode_adsb(gen.modulate(
        [gen.velocity(0x4CA1FA, 250, -180, 1216)]), RATE)
    craft = registry.aircraft["4CA1FA"]
    assert craft.ground_speed_kt == pytest.approx(308, abs=3)
    assert craft.track_deg == pytest.approx(126, abs=2)
    assert craft.vertical_rate_fpm == pytest.approx(1216, abs=64)


@pytest.mark.parametrize("noise", [0.0, 0.05, 0.15, 0.30])
def test_every_frame_survives_a_noisy_receiver(noise):
    frames = [gen.identification(0x4CA1FA, "RYR1234"),
              gen.airborne_position(0x4CA1FA, 51.5, -0.12, 35000, odd=False),
              gen.airborne_position(0x4CA1FA, 51.5, -0.12, 35000, odd=True),
              gen.velocity(0x4CA1FA, 250, -180, 1216)]
    got, registry = decode_adsb(gen.modulate(frames, noise=noise, seed=3),
                                RATE)
    assert len(got) == len(frames)
    assert registry.aircraft["4CA1FA"].located


def test_a_frame_at_the_very_end_of_the_block_is_not_half_read():
    """The bit reader runs off the end of the samples, and a long frame that
    does not fit must not be read as the short frame that does."""
    frame = gen.identification(0x4CA1FA, "RYR1234")
    samples = gen.modulate([frame], gap_us=8.0)
    cut = samples[:samples.size - int(0.5 * RATE / 1e6)]     # half a bit short
    got = decode_frames(cut, RATE)
    assert got == [] or got[0].icao == "4CA1FA"


def test_a_second_of_sky_is_decoded_in_less_than_a_second():
    """A capture mode that cannot keep up is not capturing.  Timed loosely --
    this is about the shape of the work, not the speed of the machine."""
    import time

    from bandsaunter.adsb import SAMPLE_RATE, SimulatedSky, default_sky

    sky = SimulatedSky(default_sky(), noise=0.02, seed=1)
    block = sky.read_samples(int(SAMPLE_RATE))
    started = time.perf_counter()
    got = decode_frames(block, SAMPLE_RATE)
    took = time.perf_counter() - started
    assert len(got) >= 18                     # six aircraft, three frames each
    assert took < 3.0, f"a second of sky took {took:.1f} s to decode"


def test_many_aircraft_are_kept_apart():
    frames = []
    for i, icao in enumerate((0x4CA1FA, 0xA0B1C2, 0x3C6444, 0x780102)):
        frames.append(gen.identification(icao, f"FLT{i}"))
        frames.append(gen.airborne_position(icao, 50 + i, -1 - i, 30000, False))
        frames.append(gen.airborne_position(icao, 50 + i, -1 - i, 30000, True))
    _, registry = decode_adsb(gen.modulate(frames), RATE)
    assert len(registry) == 4
    assert all(craft.located for craft in registry.aircraft.values())


@pytest.mark.parametrize("seed", range(6))
def test_silence_does_not_become_aircraft(seed):
    """A frame of all zeros satisfies the checksum, and silence is one."""
    rng = np.random.default_rng(seed)
    for samples in (np.zeros(int(RATE // 4), dtype=np.complex64),
                    (rng.standard_normal(int(RATE // 4))
                     + 1j * rng.standard_normal(int(RATE // 4))
                     ).astype(np.complex64) * 0.1):
        assert decode_frames(samples, RATE) == []


def test_a_rate_too_low_to_see_a_bit_is_refused():
    """One megabit a second cannot be read at one megasample a second."""
    iq = gen.modulate([gen.identification(0x4CA1FA, "TEST")])
    assert decode_frames(iq, SAMPLE_RATE / 2) == []


def test_a_corrupted_frame_is_dropped_rather_than_reported():
    frame = bytearray(gen.identification(0x4CA1FA, "RYR1234"))
    frame[6] ^= 0xFF
    assert decode_frames(gen.modulate([bytes(frame)]), RATE) == []


def test_a_position_straddling_a_zone_boundary_is_refused():
    """Two frames from different latitude bands cannot be combined.

    The longitude zones get wider towards the poles, so a pair that came from
    either side of a boundary would be resolved against two different grids
    and land somewhere neither of them was.
    """
    from bandsaunter.adsb import Frame
    # These two resolve to latitudes with a different number of longitude
    # zones, which is exactly the case the standard says cannot be combined.
    even = Frame(cpr_lat=2048, cpr_lon=0, cpr_odd=False)
    odd = Frame(cpr_lat=12288, cpr_lon=0, cpr_odd=True)
    assert global_position(even, odd) is None
    # And a pair that does not straddle one still resolves.
    lat, lon = gen._cpr(51.5, -0.12, odd=False)
    olat, olon = gen._cpr(51.5, -0.12, odd=True)
    assert global_position(Frame(cpr_lat=lat, cpr_lon=lon),
                           Frame(cpr_lat=olat, cpr_lon=olon,
                                 cpr_odd=True)) is not None


# ---------------------------------------------------------------------------
# Utility meters
# ---------------------------------------------------------------------------

@pytest.mark.parametrize("meter,reading,ert", [
    (12345678, 987654, 4), (0x3FFFFFF, 16777215, 5), (1, 0, 11),
    (555555, 42, 12), (67108863, 1, 13),
])
def test_a_meter_reading_comes_back_as_it_was_sent(meter, reading, ert):
    got = decode_scm(scm_frame(meter, reading, ert))
    assert got is not None
    assert got.identifier == str(meter)
    assert ("reading", str(reading)) in got.fields


@pytest.mark.parametrize("ert,name", [(4, "electricity"), (5, "gas"),
                                      (11, "water"), (12, "gas")])
def test_the_kind_of_meter_is_named(ert, name):
    got = decode_scm(scm_frame(7, 7, ert))
    assert got.device == f"{name} meter"


def test_a_meter_type_that_is_not_known_is_numbered_not_guessed():
    got = decode_scm(scm_frame(7, 7, ert_type=2))
    assert "type 2" in got.device


def test_the_tamper_flags_are_reported():
    got = decode_scm(scm_frame(7, 7, tamper_physical=2, tamper_encoder=1))
    names = dict(got.fields)
    assert names["physical tamper"] == "2"
    assert names["encoder tamper"] == "1"


def test_a_message_is_found_after_whatever_came_before_it():
    bits = "0101101" + scm_frame(4242, 999) + "1101"
    assert decode_scm(bits).identifier == "4242"


def test_a_meter_message_with_a_bit_wrong_is_refused():
    bits = list(scm_frame(12345678, 987654))
    bits[40] = "1" if bits[40] == "0" else "0"
    assert decode_scm("".join(bits)) is None


def test_a_meter_number_too_large_is_an_error_not_a_wrong_reading():
    with pytest.raises(ValueError):
        scm_frame(1 << 27, 5)
    with pytest.raises(ValueError):
        scm_frame(5, 1 << 25)


def test_random_bits_behind_a_real_preamble_are_refused():
    """The preamble is 21 bits and turns up; the checksum is what matters."""
    rng = np.random.default_rng(0)
    accepted = 0
    for _ in range(3000):
        body = "".join(rng.integers(0, 2, 75).astype(str))
        if decode_scm(SCM_PREAMBLE + body) is not None:
            accepted += 1
    assert accepted == 0


# ---------------------------------------------------------------------------
# AcuRite
# ---------------------------------------------------------------------------

@pytest.mark.parametrize("sensor,celsius,humidity,channel", [
    (0x1234, 21.5, 48, "A"), (0x0001, -20.0, 5, "C"), (0x3FFF, 45.3, 100, "B"),
    (0x2AAA, 0.0, 50, "D"), (0x0555, -39.9, 1, "A"),
])
def test_a_sensor_reading_comes_back_as_it_was_sent(sensor, celsius,
                                                    humidity, channel):
    got = decode_acurite(acurite_frame(sensor, celsius, humidity, channel))
    assert got is not None
    assert got.identifier == f"{sensor:04X}"
    fields = dict(got.fields)
    assert fields["temperature"] == f"{celsius:.1f} C"
    assert fields["humidity"] == f"{humidity}%"
    assert fields["channel"] == channel


def test_a_flat_battery_is_reported_and_a_good_one_is_not():
    good = decode_acurite(acurite_frame(0x1234, 20.0, 50, "A", False))
    flat = decode_acurite(acurite_frame(0x1234, 20.0, 50, "A", True))
    assert "battery" not in dict(good.fields)
    assert dict(flat.fields)["battery"] == "low"


def test_a_message_is_found_wherever_in_the_burst_it_starts():
    bits = "10110" + acurite_frame(0x0ABC, 12.3, 77, "B") + "0011"
    assert decode_acurite(bits).identifier == "0ABC"


def test_a_sensor_message_with_a_bit_wrong_is_refused():
    bits = list(acurite_frame(0x1234, 21.5, 48, "A"))
    bits[20] = "1" if bits[20] == "0" else "0"
    assert decode_acurite("".join(bits)) is None


def test_a_reading_outside_what_the_sensor_can_report_is_refused():
    """The checksum can be satisfied by a message the hardware cannot send."""
    from bandsaunter.ism import _parity
    data = [0x00, 0x01, 0x04, 0x7F, 0x0F, 0x7F]
    for i in range(2, 6):
        if _parity(data[i]) != 1:
            data[i] |= 0x80
    data.append(sum(data[:6]) & 0xFF)
    bits = "".join(format(b, "08b") for b in data)
    assert decode_acurite(bits) is None


def test_the_band_decides_which_is_tried_first_and_nothing_else():
    meter = scm_frame(4242, 999)
    assert decode_ism(meter, frequency=915e6).kind == "SCM"
    assert decode_ism(meter, frequency=433.92e6).kind == "SCM"


def test_neither_reads_a_message_out_of_nothing():
    rng = np.random.default_rng(1)
    accepted = sum(1 for _ in range(5000)
                   if decode_ism("".join(rng.integers(0, 2, 96).astype(str)))
                   is not None)
    # The seven-byte sensor message is checked at every offset, so the bar is
    # a rate rather than zero.  What reaches this in the scanner has already
    # had to arrive identically several times over.
    assert accepted / 5000 < 0.01


# ---------------------------------------------------------------------------
# Reading a payload
# ---------------------------------------------------------------------------

def _encode(text: str, width: int = 8, msb: bool = True, before: int = 0,
            after: int = 0, seed: int = 0) -> str:
    rng = np.random.default_rng(seed)
    bits = "".join(rng.integers(0, 2, before).astype(str))
    for ch in text:
        chunk = format(ord(ch), f"0{width}b")
        bits += chunk if msb else chunk[::-1]
    return bits + "".join(rng.integers(0, 2, after).astype(str))


@pytest.mark.parametrize("message", [
    "HELLO WORLD", "ENGINE 4 RESPOND", "the quick brown fox",
    "Meeting at seven", "BATTERY LOW", "unit twelve en route",
])
@pytest.mark.parametrize("width,msb", [(8, True), (8, False), (7, True)])
def test_text_in_a_packet_is_read_out(message, width, msb):
    got = read_text(_encode(message, width, msb))
    assert got is not None, message
    assert got.text.lower() in message.lower()


def test_text_is_found_behind_a_preamble_and_an_address():
    got = read_text(_encode("STATION OPEN", before=13, after=7, seed=4))
    assert got is not None and "STATION OPEN" in got.text


def test_random_payloads_are_almost_never_read_as_text():
    """Printability is not evidence, and this is the measurement that says so.

    Every framing at every offset in both bit orders is about forty readings
    of each packet, and seven-bit values are printable three times in four, so
    a bar set on printability alone called 64% of random payloads text.
    """
    rng = np.random.default_rng(5)
    hits = 0
    total = 0
    for n_bits in (24, 32, 48, 64, 96, 128, 192, 256, 512):
        for _ in range(250):
            total += 1
            if read_text("".join(rng.integers(0, 2, n_bits).astype(str))):
                hits += 1
    assert hits / total < 0.01, f"{100 * hits / total:.1f}% read as text"


def test_a_run_of_one_case_and_no_vowels_is_not_text():
    assert read_text(_encode("XKCDZQRT")) is None


def test_the_bytes_are_laid_out_with_their_characters_beside_them():
    lines = hexdump(_encode("ABCDEFGH"))
    assert lines[0].startswith("0000")
    assert "|ABCDEFGH|" in lines[0]


def test_a_trailing_part_byte_is_shown_as_bits_not_padded():
    """Padding four bits to a byte invents four zeroes nobody sent."""
    lines = hexdump("1" * 12)
    assert "4 bit(s): 1111" in lines[-1]


def test_a_remote_control_gets_its_fields_named():
    readings = interpret("1" * 20 + "0100", encoding="PWM")
    fields = [r for r in readings if r.kind == "fields"]
    assert fields and fields[0].how.startswith("EV1527")
    assert dict(fields[0].fields)["button"] == "B"


def test_every_payload_gets_at_least_its_bytes_back():
    lines = readable("10110010" * 4)
    assert lines and lines[0].startswith("0000")


def test_a_payload_that_says_what_it_is_is_named():
    """GRIB gets asked about as though it were a modulation.

    It is not: it is the format weather models are published in, and it
    arrives by satellite data link and by e-mail.  Where a byte stream begins
    with its magic number that is worth saying; nothing here renders one.
    """
    bits = "".join(format(b, "08b") for b in b"GRIB\x00\x00\x00\x02payload!")
    named = [r for r in interpret(bits) if r.kind == "fields"]
    assert named and "GRIB" in named[0].line()


def test_an_ordinary_payload_claims_no_format():
    bits = "".join(format(b, "08b") for b in b"\x01\x02\x03\x04\x05\x06\x07\x08")
    assert not [r for r in interpret(bits) if r.kind == "fields"]


# ---------------------------------------------------------------------------
# A position is only as good as the pair it was decoded from
# ---------------------------------------------------------------------------

def _position_frame(icao, lat, lon, odd, when, altitude=35000):
    """One position frame, read back the way the decoder would read it."""
    from bandsaunter.adsb import _read, encode_position

    data = encode_position(icao, lat, lon, altitude, odd=odd)
    frame = _read("".join(format(b, "08b") for b in data), data)
    frame.received_at = when
    return frame


def test_a_stale_pair_is_not_a_position():
    """Compact position reporting sends a fraction of a zone, so an even
    frame from ten minutes ago read against a fresh odd one puts the
    aircraft on the wrong side of the world.  Measured against one night's
    recording it was doing exactly that to two aircraft in three."""
    registry = AircraftRegistry()
    registry.add(_position_frame(0xABCDEF, 32.55, -111.16, False, 1000.0),
                 when=1000.0)
    registry.add(_position_frame(0xABCDEF, 33.22, -111.16, True, 1300.0),
                 when=1300.0)
    assert not registry.aircraft["ABCDEF"].located


def test_a_fresh_pair_still_places_it_exactly():
    registry = AircraftRegistry()
    registry.add(_position_frame(0xABCDEF, 33.22, -111.16, False, 1000.0),
                 when=1000.0)
    registry.add(_position_frame(0xABCDEF, 33.22, -111.16, True, 1000.5),
                 when=1000.5)
    craft = registry.aircraft["ABCDEF"]
    assert craft.latitude == pytest.approx(33.22, abs=0.01)
    assert craft.longitude == pytest.approx(-111.16, abs=0.01)


@pytest.mark.parametrize("gap", [0.0, 0.5, 4.0, 9.5])
def test_a_pair_inside_the_allowed_gap_is_used(gap):
    registry = AircraftRegistry()
    registry.add(_position_frame(0xA0B1C2, 51.5, -0.12, False, 100.0),
                 when=100.0)
    registry.add(_position_frame(0xA0B1C2, 51.5, -0.12, True, 100.0 + gap),
                 when=100.0 + gap)
    assert registry.aircraft["A0B1C2"].located


def test_a_position_that_is_not_on_earth_is_refused():
    """A latitude of 240 degrees is not a place.  One night's log held
    eleven of them."""
    from bandsaunter.adsb import _on_earth

    assert not _on_earth(239.6, -111.0)
    assert not _on_earth(45.0, 200.0)
    assert _on_earth(-33.9, 151.2)


def test_an_aircraft_cannot_cross_a_continent_between_two_frames():
    """A position needing nine hundred thousand knots to reach is not a
    position, whatever the checksum said about the frames it came from."""
    registry = AircraftRegistry()
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 51.5, -0.12, odd, 100.0),
                     when=100.0)
    craft = registry.aircraft["A0B1C2"]
    assert craft.located
    was = (craft.latitude, craft.longitude)
    # Two seconds later, a pair that decodes to the far side of the Atlantic.
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 40.7, -74.0, odd, 102.0),
                     when=102.0)
    assert (craft.latitude, craft.longitude) == was


def test_an_aircraft_that_really_moved_is_still_followed():
    """The bar has to be above anything that flies, or a fast aircraft is
    called an error."""
    registry = AircraftRegistry()
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 51.5, -0.12, odd, 100.0),
                     when=100.0)
    # Sixty seconds on and eight miles further east: 480 knots.
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 51.5, 0.08, odd, 160.0),
                     when=160.0)
    assert registry.aircraft["A0B1C2"].longitude == pytest.approx(0.08,
                                                                  abs=0.02)


def test_a_gap_in_reception_is_not_treated_as_an_error():
    """Nothing heard for an hour, then a position a long way off: that is an
    aircraft that flew away and came back, not a bad decode."""
    registry = AircraftRegistry()
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 51.5, -0.12, odd, 100.0),
                     when=100.0)
    for odd in (False, True):
        registry.add(_position_frame(0xA0B1C2, 48.8, 2.3, odd, 4000.0),
                     when=4000.0)
    assert registry.aircraft["A0B1C2"].latitude == pytest.approx(48.8,
                                                                 abs=0.05)