bandsaunter/tests/test_callsign.py

"""Finding callsigns in a transcript, and saying who they belong to.

The hard part is not the regular expression, it is everything the speech
recogniser does to a callsign on the way: it breaks it wherever the speaker
paused, and it writes the phonetic alphabet down as words.  So most of these
are about text that came out of a real recogniser rather than text anyone
would type.

Nothing here touches the network.  The lookup is tested against a stub, and
one test asserts that no request is made when it is switched off.
"""
import json
import time

import pytest

from bandsaunter.callsign import (Callsign, CallsignBook, FILLERS, HEADING,
                                  NOT_CALLSIGNS, PHONETIC, SHAPE,
                                  _phonetic_parts, describe_prefix,
                                  find_callsigns, person_case, report,
                                  split_postcode)


@pytest.fixture(autouse=True)
def no_network(monkeypatch):
    """Nothing here may contact the licence database for real.

    Every lookup test uses a stub; this makes forgetting one fail loudly
    rather than quietly pass with somebody's real address in the output.
    """
    def refuse(self, call):
        raise AssertionError(f"a test tried to look up {call} for real")

    monkeypatch.setattr(CallsignBook, "_request", refuse)


# -- finding them ------------------------------------------------------------

@pytest.mark.parametrize("text,want", [
    ("This is W1AW calling CQ", ["W1AW"]),
    ("K7RA this is N7XYZ over", ["K7RA", "N7XYZ"]),
    ("VE3ABC and G0ABC and 2E0XYZ", ["VE3ABC", "G0ABC", "2E0XYZ"]),
    ("net control, W1AW, W1AW again", ["W1AW"]),
])
def test_a_callsign_written_properly_is_found(text, want):
    assert find_callsigns(text) == want


@pytest.mark.parametrize("text,want", [
    ("Alright, KU 0W. And the HF net will be a couple minutes.", ["KU0W"]),
    ("N7 XYZ this is K7 RA", ["N7XYZ", "K7RA"]),
    ("W 1 A W standing by", ["W1AW"]),
])
def test_a_callsign_broken_across_words_is_still_one_callsign(text, want):
    """A recogniser breaks it wherever the speaker paused.  "KU 0W" is what
    came out of a real recording of somebody saying KU0W."""
    assert find_callsigns(text) == want


@pytest.mark.parametrize("text,want", [
    ("kilo uniform zero whiskey, are you there", ["KU0W"]),
    ("whiskey one alpha whiskey this is kilo seven romeo alpha",
     ["W1AW", "K7RA"]),
    ("november seven x-ray yankee zulu", ["N7XYZ"]),
    ("kilo uniform zero whisky", ["KU0W"]),        # both spellings
    ("niner", []),
])
def test_the_phonetic_alphabet_is_read_back(text, want):
    """Spelled out, a callsign arrives as one token per character."""
    assert find_callsigns(text) == want


@pytest.mark.parametrize("text", [
    "7 on alleviate or 3. Can you open 4 and close the rest.",
    "CC1 boy",
    "I bought 3 of them for $5 and a B4U model MP3 player",
    "the 4th Saturday at 1.30pm on 7.242 megahertz",
    "Check out communication of group activities and training.",
    "we need 2 of them and 4 spare",
    "it was on channel 3 for a while",
])
def test_ordinary_speech_produces_no_callsigns(text):
    """Every one of these came out of a real transcript or is one word away
    from something that did.  A browser that invents callsigns is worse than
    one that finds none."""
    assert find_callsigns(text) == []


def test_a_join_is_refused_when_a_part_is_an_english_word():
    """"or 3. Can you" fits the shape once the punctuation is gone.  Nobody
    spells a callsign with English words."""
    assert find_callsigns("or 3 can") == []
    assert find_callsigns("CC1 boy") == []
    # ... but a single token said in one breath is trusted.
    assert find_callsigns("W1BOY is on frequency") == ["W1BOY"]


def test_a_lone_article_is_not_a_prefix():
    assert find_callsigns("a B4U player") == []
    assert find_callsigns("alpha bravo four uniform") == ["AB4U"]


def test_the_same_callsign_is_reported_once():
    assert find_callsigns("W1AW W1AW W1AW") == ["W1AW"]


def test_they_come_back_in_the_order_they_were_said():
    assert find_callsigns("N7XYZ this is W1AW") == ["N7XYZ", "W1AW"]


def test_the_count_is_capped():
    text = " ".join(f"W{i}AB" for i in range(0, 9)) + " K7RA N7XYZ VE3ABC"
    assert len(find_callsigns(text, max_found=4)) == 4


def test_empty_text_is_not_an_error():
    assert find_callsigns("") == []
    assert find_callsigns("   \n  ") == []


def test_every_excluded_word_could_actually_have_matched():
    """A list of words that could not match anyway would only suggest the
    filter was doing more work than it is."""
    for word in NOT_CALLSIGNS:
        assert SHAPE.match(word), f"{word} never matched in the first place"


# -- the phonetic alphabet, as a recogniser writes it down --------------------
#
# It is never written the way the ITU prints it.  A recogniser hears the
# words and writes what they sounded like, joins them up, hyphenates them, or
# drops a hesitation in the middle of the run.

@pytest.mark.parametrize("text,want", [
    # Hyphenated, which is how a recogniser writes anything spelled out.
    ("Whiskey-One-Alpha-Whiskey", ["W1AW"]),
    ("this is whiskey-one-alpha-whiskey clear", ["W1AW"]),
    # Run together, when it was said quickly.
    ("WhiskeyOneAlphaWhiskey", ["W1AW"]),
    ("The station is Whiskey1AlphaWhiskey", ["W1AW"]),
    ("kilouniformzerowhiskey", ["KU0W"]),
    # Said with a hesitation in the middle of it.
    ("whiskey one alpha, uh, whiskey", ["W1AW"]),
    ("kilo uniform um zero whiskey", ["KU0W"]),
    # The spellings that are not the official ones.
    ("wiskey one alfa whisky", ["W1AW"]),
    ("juliette alpha one november golf", ["JA1NG"]),
    ("oskar viktor two charley romeo", ["OV2CR"]),
])
def test_a_phonetic_spelling_is_read_however_it_was_written(text, want):
    assert find_callsigns(text) == want


@pytest.mark.parametrize("word", [
    "kilometre", "kilometer", "victorious", "november", "onetime", "papaya",
    "hotelier", "echoes", "deltas", "golfing", "oneself", "foxtrotting",
    "sierras", "alphabet", "zeroed", "information", "uniformity", "tangos",
    "twofold", "fivefold", "sixty", "seventeen", "nineteen", "charlies",
    "oscars", "limas", "romeos", "echoing", "novembers", "onto", "golfer",
])
def test_an_ordinary_word_is_not_taken_apart(word):
    """Splitting only fires on a word that is phonetic all the way through.

    A partial match is no match, which is the whole of what keeps this away
    from English: "kilometre" begins with a phonetic word and "victorious"
    contains one, and neither can be consumed to the end.
    """
    assert _phonetic_parts(word) is None


def test_x_ray_is_one_letter_not_two_words():
    """The one phonetic word with a hyphen in it of its own."""
    assert find_callsigns("x-ray echo two delta") == ["XE2D"]
    assert find_callsigns("xray echo two delta") == ["XE2D"]


def test_every_filler_is_something_a_recogniser_writes():
    """Each of these has to be a word, not a letter it would swallow."""
    for word in FILLERS:
        assert word.lower() not in PHONETIC, word
        assert not SHAPE.match(word), word


# -- suffixes, which are not part of the callsign ----------------------------

@pytest.mark.parametrize("text,want", [
    ("W1AW/4", ["W1AW"]),
    ("DL/W1AW", ["W1AW"]),
    ("This is W1AW-4", ["W1AW"]),
    ("VVV DE W1AW/B FN31", ["W1AW"]),
    ("VE3XYZ/M mobile", ["VE3XYZ"]),
])
def test_a_suffix_is_not_glued_onto_the_callsign(text, want):
    """A slash means "somewhere else", not another letter.

    The beacon W1AW/B used to come back as W1AWB, which belongs to nobody
    and would have been looked up and pinned to a map.
    """
    assert find_callsigns(text) == want


# -- where the spacing is real ----------------------------------------------

def test_words_are_joined_where_a_recogniser_put_the_gaps_in():
    """A transcript's spacing is a guess, so it is allowed to be wrong."""
    assert find_callsigns("KU 0W") == ["KU0W"]
    assert find_callsigns("K7 RA") == ["K7RA"]


@pytest.mark.parametrize("text,want", [
    ("KU0W K", ["KU0W"]),          # a station signing off, not KU0WK
    ("CQ CQ DE KU0W K", ["KU0W"]),
    ("DE W1AW K", ["W1AW"]),
    ("VVV DE W1AW/B FN31", ["W1AW"]),
])
def test_nothing_is_joined_where_the_sender_put_the_gaps_in(text, want):
    """A word gap in Morse is seven dot units the sender chose to send.

    Joining across one turns a station signing off with K -- "over" -- into a
    callsign one letter longer that belongs to somebody else entirely.
    """
    assert find_callsigns(text, join_words=False) == want


# -- what the callsign says about itself -------------------------------------

@pytest.mark.parametrize("call,country", [
    ("W1AW", "United States"), ("KU0W", "United States"),
    ("N7XYZ", "United States"), ("AA1AB", "United States"),
    ("VE3ABC", "Canada"), ("XE1ABC", "Mexico"),
    ("G0ABC", "United Kingdom"), ("2E0XYZ", "United Kingdom"),
    ("JA1ABC", "Japan"), ("VK2DEF", "Australia"),
    ("IK4ABC", "Italy"), ("DL1ABC", "Germany"),
    ("4X4ABC", "Israel"), ("ZZ9ZZZ", ""),
])
def test_the_prefix_names_the_country_without_a_database(call, country):
    """This works offline and for callsigns no database holds."""
    assert describe_prefix(call)[0] == country


def test_a_one_letter_prefix_does_not_claim_everything():
    """"I" is Italy, but only when what follows is the rest of a prefix."""
    assert describe_prefix("IK4ABC")[0] == "Italy"
    assert describe_prefix("KU0W")[0] == "United States"   # not Italy-shaped


def test_the_digit_gives_the_us_district():
    assert "New England" in describe_prefix("W1AW")[1]
    assert "California" in describe_prefix("K6ABC")[1]
    assert describe_prefix("VE3ABC")[1] == "", "districts are a US idea"


# -- tidying the licence record ----------------------------------------------

@pytest.mark.parametrize("raw,want", [
    ("ROD R GOWDY", "Rod R Gowdy"),
    ("PAUL PAKES COOK, III.", "Paul Pakes Cook, III."),
    ("ARRL HQ OPERATORS CLUB", "ARRL HQ Operators Club"),
    ("JOHN SMITH JR", "John Smith JR"),
    ("de Vries, Anna", "de Vries, Anna"),          # already mixed: left alone
])
def test_licence_names_are_made_readable_without_being_corrupted(raw, want):
    assert person_case(raw) == want


@pytest.mark.parametrize("raw,town,code", [
    ("TUCSON, AZ 85742", "Tucson, AZ", "85742"),
    ("SEATTLE, WA 98105-3505", "Seattle, WA", "98105-3505"),
    ("NEWINGTON, CT 06111", "Newington, CT", "06111"),
    ("SOMEWHERE ABROAD", "Somewhere Abroad", ""),
])
def test_the_postcode_is_separated_from_the_place(raw, town, code):
    """The town and state say where somebody is; the postcode is detail."""
    assert split_postcode(raw) == (town, code)


# -- looking them up ---------------------------------------------------------

VALID = {
    "status": "VALID", "type": "PERSON", "name": "ROD R GOWDY",
    "current": {"callsign": "KU0W", "operClass": "EXTRA"},
    "previous": {"callsign": "KK7QPA"},
    "trustee": {"callsign": ""},
    "address": {"line1": "5455 W OASIS RD", "line2": "TUCSON, AZ 85742"},
    "location": {"gridsquare": "DM42lj"},
    "otherInfo": {"expiryDate": "08/09/2034"},
}


class StubBook(CallsignBook):
    """A book whose lookups are answered from a dict, not the network."""

    def __init__(self, answers, **kw):
        self.answers = answers
        self.requested = []
        super().__init__(**kw)

    def _request(self, call):
        self.requested.append(call)
        if call not in self.answers:
            raise OSError("no route to host")
        return self.answers[call]


def book(tmp_path, answers=None, **kw):
    return StubBook(answers if answers is not None else {"KU0W": VALID},
                    cache=tmp_path / "cache.json", **kw)


def test_a_lookup_fills_in_the_name_and_the_place(tmp_path):
    b = book(tmp_path)
    b.get("KU0W")
    b.wait(5)
    entry = b.get("KU0W")
    assert entry.known
    assert entry.name == "Rod R Gowdy"
    assert entry.location == "Tucson, AZ"
    assert entry.grid == "DM42lj"
    assert entry.oper_class == "EXTRA"
    assert entry.previous == "KK7QPA"


def test_rendering_never_waits_for_the_network(tmp_path):
    """A lookup returns at once with what is known, and fills itself in.

    The stub is deliberately slow: a fast one would finish before the
    assertion and the test would pass whether or not the call blocked.
    """
    class Slow(StubBook):
        def _request(self, call):
            time.sleep(1.0)
            return super()._request(call)

    b = Slow({"KU0W": VALID}, cache=tmp_path / "cache.json")
    started = time.time()
    entry = b.get("KU0W")
    assert time.time() - started < 0.3, "the lookup blocked the caller"
    assert entry.status == "pending"
    assert entry.summary() == "looking up…"
    b.wait(10)
    assert b.get("KU0W").known


def test_an_unknown_callsign_says_what_the_prefix_says(tmp_path):
    """Not in the database is not nothing: the prefix is still information."""
    b = book(tmp_path, {"G0ABC": {"status": "INVALID"}})
    b.get("G0ABC")
    b.wait(5)
    summary = b.get("G0ABC").summary()
    assert "unlisted" in summary and "United Kingdom" in summary


def test_a_lookup_that_fails_is_not_an_error(tmp_path):
    """Offline, blocked or rate-limited, the transcript still has to show."""
    b = book(tmp_path, {})
    b.get("W1AW")
    b.wait(5)
    entry = b.get("W1AW")
    assert entry.status == "offline"
    assert "United States" in entry.summary()


def test_nothing_is_requested_when_lookup_is_switched_off(tmp_path):
    b = book(tmp_path, online=False)
    entry = b.get("KU0W")
    b.wait(2)
    assert b.requested == [], "a request was made with lookup off"
    assert entry.status == "offline"
    assert "United States" in entry.summary()


def test_a_callsign_is_looked_up_once(tmp_path):
    b = book(tmp_path)
    for _ in range(5):
        b.get("KU0W")
    b.wait(5)
    assert b.requested == ["KU0W"]


def test_results_are_cached_between_runs(tmp_path):
    """The same net, logged night after night, is looked up once."""
    first = book(tmp_path)
    first.get("KU0W")
    first.wait(5)
    first.save()
    assert (tmp_path / "cache.json").exists()

    second = book(tmp_path)
    entry = second.get("KU0W")
    assert entry.known, "the cache was not read"
    assert second.requested == [], "a cached callsign was looked up again"


def test_a_stale_cache_entry_is_looked_up_again(tmp_path):
    first = book(tmp_path)
    first.get("KU0W")
    first.wait(5)
    first.save()
    body = json.loads((tmp_path / "cache.json").read_text())
    body["KU0W"]["fetched_at"] = time.time() - 400 * 86_400
    (tmp_path / "cache.json").write_text(json.dumps(body))

    second = book(tmp_path)
    second.get("KU0W")
    second.wait(5)
    assert second.requested == ["KU0W"]


def test_a_cache_from_another_version_is_ignored_not_fatal(tmp_path):
    (tmp_path / "cache.json").write_text(
        json.dumps({"KU0W": {"call": "KU0W", "somethingelse": 1}}))
    b = book(tmp_path)
    b.get("KU0W")
    b.wait(5)
    assert b.requested == ["KU0W"], "an unusable cache entry was trusted"
    assert b.get("KU0W").known


def test_an_unreadable_cache_is_ignored_not_fatal(tmp_path):
    (tmp_path / "cache.json").write_text("{not json")
    b = book(tmp_path)
    b.get("KU0W")
    b.wait(5)
    assert b.get("KU0W").known


def test_only_resolved_entries_are_cached(tmp_path):
    """Caching a failure would make one flaky lookup permanent."""
    b = book(tmp_path, {})
    b.get("W1AW")
    b.wait(5)
    b.save()
    body = json.loads((tmp_path / "cache.json").read_text()) \
        if (tmp_path / "cache.json").exists() else {}
    assert "W1AW" not in body


# -- the block that goes under the transcript --------------------------------

def test_the_report_is_headed_the_way_it_was_asked_for():
    entry = Callsign(call="W1AW", name="ARRL", location="Newington, CT",
                     status="found")
    lines = report([entry])
    assert lines[0] == HEADING == "DETECTED CALLSIGNS:"
    assert "W1AW" in lines[1] and "Newington" in lines[1]


def test_nothing_found_means_no_block_at_all():
    assert report([]) == []


def test_the_report_carries_the_extra_detail():
    entry = Callsign(call="KU0W", name="Rod R Gowdy", location="Tucson, AZ",
                     oper_class="EXTRA", grid="DM42lj", status="found")
    line = report([entry], width=0)[1]
    assert "Extra" in line and "DM42lj" in line


# ---------------------------------------------------------------------------
# Coordinates, and the cache that holds them
# ---------------------------------------------------------------------------

def test_a_lookup_records_where_the_licence_says_the_station_is(monkeypatch):
    """The map is built from these, so they have to survive the parse."""
    body = {"status": "VALID", "name": "ARRL HQ OPERATORS CLUB",
            "address": {"line2": "NEWINGTON, CT 06111"},
            "location": {"latitude": "41.714775", "longitude": "-72.727260",
                         "gridsquare": "FN31pr"}}
    entry = Callsign(call="W1AW")
    CallsignBook._apply(entry, body)
    assert entry.position == pytest.approx((41.714775, -72.727260))
    assert not entry.coarse_position


def test_a_licence_with_only_a_grid_square_falls_back_to_it(monkeypatch):
    body = {"status": "VALID", "name": "SOMEONE",
            "address": {"line2": "SOMEWHERE, AZ 85742"},
            "location": {"gridsquare": "DM42lj"}}
    entry = Callsign(call="KU0W")
    CallsignBook._apply(entry, body)
    assert entry.position is not None
    # and it says the position is only as good as the square
    assert entry.coarse_position


def test_a_licence_with_no_position_at_all_has_none():
    body = {"status": "VALID", "name": "SOMEONE", "address": {}, "location": {}}
    entry = Callsign(call="K7XYZ")
    CallsignBook._apply(entry, body)
    assert entry.position is None
    assert not entry.coarse_position


def test_an_entry_cached_before_coordinates_is_looked_up_again(tmp_path):
    """Otherwise a warm cache pins every station to its grid square for good."""
    from bandsaunter.callsign import CACHE_VERSION
    cache = tmp_path / "callsigns.json"
    cache.write_text(json.dumps({"W1AW": {
        "call": "W1AW", "name": "Arrl Hq Operators Club", "status": "found",
        "grid": "FN31pr", "fetched_at": time.time()}}))
    book = CallsignBook(online=False, cache=cache)
    assert "W1AW" not in book._entries

    cache.write_text(json.dumps({"W1AW": {
        "call": "W1AW", "name": "Arrl Hq Operators Club", "status": "found",
        "grid": "FN31pr", "fetched_at": time.time(),
        "version": CACHE_VERSION}}))
    assert "W1AW" in CallsignBook(online=False, cache=cache)._entries


def test_an_unlisted_callsign_is_cached_too(tmp_path, monkeypatch):
    """Asking again every run about a callsign with no licence is waste."""
    from bandsaunter.callsign import CACHE_VERSION
    entry = Callsign(call="XX9ZZ")
    CallsignBook._apply(entry, {"status": "INVALID"})
    assert entry.status == "unlisted"
    assert entry.version == CACHE_VERSION


# -- the whole address -------------------------------------------------------
#
# US amateur licence records are public by law and carry the street the
# licence was issued to.  It is recorded because holding it and not saying so
# would be worse than either showing it or not asking for it, and --no-lookup
# asks for none of it.

def test_the_street_is_read_off_the_licence(tmp_path):
    b = book(tmp_path, {"W1AW": {
        "status": "VALID", "current": {"callsign": "W1AW"},
        "name": "ARRL HQ OPERATORS CLUB",
        "address": {"line1": "225 MAIN ST", "line2": "NEWINGTON, CT 06111"},
        "location": {"latitude": "41.714", "longitude": "-72.727"}}})
    entry = b.get("W1AW")
    b.wait(5.0)
    assert entry.street == "225 Main St"
    assert entry.location == "Newington, CT"
    assert entry.postcode == "06111"
    assert entry.address == "225 Main St, Newington, CT, 06111"


def test_the_street_is_shown_under_the_callsign(tmp_path):
    b = book(tmp_path, {"W1AW": {
        "status": "VALID", "current": {"callsign": "W1AW", "operClass": "CLUB"},
        "name": "ARRL", "address": {"line1": "225 MAIN ST",
                                    "line2": "NEWINGTON, CT 06111"},
        "location": {"gridsquare": "FN31pr"}}})
    entry = b.get("W1AW")
    b.wait(5.0)
    assert "225 Main St" in entry.details()


def test_an_address_with_nothing_in_it_is_an_empty_string(tmp_path):
    b = book(tmp_path, {"W1AW": {"status": "INVALID"}})
    entry = b.get("W1AW")
    b.wait(5.0)
    assert entry.address == ""


def test_a_record_cached_before_the_street_is_asked_about_again(tmp_path):
    """The cache is versioned so an old entry is not kept with a field empty."""
    import json
    import time as _time
    cache = tmp_path / "calls.json"
    cache.write_text(json.dumps({"W1AW": {
        "call": "W1AW", "name": "ARRL", "status": "found",
        "fetched_at": _time.time(), "version": 2}}))
    b = StubBook({"W1AW": {
        "status": "VALID", "current": {"callsign": "W1AW"}, "name": "ARRL",
        "address": {"line1": "225 MAIN ST", "line2": "NEWINGTON, CT 06111"},
        "location": {}}}, cache=cache)
    entry = b.get("W1AW")
    b.wait(5.0)
    assert entry.street == "225 Main St"