bandsaunter/bandsaunter/callsign.py

"""Finding callsigns in a transcript, and saying who they belong to.

Speech recognisers are poor at callsigns.  They are not words, they are said
one character at a time, and the recogniser reaches for whatever English is
nearest: ``KU0W`` comes out as "KU 0W", ``K7RA`` as "K7 RA", and an operator
who spells it out in the phonetic alphabet gets "kilo uniform zero whiskey"
written down verbatim.  So finding one is not a matter of a single regular
expression over the text as written.

What comes out is a shape -- one or two letters, a digit, one to three
letters -- which is checked against the licence database.  A callsign that
resolves to a licence is real; one that does not is reported as unverified
rather than silently dropped, because a mangled callsign is still a signal
that someone identified themselves.
"""

from __future__ import annotations

import json
import os
import re
import threading
import time
import urllib.parse
import urllib.request
from dataclasses import dataclass
from pathlib import Path

__all__ = ["Callsign", "CallsignBook", "find_callsigns", "describe_prefix",
           "PHONETIC", "LOOKUP_URL"]

# The FCC's own licence data, served as JSON without an account or a key.
# US callsigns only; everything else resolves to what the prefix alone says.
LOOKUP_URL = "https://callook.info/{call}/json"

PHONETIC = {
    "alpha": "A", "alfa": "A", "bravo": "B", "charlie": "C", "delta": "D",
    "echo": "E", "foxtrot": "F", "fox": "F", "golf": "G", "hotel": "H",
    "india": "I", "juliet": "J", "juliett": "J", "julliet": "J",
    "kilo": "K", "lima": "L", "mike": "M", "november": "N", "oscar": "O",
    "papa": "P", "quebec": "Q", "romeo": "R", "sierra": "S", "tango": "T",
    "uniform": "U", "victor": "V", "whiskey": "W", "whisky": "W",
    "xray": "X", "x-ray": "X", "yankee": "Y", "zulu": "Z",
    "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
    "five": "5", "fife": "5", "six": "6", "seven": "7", "eight": "8",
    "nine": "9", "niner": "9",
}

# A prefix, a digit, then one to three letters.  The prefix is one or two
# letters (W, KU, VE) or a digit and a letter (2E, 4X, 9A) -- the latter is why
# a plain "letters then digit" pattern is not enough, and why every British
# 2E0 callsign would otherwise be missed.
SHAPE = re.compile(r"^(?:[A-Z]{1,2}|[0-9][A-Z])[0-9][A-Z]{1,3}$")

# Things that fit the shape and are never callsigns.  Every entry here has
# been checked against SHAPE: a list of words that could not match anyway
# would only suggest the filter was doing more than it is.
NOT_CALLSIGNS = frozenset({"B2B", "B4U", "H2O", "M4A", "P2P", "Y2K", "W2K",
                           "H2S", "N2O", "C2C", "D2D", "F2F"})

# Ordinary English words, short enough to be mistaken for part of a callsign.
# They only disqualify a *join*: "or 3. Can you open 4" became OR3CAN, and
# "CC1 boy" became CC1BOY, because both fit the shape once the punctuation is
# gone.  Nobody spells a callsign with English words -- they say the letters,
# or the phonetic alphabet, which is converted before this is consulted -- so
# a component that is a word means the run is not a callsign.  A single token
# is still trusted: W1BOY is a perfectly good callsign, said in one breath.
COMMON_WORDS = frozenset("""
a i o an as at be by do go he hi if in is it me my no of on or so to up us we
add age ago air all and any are arm art ask bad bag bar bed bet big bit box
boy bus but buy can car cat cop cup cut dad day did die dog dry due eat egg
end eye far few fit fix fly for fun gas get god got gun guy had has hat her
him his hit hot how ice its job key kid law lay leg let lie lot low man map
may men met mom nor not now odd off oil old one out own pay per pop put ran
red rid row run sat saw say sea see set she sir sit six son sun tax tea ten
the tie tip toe ton too top try two use van war was way wet who why win yes
yet you
able also away back bad been beer bell best bill blue boat body book both
call came care case city club cold come cost dark data date days dead deal
does done door down draw drop each easy else even ever face fact fall feel
feet fell file fill find fire fish five flat food foot form four free from
full game gave give goes gold gone good grew grey guys half hall hand hard
have head hear held help here hers high hold hole home hope hour huge idea
into item join just keep kept kind knew know land last late lead left less
life like line list live long look lose lost love made mail main make many
mark mean meet mile mind mine miss mode more most move much must name near
neck need news next nice nine none note okay once only open over page paid
part pass past path pick plan play plus poor post pull pure push race rain
read real rest ride ring rise risk road rock role roll room rule safe said
sale same save says seem seen self sell send sent ship shop shot show shut
side sign site size skin slow snow sold some song soon sort soul stay step
stop such sure take talk tall team tell test text than that them then they
thin this thus time tiny told took town tree trip true turn type unit upon
used user very view vote wait walk wall want ward warm wash wave ways weak
wear week well went were west what when whom wide wife wild will wind wine
wire wish with wood word work yard yeah year your zone
""".upper().split())

# Single letters that are English words far more often than they are the start
# of a callsign.  They are only rejected as the *first* token of a join --
# "a B4U" must not become AB4U -- and never when the speaker said them in the
# phonetic alphabet, where "alpha" is unambiguous.
LONE_WORDS = frozenset({"A", "I", "O"})


# ---------------------------------------------------------------------------
# What a callsign says about itself
# ---------------------------------------------------------------------------

# ITU prefix blocks, coarsely.  A full table runs to several hundred entries
# and most of it never appears in a scanner's transcripts; this covers what a
# receiver in North America actually hears, and says "unknown" rather than
# guessing for the rest.
_PREFIXES: tuple[tuple[tuple[str, ...], str], ...] = (
    (("K", "N", "W", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AI", "AJ",
      "AK", "AL", "KG", "KH", "KL", "KP", "NH", "NL", "NP", "WH", "WL",
      "WP"), "United States"),
    (("VA", "VE", "VO", "VY", "CF", "CG", "CH", "CI", "CJ", "CK", "CY",
      "CZ", "XJ", "XK", "XL", "XM", "XN", "XO"), "Canada"),
    (("XE", "XF", "4A", "6D", "6E"), "Mexico"),
    (("G", "M", "2E", "GM", "GW", "GI", "GD", "GJ", "GU"), "United Kingdom"),
    (("VK", "AX"), "Australia"),
    (("ZL", "ZM"), "New Zealand"),
    (("JA", "JE", "JF", "JG", "JH", "JI", "JJ", "JK", "JL", "JM", "JN",
      "JO", "JP", "JQ", "JR", "JS", "7J", "7K", "7L", "7M", "7N"), "Japan"),
    (("DL", "DK", "DJ", "DB", "DC", "DD", "DF", "DG", "DH", "DO"), "Germany"),
    (("F",), "France"),
    (("I",), "Italy"),
    (("EA", "EB", "EC"), "Spain"),
    (("PY", "PP", "PQ", "PR", "PS", "PT", "PU", "PV", "PW"), "Brazil"),
    (("LU",), "Argentina"),
    (("CE", "CA", "CB", "CC"), "Chile"),
    (("HK",), "Colombia"),
    (("CO", "CM"), "Cuba"),
    (("UA", "UB", "R", "RA", "RK", "RN", "RU", "RV", "RW", "RX", "RZ"),
     "Russia"),
    (("BY", "BA", "BD", "BG", "BH", "BI"), "China"),
    (("HL", "DS"), "South Korea"),
    (("VU",), "India"),
    (("ZS", "ZR", "ZT", "ZU"), "South Africa"),
    (("4X", "4Z"), "Israel"),
    (("SM", "SA", "SB", "SC", "SD", "SE", "SF", "SG", "SH", "SI", "SJ",
      "SK", "SL", "8S"), "Sweden"),
    (("LA", "LB", "LC", "LD", "LE", "LF", "LG", "LH", "LI", "LJ", "LK",
      "LL", "LM", "LN"), "Norway"),
    (("OH", "OF", "OG", "OI"), "Finland"),
    (("OZ", "5P", "5Q", "OU", "OV"), "Denmark"),
    (("PA", "PB", "PC", "PD", "PE", "PF", "PG", "PH", "PI"), "Netherlands"),
    (("ON", "OO", "OP", "OQ", "OR", "OS", "OT"), "Belgium"),
    (("HB", "HE"), "Switzerland"),
    (("OE",), "Austria"),
    (("SP", "SN", "SO", "SQ", "SR", "3Z"), "Poland"),
    (("CT", "CQ", "CR", "CS"), "Portugal"),
    (("EI", "EJ"), "Ireland"),
)

# US call districts.  A callsign's digit says where the licence was issued,
# which for older licences is often not where the operator now lives -- so
# this is phrased as the district, not as an address.
_US_DISTRICTS = {
    "0": "district 0 (CO IA KS MN MO NE ND SD)",
    "1": "district 1 (New England)",
    "2": "district 2 (NY NJ)",
    "3": "district 3 (DE MD PA)",
    "4": "district 4 (Southeast)",
    "5": "district 5 (South Central)",
    "6": "district 6 (California)",
    "7": "district 7 (Northwest and Mountain)",
    "8": "district 8 (MI OH WV)",
    "9": "district 9 (IL IN WI)",
}


def describe_prefix(call: str) -> tuple[str, str]:
    """``(country, district)`` from the callsign's own structure.

    Needs no database and no network: the prefix is allocated by the ITU and
    the digit is the licensing district, so a callsign carries this much about
    itself wherever it is heard.
    """
    call = call.upper()
    country = ""
    for prefixes, name in _PREFIXES:
        for p in sorted(prefixes, key=len, reverse=True):
            if not call.startswith(p):
                continue
            # A one-letter prefix has to be followed by the rest of a real
            # prefix -- an optional second letter, then the district digit.
            # Without that, "I" claims every callsign beginning with I, and
            # with too strict a version ("K" then a digit) it fails to claim
            # KU0W, whose prefix is two letters long.
            if len(p) == 1 and not re.match(r"^[A-Z]?[0-9]", call[1:]):
                continue
            country = name
            break
        if country:
            break
    district = ""
    digit = next((c for c in call if c.isdigit()), "")
    if country == "United States" and digit in _US_DISTRICTS:
        district = _US_DISTRICTS[digit]
    return country, district


# ---------------------------------------------------------------------------
# Finding them in text
# ---------------------------------------------------------------------------

def _tokens(text: str) -> list[tuple[str, bool]]:
    """``(text, was_phonetic)`` per word, punctuation dropped.

    Whether a token was spelled out matters: "alpha" is a letter beyond doubt,
    while a bare "a" is almost always the English article.
    """
    out = []
    for raw in re.split(r"[^0-9A-Za-z\-]+", text):
        if not raw:
            continue
        spoken = PHONETIC.get(raw.lower())
        out.append((spoken, True) if spoken else (raw.upper(), False))
    return out


# A callsign is at most six characters, so a run of more than six tokens can
# never be one however it was broken up.
MAX_SPAN = 6


def find_callsigns(text: str, max_found: int = 12) -> list[str]:
    """Every callsign-shaped run in ``text``, in the order they appear.

    Candidates are built from runs of adjacent tokens, because a recogniser
    breaks a callsign wherever the speaker paused -- "KU 0W" and "K7 RA" are
    one callsign each, written as two words -- and an operator who spells it
    out phonetically produces one token per character: "kilo uniform zero
    whiskey" is four.  Longest first, so "KU 0W" wins over the "U0W" hiding
    inside it.
    """
    tokens = _tokens(text)
    found: list[str] = []
    seen: set[str] = set()
    i = 0
    while i < len(tokens):
        for span in range(MAX_SPAN, 0, -1):
            if i + span > len(tokens):
                continue
            run = tokens[i:i + span]
            if span > 1:
                if run[0][0] in LONE_WORDS and not run[0][1]:
                    continue      # "a B4U" is not AB4U
                # Only words of two letters or more.  A callsign spelled out
                # one character at a time -- "W 1 A W" -- has a bare "A" in
                # the middle of it, and rejecting that would lose exactly the
                # case this join exists to catch.  The first position is
                # guarded separately, above.
                if any(len(word) > 1 and word in COMMON_WORDS and not phonetic
                       for word, phonetic in run):
                    continue      # "or 3. Can you" is not OR3CAN
            joined = "".join(word for word, _ in run)
            if not SHAPE.match(joined) or joined in NOT_CALLSIGNS:
                continue
            if joined not in seen:
                seen.add(joined)
                found.append(joined)
            i += span
            break
        else:
            i += 1
        if len(found) >= max_found:
            break
    return found


# ---------------------------------------------------------------------------
# Looking them up
# ---------------------------------------------------------------------------

@dataclass
class Callsign:
    """What is known about one callsign."""

    call: str
    name: str = ""
    location: str = ""            # town and state, as licensed
    country: str = ""
    district: str = ""
    oper_class: str = ""
    grid: str = ""
    licence_type: str = ""        # PERSON, CLUB, ...
    expires: str = ""
    previous: str = ""
    trustee: str = ""
    postcode: str = ""
    status: str = "pending"       # pending / found / unlisted / offline
    fetched_at: float = 0.0

    @property
    def known(self) -> bool:
        return self.status == "found"

    def summary(self) -> str:
        """One line: who and where, falling back to what the prefix says."""
        if self.known:
            where = self.location or self.country
            bits = [b for b in (self.name, where) if b]
            return " — ".join(bits) if bits else self.call
        if self.status == "pending":
            return "looking up…"
        # Not in the database: say what the callsign says about itself, which
        # is real information, rather than nothing at all.
        bits = [b for b in (self.country, self.district) if b]
        tail = ", ".join(bits) if bits else "no matching licence"
        if self.status == "offline":
            return f"not looked up ({tail})" if bits else "not looked up"
        return f"unlisted ({tail})" if bits else "no matching licence"

    def details(self) -> list[str]:
        """The rest of it, for anywhere with room to show more than one line."""
        out = []
        if self.oper_class:
            out.append(self.oper_class.title())
        if self.licence_type and self.licence_type.upper() != "PERSON":
            out.append(self.licence_type.title())
        if self.grid:
            out.append(self.grid)
        if self.postcode:
            out.append(self.postcode)
        if self.previous:
            out.append(f"ex {self.previous}")
        if self.trustee:
            out.append(f"trustee {self.trustee}")
        if self.expires:
            out.append(f"expires {self.expires}")
        return out


# Words the licence database writes in capitals that are not names, and would
# be wrong in title case: "PAUL PAKES COOK, III." must not become "Iii.".
_KEEP_CAPS = frozenset({"II", "III", "IV", "V", "VI", "VII", "VIII", "JR",
                        "SR", "MD", "DDS", "PHD", "HQ", "ARRL", "USA", "US",
                        "LLC", "INC", "LTD", "ARC", "EMA", "RACES", "ARES"})


def person_case(name: str) -> str:
    """Licence records are all capitals; make them readable without lying.

    Straight title case turns "III." into "Iii." and "ARRL" into "Arrl", so
    anything that is an acronym, a Roman numeral or a suffix is left alone.
    A name that was not in capitals to begin with is not touched at all --
    somebody else's idea of how their name is spelled is not ours to correct.
    """
    if not name or not name.isupper():
        return name
    out = []
    for word in name.split():
        bare = word.strip(".,")
        out.append(word if bare in _KEEP_CAPS else word.title())
    return " ".join(out)


def split_postcode(line: str) -> tuple[str, str]:
    """``"TUCSON, AZ 85742"`` -> ``("Tucson, AZ", "85742")``.

    The town and state are the useful half; the postcode is detail that
    belongs further down the entry, not in the one line that has to say where
    somebody is.
    """
    m = re.match(r"^(.*?)[ ,]+(\d{5}(?:-\d{4})?)$", line.strip())
    if not m:
        return person_case(line.strip()), ""
    town = m.group(1).strip().rstrip(",")
    # State abbreviations stay in capitals; the town is title-cased.
    parts = [p.strip() for p in town.split(",")]
    if len(parts) == 2 and len(parts[1]) == 2:
        town = f"{person_case(parts[0])}, {parts[1].upper()}"
    else:
        town = person_case(town)
    return town, m.group(2)


def _cache_path() -> Path:
    root = os.environ.get("XDG_CACHE_HOME") or "~/.cache"
    return Path(root).expanduser() / "bandsaunter" / "callsigns.json"


class CallsignBook:
    """Resolves callsigns, in the background, once each.

    Rendering must never wait on a network request, so a lookup returns
    immediately with whatever is known and fills itself in later; the display
    redraws several times a second anyway.  Results are cached on disk, so the
    same net logged night after night is looked up once.
    """

    def __init__(self, online: bool = True, cache: Path | None = None,
                 timeout: float = 5.0, url: str = LOOKUP_URL,
                 max_age: float = 30 * 86_400):
        self.online = online
        self.timeout = timeout
        self.url = url
        self.max_age = max_age
        self.cache_path = Path(cache) if cache is not None else _cache_path()
        self._lock = threading.Lock()
        self._entries: dict[str, Callsign] = {}
        self._threads: list[threading.Thread] = []
        self._dirty = False
        self._load()

    # -- cache ------------------------------------------------------------
    def _load(self) -> None:
        try:
            raw = json.loads(self.cache_path.read_text())
        except (OSError, ValueError):
            return
        now = time.time()
        for call, body in (raw or {}).items():
            try:
                entry = Callsign(**body)
            except TypeError:
                continue           # written by a version with other fields
            if entry.status in ("found", "unlisted") and \
                    now - entry.fetched_at < self.max_age:
                self._entries[call] = entry

    def save(self) -> None:
        """Write the cache.  Failing to is never worth an error."""
        with self._lock:
            if not self._dirty:
                return
            body = {c: e.__dict__ for c, e in self._entries.items()
                    if e.status in ("found", "unlisted")}
            self._dirty = False
        try:
            self.cache_path.parent.mkdir(parents=True, exist_ok=True)
            tmp = self.cache_path.with_suffix(".tmp")
            tmp.write_text(json.dumps(body, indent=1, sort_keys=True))
            tmp.replace(self.cache_path)
        except OSError:
            pass

    # -- lookup -----------------------------------------------------------
    def get(self, call: str) -> Callsign:
        """What is known about ``call`` right now, starting a fetch if needed."""
        call = call.upper()
        with self._lock:
            entry = self._entries.get(call)
            if entry is not None:
                return entry
            country, district = describe_prefix(call)
            entry = Callsign(call=call, country=country, district=district,
                             status="pending" if self.online else "offline")
            self._entries[call] = entry
        if self.online:
            thread = threading.Thread(target=self._fetch, args=(entry,),
                                      daemon=True)
            self._threads.append(thread)
            thread.start()
        return entry

    def get_all(self, calls) -> list[Callsign]:
        return [self.get(c) for c in calls]

    def wait(self, timeout: float = 10.0) -> None:
        """Block until the outstanding lookups finish.  For scripts, not the UI."""
        deadline = time.time() + timeout
        for thread in list(self._threads):
            thread.join(max(0.0, deadline - time.time()))

    def _fetch(self, entry: Callsign) -> None:
        try:
            body = self._request(entry.call)
        except Exception:
            # Offline, blocked, rate-limited, or the service moved.  The
            # prefix still says something, and a browser that cannot look a
            # callsign up must still show the transcript.
            with self._lock:
                entry.status = "offline"
            return
        self._apply(entry, body)
        with self._lock:
            self._dirty = True

    def _request(self, call: str) -> dict:
        req = urllib.request.Request(
            self.url.format(call=urllib.parse.quote(call)),
            headers={"User-Agent": "bandsaunter"})
        with urllib.request.urlopen(req, timeout=self.timeout) as response:
            return json.loads(response.read(64_000).decode("utf8", "replace"))

    @staticmethod
    def _apply(entry: Callsign, body: dict) -> None:
        entry.fetched_at = time.time()
        if not isinstance(body, dict) or body.get("status") != "VALID":
            entry.status = "unlisted"
            return
        address = body.get("address") or {}
        location = body.get("location") or {}
        other = body.get("otherInfo") or {}
        current = body.get("current") or {}
        previous = body.get("previous") or {}
        trustee = body.get("trustee") or {}
        entry.name = person_case(str(body.get("name") or ""))
        entry.location, entry.postcode = split_postcode(
            str(address.get("line2") or ""))
        entry.grid = str(location.get("gridsquare") or "")
        entry.oper_class = str(current.get("operClass") or "")
        entry.licence_type = str(body.get("type") or "")
        entry.expires = str(other.get("expiryDate") or "")
        entry.previous = str(previous.get("callsign") or "")
        entry.trustee = str(trustee.get("callsign") or "")
        if not entry.country:
            entry.country = "United States"
        entry.status = "found"


# ---------------------------------------------------------------------------
# Rendering
# ---------------------------------------------------------------------------

HEADING = "DETECTED CALLSIGNS:"


def report(entries: list[Callsign], width: int = 78) -> list[str]:
    """The block that goes at the bottom of a transcript, as plain lines."""
    if not entries:
        return []
    out = [HEADING]
    pad = max((len(e.call) for e in entries), default=4)
    for entry in entries:
        line = f"  {entry.call:<{pad}}  {entry.summary()}"
        extra = entry.details()
        if extra:
            line += "  ·  " + "  ·  ".join(extra)
        out.append(line[:width] if width and len(line) > width
                   else line)
    return out