diff --git a/README.md b/README.md index 4d64dfd..45aad87 100644 --- a/README.md +++ b/README.md @@ -773,7 +773,51 @@ reliably: ``` The decoder runs its own CW detector over the captured IQ, so Morse is found -even when the recording itself was made in FM or SSB. +even when the recording itself was made in FM or SSB — and it is run over +**every** capture once it has finished, whatever the classifier called it. + +#### Short bursts, which is most of it + +Most of the Morse on the air is not a conversation. It is a repeater, a beacon +or an unattended transmitter saying who it is and stopping — four to six +characters, over in a second or two: + +``` +147.06 MHz 5.3s SNR 31.2 dB CW / Morse at 20 WPM CW "DE K1AA" + K1AA Newington Radio Club — Newington, CT · FN31pr +``` + +That burst is a fraction of a capture the classifier named after whatever +filled the rest of it, so waiting for the label to say "CW" missed it. Short +is now the normal case rather than the awkward one, and a decode of two or +three characters gets in on its timing alone: + +- every element within a third of a unit of one or three +- every character resolving to something in the table +- **and the keyed tone at least 20 dB above the rest of its band** + +The last one is what separates an ident from a blip, and it is not +decoration. With four elements the dot length is fitted to those very +elements, so they land on the grid whatever produced them — a third of a +second of white noise decodes as a perfectly timed `V`. Measured over 200 +noise blocks the loudest bin never rose 13 dB above the median of its band, +while keying at 3 dB SNR sits above 40, so 20 dB has room on both sides. One +keyed element is refused outright: a single pulse is an `E` or a `T` whether a +person sent it or the squelch opened on a click. + +#### What the capture window cut off + +A capture opens when the squelch does, which is in the middle of an element as +often as not. Half a character is not a smaller reading of what was sent — it +is a different one. A `K` missing its first dash is an `A`; a `W` missing its +first dot is an `M`. + +So the character at a sliced end is dropped, and so is the rest of the word it +was in, because what is left of that word can read as a whole one: **`K1AA` +caught halfway through is `K1A`, which belongs to somebody else.** The full +text is still shown; it is the *identification* that is held to the stricter +standard. Over 1805 truncated captures of four different messages, that turns +107 invented callsigns into none, while still recovering 550 correct ones. ### Single sideband @@ -1209,7 +1253,35 @@ Note what the recogniser actually wrote: **"KU 0W"**, with a space. Speech recognisers are poor at callsigns — they are not words, they are said one character at a time — so a callsign arrives broken wherever the speaker paused, and an operator who spells it out gets *"kilo uniform zero whiskey"* -written down verbatim. All three forms read back to `KU0W`. +written down verbatim. + +A recogniser has never heard of the phonetic alphabet, so it writes what the +words sounded like and does whatever it likes with the spacing. All of these +are one callsign, and all of them read back correctly: + +| What the recogniser wrote | Why | +|---|---| +| `KU 0W`, `K7 RA` | broken where the speaker paused | +| `kilo uniform zero whiskey` | spelled out, one word per character | +| `Whiskey-One-Alpha-Whiskey` | spelled out and hyphenated | +| `WhiskeyOneAlphaWhiskey`, `Whiskey1AlphaWhiskey` | run together | +| `wiskey one alfa whisky` | spelled the way it sounded | +| `whiskey one alpha, uh, whiskey` | said with a hesitation in the middle | +| `W1AW-4`, `W1AW/B`, `DL/W1AW` | a suffix, which is not part of the callsign | + +A word is only taken apart when it is phonetic *all the way through*, which is +what keeps this away from English: "kilometre" begins with a phonetic word and +"victorious" contains one, and neither can be consumed to the end. + +Callsigns arrive from three directions and all three end up in the same list +and on the same map: **spoken and transcribed, sent in Morse, or carried in +the header of an APRS packet.** Neither of the last two involves a speech +recogniser, so a machine with none installed still builds a map. + +Where the gaps came from decides whether they can be closed. A transcript's +spacing is the recogniser's guess, so `KU 0W` may be joined; a word gap in +Morse is seven dot units the sender chose, so `KU0W K` is a station signing +off, not a callsign one letter longer. The other half of the problem is not inventing them. A browser that reports callsigns nobody said is worse than one that reports none, so a run of words diff --git a/bandsaunter/__init__.py b/bandsaunter/__init__.py index 1a31259..629f490 100755 --- a/bandsaunter/__init__.py +++ b/bandsaunter/__init__.py @@ -9,7 +9,7 @@ and transcribing speech. # 2026-08-21_02 is the second build made on the 21st. The revision is padded # to two digits so versions sort as text. VERSION_DATE = "2026-08-29" -VERSION_REVISION = 1 +VERSION_REVISION = 2 __version__ = f"{VERSION_DATE}_{VERSION_REVISION:02d}" diff --git a/bandsaunter/browse.py b/bandsaunter/browse.py index e2a6f2c..ec609a6 100644 --- a/bandsaunter/browse.py +++ b/bandsaunter/browse.py @@ -187,11 +187,57 @@ class Capture: bits.append(str(checks[0])) return " ".join(bits) + @property + def morse(self) -> str: + """What a CW capture was keying, if anything.""" + return str(self.meta.get("morse_text", "")).strip() + + @property + def morse_complete(self) -> str: + """The part of it a station can be identified from. + + A capture that opened partway through an ident lost the start of the + word it opened on, and what is left of that word can read as a whole + one: "K1AA" caught halfway through is "K1A", which belongs to + somebody else. Recordings made before this was written down carry no + such distinction, so for those it is the whole text or nothing. + """ + # Presence, not truthiness. An empty value means the decoder looked + # and found nothing safe to identify from, which is a different thing + # from a sidecar that predates the question being asked. + if "morse_complete" in self.meta: + return str(self.meta["morse_complete"]).strip() + return self.morse + @property def callsigns(self) -> list[str]: - """Callsign-shaped runs in the transcript, found once and kept.""" + """Callsign-shaped runs in everything this capture said. + + Not only the transcript. A beacon, a repeater or an unattended + transmitter identifies itself in Morse and says nothing else at all, + and an APRS packet carries the sender's callsign in its first field -- + those are the same people as the ones on the net, and belong in the + same list and on the same map. + + Packet lines are only searched where they look like AX.25. A hex + dump is a string of two-character groups, and enough of those in a row + will join into something callsign-shaped that nobody transmitted. + """ if self._calls is None: - self._calls = find_callsigns(self.transcript) + # Only the transcript has spacing a recogniser invented; the + # gaps in Morse and in a packet header were put there by the + # sender, so nothing is joined across those. + sources = [(self.transcript, True), (self.morse_complete, False)] + sources += [(line, False) for line in self.decoded if ">" in line] + seen: set[str] = set() + found: list[str] = [] + for text, join in sources: + for call in (find_callsigns(text, join_words=join) + if text else []): + if call not in seen: + seen.add(call) + found.append(call) + self._calls = found return self._calls # -- derived ---------------------------------------------------------- @@ -601,9 +647,13 @@ class Browser: if any(q in band.lower() for band in cap.bands): return True # And in what a data capture said: a pager message is as searchable - # as a spoken one, and the reason for searching is the same. + # as a spoken one, and the reason for searching is the same. So is + # what a station keyed: "/W1AW" should find the CW ident as readily + # as the net that mentioned it. if any(q in line.lower() for line in cap.decoded): return True + if q in cap.morse.lower(): + return True return q in cap.transcript.lower() @property @@ -825,6 +875,22 @@ class Browser: return [line.plain.rstrip() for line in text.wrap(self.console, width)] + def _morse_lines(self, pad: int = 3) -> list[str]: + """What a CW capture keyed, wrapped to the width it is drawn at. + + Morse gets a panel of its own rather than the "no transcript" notice + it used to share with silence and static. A station that identifies + itself in CW has said the one thing worth reading, and the callsigns + found in it belong under it exactly as they do under speech. + """ + cap = self.current + if cap is None or cap.transcript or not cap.morse: + return [] + width = max(20, (self.console.size.width or 80) - 2 - 2 * pad) + text = Text(cap.morse, style="bold white") + return [line.plain.rstrip() + for line in text.wrap(self.console, width)] + def _transcript_height(self) -> int: """Tall enough for the words, but never more than a third of the screen. @@ -839,8 +905,9 @@ class Browser: # ceiling when there are any, and never fewer than enough to show them. ceiling = max(6, min(20 if calls else 16, screen // (2 if calls else 3))) cap = self.current - body = len(self._transcript_lines()) or ( - len(cap.decoded) + 2 if cap is not None and cap.decoded else 0) + body = (len(self._transcript_lines()) or len(self._morse_lines()) + or (len(cap.decoded) + 2 + if cap is not None and cap.decoded else 0)) needed = body + len(calls) + 4 if calls: needed += 1 # the blank line above @@ -888,6 +955,26 @@ class Browser: height = self._transcript_height() if cap is None: return Panel("", border_style="bright_black", height=height) + morse = self._morse_lines() + if morse: + calls = self._callsign_lines() + # Never below one: on a very short window the callsigns can eat + # the whole panel, and a negative slice would take lines off the + # wrong end of the message. + room = max(1, height - 4 - (len(calls) + 1 if calls else 0)) + shown = morse[:room - 1] if len(morse) > room else morse + body = Text("\n".join(shown), style="bold white") + if len(morse) > len(shown): + body.append(f"\n… {len(morse) - len(shown)} more line(s) — " + "press t to read it all", style="not bold yellow") + if calls: + body.append("\n\n") + body.append(calls[0], style="not bold bold cyan") + for line in calls[1:]: + body.append("\n") + body.append(line, style="not bold white") + return Panel(body, title="Morse", title_align="left", + border_style="yellow", padding=(1, 3), height=height) if not cap.transcript and cap.decoded: # A data capture has no words, but it does have content, and it # belongs in the same place a transcript would be: at the top, @@ -950,8 +1037,7 @@ class Browser: return ("one file per frequency\n" "transcripts for these are appended to a .txt beside it") cat = cap.category - if cap.meta.get("morse_text"): - return f'Morse, decoded as:\n"{cap.meta["morse_text"].strip()}"' + # Morse that decoded never reaches here: it has a panel of its own. if cat == "cw": return "Morse -- nothing for a speech recogniser to hear" if cat in ("digital", "trunk"): @@ -1039,6 +1125,7 @@ class Browser: cat = CATEGORY_STYLE.get(cap.category, "white") when = cap.when summary = (cap.transcript.replace("\n", " ") + or cap.morse or (cap.decoded[0] if cap.decoded else "") or cap.classification) t.add_row( @@ -1192,6 +1279,9 @@ class Browser: room = max(3, screen - 4 - (len(calls) + 1 if calls else 0)) lines = self._transcript_lines(pad=4) title_word = "transcript" + if not lines and cap is not None and cap.morse: + lines = self._morse_lines(pad=4) + title_word = "Morse" if not lines and cap is not None and cap.decoded: # A long paging capture is as worth reading in full as a long # net, and there is nowhere else to read it. @@ -1309,7 +1399,8 @@ class Browser: # Either kind of content: the decoded panel says "press t to read # it all" when a long paging capture overflows it, and the key has # to mean what the panel says it means. - if cap is not None and (cap.transcript or cap.decoded): + if cap is not None and (cap.transcript or cap.morse + or cap.decoded): self.reading = True self.read_top = 0 else: @@ -1483,6 +1574,8 @@ def _first_line(cap: Capture) -> str: """The most informative thing about a capture, in one line.""" if cap.transcript: return cap.transcript.splitlines()[0] + if cap.morse: + return cap.morse if cap.decoded: # A pager message speaks for itself; a packet of hex does not, so # that one is introduced by what kind of packet it is. diff --git a/bandsaunter/callsign.py b/bandsaunter/callsign.py index cef427a..9555fe3 100644 --- a/bandsaunter/callsign.py +++ b/bandsaunter/callsign.py @@ -40,19 +40,49 @@ LOOKUP_URL = "https://callook.info/{call}/json" # good after coordinates were added. CACHE_VERSION = 2 +# The NATO alphabet, with the spellings a speech recogniser actually writes +# down beside the ones the ITU prints. A recogniser has never heard of the +# phonetic alphabet: it writes what the word sounded like, so "alfa", +# "juliett" and "whisky" are as common as the official forms and cost nothing +# to accept. PHONETIC = { - "alpha": "A", "alfa": "A", "bravo": "B", "charlie": "C", "delta": "D", - "echo": "E", "foxtrot": "F", "fox": "F", "golf": "G", "hotel": "H", - "india": "I", "juliet": "J", "juliett": "J", "julliet": "J", - "kilo": "K", "lima": "L", "mike": "M", "november": "N", "oscar": "O", - "papa": "P", "quebec": "Q", "romeo": "R", "sierra": "S", "tango": "T", - "uniform": "U", "victor": "V", "whiskey": "W", "whisky": "W", - "xray": "X", "x-ray": "X", "yankee": "Y", "zulu": "Z", + "alpha": "A", "alfa": "A", + "bravo": "B", + "charlie": "C", "charley": "C", + "delta": "D", + "echo": "E", + "foxtrot": "F", "fox": "F", + "golf": "G", + "hotel": "H", + "india": "I", + "juliet": "J", "juliett": "J", "julliet": "J", "juliette": "J", + "kilo": "K", + "lima": "L", + "mike": "M", + "november": "N", + "oscar": "O", "oskar": "O", + "papa": "P", + "quebec": "Q", + "romeo": "R", + "sierra": "S", + "tango": "T", + "uniform": "U", + "victor": "V", "viktor": "V", + "whiskey": "W", "whisky": "W", "wiskey": "W", + "xray": "X", "x-ray": "X", + "yankee": "Y", + "zulu": "Z", "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", "fife": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9", "niner": "9", } +# Noises a recogniser writes down as words. Dropping them keeps a run +# together: "whiskey one alpha, uh, whiskey" is one callsign said with a +# hesitation in the middle of it, and it used to be none. +FILLERS = frozenset({"UH", "UM", "UHM", "ER", "ERM", "AH", "HMM", "MM", + "MHM", "EH"}) + # A prefix, a digit, then one to three letters. The prefix is one or two # letters (W, KU, VE) or a digit and a letter (2E, 4X, 9A) -- the latter is why # a plain "letters then digit" pattern is not enough, and why every British @@ -213,18 +243,85 @@ def describe_prefix(call: str) -> tuple[str, str]: # Finding them in text # --------------------------------------------------------------------------- +def _hyphen_parts(raw: str) -> list[str]: + """A hyphenated word, split -- unless the whole of it is a phonetic word. + + "x-ray" is one letter. "Whiskey-One-Alpha-Whiskey" is four, and a + recogniser writes a spelled-out callsign that way as readily as with + spaces. The same split is what rescues ``W1AW-4``, whose suffix used to + take the callsign down with it. + """ + if "-" not in raw or raw.lower() in PHONETIC: + return [raw] + return [part for part in raw.split("-") if part] + + +def _phonetic_parts(word: str) -> list[str] | None: + """Split a run-together phonetic spelling, or None if it is not one. + + A recogniser that hears the alphabet said quickly writes it as one word: + "WhiskeyOneAlphaWhiskey". Only a word that is phonetic *all the way + through* is split, which is what keeps this away from ordinary English -- + "kilometre" begins with a phonetic word and "victorious" contains one, + and neither can be consumed to the end, so neither is touched. + """ + lowered = word.lower() + n = len(lowered) + if n < 4: # nothing shorter holds two of anything + return None + memo: dict[int, list[str] | None] = {n: []} + + def solve(at: int) -> list[str] | None: + if at in memo: + return memo[at] + found = None + # Longest first, so "one" is not read out of "november". Every + # branch only ever moves forward, so this terminates. + for end in range(n, at, -1): + piece = lowered[at:end] + spoken = PHONETIC.get(piece) + if spoken is None and end == at + 1 and piece.isdigit(): + spoken = piece # "Whiskey1AlphaWhiskey" + if spoken is None: + continue + rest = solve(end) + if rest is not None: + found = [spoken] + rest + break + memo[at] = found + return found + + parts = solve(0) + return parts if parts and len(parts) > 1 else None + + def _tokens(text: str) -> list[tuple[str, bool]]: """``(text, was_phonetic)`` per word, punctuation dropped. Whether a token was spelled out matters: "alpha" is a letter beyond doubt, while a bare "a" is almost always the English article. + + Three things happen to a word before it is taken as written: a hyphenated + one is split, a phonetic spelling run together is split into its letters, + and a filler is dropped so a hesitation in the middle of a callsign does + not break the run. """ - out = [] + out: list[tuple[str, bool]] = [] for raw in re.split(r"[^0-9A-Za-z\-]+", text): if not raw: continue - spoken = PHONETIC.get(raw.lower()) - out.append((spoken, True) if spoken else (raw.upper(), False)) + for piece in _hyphen_parts(raw): + spoken = PHONETIC.get(piece.lower()) + if spoken: + out.append((spoken, True)) + continue + parts = _phonetic_parts(piece) + if parts: + out.extend((part, True) for part in parts) + continue + word = piece.upper() + if word not in FILLERS: + out.append((word, False)) return out @@ -233,7 +330,8 @@ def _tokens(text: str) -> list[tuple[str, bool]]: MAX_SPAN = 6 -def find_callsigns(text: str, max_found: int = 12) -> list[str]: +def find_callsigns(text: str, max_found: int = 12, + join_words: bool = True) -> list[str]: """Every callsign-shaped run in ``text``, in the order they appear. Candidates are built from runs of adjacent tokens, because a recogniser @@ -242,13 +340,40 @@ def find_callsigns(text: str, max_found: int = 12) -> list[str]: out phonetically produces one token per character: "kilo uniform zero whiskey" is four. Longest first, so "KU 0W" wins over the "U0W" hiding inside it. + + ``join_words`` is what makes that safe to do only where the spacing is a + guess. In a transcript it is: the recogniser put the gaps in, and they + mean nothing. In Morse it is not -- a word gap is seven dot units of + silence that the sender chose -- and joining across one turns "KU0W K", + a station signing off, into KU0WK, which belongs to nobody. The same + goes for the callsign in a packet header. Pass False for those and each + word is judged on its own. """ + # A slash is a hard boundary, never a join. It is how an operator says + # "somewhere else" -- W1AW/4, DL/W1AW, and the /B a beacon signs with -- + # so the two sides are a callsign and a qualifier, not one longer + # callsign. Joining across it turned the beacon W1AW/B into W1AWB, which + # belongs to nobody. + if "/" in text: + found: list[str] = [] + seen: set[str] = set() + for part in text.split("/"): + for call in find_callsigns(part, max_found=max_found, + join_words=join_words): + if call not in seen: + seen.add(call) + found.append(call) + if len(found) >= max_found: + return found + return found + tokens = _tokens(text) found: list[str] = [] seen: set[str] = set() + widest = MAX_SPAN if join_words else 1 i = 0 while i < len(tokens): - for span in range(MAX_SPAN, 0, -1): + for span in range(widest, 0, -1): if i + span > len(tokens): continue run = tokens[i:i + span] diff --git a/bandsaunter/classify.py b/bandsaunter/classify.py index e4c4aab..1092c56 100755 --- a/bandsaunter/classify.py +++ b/bandsaunter/classify.py @@ -271,6 +271,12 @@ def _psk_order(x: np.ndarray, fs: float): spec = np.abs(np.fft.fftshift(np.fft.fft(y[:nfft] * np.hanning(nfft), nfft))) peak = spec.max() med = np.median(spec) + 1e-12 + if peak <= 0.0: + # Nothing in the window at all. A real receiver always has + # noise, so this only happens on a synthetic silence -- but the + # logarithm of zero is not a number and the caller was left with + # a math domain error where it expected a measurement. + continue strength = 20.0 * math.log10(peak / med) if strength > best[1]: best = (m, strength) diff --git a/bandsaunter/cli.py b/bandsaunter/cli.py index db22da0..7440c16 100755 --- a/bandsaunter/cli.py +++ b/bandsaunter/cli.py @@ -331,6 +331,18 @@ def cmd_scan(args) -> int: # happened to land near a made-up frequency. Locking out still works # for the run in hand; it is only the writing back that is refused. cfg.save_lockouts = False + if args.simulate and (cfg.callsign_lookup or cfg.kml_file): + # Same reason, and a sharper one. The demo band is made up but the + # callsigns in it are real people -- the beacon identifies itself as + # W1AW, which is the ARRL's own station -- so a simulated run would + # look up a licence nobody heard and pin it to the same map a real + # scan writes. Callsigns are still found and shown; it is the + # contacting and the recording that are refused. + cfg.callsign_lookup = False + cfg.kml_file = "" + console.print("[magenta]Callsigns in the demo band belong to real " + "stations, so they are not looked up and not " + "mapped.[/magenta]") scanner = Scanner(cfg, device=device, callbacks=ScannerCallbacks()) try: diff --git a/bandsaunter/morse.py b/bandsaunter/morse.py index db198c8..be0a504 100755 --- a/bandsaunter/morse.py +++ b/bandsaunter/morse.py @@ -14,7 +14,8 @@ from dataclasses import dataclass, field import numpy as np from scipy import signal as sps -__all__ = ["decode_morse", "MorseResult", "MORSE_TABLE", "encode_morse"] +__all__ = ["decode_morse", "MorseResult", "MORSE_TABLE", "encode_morse", + "SHORT_TONE_DB", "MIN_SAMPLES"] MORSE_TABLE: dict[str, str] = { @@ -41,6 +42,21 @@ _REVERSE = {v: k for k, v in MORSE_TABLE.items() if len(v) == 1} # Morse timing is defined against PARIS = 50 dot units per word. _DOTS_PER_WORD = 50.0 +# How far the keyed tone has to stand above the rest of its band before a +# decode of only a character or two is believed. See MorseResult.is_morse. +SHORT_TONE_DB = 20.0 + +# What _find_tone needs before it can measure anything: one FFT's worth of +# samples. It is also the floor on the whole decode, because a station that +# identifies itself in Morse and nothing else is on the air for well under a +# second, and a floor set for comfort throws those away unheard. +MIN_SAMPLES = 1024 + +# Silence of this many dot units or more separates words. ITU says seven; +# five is where the decoder splits them, because a hand key stretches the +# short gaps and shortens the long ones. +_WORD_GAP_UNITS = 5.0 + @dataclass class MorseResult: @@ -54,19 +70,66 @@ class MorseResult: timing_fit: float = 0.0 undecoded: int = 0 snr_db: float = 0.0 + head_cut: bool = False # the capture opened mid-character + tail_cut: bool = False # and/or closed mid-character notes: list[str] = field(default_factory=list) + @property + def complete_text(self) -> str: + """The part of the text that is certainly what was sent. + + The character sliced by the capture window has already been dropped, + but what is left of the word it was in can still read as a whole one: + "K1AA" cut short is "K1A", which is somebody else's callsign, and + this text is looked at for callsigns. So a word touching a cut end + is not reported either. What comes back is what a station can be + identified from; ``text`` remains everything that was read. + """ + if not (self.head_cut or self.tail_cut): + return self.text + words = self.text.split() + if self.head_cut and words: + words = words[1:] + if self.tail_cut and words: + words = words[:-1] + return " ".join(words) + @property def is_morse(self) -> bool: - # Nobody sends below 5 or above 60 WPM. A "decode" outside that range - # is the timing estimator latching onto something that is not Morse -- - # speech syllables, for instance. - return (self.confidence >= 0.5 and self.n_characters >= 3 - and 5.0 <= self.wpm <= 60.0 - # Enough elements to be a transmission rather than a handful - # of noise crossings, and timing that genuinely fits Morse. - and self.n_elements >= 8 and self.timing_fit >= 0.7 - and self.undecoded <= 0.3 * max(1, self.n_characters)) + """Whether this decode should be believed. + + Two bars, not one. A few seconds of text clears the ordinary one. + Anything shorter has to be cleaner than that, because at two or three + characters there is not enough of it for a wrong reading to + contradict itself -- but shorter is exactly what a station giving + nothing but its callsign sends, and a bar it could never clear would + throw away the transmissions most worth having. + + Below three elements there is nothing left to be right about: one + keyed pulse is an E or a T whether a person sent it or the squelch + opened on a click, so that is where this stops. + """ + # Nobody sends below 5 or above 60 WPM. A "decode" outside that + # range is the timing estimator latching onto something that is not + # Morse -- speech syllables, for instance. + if not 5.0 <= self.wpm <= 60.0: + return False + if self.undecoded > 0.3 * max(1, self.n_characters): + return False + if self.n_characters >= 3 and self.n_elements >= 8: + return self.confidence >= 0.5 and self.timing_fit >= 0.7 + # Timing alone is not enough down here. With three or four elements + # the dot length is fitted to those very elements, so they land on the + # grid whatever produced them -- a third of a second of white noise + # decodes as a perfectly timed V. So a short burst has to have been + # a tone as well: measured over two hundred noise blocks the loudest + # bin never rose 13 dB above the median of its band, while keying at + # 3 dB SNR sits above 40. Twenty is between the two with room to + # spare on both sides. + return (self.n_characters >= 1 and self.n_elements >= 3 + and self.timing_fit >= 0.95 and self.undecoded == 0 + and self.snr_db >= SHORT_TONE_DB + and self.confidence >= 0.4) def summary(self) -> str: if not self.text: @@ -215,16 +278,21 @@ def _estimate_dot(on_lengths: list[int], off_lengths: list[int]) -> float: def decode_morse(audio: np.ndarray, sample_rate: float, - min_elements: int = 6) -> MorseResult: + min_elements: int = 3) -> MorseResult: """Decode CW from a block of demodulated audio. ``audio`` should be real audio containing the beat note (what the ``cw`` demodulator produces). Speed is estimated from the signal itself, so no WPM setting is needed. + + ``min_elements`` counts key transitions, up and down together, after the + partial runs at each end have been dropped. Three is one character with + structure of its own -- a K, an R, a digit -- which is the shortest thing + that can be told from a click. """ res = MorseResult() audio = np.asarray(audio, dtype=np.float64).ravel() - if audio.size < int(sample_rate * 0.3): + if audio.size < MIN_SAMPLES: res.notes.append("too short to decode") return res @@ -247,8 +315,14 @@ def decode_morse(audio: np.ndarray, sample_rate: float, on = _despeckle(on, max(2, int(0.006 * fs_env))) runs = _runs(on) # Drop the leading and trailing partial runs -- they are cut off by the - # capture window and would corrupt the timing estimate. + # capture window and would corrupt the timing estimate. Keep them, + # though: once the dot length is known they say whether the character at + # each end is cut off too, and half a character decodes to a different + # one rather than to less of the same. A K missing its first dash is an + # A; a W missing its first dot is an M. + edges: tuple = (None, None) if len(runs) >= 3: + edges = (runs[0], runs[-1]) runs = runs[1:-1] if len(runs) < min_elements: res.notes.append("not enough keying transitions") @@ -270,43 +344,71 @@ def decode_morse(audio: np.ndarray, sample_rate: float, return res res.wpm = 1.2 / res.dot_seconds * (_DOTS_PER_WORD / 50.0) + def _cut(edge) -> bool: + """Whether the character at this end of the window is incomplete. + + Two ways for it to be. Key-down at the boundary is the obvious one: + the element itself is sliced. Silence too short to be a word gap is + the other, and the one that is easy to miss -- the window opened + partway through a word, so the rest of that word is outside it, and + what is left of it can read as a whole word of its own. Only silence + long enough to be a gap between words says that what follows really + did begin there. + """ + if edge is None: + return False + state, length = edge + return bool(state) or (length / dot) < _WORD_GAP_UNITS + + head_cut, tail_cut = _cut(edges[0]), _cut(edges[1]) + res.head_cut, res.tail_cut = head_cut, tail_cut + # ---- element decision -------------------------------------------- - symbols: list[str] = [] - text_parts: list[str] = [] + # Characters are carried with the number of elements each one took, so + # that dropping a character cut off by the capture window drops its + # elements with it and the counts stay true. + decoded: list[tuple[str, int]] = [] current = "" - undecoded = 0 def flush(): - nonlocal current, undecoded + nonlocal current if not current: return - ch = MORSE_TABLE.get(current) - if ch is None: - undecoded += 1 - text_parts.append("") - else: - text_parts.append(ch) + decoded.append((MORSE_TABLE.get(current, ""), len(current))) current = "" for state, n in runs: units = n / dot if state: - symbols.append("." if units < 2.0 else "-") - current += symbols[-1] + current += "." if units < 2.0 else "-" else: if units < 2.0: continue # gap between elements of a letter flush() - if units >= 5.0: - text_parts.append(" ") # word gap + if units >= _WORD_GAP_UNITS: + decoded.append((" ", 0)) # word gap flush() + # The characters at a truncated end are incomplete, so they are dropped + # rather than reported: a partial character is not a smaller reading of + # what was sent, it is a different one, and this text is looked at for + # callsigns. A station heard through half its ident is better reported + # as half an ident than as a different station. + if head_cut and decoded: + decoded = decoded[1:] + res.notes.append("first character was cut off and dropped") + if tail_cut and decoded: + decoded = decoded[:-1] + res.notes.append("last character was cut off and dropped") + text_parts = [ch for ch, _ in decoded] + text = "".join(text_parts) # Collapse runs of spaces the timing may have produced. - text = " ".join(text.split(" ")) if text else "" + text = " ".join(text.split(" ")).strip() if text else "" + undecoded = sum(1 for ch, _ in decoded if ch == "") res.text = text - res.n_elements = len(symbols) - res.n_characters = sum(1 for p in text_parts if p not in (" ",)) + res.n_elements = sum(n for _, n in decoded) + res.n_characters = sum(1 for ch, _ in decoded if ch != " ") res.undecoded = undecoded # ---- confidence ---------------------------------------------------- @@ -337,8 +439,15 @@ def decode_morse(audio: np.ndarray, sample_rate: float, res.notes.append("element mix is not typical of Morse text") distinct = {p for p in text_parts if p not in (" ", "")} if len(distinct) < 2: - conf *= 0.35 - res.notes.append("only one distinct character decoded") + # One repeated character is only suspicious when the character is one + # element long: "EEEE" and "TTTT" are what a uniform pulse train + # decodes to, and they are not text. "VVV" is text -- it is the + # oldest thing anyone sends -- and the character has structure of its + # own, which a train of identical pulses cannot produce. + only = next(iter(distinct), "") + if len(_REVERSE.get(only, "")) <= 1: + conf *= 0.35 + res.notes.append("one repeated element, not text") res.confidence = round(min(0.99, max(0.0, conf)), 3) if res.confidence < 0.5: res.notes.append("timing does not fit Morse cleanly") diff --git a/bandsaunter/quality.py b/bandsaunter/quality.py index b126ef7..8242fa6 100644 --- a/bandsaunter/quality.py +++ b/bandsaunter/quality.py @@ -327,7 +327,17 @@ def assess(classification, audio: np.ndarray, audio_rate: float, a.reason = f"{c.describe()}: {c.reason}" # 2. Morse that actually decoded. - elif morse is not None and morse.is_morse: + # + # Every capture is offered to the CW decoder now, not only the ones + # that looked keyed, because a station identifying itself in Morse + # sends a burst of a second or two and the classifier names the + # capture after whatever fills the rest of it. That is worth + # decoding, and it is not worth relabelling a conversation over: where + # the modulation was never keyed and there is speech in the audio, the + # speech is what the capture is, and the Morse text is recorded beside + # it either way. + elif (morse is not None and morse.is_morse + and (fam in ("cw", "ook", "carrier") or vm.score < min_voice)): a.category = "cw" a.score = float(morse.confidence) a.reason = f'Morse decoded at {morse.wpm:.0f} WPM: "{morse.text.strip()[:40]}"' diff --git a/bandsaunter/recorder.py b/bandsaunter/recorder.py index 168a5a0..21578b2 100755 --- a/bandsaunter/recorder.py +++ b/bandsaunter/recorder.py @@ -64,6 +64,10 @@ class HitRecord: alternatives: list = field(default_factory=list) morse_text: str = "" + # The part of it whose words were not cut in half by the capture window, + # which is the only part a station can be identified from: "K1AA" caught + # halfway through reads as "K1A", and that is somebody else. + morse_complete: str = "" morse_wpm: float = 0.0 ctcss_hz: float = 0.0 baud: float = 0.0 diff --git a/bandsaunter/scanner.py b/bandsaunter/scanner.py index 66515ff..5a071c9 100755 --- a/bandsaunter/scanner.py +++ b/bandsaunter/scanner.py @@ -117,6 +117,10 @@ class Scanner: self.callsigns: CallsignBook | None = None self.kml: KmlLog | None = None self.heard: dict[str, int] = {} # callsign -> times heard this run + self._lookups: list[threading.Thread] = [] # Morse callsign lookups + # Three threads can now be writing the map: the transcription + # worker, and one per Morse ident while its licence is looked up. + self._heard_lock = threading.Lock() self.hits: list[HitRecord] = [] self.nfft = 1024 self.detector_bias = 0.0 @@ -235,15 +239,21 @@ class Scanner: on_error=self._error) self.transcriber.start() self._status(f"transcribing speech with {engine}") - # Callsigns come out of transcripts, so both of these are - # only worth setting up where there will be transcripts. - self.callsigns = CallsignBook(online=self.cfg.callsign_lookup) - if self.cfg.kml_file: - self.kml = KmlLog( - root / self.cfg.kml_file, - title="bandsaunter — stations heard", - description="Every callsign heard during a scan, " - "placed where its licence says it is.") + + # Not inside the transcription branch. Callsigns were once only + # going to arrive in a transcript, and these were built where the + # transcripts were. They also arrive in Morse and inside APRS + # packets, neither of which involves a speech recogniser, and a + # station that identifies itself in CW should reach the map on a + # machine with no recogniser installed at all. + if self.cfg.classify: + self.callsigns = CallsignBook(online=self.cfg.callsign_lookup) + if self.cfg.kml_file: + self.kml = KmlLog( + root / self.cfg.kml_file, + title="bandsaunter — stations heard", + description="Every callsign heard during a scan, " + "placed where its licence says it is.") if self.cfg.combine_by_frequency: self.frequency_log = FrequencyLog( @@ -989,7 +999,14 @@ class Scanner: return None, None, None morse = None - if self.cfg.decode_morse and cls.family in ("cw", "ook", "carrier"): + # While the capture is running this is only worth the time on a + # signal that looks keyed. Once it has finished, every capture gets + # a look: a station that identifies itself in Morse does it in a + # burst of a second or two, which is not what the classifier called + # the capture -- it called it whatever filled the rest of it, or + # nothing at all. + if self.cfg.decode_morse and (final or + cls.family in ("cw", "ook", "carrier")): morse = self._decode_cw(iq, demod) # Judge the audio that was actually recorded, and only that. An @@ -1135,23 +1152,95 @@ class Scanner: self._status(f"decoded {fmt_hz(hit.frequency)}: {got.summary()}") # APRS carries callsigns, and the map already knows what to do with # one. Nothing else in a data packet is a person. - book = self.callsigns - if book is None or "AX.25" not in got.protocol: + if "AX.25" not in got.protocol: return - changed = False + calls = [] for message in got.messages: call = message.split(">", 1)[0].split("-", 1)[0].strip() - if not call or not call.isalnum(): - continue - entry = book.get(call) + if call and call.isalnum(): + calls.append(call) + self._heard(calls, hit.frequency, hit.started_at, hit.band, + hit.filename, announce=False) + + # -- who has identified themselves ------------------------------------- + # + # Callsigns arrive from three directions: spoken and transcribed, sent in + # Morse, and inside an APRS packet. They are the same people, wanted in + # the same book and on the same map, so they go through one place. + + def _heard(self, calls, frequency: float, when: float, band: str, + recording: str, announce: bool = True, + wait: float = 0.0) -> None: + """Record who was heard, say so, and put them on the map. + + ``wait`` is how long to hold for the licence lookups. Zero is right + on any thread the sweep is waiting on: the lookups have been started + and will fill themselves in, and the map is written again when they + do. Only a caller already off the scan loop should wait. + """ + book = self.callsigns + if book is None or not calls: + return + entries = book.get_all(calls) + if wait > 0: + book.wait(timeout=wait) + book.save() + with self._heard_lock: + self._record_heard(entries, frequency, when, band, recording, + announce) + + def _record_heard(self, entries, frequency: float, when: float, + band: str, recording: str, announce: bool) -> None: + before = len(self.kml) if self.kml is not None else 0 + changed = False + for entry in entries: self.heard[entry.call] = self.heard.get(entry.call, 0) + 1 + if announce: + self._announce_callsign(entry) if self.kml is not None: - changed |= self.kml.add(entry, hit.frequency, hit.started_at, - hit.band, hit.filename) + changed |= self.kml.add(entry, frequency, when, band, + recording) if changed and self.kml is not None: # Saved as we go, not only at the end: a scan left running # overnight and stopped with a signal should still have its map. - self.kml.save() + written = self.kml.save() + # Announced only when the map gained a station. Every over of a + # long net adds a line to a pin that is already there, and saying + # so each time would push everything else off the status line. + if written is not None and len(self.kml) > before: + self._status(f"{len(self.kml)} station(s) on the map in " + f"{written.name}") + + def _heard_in_morse(self, hit: HitRecord, text: str) -> None: + """Look for a callsign in what a station sent in CW. + + This is the whole reason short bursts are worth decoding. A beacon, + a repeater and an unattended transmitter all say who they are in + Morse and nothing else, and until this existed the text was written + into the sidecar and read by nobody. + + On a thread of its own, because the licence lookup goes over the + network and this is called from the scan loop. + """ + # Word gaps in Morse are real, so nothing is joined across one. + calls = find_callsigns(text, join_words=False) + if not calls or self.callsigns is None: + return + + def work(): + try: + self._heard(calls, hit.frequency, hit.started_at, hit.band, + hit.filename, wait=8.0) + except Exception as exc: + self._error(exc) + + thread = threading.Thread(target=work, daemon=True, + name="callsign-morse") + # Pruned as they are added, so an evening on a CW band does not end + # with a list of ten thousand finished threads in it. + self._lookups = [t for t in self._lookups if t.is_alive()] + self._lookups.append(thread) + thread.start() def _on_transcript(self, path, result, job) -> None: """Pull callsigns out of a finished transcript and map them. @@ -1161,40 +1250,15 @@ class Scanner: is best-effort: a scan must not fail because a website did not answer. """ - book = self.callsigns - if book is None: + if self.callsigns is None: return try: - calls = find_callsigns(result.text) - if not calls: - return - entries = book.get_all(calls) - # The lookups were started by get_all and run in their own - # threads; waiting here is what turns "pending" into a name. A - # short wait, because a slow answer is not worth holding the - # queue for -- the callsign is already in the cache request and - # will be filled in by the time the next one is looked up. - book.wait(timeout=8.0) - book.save() - when = job.when.timestamp() - band = label_for(job.frequency).name - before = len(self.kml) if self.kml is not None else 0 - changed = False - for entry in entries: - self.heard[entry.call] = self.heard.get(entry.call, 0) + 1 - self._announce_callsign(entry) - if self.kml is not None: - changed |= self.kml.add(entry, job.frequency, when, band, - job.recording) - if changed and self.kml is not None: - written = self.kml.save() - # Only when the map gained a station. Every over of a long - # net adds a line to a pin that is already there, and saying - # so each time would push everything else off the status - # line. - if written is not None and len(self.kml) > before: - self._status(f"{len(self.kml)} station(s) on the map in " - f"{written.name}") + # Already off the scan loop, so this is the one caller that can + # afford to wait for the lookups -- which is what turns "pending" + # into a name before the map is written. + self._heard(find_callsigns(result.text), job.frequency, + job.when.timestamp(), label_for(job.frequency).name, + job.recording, wait=8.0) except Exception as exc: self._error(exc) @@ -1240,11 +1304,18 @@ class Scanner: if morse is not None and morse.is_morse: hit.morse_text = morse.text + hit.morse_complete = morse.complete_text hit.morse_wpm = round(morse.wpm, 1) - hit.classification = f"CW / Morse at {morse.wpm:.0f} WPM" - hit.family = "cw" - hit.confidence = max(hit.confidence, morse.confidence) hit.reasons.insert(0, f'decoded Morse: "{morse.text.strip()}"') + # The label only changes when the content check agrees that Morse + # is what the capture *is*. A repeater identifying itself in CW + # over the top of a conversation is both, and the conversation is + # the thing that was recorded. + if verdict is None or verdict.category == "cw": + hit.classification = f"CW / Morse at {morse.wpm:.0f} WPM" + hit.family = "cw" + hit.confidence = max(hit.confidence, morse.confidence) + self._heard_in_morse(hit, morse.complete_text) elif morse is not None and cls.family == "cw": hit.reasons.append( "keyed carrier but the Morse timing did not resolve") @@ -1380,7 +1451,13 @@ class Scanner: self._status(f"finishing {pending} transcription(s)") self.transcriber.close() # After the worker has stopped, so the last transcript's callsigns - # are on the map before it is written for the last time. + # are on the map before it is written for the last time. The + # Morse lookups are their own threads and get the same courtesy, + # briefly: a station heard in the last seconds of a scan belongs + # on the map as much as one heard in the first. + for thread in self._lookups: + thread.join(timeout=2.0) + self._lookups = [t for t in self._lookups if t.is_alive()] if self.callsigns is not None: self.callsigns.save() if self.kml is not None: diff --git a/bandsaunter/settings.py b/bandsaunter/settings.py index c7681cc..93b4151 100644 --- a/bandsaunter/settings.py +++ b/bandsaunter/settings.py @@ -711,7 +711,11 @@ _GUIDANCE: dict[str, str] = { "Turn keyed carriers into readable text, with the sending speed. " "Morse is still in daily use by amateurs and by beacons, and this " "saves you learning to read it by ear. It costs almost nothing when " - "there is no Morse about.", + "there is no Morse about. Every capture is tried once it has " + "finished, whatever the modulation was called, because most of the " + "Morse on the air is a repeater or a beacon giving its callsign in a " + "burst of a second or two; a callsign read out of one goes to the " + "same lookup and the same map as a spoken one.", "log_file": "The name of the run log inside the output directory. It records " "every recording with its time, frequency, duration and " diff --git a/bandsaunter/simulator.py b/bandsaunter/simulator.py index 11f914d..479e752 100755 --- a/bandsaunter/simulator.py +++ b/bandsaunter/simulator.py @@ -327,7 +327,17 @@ class VirtualTransmitter: key = self._morse_key(self.message, self.wpm) dot = 1.2 / self.wpm total = len(key) * dot - pos = ((t % total) / dot).astype(np.int64) + # A transmitter that keys up for a scheduled burst sends its message + # from the beginning, the way a repeater's ident does. Running the + # loop off absolute time instead started it wherever the clock + # happened to be, so a four-second burst held the middle of a repeat + # and a receiver was handed half a callsign at each end. + when = t + if self.period_seconds > 0 and self.on_seconds > 0: + begin = (((t + self.phase_offset) // self.period_seconds) + * self.period_seconds - self.phase_offset) + when = t - begin + pos = ((when % total) / dot).astype(np.int64) env = key[np.clip(pos, 0, len(key) - 1)].astype(np.float64) # Soften the edges the way real keying shaping does. Two box filters # give a triangular window in O(n) -- a direct convolution with a @@ -341,7 +351,7 @@ class VirtualTransmitter: @staticmethod def _morse_key(msg: str, wpm: float) -> np.ndarray: """One element per dot-unit: 1 = key down, 0 = key up.""" - units: list[int] = [0] * 2 + units: list[int] = [] toks = encode_morse(msg).split(" ") for i, tok in enumerate(toks): if tok == "/": @@ -353,7 +363,13 @@ class VirtualTransmitter: if j > 0: units += [0] units += [1] * (1 if el == "." else 3) - units += [0] * 2 + # A word gap at the end, because this pattern is looped and two units + # ran the last character of one repeat into the first of the next: a + # receiver reading "K1AA K1AA" off the air got "AK1AAN". At the end + # only -- it wraps round to the beginning of the next repeat, and + # silence at the front would mean the transmitter kept the channel + # for a second before saying anything. + units += [0] * 7 return np.array(units, dtype=np.float64) @@ -367,6 +383,18 @@ def default_transmitters() -> list[VirtualTransmitter]: period_seconds=19, on_seconds=5, phase_offset=7), V(144_100_000, "cw", 0.30, 500, "2 m CW beacon", wpm=18, message="VVV DE W1AW/B FN31"), + # A repeater giving its callsign in Morse and nothing else, which is + # what most CW on the air actually is: a few characters, once every + # few minutes, over in seconds. Here it comes round more often, + # because a demo that has to be left running for ten minutes + # demonstrates nothing. "DE" in front of it is not decoration: a + # scanner spends a moment detecting, retuning and probing before it + # records, so the first half-second is always lost, and every real + # ident is sent with something in front of the callsign for the same + # reason. + V(147_060_000, "cw", 0.32, 400, "2 m repeater CW ident", wpm=20, + message="DE K1AA", period_seconds=14, on_seconds=5.0, + phase_offset=9), V(162_400_000, "nfm", 0.35, 12_500, "NOAA weather radio"), V(121_500_000, "am", 0.30, 8_000, "airband AM", period_seconds=17, on_seconds=5, phase_offset=3), diff --git a/bandsaunter/tui.py b/bandsaunter/tui.py index cbcdaf8..0cd7ff8 100644 --- a/bandsaunter/tui.py +++ b/bandsaunter/tui.py @@ -591,6 +591,12 @@ It opens in Google Earth, QGIS, Marble and OsmAnd, and later scans add to it rather than starting it over, so it fills in as a picture of what your aerial can reach. +Callsigns arrive from three directions and all three end up on the same map: +spoken and transcribed, sent in Morse, or carried in the header of an APRS +packet. Neither of the last two involves a speech recogniser, so a receiver +with none installed still builds a map -- most of the stations on the air +never say a word, and identify themselves in a short burst of CW instead. + Turn 'Look callsigns up' off to keep the scan entirely offline; callsigns are still found, and named by country from their prefix. Clear 'Map file' to stop writing the map. US licence records are public, and include addresses."""), diff --git a/packaging/bandsaunter.1 b/packaging/bandsaunter.1 index fba8a1c..6e8bbd2 100644 --- a/packaging/bandsaunter.1 +++ b/packaging/bandsaunter.1 @@ -540,7 +540,7 @@ Decode CW to text \[em] decode keyed carriers as Morse. Setting name \fBdecode_morse\fR, default \fByes\fR. .RS .PP -Turn keyed carriers into readable text, with the sending speed. Morse is still in daily use by amateurs and by beacons, and this saves you learning to read it by ear. It costs almost nothing when there is no Morse about. +Turn keyed carriers into readable text, with the sending speed. Morse is still in daily use by amateurs and by beacons, and this saves you learning to read it by ear. It costs almost nothing when there is no Morse about. Every capture is tried once it has finished, whatever the modulation was called, because most of the Morse on the air is a repeater or a beacon giving its callsign in a burst of a second or two; a callsign read out of one goes to the same lookup and the same map as a spoken one. .RE .TP .B --decode-data / --no-decode-data @@ -959,6 +959,54 @@ than a directory of placeholders. .BR saunterbrowse (1) reads these back, and lists any callsigns it finds in them with the licence they belong to. +.PP +A callsign in a transcript is not written the way it is printed. A recogniser +has never heard of the phonetic alphabet: it writes what the words sounded +like, breaks the callsign wherever the speaker paused, joins the words back +up, hyphenates them, or drops a hesitation into the middle of the run. So +"KU 0W", "kilo uniform zero whiskey", "Whiskey\-One\-Alpha\-Whiskey", +"WhiskeyOneAlphaWhiskey" and "whiskey one alpha, uh, whiskey" are all read +back as the callsigns they are, and "alfa", "juliett" and "whisky" count +alongside the official spellings. +.PP +Nothing is joined across a slash: a suffix says where the station is, not +what it is called, so +.I W1AW/B +is W1AW. +.SH CW AND IDENTIFICATION +Every capture is offered to a CW decoder once it has finished, whatever the +classifier made of it. Most of the Morse on the air is not a conversation: +it is a repeater, a beacon or an unattended transmitter saying who it is and +stopping, which is four to six characters and over in a second or two. That +burst is a fraction of a capture named after whatever filled the rest of it, +so waiting for the label to say "CW" missed it. +.PP +Short is therefore the normal case rather than the awkward one. A decode of +two or three characters is believed on its timing alone \[em] every element +within a third of a unit of one or three, every character resolving to +something in the table, and the keyed tone standing at least 20 dB above the +rest of its band. That last one is what separates an ident from a blip: with +four elements the dot length is fitted to those very elements, so noise lands +on the grid as neatly as keying does, and only the tone tells them apart. One +keyed element is refused, because a single pulse is an E or a T whether a +person sent it or the squelch opened on a click. +.PP +The other half of a short decode is knowing what was cut off. A capture opens +when the squelch does, which is in the middle of an element as often as not, +and half a character is not a smaller reading of what was sent \[em] it is a +different one, and a K with its first dash missing is an A. So the character +at a sliced end is dropped, and so is the rest of the word it was in, because +what is left of that word can read as a whole one: +.I K1AA +caught halfway through is +.IR K1A , +which belongs to somebody else. The full text is still reported; it is the +identification that is held to the stricter standard. +.PP +What survives goes to the same callsign lookup and the same map as a spoken +one. Word gaps in Morse are not joined across, because the sender chose them: +.I "KU0W K" +is a station signing off, not a callsign one letter longer. .SH DECODING DATA A great deal of what a scanner finds is not speech. Doorbells, tyre\-pressure sensors, weather stations, remote controls, paging and packet radio all carry @@ -1032,8 +1080,11 @@ check: a burst of keying demodulated as FM audio is a buzz, and the speech detector likes a buzz, but a frame whose own checksum came out right is not a statistic. .SH THE MAP -A callsign heard in a transcript is looked up in the FCC's published licence -data, which gives the licensee, the town, and coordinates. Those go into a +A callsign is looked up in the FCC's published licence data, which gives the +licensee, the town, and coordinates. They arrive from three directions and +all three end up in the same place: spoken and transcribed, sent in Morse, or +carried in the header of an APRS packet. None of the last two involves a +speech recogniser, so a machine with none installed still builds a map. Those go into a KML file in the output directory \[em] .I callsigns.kml unless diff --git a/packaging/make-browse-man.py b/packaging/make-browse-man.py index bde4662..feb005b 100755 --- a/packaging/make-browse-man.py +++ b/packaging/make-browse-man.py @@ -193,9 +193,17 @@ is every callsign heard in it, with the name and location on the licence. Finding them is not a matter of one regular expression over the text as written. A speech recogniser is poor at callsigns \[em] they are not words, they are said one character at a time \[em] so it breaks them wherever the -speaker paused and writes the phonetic alphabet down verbatim. "KU 0W", -"K7 RA" and "kilo uniform zero whiskey" are all one callsign each, and all -three are read back correctly. +speaker paused and writes the phonetic alphabet down verbatim. It also joins +the words back up, hyphenates them, spells them the way they sounded, and +writes down the hesitation in the middle. "KU 0W", "K7 RA", +"kilo uniform zero whiskey", "Whiskey\-One\-Alpha\-Whiskey", +"WhiskeyOneAlphaWhiskey", "wiskey one alfa whisky" and +"whiskey one alpha, uh, whiskey" are all one callsign each, and all of them +are read back correctly. +.PP +A suffix is not part of the callsign. Nothing is joined across a slash, so +.I W1AW/B +is W1AW and not W1AWB, which belongs to nobody. .PP False positives are the thing to avoid: a browser that invents callsigns is worse than one that finds none. So a run of words is only accepted when none @@ -295,6 +303,23 @@ lock-out list in your settings file. The width comes from the setting, so a lock-out is a channel rather than a single point. Locking out a frequency does not delete what has already been recorded on it \[em] the two keys are separate on purpose, and pressing both is the usual thing to do. +.SH CALLSIGNS IN MORSE +Most stations on the air never say a word. A repeater, a beacon or an +unattended transmitter sends its callsign in Morse and stops, and what it +sent takes the place of the transcript at the top of the screen, with the +licence it belongs to underneath exactly as for speech. It is searchable with +.B / +like anything else. +.PP +Only the part of it that was not cut in half is looked at for a callsign. A +capture opens when the squelch does, which is partway through a word as often +as not, and what is left of that word can read as a whole one: +.I K1AA +caught halfway through is +.IR K1A , +which belongs to somebody else. +.BR bandsaunter (1) +describes how that is decided. The full text is shown either way. .SH DECODED DATA Where a capture carried data rather than speech, what was decoded takes the place of the transcript at the top of the screen: the kind of packet, and then diff --git a/packaging/make-man.py b/packaging/make-man.py index 986a1ea..81c67c9 100755 --- a/packaging/make-man.py +++ b/packaging/make-man.py @@ -381,6 +381,54 @@ than a directory of placeholders. .BR saunterbrowse (1) reads these back, and lists any callsigns it finds in them with the licence they belong to. +.PP +A callsign in a transcript is not written the way it is printed. A recogniser +has never heard of the phonetic alphabet: it writes what the words sounded +like, breaks the callsign wherever the speaker paused, joins the words back +up, hyphenates them, or drops a hesitation into the middle of the run. So +"KU 0W", "kilo uniform zero whiskey", "Whiskey\-One\-Alpha\-Whiskey", +"WhiskeyOneAlphaWhiskey" and "whiskey one alpha, uh, whiskey" are all read +back as the callsigns they are, and "alfa", "juliett" and "whisky" count +alongside the official spellings. +.PP +Nothing is joined across a slash: a suffix says where the station is, not +what it is called, so +.I W1AW/B +is W1AW. +.SH CW AND IDENTIFICATION +Every capture is offered to a CW decoder once it has finished, whatever the +classifier made of it. Most of the Morse on the air is not a conversation: +it is a repeater, a beacon or an unattended transmitter saying who it is and +stopping, which is four to six characters and over in a second or two. That +burst is a fraction of a capture named after whatever filled the rest of it, +so waiting for the label to say "CW" missed it. +.PP +Short is therefore the normal case rather than the awkward one. A decode of +two or three characters is believed on its timing alone \[em] every element +within a third of a unit of one or three, every character resolving to +something in the table, and the keyed tone standing at least 20 dB above the +rest of its band. That last one is what separates an ident from a blip: with +four elements the dot length is fitted to those very elements, so noise lands +on the grid as neatly as keying does, and only the tone tells them apart. One +keyed element is refused, because a single pulse is an E or a T whether a +person sent it or the squelch opened on a click. +.PP +The other half of a short decode is knowing what was cut off. A capture opens +when the squelch does, which is in the middle of an element as often as not, +and half a character is not a smaller reading of what was sent \[em] it is a +different one, and a K with its first dash missing is an A. So the character +at a sliced end is dropped, and so is the rest of the word it was in, because +what is left of that word can read as a whole one: +.I K1AA +caught halfway through is +.IR K1A , +which belongs to somebody else. The full text is still reported; it is the +identification that is held to the stricter standard. +.PP +What survives goes to the same callsign lookup and the same map as a spoken +one. Word gaps in Morse are not joined across, because the sender chose them: +.I "KU0W K" +is a station signing off, not a callsign one letter longer. .SH DECODING DATA A great deal of what a scanner finds is not speech. Doorbells, tyre\-pressure sensors, weather stations, remote controls, paging and packet radio all carry @@ -454,8 +502,11 @@ check: a burst of keying demodulated as FM audio is a buzz, and the speech detector likes a buzz, but a frame whose own checksum came out right is not a statistic. .SH THE MAP -A callsign heard in a transcript is looked up in the FCC's published licence -data, which gives the licensee, the town, and coordinates. Those go into a +A callsign is looked up in the FCC's published licence data, which gives the +licensee, the town, and coordinates. They arrive from three directions and +all three end up in the same place: spoken and transcribed, sent in Morse, or +carried in the header of an APRS packet. None of the last two involves a +speech recogniser, so a machine with none installed still builds a map. Those go into a KML file in the output directory \[em] .I callsigns.kml unless diff --git a/packaging/saunterbrowse.1 b/packaging/saunterbrowse.1 index ba8874b..20bfb9a 100644 --- a/packaging/saunterbrowse.1 +++ b/packaging/saunterbrowse.1 @@ -167,9 +167,17 @@ is every callsign heard in it, with the name and location on the licence. Finding them is not a matter of one regular expression over the text as written. A speech recogniser is poor at callsigns \[em] they are not words, they are said one character at a time \[em] so it breaks them wherever the -speaker paused and writes the phonetic alphabet down verbatim. "KU 0W", -"K7 RA" and "kilo uniform zero whiskey" are all one callsign each, and all -three are read back correctly. +speaker paused and writes the phonetic alphabet down verbatim. It also joins +the words back up, hyphenates them, spells them the way they sounded, and +writes down the hesitation in the middle. "KU 0W", "K7 RA", +"kilo uniform zero whiskey", "Whiskey\-One\-Alpha\-Whiskey", +"WhiskeyOneAlphaWhiskey", "wiskey one alfa whisky" and +"whiskey one alpha, uh, whiskey" are all one callsign each, and all of them +are read back correctly. +.PP +A suffix is not part of the callsign. Nothing is joined across a slash, so +.I W1AW/B +is W1AW and not W1AWB, which belongs to nobody. .PP False positives are the thing to avoid: a browser that invents callsigns is worse than one that finds none. So a run of words is only accepted when none @@ -283,6 +291,23 @@ lock-out list in your settings file. The width comes from the setting, so a lock-out is a channel rather than a single point. Locking out a frequency does not delete what has already been recorded on it \[em] the two keys are separate on purpose, and pressing both is the usual thing to do. +.SH CALLSIGNS IN MORSE +Most stations on the air never say a word. A repeater, a beacon or an +unattended transmitter sends its callsign in Morse and stops, and what it +sent takes the place of the transcript at the top of the screen, with the +licence it belongs to underneath exactly as for speech. It is searchable with +.B / +like anything else. +.PP +Only the part of it that was not cut in half is looked at for a callsign. A +capture opens when the squelch does, which is partway through a word as often +as not, and what is left of that word can read as a whole one: +.I K1AA +caught halfway through is +.IR K1A , +which belongs to somebody else. +.BR bandsaunter (1) +describes how that is decided. The full text is shown either way. .SH DECODED DATA Where a capture carried data rather than speech, what was decoded takes the place of the transcript at the top of the screen: the kind of packet, and then diff --git a/tests/conftest.py b/tests/conftest.py index b3bde23..b923f73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Fixtures every test gets. -Both of them are about not touching the machine the tests run on. +All three are about not touching the machine the tests run on, or anyone +else's. The cache: a lookup writes to ``~/.cache`` by default, and a test run that touches the real one leaves entries behind and reads back entries an earlier @@ -10,10 +11,14 @@ The settings: locking a frequency out writes it into ``config.yaml``, and a test that reached the real one would silently change what the next real scan does. The constant is replaced in every module that holds a copy, so that forgetting to pass a directory somewhere cannot end in someone's own settings. + +The network: a licence lookup goes to a public database and returns a real +person's name and address. No test has any business doing that. """ import pytest import bandsaunter.browse +import bandsaunter.callsign import bandsaunter.cli import bandsaunter.config import bandsaunter.tui @@ -25,6 +30,21 @@ def isolated_cache(tmp_path_factory, monkeypatch): str(tmp_path_factory.mktemp("cache"))) +@pytest.fixture(autouse=True) +def no_licence_lookups(monkeypatch): + """No test may contact the licence database. + + One did, silently, and passed -- it was only visible because the + assertion it failed printed a real operator's address. A test that wants + answers stubs this itself; anything else fails loudly rather than going + to the network and being slow, flaky and rude about it. + """ + def refuse(self, call): + raise AssertionError(f"a test tried to look up {call} for real") + + monkeypatch.setattr(bandsaunter.callsign.CallsignBook, "_request", refuse) + + @pytest.fixture(autouse=True) def isolated_settings(tmp_path_factory, monkeypatch): where = tmp_path_factory.mktemp("config") diff --git a/tests/test_callsign.py b/tests/test_callsign.py index 73d9b48..398b63f 100644 --- a/tests/test_callsign.py +++ b/tests/test_callsign.py @@ -14,8 +14,9 @@ import time import pytest -from bandsaunter.callsign import (Callsign, CallsignBook, HEADING, - NOT_CALLSIGNS, SHAPE, describe_prefix, +from bandsaunter.callsign import (Callsign, CallsignBook, FILLERS, HEADING, + NOT_CALLSIGNS, PHONETIC, SHAPE, + _phonetic_parts, describe_prefix, find_callsigns, person_case, report, split_postcode) @@ -124,6 +125,103 @@ def test_every_excluded_word_could_actually_have_matched(): assert SHAPE.match(word), f"{word} never matched in the first place" +# -- the phonetic alphabet, as a recogniser writes it down -------------------- +# +# It is never written the way the ITU prints it. A recogniser hears the +# words and writes what they sounded like, joins them up, hyphenates them, or +# drops a hesitation in the middle of the run. + +@pytest.mark.parametrize("text,want", [ + # Hyphenated, which is how a recogniser writes anything spelled out. + ("Whiskey-One-Alpha-Whiskey", ["W1AW"]), + ("this is whiskey-one-alpha-whiskey clear", ["W1AW"]), + # Run together, when it was said quickly. + ("WhiskeyOneAlphaWhiskey", ["W1AW"]), + ("The station is Whiskey1AlphaWhiskey", ["W1AW"]), + ("kilouniformzerowhiskey", ["KU0W"]), + # Said with a hesitation in the middle of it. + ("whiskey one alpha, uh, whiskey", ["W1AW"]), + ("kilo uniform um zero whiskey", ["KU0W"]), + # The spellings that are not the official ones. + ("wiskey one alfa whisky", ["W1AW"]), + ("juliette alpha one november golf", ["JA1NG"]), + ("oskar viktor two charley romeo", ["OV2CR"]), +]) +def test_a_phonetic_spelling_is_read_however_it_was_written(text, want): + assert find_callsigns(text) == want + + +@pytest.mark.parametrize("word", [ + "kilometre", "kilometer", "victorious", "november", "onetime", "papaya", + "hotelier", "echoes", "deltas", "golfing", "oneself", "foxtrotting", + "sierras", "alphabet", "zeroed", "information", "uniformity", "tangos", + "twofold", "fivefold", "sixty", "seventeen", "nineteen", "charlies", + "oscars", "limas", "romeos", "echoing", "novembers", "onto", "golfer", +]) +def test_an_ordinary_word_is_not_taken_apart(word): + """Splitting only fires on a word that is phonetic all the way through. + + A partial match is no match, which is the whole of what keeps this away + from English: "kilometre" begins with a phonetic word and "victorious" + contains one, and neither can be consumed to the end. + """ + assert _phonetic_parts(word) is None + + +def test_x_ray_is_one_letter_not_two_words(): + """The one phonetic word with a hyphen in it of its own.""" + assert find_callsigns("x-ray echo two delta") == ["XE2D"] + assert find_callsigns("xray echo two delta") == ["XE2D"] + + +def test_every_filler_is_something_a_recogniser_writes(): + """Each of these has to be a word, not a letter it would swallow.""" + for word in FILLERS: + assert word.lower() not in PHONETIC, word + assert not SHAPE.match(word), word + + +# -- suffixes, which are not part of the callsign ---------------------------- + +@pytest.mark.parametrize("text,want", [ + ("W1AW/4", ["W1AW"]), + ("DL/W1AW", ["W1AW"]), + ("This is W1AW-4", ["W1AW"]), + ("VVV DE W1AW/B FN31", ["W1AW"]), + ("VE3XYZ/M mobile", ["VE3XYZ"]), +]) +def test_a_suffix_is_not_glued_onto_the_callsign(text, want): + """A slash means "somewhere else", not another letter. + + The beacon W1AW/B used to come back as W1AWB, which belongs to nobody + and would have been looked up and pinned to a map. + """ + assert find_callsigns(text) == want + + +# -- where the spacing is real ---------------------------------------------- + +def test_words_are_joined_where_a_recogniser_put_the_gaps_in(): + """A transcript's spacing is a guess, so it is allowed to be wrong.""" + assert find_callsigns("KU 0W") == ["KU0W"] + assert find_callsigns("K7 RA") == ["K7RA"] + + +@pytest.mark.parametrize("text,want", [ + ("KU0W K", ["KU0W"]), # a station signing off, not KU0WK + ("CQ CQ DE KU0W K", ["KU0W"]), + ("DE W1AW K", ["W1AW"]), + ("VVV DE W1AW/B FN31", ["W1AW"]), +]) +def test_nothing_is_joined_where_the_sender_put_the_gaps_in(text, want): + """A word gap in Morse is seven dot units the sender chose to send. + + Joining across one turns a station signing off with K -- "over" -- into a + callsign one letter longer that belongs to somebody else entirely. + """ + assert find_callsigns(text, join_words=False) == want + + # -- what the callsign says about itself ------------------------------------- @pytest.mark.parametrize("call,country", [ diff --git a/tests/test_ident.py b/tests/test_ident.py new file mode 100644 index 0000000..9af136f --- /dev/null +++ b/tests/test_ident.py @@ -0,0 +1,340 @@ +"""A station that identifies itself, and what happens to it after that. + +Most stations on the air never say a word. A repeater, a beacon or an +unattended transmitter sends its callsign in Morse and stops, and until this +existed the text was decoded, written into the sidecar, and read by nobody: +the callsign book and the map were built where the transcripts were, and a +CW ident produces no transcript. + +So these are about the whole path -- decode, find the callsign, look it up, +say so, put it on the map -- and about the two places it is allowed to say +nothing rather than say the wrong thing. +""" +import json +import wave +from pathlib import Path + +import numpy as np +import pytest +from rich.console import Console + +from bandsaunter.browse import Browser, Player +from bandsaunter.callsign import CallsignBook +from bandsaunter.config import ScanConfig +from bandsaunter.morse import decode_morse +from bandsaunter.quality import assess +from bandsaunter.ranges import parse_range_list +from bandsaunter.scanner import Scanner +from bandsaunter.simulator import SimulatedDevice, default_transmitters + +from morse_gen import morse_audio + +FS = 16000 + + +class StubBook(CallsignBook): + """Answers every lookup from itself, so no test goes to the network.""" + + def __init__(self, tmp, **kw): + self.asked: list[str] = [] + super().__init__(cache=Path(tmp) / "calls.json", **kw) + + def _request(self, call): + self.asked.append(call) + return {"status": "VALID", "current": {"callsign": call}, + "name": "Newington Radio Club", + "address": {"line2": "Newington, CT"}, + "location": {"latitude": "41.71", "longitude": "-72.72", + "gridsquare": "FN31pr"}} + + +# --------------------------------------------------------------------------- +# The scan +# --------------------------------------------------------------------------- + +def _scanner(tmp_path, mhz: str, **over) -> Scanner: + cfg = ScanConfig(ranges=parse_range_list(mhz), + output_dir=str(tmp_path), transcribe=False, + record_seconds=10, hang_seconds=1.5, + max_runtime_seconds=over.pop("seconds", 45), + **over) + scanner = Scanner(cfg, device=SimulatedDevice(realtime=False).open()) + scanner.prepare() + scanner.callsigns = StubBook(tmp_path) + return scanner + + +def test_the_book_and_the_map_exist_without_a_speech_recogniser(tmp_path): + """They used to be built inside the transcription branch. + + Callsigns arrive in Morse and in packets as well as in speech, neither of + which involves a recogniser, so a machine with none installed found none + of them -- which is most of the machines this runs on. + """ + scanner = _scanner(tmp_path, "147.0M-147.1M") + assert scanner.transcriber is None + assert scanner.callsigns is not None + assert scanner.kml is not None + + +def test_a_repeater_identifying_in_morse_reaches_the_map(tmp_path): + scanner = _scanner(tmp_path, "147.0M-147.1M", seconds=45) + scanner.run() + assert scanner.hits, "the ident was not recorded at all" + assert any(h.morse_text for h in scanner.hits), \ + "recorded it and read nothing out of it" + assert "K1AA" in scanner.heard + assert "K1AA" in scanner.kml.contacts + contact = scanner.kml.contacts["K1AA"] + assert contact.name == "Newington Radio Club" + assert contact.located + + +def test_the_map_is_written_out(tmp_path): + scanner = _scanner(tmp_path, "147.0M-147.1M", seconds=45) + scanner.run() + written = (tmp_path / scanner.cfg.kml_file).read_text() + assert "K1AA" in written and "" in written + + +def test_a_beacon_is_identified_from_the_words_that_survived(tmp_path): + """A continuous beacon is always caught partway through. + + Every capture of one begins and ends in the middle of the message, so + what can be said about it is whatever lies between two word gaps. + """ + scanner = _scanner(tmp_path, "144.05M-144.15M", seconds=30) + scanner.run() + assert "W1AW" in scanner.heard + assert set(scanner.heard) == {"W1AW"}, \ + f"invented a station: {sorted(scanner.heard)}" + + +def test_the_hit_keeps_both_the_text_and_the_part_it_can_be_identified_from( + tmp_path): + scanner = _scanner(tmp_path, "144.05M-144.15M", seconds=30) + scanner.run() + cw = [h for h in scanner.hits if h.morse_text] + assert cw + for hit in cw: + assert hit.morse_complete in hit.morse_text or not hit.morse_complete + + +def test_the_sidecar_carries_them(tmp_path): + scanner = _scanner(tmp_path, "147.0M-147.1M", seconds=45) + scanner.run() + sidecars = [json.loads(p.read_text()) for p in tmp_path.glob("*.json")] + hits = [doc.get("hit", doc) for doc in sidecars] + assert any(h.get("morse_text") for h in hits) + assert any(h.get("morse_complete") for h in hits) + + +# --------------------------------------------------------------------------- +# Every capture is offered to the decoder, not only the ones that looked keyed +# --------------------------------------------------------------------------- + +def test_morse_under_a_label_that_is_not_cw_is_still_decoded(tmp_path): + """A burst of CW is a second or two of a capture the classifier named + after whatever filled the rest of it, or after nothing at all.""" + seen = {} + scanner = _scanner(tmp_path, "147.0M-147.1M", seconds=45) + real = scanner._decode_cw + + def watch(iq, demod): + got = real(iq, demod) + seen.setdefault("calls", 0) + seen["calls"] += 1 + return got + + scanner._decode_cw = watch + scanner.run() + assert seen.get("calls"), "nothing was offered to the CW decoder" + + +def _on_the_air(mode: str, seconds: float = 3.0, freq: float = 147.06e6): + """A real signal, classified and demodulated the way the scanner does. + + Not a stand-in feature object: what `assess` does with a Morse decode + depends on measurements taken off the signal, and a hand-written set of + them would only ever prove that the test agreed with itself. + """ + from bandsaunter.classify import classify + from bandsaunter.demod import make_demodulator + from bandsaunter.simulator import VirtualTransmitter + + fs = 240_000 + voice = mode == "nfm" + tx = VirtualTransmitter(freq, mode, 0.5, 12_500 if voice else 500, + f"test {mode}", message="DE W1AW", wpm=20) + iq = tx.generate(0.0, int(fs * seconds), fs) + # A receiver always has some. Without it the key-up stretches of a CW + # signal are exactly zero, which is not a thing any aerial produces. + rng = np.random.default_rng(3) + iq = iq + (0.004 * (rng.standard_normal(iq.size) + + 1j * rng.standard_normal(iq.size))).astype("complex64") + cls = classify(iq, fs, freq_hz=freq) + demod = make_demodulator("nfm" if voice else "cw", fs, + 12_500.0 if voice else 800.0, FS) + return cls, demod.process(iq), demod.audio_rate + + +def test_speech_is_not_relabelled_by_a_morse_decode(): + """Both can be true at once, and the conversation is what was recorded. + + Every capture is offered to the CW decoder now, not only the ones that + looked keyed, so a decode can land on a capture full of speech. The text + is recorded either way; the label follows the speech. + """ + cls, audio, rate = _on_the_air("nfm") + spoken = assess(cls, audio, rate, morse=None) + if spoken.category != "voice": + pytest.skip("the speech detector did not hear the synthetic talker") + + _, keyed, keyed_rate = _on_the_air("cw") + morse = decode_morse(keyed, keyed_rate) + assert morse.is_morse, "the fixture did not produce a Morse decode" + assert assess(cls, audio, rate, morse=morse).category == "voice" + + +def test_a_keyed_carrier_is_still_labelled_cw(): + """The families that were always decoded keep their behaviour exactly.""" + cls, audio, rate = _on_the_air("cw") + morse = decode_morse(audio, rate) + assert morse.is_morse + assert assess(cls, audio, rate, morse=morse).category == "cw" + + +# --------------------------------------------------------------------------- +# Reading it back +# --------------------------------------------------------------------------- + +def _capture(directory: Path, meta: dict) -> Path: + stem = "0147.060000MHz--2026-08-29_10_00_00-cw" + with wave.open(str(directory / f"{stem}.wav"), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(FS) + w.writeframes(b"\0\0" * FS) + (directory / f"{stem}.json").write_text(json.dumps({"hit": meta})) + return directory / f"{stem}.wav" + + +def _browser(directory) -> Browser: + console = Console(width=100, height=30, force_terminal=True) + return Browser(directory, console=console, player=Player([]), + book=StubBook(directory)) + + +def frame(browser) -> str: + with browser.console.capture() as cap: + browser.console.print(browser.render()) + return cap.get() + + +@pytest.fixture +def keyed(tmp_path): + _capture(tmp_path, {"frequency": 147.06e6, "category": "cw", + "classification": "CW / Morse at 20 WPM", + "morse_text": "E DE K1AA", + "morse_complete": "DE K1AA", + "morse_wpm": 20.0, "confidence": 0.9}) + return tmp_path + + +def test_the_morse_gets_the_top_of_the_screen(keyed): + shown = frame(_browser(keyed)) + assert "Morse" in shown and "DE K1AA" in shown + + +def test_the_callsign_is_listed_under_it(keyed): + browser = _browser(keyed) + browser.book.get_all(browser.current.callsigns) + browser.book.wait(5.0) + shown = frame(browser) + assert "K1AA" in shown and "Newington Radio Club" in shown + + +def test_a_cw_capture_can_be_searched_for_by_what_it_keyed(keyed): + browser = _browser(keyed) + browser.query = "k1aa" + browser.apply() + assert len(browser.view) == 1 + + +def test_the_reader_opens_on_it(keyed): + browser = _browser(keyed) + assert browser.handle("t") + assert browser.reading + assert "DE K1AA" in frame(browser) + + +def test_only_the_part_it_can_be_identified_from_is_searched(tmp_path): + """The sidecar keeps both, and the callsign comes out of the safe one.""" + _capture(tmp_path, {"frequency": 147.06e6, "category": "cw", + "morse_text": "K1A", "morse_complete": "", + "classification": "CW / Morse at 20 WPM"}) + browser = _browser(tmp_path) + assert browser.current.morse == "K1A" + assert browser.current.callsigns == [] + + +def test_an_older_recording_falls_back_to_the_whole_text(tmp_path): + """Sidecars written before this distinction existed carry only the text.""" + _capture(tmp_path, {"frequency": 147.06e6, "category": "cw", + "morse_text": "VVV DE W1AW", + "classification": "CW / Morse at 20 WPM"}) + assert _browser(tmp_path).current.callsigns == ["W1AW"] + + +def test_a_hex_dump_is_not_searched_for_callsigns(tmp_path): + """Enough two-character groups in a row join into something shaped like + a callsign that nobody transmitted.""" + _capture(tmp_path, {"frequency": 147.06e6, "category": "digital", + "data_messages": ["4A 3F 1B 22 9C 04"], + "classification": "OOK data"}) + assert _browser(tmp_path).current.callsigns == [] + + +def test_a_packet_header_is(tmp_path): + _capture(tmp_path, {"frequency": 144.39e6, "category": "digital", + "data_messages": ["W1AW-1>APRS,TCPIP*:=4123.45N/" + "07234.56W-"], + "classification": "AX.25 / APRS"}) + assert _browser(tmp_path).current.callsigns == ["W1AW"] + + +# --------------------------------------------------------------------------- +# The demo band is invented; the callsigns in it are not +# --------------------------------------------------------------------------- + +def test_the_simulator_identifies_itself_the_way_a_repeater_does(): + idents = [t for t in default_transmitters() + if t.mode == "cw" and t.period_seconds] + assert idents, "nothing in the demo band sends a short CW ident" + assert idents[0].on_seconds < 8.0 + + +def test_a_simulated_run_neither_looks_up_nor_maps_anything(monkeypatch, + tmp_path): + """W1AW is the ARRL's own station, and the demo band is made up. + + Looking it up would put a licence nobody heard on the same map a real + scan writes. + """ + from bandsaunter import cli + seen = {} + + class Stop(Exception): + pass + + def fake_scanner(cfg, **kw): + seen["cfg"] = cfg + raise Stop + + monkeypatch.setattr(cli, "Scanner", fake_scanner) + monkeypatch.setattr(cli, "console", Console(file=open("/dev/null", "w"))) + with pytest.raises(Stop): + cli.main(["scan", "--simulate", "--no-config", "-r", "144M-148M", + "-o", str(tmp_path)]) + assert seen["cfg"].callsign_lookup is False + assert seen["cfg"].kml_file == "" diff --git a/tests/test_morse.py b/tests/test_morse.py index fb5d5d0..0dc5093 100644 --- a/tests/test_morse.py +++ b/tests/test_morse.py @@ -48,3 +48,152 @@ def test_silence_is_rejected(): def test_encode_round_trip(): assert encode_morse("SOS") == "... --- ..." assert encode_morse("A B") == ".- / -..." + + +# --------------------------------------------------------------------------- +# Short bursts +# +# Most of the CW on the air is not a conversation. It is a repeater, a +# beacon or an unattended transmitter saying who it is and stopping, which is +# four to six characters and over in a second or two. Those used to be read +# correctly and then thrown away by gates written for a paragraph of text. +# --------------------------------------------------------------------------- + +SHORT = ["W1AW", "K1AA", "KU0W", "N0CALL", "VVV", "DE", "AR", "K", "73", "QRZ"] + + +@pytest.mark.parametrize("msg", SHORT) +@pytest.mark.parametrize("wpm", [12, 20, 30, 45]) +def test_a_short_burst_is_read_and_believed(msg, wpm): + r = decode_morse(morse_audio(msg, wpm, FS, 18), FS) + assert r.text.strip() == msg + assert r.is_morse, f"read {msg} correctly and then refused it: {r.notes}" + + +def test_a_callsign_on_its_own_is_under_two_seconds_at_thirty_words(): + """The length this is all about, stated so a regression is obvious.""" + from morse_gen import morse_keying + keyed = np.flatnonzero(morse_keying("W1AW", 30, FS) > 0) + assert (keyed[-1] - keyed[0]) / FS < 2.0 + assert decode_morse(morse_audio("W1AW", 30, FS, 18), FS).is_morse + + +@pytest.mark.parametrize("msg", ["E", "T"]) +def test_one_keyed_element_is_not_an_identification(msg): + """A single pulse is an E or a T whether a person sent it or not. + + This is where "no matter how short" stops, and it stops here because + below it there is nothing left to be right about. + """ + assert not decode_morse(morse_audio(msg, 20, FS, 20), FS).is_morse + + +def test_a_repeated_character_with_structure_is_still_text(): + """VVV is the oldest thing anyone sends, and it is not a pulse train. + + A uniform train of identical pulses decodes to EEEE or TTTT, which is + what the check against one repeated character is for -- but V has dots + and a dash of its own, which no train of identical pulses can produce. + """ + r = decode_morse(morse_audio("VVV", 20, FS, 20), FS) + assert r.text.strip() == "VVV" and r.is_morse + + +def test_a_uniform_pulse_train_is_still_refused(): + dot = 1.2 / 20.0 + env, fs = [], FS + for _ in range(14): + env += [1.0] * int(dot * fs) + [0.0] * int(3 * dot * fs) + env = np.array(env) + t = np.arange(env.size) / fs + audio = env * np.sin(2 * np.pi * 700 * t) + assert not decode_morse(audio, fs).is_morse + + +# -- what the capture window cut off ---------------------------------------- + +def _clipped(msg, wpm, head, tail): + audio = morse_audio(msg, wpm, FS, 20) + return audio[int(head * FS):audio.size - int(tail * FS)] + + +def test_a_character_sliced_by_the_window_is_dropped_not_guessed(): + """A K with its first dash missing is an A, not a worse K.""" + r = decode_morse(_clipped("K1AA", 20, 0.55, 0.55), FS) + assert "A1AA" not in r.text, "half a character was reported as a whole one" + assert any("cut off" in note for note in r.notes) + + +def test_what_is_left_of_a_cut_word_is_not_reported_as_a_whole_one(): + """K1AA caught halfway through reads as K1A, which is somebody else.""" + r = decode_morse(_clipped("K1AA", 20, 0.05, 0.9), FS) + assert r.tail_cut + assert "K1A" not in r.complete_text.split() + + +def test_the_words_that_survived_are_still_reported(): + r = decode_morse(_clipped("VVV DE W1AW", 20, 0.6, 0.1), FS) + assert r.head_cut and not r.tail_cut + assert "W1AW" in r.complete_text.split() + + +def test_a_complete_transmission_keeps_every_word(): + r = decode_morse(morse_audio("VVV DE W1AW", 20, FS, 20), FS) + assert not r.head_cut and not r.tail_cut + assert r.complete_text == r.text == "VVV DE W1AW" + + +@pytest.mark.parametrize("head,tail", [(0.05, 0.4), (0.4, 0.05), (0.7, 0.7), + (1.1, 0.3), (0.3, 1.1)]) +def test_a_truncated_capture_never_invents_a_callsign(head, tail): + """The point of all of the above: this text is looked at for callsigns. + + A station heard through half its ident is better reported as half an + ident than as a different station, because the different station gets + looked up and pinned to a map. + """ + from bandsaunter.callsign import find_callsigns + r = decode_morse(_clipped("VVV DE W1AW/B FN31", 20, head, tail), FS) + if not r.is_morse: + return + found = find_callsigns(r.complete_text, join_words=False) + assert set(found) <= {"W1AW"}, f"invented {found} from {r.text!r}" + + +# -- and nothing that was not sent ------------------------------------------ + +@pytest.mark.parametrize("seed", range(10)) +def test_a_blip_of_noise_is_not_a_short_transmission(seed): + """The gates that let a two-character ident in must not let this in. + + With three or four elements the dot length is fitted to those very + elements, so they land on the grid whatever produced them: a third of a + second of noise decodes as a perfectly timed V. What separates them is + that keying is a tone and noise is not. + """ + rng = np.random.default_rng(seed) + for size in (1200, int(FS * 0.3), FS * 2): + assert not decode_morse(rng.standard_normal(size), FS).is_morse + + +@pytest.mark.parametrize("seed", range(6)) +def test_a_squelch_click_is_not_a_transmission(seed): + rng = np.random.default_rng(seed) + t = np.arange(int(FS * 0.5)) / FS + for edges in ([(0.20, 0.25)], [(0.10, 0.14), (0.30, 0.34)]): + env = np.zeros_like(t) + for a, b in edges: + env[int(a * FS):int(b * FS)] = 1.0 + audio = env * np.sin(2 * np.pi * 700 * t) + 0.02 * rng.standard_normal(t.size) + assert not decode_morse(audio, FS).is_morse + + +def test_a_tone_that_never_keys_is_not_morse(): + t = np.arange(FS * 2) / FS + assert not decode_morse(np.sin(2 * np.pi * 700 * t), FS).is_morse + + +@pytest.mark.parametrize("seed", range(4)) +def test_speech_is_not_morse(seed): + from speech import synth_speech + assert not decode_morse(synth_speech(2.5, FS, seed=seed), FS).is_morse