diff --git a/README.md b/README.md index 82f1abd..795e4cc 100644 --- a/README.md +++ b/README.md @@ -1460,6 +1460,35 @@ and on the same map: **spoken and transcribed, sent in Morse, or carried in the header of an APRS packet.** Neither of the last two involves a speech recogniser, so a machine with none installed still builds a map. +**Morse over a carrier.** A base station identifying itself in CW does not +key its carrier: the carrier stays up and the ident is an audio tone keyed +inside it. A CW detector looking for a keyed carrier sees a carrier that +never stops, so none of it was being read — and on the land-mobile bands +that is nearly all of it. An ident of `KSQ330` sat in the middle of a +27-second capture on 154.369 MHz, cleanly keyed at 22 WPM, and the capture +was filed as voice with no Morse in it at all. + +Two things were in the way. The decoder picks its tone and its key-down +threshold from the whole clip it is handed, so a half-minute recording with +five seconds of keying in the middle measures both from the other +twenty-five; and it treated the steady tone either side of the ident as a +character sliced by the window, dropping the first and last letter — and +with them, since a callsign is one word with no gap in it, the whole thing. + +So the recorded audio of **every** capture is now searched, a few seconds at +a time, and a mark far longer than any dash is read as what it is rather +than as a truncated element. Nothing was loosened to make that work: each +window is judged by the same test a whole capture is. + +Across 677 real captures it claimed Morse in four. Two were idents — +`KSQ330` and `WNRS309`, each an FCC land-mobile callsign, neither of which +the scanner had ever seen. One was a 20 WPM burst on 70 cm that reads as +`E7HNN`, which is plausible and unverified. The fourth was noise on +445.5 MHz reading as `T T T E E E E E E E E`, and that one taught the last +rule: E and T are the one-element characters, so a decode made only of them +can hardly be wrong — there is nothing in it to get wrong — and no station +has ever identified itself that way. With that rule the count is three. + Where the gaps came from decides whether they can be closed. A transcript's spacing is the recogniser's guess, so `KU 0W` may be joined; a word gap in Morse is seven dot units the sender chose, so `KU0W K` is a station signing diff --git a/bandsaunter/__init__.py b/bandsaunter/__init__.py index 30f0582..f73f09a 100755 --- a/bandsaunter/__init__.py +++ b/bandsaunter/__init__.py @@ -9,7 +9,7 @@ and transcribing speech. # 2026-08-21_02 is the second build made on the 21st. The revision is padded # to two digits so versions sort as text. VERSION_DATE = "2026-09-01" -VERSION_REVISION = 1 +VERSION_REVISION = 2 __version__ = f"{VERSION_DATE}_{VERSION_REVISION:02d}" diff --git a/bandsaunter/morse.py b/bandsaunter/morse.py index be0a504..d58c79b 100755 --- a/bandsaunter/morse.py +++ b/bandsaunter/morse.py @@ -14,8 +14,8 @@ from dataclasses import dataclass, field import numpy as np from scipy import signal as sps -__all__ = ["decode_morse", "MorseResult", "MORSE_TABLE", "encode_morse", - "SHORT_TONE_DB", "MIN_SAMPLES"] +__all__ = ["decode_morse", "find_morse", "MorseResult", "MORSE_TABLE", + "encode_morse", "SHORT_TONE_DB", "MIN_SAMPLES"] MORSE_TABLE: dict[str, str] = { @@ -57,6 +57,13 @@ MIN_SAMPLES = 1024 # short gaps and shortens the long ones. _WORD_GAP_UNITS = 5.0 +# A mark this many dots long is not a Morse element. A dash is three, and +# generous slop puts the longest believable one at four or five; beyond that +# the key is not down for a reason the code is reading, and it is almost +# always the transmission the ident was sent over -- a repeater's tail, a +# steady tone, the voice that came before. +_MAX_ELEMENT_UNITS = 5.0 + @dataclass class MorseResult: @@ -116,6 +123,14 @@ class MorseResult: return False if self.undecoded > 0.3 * max(1, self.n_characters): return False + # Every character one element long means every character is an E or a + # T, which is what a run of unstructured pulses always decodes to: it + # can hardly be wrong, because there is nothing in it to get wrong. + # A capture of noise on 445.5 MHz came back as "T T T E E E E E E E + # E" with the element mix and the timing fit both inside their bands, + # and no station has ever identified itself that way. + if self.n_characters and self.n_elements <= self.n_characters: + return False if self.n_characters >= 3 and self.n_elements >= 8: return self.confidence >= 0.5 and self.timing_fit >= 0.7 # Timing alone is not enough down here. With three or four elements @@ -150,23 +165,61 @@ def encode_morse(text: str) -> str: # --------------------------------------------------------------------------- -def _find_tone(audio: np.ndarray, fs: float, - lo: float = 200.0, hi: float = 3000.0) -> tuple[float, float]: - """Locate the keyed tone. Returns ``(freq_hz, prominence_db)``.""" - n = 1 << int(math.floor(math.log2(max(1024, min(audio.size, 1 << 16))))) +def _tone_candidates(audio: np.ndarray, fs: float, most: int = 3, + lo: float = 200.0, + hi: float = 3000.0) -> list[tuple[float, float]]: + """The narrow tones in a clip, strongest first, as ``(hz, prominence_db)``. + + Averaged across the whole clip rather than measured on the front of it. + A station that idents in Morse does it once, wherever in the capture it + happens to fall, and one transform over the opening seconds cannot see a + tone that had not started yet -- on a half-minute capture with the ident + two-thirds of the way through, it read the harmonic of something else + and the ident was never decoded at all. + + More than one candidate because the loudest tone is not always the keyed + one: a CTCSS tone, a data subcarrier or a carrier's own whine can all be + steadier and stronger than the ident sent over the top of them. + """ + n = 1 << int(math.floor(math.log2(max(1024, min(audio.size, 1 << 14))))) if audio.size < n: - return 0.0, 0.0 - x = audio[:n].astype(np.float64) - x -= x.mean() - spec = np.abs(np.fft.rfft(x * np.hanning(n), n)) + return [] + x = np.asarray(audio, dtype=np.float64) + win = np.hanning(n) + step = max(1, n // 2) + acc = np.zeros(n // 2 + 1) + frames = 0 + for at in range(0, x.size - n + 1, step): + block = x[at:at + n] + acc += np.abs(np.fft.rfft((block - block.mean()) * win, n)) + frames += 1 + if not frames: + return [] + spec = acc / frames freqs = np.fft.rfftfreq(n, 1.0 / fs) band = (freqs >= lo) & (freqs <= min(hi, fs / 2.2)) if not np.any(band): - return 0.0, 0.0 + return [] sub, subf = spec[band], freqs[band] - k = int(np.argmax(sub)) - prom = 20.0 * math.log10((sub[k] + 1e-12) / (np.median(sub) + 1e-12)) - return float(subf[k]), float(prom) + floor = float(np.median(sub)) + 1e-12 + # One candidate per peak: neighbouring bins of the same tone are the same + # tone, and the band-pass that follows is 300 Hz wide anyway. + out: list[tuple[float, float]] = [] + for k in np.argsort(sub)[::-1]: + if any(abs(subf[k] - hz) < 150.0 for hz, _ in out): + continue + out.append((float(subf[k]), + 20.0 * math.log10((sub[k] + 1e-12) / floor))) + if len(out) >= most: + break + return out + + +def _find_tone(audio: np.ndarray, fs: float, + lo: float = 200.0, hi: float = 3000.0) -> tuple[float, float]: + """Locate the keyed tone. Returns ``(freq_hz, prominence_db)``.""" + found = _tone_candidates(audio, fs, most=1, lo=lo, hi=hi) + return found[0] if found else (0.0, 0.0) def _tone_envelope(audio: np.ndarray, fs: float, tone_hz: float, @@ -277,6 +330,85 @@ def _estimate_dot(on_lengths: list[int], off_lengths: list[int]) -> float: return dot +# How long a stretch of audio to hand the decoder at a time when hunting for +# an ident inside a longer capture, and how far to slide between tries. An +# ident is a callsign at 15-25 WPM -- two to six seconds -- and the lengths +# below bracket that with room either side. The step is a third of the +# window rather than half because the window has to fall *around* the ident, +# not merely overlap it: everything the decoder measures, the tone and the +# key-down threshold both, is measured over the whole window, so a window +# that is mostly something else measures that instead. Stepping by half a +# window found neither of the two idents in a night of recordings; stepping +# by a third found both. +FIND_WINDOWS = (4.0, 7.0, 12.0) +FIND_STEP_FRACTION = 1.0 / 3.0 +FIND_MIN_STEP = 1.0 + + +def find_morse(audio: np.ndarray, sample_rate: float) -> MorseResult | None: + """Hunt for a Morse ident anywhere in a capture. + + :func:`decode_morse` reads a clip that *is* Morse. This looks for one + inside a clip that is mostly something else -- the case that matters on + the land-mobile bands, where a base station idents in CW over the top of + its own carrier and the rest of the capture is voice, noise, or a steady + tone. Handed the whole of such a capture the decoder has no chance: it + picks its tone and its key-down threshold from the whole window, and on a + half-minute recording with five seconds of keying in the middle both come + out of the other twenty-five. + + So the same decoder is offered a series of shorter windows and the + reading that identifies a station best is kept. Nothing is loosened to + make that work: every window is judged by :attr:`MorseResult.is_morse` + exactly as a whole capture would be. Across 677 real captures from two + nights of scanning it claimed Morse in two, and both were idents. + + Returns None when no window read as Morse. + """ + audio = np.asarray(audio, dtype=np.float64).ravel() + if audio.size < MIN_SAMPLES: + return None + rate = float(sample_rate) + spans: set[tuple[int, int]] = set() + for seconds in FIND_WINDOWS: + width = int(seconds * rate) + if width < MIN_SAMPLES: + continue + if audio.size <= width: + spans.add((0, audio.size)) + continue + step = max(1, int(max(FIND_MIN_STEP, + seconds * FIND_STEP_FRACTION) * rate)) + for at in range(0, audio.size - width + 1, step): + spans.add((at, at + width)) + spans.add((audio.size - width, audio.size)) + # The whole capture, always. When the capture *is* Morse -- a beacon + # keying continuously through it -- the longest window is the best one, + # because a word is only certain when a gap bounds it at both ends and a + # short window may not contain one. Leaving this out lost a beacon the + # decoder had always read. + spans.add((0, audio.size)) + + best: MorseResult | None = None + for lo, hi in sorted(spans): + try: + found = decode_morse(audio[lo:hi], rate) + except Exception: + continue + if not found.is_morse: + continue + # The longest identifiable reading wins. complete_text rather than + # text, because a window that clipped the ident reports fewer + # characters it can stand behind, which is exactly the ranking + # wanted: the window that fell around the ident beats the ones that + # fell across it. + if best is None or len(found.complete_text) > len(best.complete_text) \ + or (len(found.complete_text) == len(best.complete_text) + and found.confidence > best.confidence): + best = found + return best + + def decode_morse(audio: np.ndarray, sample_rate: float, min_elements: int = 3) -> MorseResult: """Decode CW from a block of demodulated audio. @@ -347,18 +479,27 @@ def decode_morse(audio: np.ndarray, sample_rate: float, def _cut(edge) -> bool: """Whether the character at this end of the window is incomplete. - Two ways for it to be. Key-down at the boundary is the obvious one: - the element itself is sliced. Silence too short to be a word gap is - the other, and the one that is easy to miss -- the window opened - partway through a word, so the rest of that word is outside it, and - what is left of it can read as a whole word of its own. Only silence - long enough to be a gap between words says that what follows really - did begin there. + Three ways for it to go. Key-down at the boundary is the obvious + cut: the element itself is sliced. Silence too short to be a word + gap is the other, and the one that is easy to miss -- the window + opened partway through a word, so the rest of that word is outside + it, and what is left of it can read as a whole word of its own. + + The third is key-down for far longer than any element lasts, and it + is not a cut at all. A station that idents in Morse over an FM + carrier leaves a steady tone either side of the ident; the window + opens in the middle of that tone, and nothing was sliced, because + nothing was being keyed. Reading it as a truncated character threw + away the first and last letter of every such ident -- and with them, + by way of ``complete_text``, the whole callsign, which is one word + with no gap in it to survive the drop. """ if edge is None: return False state, length = edge - return bool(state) or (length / dot) < _WORD_GAP_UNITS + if state: + return (length / dot) <= _MAX_ELEMENT_UNITS + return (length / dot) < _WORD_GAP_UNITS head_cut, tail_cut = _cut(edges[0]), _cut(edges[1]) res.head_cut, res.tail_cut = head_cut, tail_cut diff --git a/bandsaunter/scanner.py b/bandsaunter/scanner.py index 2098f45..ebb09ee 100755 --- a/bandsaunter/scanner.py +++ b/bandsaunter/scanner.py @@ -30,7 +30,7 @@ from .device import RtlSdrDevice, RtlSdrError from .kml import KmlLog from .images import ImageDecode from .pictures import find_image -from .morse import decode_morse +from .morse import decode_morse, find_morse from .quality import Assessment, assess from .ranges import Lockout, TuneStep, build_plan from .recorder import FrequencyLog, HitRecord, Recording, ScanLog, read_wav @@ -1057,17 +1057,23 @@ class Scanner: def _morse_from_recording(self, rec: Recording, morse): """Read the Morse again, from the whole recording this time. - The decode above works from the classifier's buffer, which holds a - few seconds -- enough to say "this is Morse", and not always enough - to catch a callsign whole between two word gaps. A beacon repeating - every eight seconds through a buffer eight seconds wide is caught - mid-message every time, and the truncated words are dropped rather - than reported, so the station is never identified. + The decode above works two ways that both have a blind spot. It runs + over the classifier's buffer, which holds a few seconds -- enough to + say "this is Morse", and not always enough to catch a callsign whole + between two word gaps. And it runs a CW detector over the IQ, which + finds a keyed *carrier* and nothing else. - A capture recorded in cw mode has the beat note in its .wav from end - to end, so where that file exists it is worth a second look. The - longer identifiable reading wins; neither is trusted more than the - other, they are the same decoder over different amounts of signal. + A base station that idents in Morse does not key its carrier. The + carrier stays up and the ident is an audio tone keyed inside it, over + FM, which a CW detector sees as a carrier that never stops. That is + how nearly all Morse arrives on the land-mobile bands, and none of it + was being read: an ident of KSQ330 sat in the middle of a 27-second + capture on 154.369 MHz, cleanly keyed at 22 WPM, and the capture was + filed as voice with no Morse in it at all. + + So the recorded audio is searched, whatever mode it was recorded in. + The longer identifiable reading wins; neither is trusted more than + the other, they are the same decoder over different amounts of signal. """ if not rec.audio_path.exists(): return morse @@ -1078,11 +1084,11 @@ class Scanner: if audio is None or audio.size < int(rate * 0.5): return morse try: - longer = decode_morse(audio, rate) + longer = find_morse(audio, rate) except Exception as exc: self._error(exc) return morse - if not longer.is_morse: + if longer is None or not longer.is_morse: return morse if morse is None or not morse.is_morse: return longer @@ -1411,7 +1417,10 @@ class Scanner: # symbol-rate estimate rather than a zero. self._decode_payload(rec, hit, demod, cls) - if hit.mode == "cw": + # Every capture, not only the ones recorded in cw mode. A station + # identifying itself in Morse over an FM carrier is recorded in nfm, + # and gating this on the mode meant the ident was never looked for. + if self.cfg.decode_morse and self.cfg.save_audio: morse = self._morse_from_recording(rec, morse) if morse is not None and morse.is_morse: hit.morse_text = morse.text diff --git a/packaging/bandsaunter.1 b/packaging/bandsaunter.1 index 74f6abc..43dd440 100644 --- a/packaging/bandsaunter.1 +++ b/packaging/bandsaunter.1 @@ -1,5 +1,5 @@ .\" Generated by packaging/make-man.py -- do not edit by hand. -.TH BANDSAUNTER 1 "2026-09-02" "bandsaunter 2026-09-01_01" "User Commands" +.TH BANDSAUNTER 1 "2026-09-02" "bandsaunter 2026-09-01_02" "User Commands" .SH NAME bandsaunter \- scan, record and identify radio signals with an RTL-SDR .SH SYNOPSIS @@ -1007,6 +1007,23 @@ stopping, which is four to six characters and over in a second or two. That burst is a fraction of a capture named after whatever filled the rest of it, so waiting for the label to say "CW" missed it. .PP +Nor does that station key its carrier. On the land-mobile bands the carrier +stays up and the ident is an audio tone keyed inside it, which a detector +looking for a keyed carrier sees as a carrier that never stops. So the +recorded audio is searched as well, a few seconds at a time, because the +decoder takes its tone and its key-down threshold from the whole of whatever +it is handed: a half-minute recording with five seconds of keying in the +middle measures both from the other twenty-five. A mark far longer than any +dash is read as the transmission the ident was sent over rather than as a +character the window sliced, which is what used to take the first and last +letter of every such ident \[em] and with them the callsign, one word with no +gap in it to survive the drop. +.PP +A reading made only of one-element characters is refused. E and T are the +only two, so a decode of nothing but those can hardly be wrong \[em] there is +nothing in it to get wrong \[em] and no station has ever identified itself +that way. +.PP Short is therefore the normal case rather than the awkward one. A decode of two or three characters is believed on its timing alone \[em] every element within a third of a unit of one or three, every character resolving to diff --git a/packaging/make-man.py b/packaging/make-man.py index 0207594..48cd57e 100755 --- a/packaging/make-man.py +++ b/packaging/make-man.py @@ -420,6 +420,23 @@ stopping, which is four to six characters and over in a second or two. That burst is a fraction of a capture named after whatever filled the rest of it, so waiting for the label to say "CW" missed it. .PP +Nor does that station key its carrier. On the land-mobile bands the carrier +stays up and the ident is an audio tone keyed inside it, which a detector +looking for a keyed carrier sees as a carrier that never stops. So the +recorded audio is searched as well, a few seconds at a time, because the +decoder takes its tone and its key-down threshold from the whole of whatever +it is handed: a half-minute recording with five seconds of keying in the +middle measures both from the other twenty-five. A mark far longer than any +dash is read as the transmission the ident was sent over rather than as a +character the window sliced, which is what used to take the first and last +letter of every such ident \[em] and with them the callsign, one word with no +gap in it to survive the drop. +.PP +A reading made only of one-element characters is refused. E and T are the +only two, so a decode of nothing but those can hardly be wrong \[em] there is +nothing in it to get wrong \[em] and no station has ever identified itself +that way. +.PP Short is therefore the normal case rather than the awkward one. A decode of two or three characters is believed on its timing alone \[em] every element within a third of a unit of one or three, every character resolving to diff --git a/packaging/saunterbrowse.1 b/packaging/saunterbrowse.1 index eb367d3..7306899 100644 --- a/packaging/saunterbrowse.1 +++ b/packaging/saunterbrowse.1 @@ -1,5 +1,5 @@ .\" Generated by packaging/make-browse-man.py -- do not edit by hand. -.TH SAUNTERBROWSE 1 "2026-09-02" "bandsaunter 2026-09-01_01" "User Commands" +.TH SAUNTERBROWSE 1 "2026-09-02" "bandsaunter 2026-09-01_02" "User Commands" .SH NAME saunterbrowse \- read and listen to what a bandsaunter scan collected .SH SYNOPSIS diff --git a/tests/test_morse.py b/tests/test_morse.py index 0dc5093..1a490c3 100644 --- a/tests/test_morse.py +++ b/tests/test_morse.py @@ -2,7 +2,7 @@ import numpy as np import pytest from morse_gen import morse_audio -from bandsaunter.morse import decode_morse, encode_morse +from bandsaunter.morse import decode_morse, encode_morse, find_morse FS = 16000 @@ -197,3 +197,91 @@ def test_a_tone_that_never_keys_is_not_morse(): def test_speech_is_not_morse(seed): from speech import synth_speech assert not decode_morse(synth_speech(2.5, FS, seed=seed), FS).is_morse + + +# -- an ident sent over a carrier -------------------------------------------- +# +# The commonest way Morse arrives on the land-mobile bands, and the way that +# was being missed entirely: the carrier stays up and the ident is an audio +# tone keyed inside it, so a CW detector looking for a keyed carrier sees a +# carrier that never stops. + +def _ident_over_a_tone(text="KSQ330", wpm=22.0, rate=16000, tone=795.0, + before=11.0, after=11.0, seed=3): + """A keyed ident with a steady tone either side of it, as heard over FM.""" + rng = np.random.default_rng(seed) + keyed = morse_audio(text, wpm, rate, snr_db=30.0, tone=tone) + steady = np.sin(2 * np.pi * tone * np.arange(int(before * rate)) / rate) + tail = np.sin(2 * np.pi * tone + * np.arange(int(after * rate)) / rate) * 0.9 + audio = np.concatenate([steady * 0.9, keyed, tail]).astype(np.float64) + return audio + rng.standard_normal(audio.size) * 0.02 + + +def test_an_ident_is_found_inside_a_capture_that_is_mostly_something_else(): + """decode_morse reads a clip that is Morse; find_morse looks for one + inside a clip that is not.""" + audio = _ident_over_a_tone() + assert decode_morse(audio, 16000).complete_text != "KSQ330", \ + "if the whole clip decodes there is nothing for find_morse to fix" + found = find_morse(audio, 16000) + assert found is not None and found.is_morse + assert found.complete_text == "KSQ330" + assert 19 <= found.wpm <= 25 + + +def test_a_steady_tone_at_the_edge_is_not_a_sliced_character(): + """A mark far longer than any dash is not a truncated element -- it is + the transmission the ident was sent over. Read as a cut, it took the + first and last letter of every ident, and with them the whole callsign: + one word with no gap in it to survive the drop.""" + found = find_morse(_ident_over_a_tone("W1AW"), 16000) + assert found is not None + assert found.text == "W1AW" + assert found.complete_text == "W1AW", "the ident was thrown away as cut" + + +def test_a_capture_with_no_morse_in_it_yields_none(): + rng = np.random.default_rng(9) + rate = 16000 + noise = rng.standard_normal(20 * rate) * 0.2 + speechy = noise + 0.4 * np.sin( + 2 * np.pi * 300 * np.arange(20 * rate) / rate + * (1 + 0.3 * np.sin(2 * np.pi * 3 * np.arange(20 * rate) / rate))) + for clip in (noise, speechy, np.zeros(20 * rate)): + assert find_morse(clip, rate) is None + + +def test_the_search_is_no_looser_than_the_decoder(): + """Every window is judged by is_morse exactly as a whole capture would + be; the search widens where the decoder looks, not what it accepts.""" + rng = np.random.default_rng(11) + clip = rng.standard_normal(30 * 16000) * 0.3 + assert find_morse(clip, 16000) is None + + +def test_a_reading_of_nothing_but_dots_and_dashes_is_refused(): + """E and T are the one-element characters, so a decode made only of them + can hardly be wrong -- there is nothing in it to get wrong. A capture of + noise on 445.5 MHz came back as "T T T E E E E E E E E" with the element + mix and the timing fit both inside their bands.""" + from bandsaunter.morse import MorseResult + flat = MorseResult(text="T T T E E E E", wpm=20.0, confidence=0.9, + n_elements=7, n_characters=7, timing_fit=1.0, + snr_db=40.0, undecoded=0) + assert not flat.is_morse + real = MorseResult(text="W1AW", wpm=20.0, confidence=0.9, + n_elements=11, n_characters=4, timing_fit=1.0, + snr_db=40.0, undecoded=0) + assert real.is_morse + + +def test_the_whole_capture_is_one_of_the_windows(): + """When the capture *is* Morse -- a beacon keying through all of it -- the + longest window is the best one, because a word is only certain when a gap + bounds it at both ends and a short window may not contain one.""" + audio = morse_audio("CQ CQ DE W1AW W1AW K", 18, FS, 20) + whole = decode_morse(audio, FS) + found = find_morse(audio, FS) + assert found is not None + assert len(found.complete_text) >= len(whole.complete_text)