bandsaunter/bandsaunter/weatherlog.py

"""Writing down the weather, and reading it back.

One line of JSON per message, written the moment it arrives.  Flushed after
every one, for the reason every log in this program is: a listening session
ends when the operator gets bored and presses control-C, and a log that only
reached the disk on a clean shutdown would be empty exactly when it was most
wanted.

Each line holds the message in hexadecimal alongside whatever was made of
it, because the message is the evidence and the rest of the line is an
opinion about it.  A future version of this program that reads a model this
one cannot will be able to go back through old logs and read them properly,
which is only possible if the bytes were kept.

There is also a way out to CSV, because weather is the one thing this
program records that people genuinely want to plot: a column per quantity, a
row per reading, the sensor's name in the second column, and nothing that
needs a program to open.
"""

from __future__ import annotations

import csv
import json
import time
from datetime import datetime
from pathlib import Path

from .acurite import Measure, Reading

__all__ = ["WeatherLog", "read_logs", "write_csv", "LOG_VERSION", "logs_in"]

LOG_VERSION = 1


class WeatherLog:
    """A JSON Lines record of every message heard, written as it arrives."""

    def __init__(self, path, receiver: str = "", frequency: float = 0.0,
                 sample_rate: float = 0.0, started: float = 0.0):
        self.path = Path(path)
        self.messages = 0
        self.started = started or time.time()
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self._file = self.path.open("a", encoding="utf8")
        self._write({"log": "bandsaunter-weather", "version": LOG_VERSION,
                     "started": round(self.started, 3),
                     "started_local": datetime.fromtimestamp(
                         self.started).strftime("%Y-%m-%d %H:%M:%S"),
                     "frequency": frequency, "sample_rate": sample_rate,
                     "receiver": receiver})

    def _write(self, body: dict) -> None:
        self._file.write(json.dumps(body, separators=(",", ":"),
                                    ensure_ascii=False) + "\n")
        self._file.flush()

    def append(self, reading: Reading, name: str = "") -> None:
        """Record one message: what arrived, and what was made of it."""
        body: dict = {"t": round(reading.at or time.time(), 3),
                      "key": reading.key, "family": reading.family,
                      "id": reading.sensor, "model": reading.model,
                      "msg": reading.message, "copies": reading.copies,
                      "hex": _hex(reading.bits)}
        if reading.channel:
            body["ch"] = reading.channel
        if name:
            # The name as it stood when the message arrived.  Kept so that a
            # log read back years later says where the sensor was, rather
            # than where a sensor with the same identity is now.
            body["name"] = name
        if reading.battery_low is not None:
            body["battery_low"] = bool(reading.battery_low)
        if reading.measures:
            body["m"] = {m.name: ([m.value, m.unit] if m.raw is None
                                  else [m.value, m.unit, m.raw])
                         for m in reading.measures}
        if reading.checks:
            body["checks"] = list(reading.checks)
        self.messages += 1
        self._write(body)

    def close(self) -> None:
        try:
            self._file.close()
        except OSError:
            pass

    def __enter__(self) -> "WeatherLog":
        return self

    def __exit__(self, *exc) -> None:
        self.close()


def _hex(bits: str) -> str:
    """A message's bits as bytes, where they make whole ones."""
    if not bits or len(bits) % 8:
        return ""
    return bytes(int(bits[i:i + 8], 2)
                 for i in range(0, len(bits), 8)).hex().upper()


def _bits(text: str) -> str:
    try:
        return "".join(format(byte, "08b") for byte in bytes.fromhex(text))
    except ValueError:
        return ""


# ---------------------------------------------------------------------------
# Reading it back
# ---------------------------------------------------------------------------

def logs_in(directory) -> list[Path]:
    """Every weather log in a directory, newest first."""
    try:
        found = list(Path(directory).expanduser().glob("weather_*.jsonl"))
    except OSError:
        return []
    return sorted(found, key=lambda p: p.stat().st_mtime, reverse=True)


def read_logs(paths) -> list[Reading]:
    """Every reading in one or more logs, in the order they were heard.

    A line that will not parse is skipped rather than fatal.  A log is
    appended to while the disk fills and the power goes off, so the last
    line of one is quite often half a line, and losing an evening's weather
    over it would be absurd.
    """
    out: list[Reading] = []
    for path in ([paths] if isinstance(paths, (str, Path)) else paths):
        try:
            text = Path(path).expanduser().read_text(encoding="utf8")
        except OSError:
            continue
        for line in text.splitlines():
            reading = _reading_from(line)
            if reading is not None:
                out.append(reading)
    out.sort(key=lambda r: r.at)
    return out


def _reading_from(line: str) -> Reading | None:
    line = line.strip()
    if not line:
        return None
    try:
        body = json.loads(line)
    except ValueError:
        return None
    if not isinstance(body, dict) or "key" not in body:
        return None            # the header line, or something else entirely
    measures = []
    for name, value in (body.get("m") or {}).items():
        if not isinstance(value, list) or not value:
            continue
        measures.append(Measure(name=name, value=float(value[0]),
                                unit=str(value[1]) if len(value) > 1 else "",
                                raw=float(value[2]) if len(value) > 2 else None))
    return Reading(model=str(body.get("model", "")),
                   family=str(body.get("family", "")),
                   sensor=str(body.get("id", "")),
                   channel=str(body.get("ch", "")),
                   battery_low=body.get("battery_low"),
                   message=int(body.get("msg", 0) or 0),
                   measures=tuple(measures),
                   bits=_bits(str(body.get("hex", ""))),
                   checks=tuple(body.get("checks") or ()),
                   at=float(body.get("t", 0.0) or 0.0),
                   copies=int(body.get("copies", 1) or 1))


# ---------------------------------------------------------------------------
# Out to a spreadsheet
# ---------------------------------------------------------------------------

def write_csv(path, readings, book=None, imperial: bool = False) -> Path:
    """A column per quantity and a row per reading.

    The columns are the union of every quantity any sensor reported, so a
    garden with a rain gauge in it has a rain column and the tower sensors
    leave it empty.  That is the shape a spreadsheet wants; the alternative,
    a file per sensor, is the shape a program wants, and this is for people.
    """
    path = Path(path).expanduser()
    path.parent.mkdir(parents=True, exist_ok=True)
    names: list[str] = []
    for reading in readings:
        for measure in reading.measures:
            if measure.name not in names:
                names.append(measure.name)
    heads = ["time", "unix", "name", "key", "model", "sensor", "channel",
             "battery"] + [_column(n, readings, imperial) for n in names]
    with open(path, "w", encoding="utf8", newline="") as fh:
        out = csv.writer(fh)
        out.writerow(heads)
        for reading in readings:
            row = [datetime.fromtimestamp(reading.at).isoformat(
                       timespec="seconds") if reading.at else "",
                   f"{reading.at:.3f}" if reading.at else "",
                   book.name_for(reading.key) if book is not None else "",
                   reading.key, reading.model, reading.sensor,
                   reading.channel,
                   "" if reading.battery_low is None else
                   ("low" if reading.battery_low else "ok")]
            values = {m.name: m for m in reading.measures}
            for name in names:
                measure = values.get(name)
                # str() rather than a format: %g turns a rain counter of a
                # million into 1e+06, which a spreadsheet reads as text.
                row.append("" if measure is None
                           else str(_converted(measure, imperial)))
            out.writerow(row)
    return path


# The units a column is written in.  Named in the heading rather than beside
# every number, because a column of "21.5 C" is text and a column of 21.5 is
# a temperature, and only one of those can be plotted.
_IMPERIAL = {"C": "F", "km/h": "mph", "mm": "in", "km": "mi"}


def _column(name: str, readings, imperial: bool) -> str:
    unit = ""
    for reading in readings:
        for measure in reading.measures:
            if measure.name == name and measure.unit:
                unit = measure.unit
                break
        if unit:
            break
    if imperial:
        unit = _IMPERIAL.get(unit, unit)
    return f"{name} ({unit})" if unit else name


def _converted(measure: Measure, imperial: bool) -> float:
    if not imperial:
        return round(measure.value, 3)
    if measure.unit == "C":
        return round(measure.value * 9 / 5 + 32, 2)
    if measure.unit == "km/h":
        return round(measure.value / 1.609344, 2)
    if measure.unit == "mm":
        return round(measure.value / 25.4, 3)
    if measure.unit == "km":
        return round(measure.value / 1.609344, 2)
    return round(measure.value, 3)