Initial commit

Author: siiptuo

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Run tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install .[dev,test]
+      - name: Run pre-commit checks
+        if: startsWith(matrix.os, 'ubuntu-')
+        run: |
+          pre-commit run --all-files --show-diff-on-failure
+      - name: Run tests
+        run: |
+          pytest

.gitignore

@@ -0,0 +1 @@
+__pycache__

.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+exclude: ^tests/data/
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+        args: ["--fix", "lf"]
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.0
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: system
+        types: [python]
+        require_serial: true
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.24.2
+    hooks:
+      - id: toml-sort-fix
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.30.2
+    hooks:
+      - id: typos

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Finnish Meteorological Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,14 @@
+# Ceilopyter
+
+Python package for reading ceilometer data.
+
+## Supported ceilometers
+
+- Campbell Scientific CS135
+- Vaisala CL31
+- Vaisala CL51
+- Vaisala CT25K
+
+## License
+
+MIT

ceilopyter/__init__.py

@@ -0,0 +1,6 @@
+from .read_cl import read_cl_file as read_cl_file
+from .read_cl import read_cl_message as read_cl_message
+from .read_cs import read_cs_file as read_cs_file
+from .read_cs import read_cs_message as read_cs_message
+from .read_ct import read_ct_file as read_ct_file
+from .read_ct import read_ct_message as read_ct_message

ceilopyter/common.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+
+
+class InvalidMessageError(Exception):
+    pass
+
+
+@dataclass
+class Message:
+    """Data message from ceilometer.
+
+    Attributes:
+        range_resolution: Range resolution (m).
+        laser_pulse_energy: Laser pulse energy (%).
+        laser_temperature: Laser temperature (degC).
+        tilt_angle: Tilt angle (deg).
+        background_light: Background light (mV).
+        n_pulses: Number of pulses.
+        sample_rate: Sampling rate (MHz).
+        beta: Backscatter coefficient (sr-1 m-1).
+    """
+
+    range_resolution: int
+    laser_pulse_energy: int
+    laser_temperature: int
+    tilt_angle: int
+    background_light: int
+    n_pulses: int
+    sample_rate: int
+    beta: npt.NDArray[np.floating]
+
+
+class Status:
+    def __repr__(self):
+        return (
+            self.__class__.__name__
+            + "("
+            + ", ".join(f"{key}=True" for key, value in vars(self).items() if value)
+            + ")"
+        )

ceilopyter/py.typed

@@ -0,0 +1,193 @@
+import datetime
+import logging
+from dataclasses import dataclass
+from os import PathLike
+from pathlib import Path
+
+from . import utils
+from .common import InvalidMessageError, Message, Status
+
+FORMAT = utils.date_format_to_regex(rb"-%Y-%m-%d %H:%M:%S\r?\n")
+
+
+class ClStatus(Status):
+    """Decoded status bits from Vaisala CL31 or CL51.
+
+    Attributes:
+        transmitter_shutoff_alarm: Transmitter shut-off.
+        transmitter_fail_alarm: Transmitter failure.
+        receiver_fail_alarm: Receiver failure.
+        voltage_fail_alarm: Voltage failure.
+        memory_error_alarm: Memory error.
+        light_path_obstruction_alarm: Light path obstruction.
+        receiver_saturation_alarm: Receiver saturation.
+        coaxial_cable_fail_alarm: Coaxial cable failure.
+        ceilometer_board_fail_alarm: Ceilometer engine board failure.
+        window_contam_warning: Window contamination.
+        battery_low_warning: Battery voltage low.
+        transmitter_expire_warning: Transmitter expires.
+        humidity_high_warning: High humidity.
+        blower_fail_warning: Blower failure.
+        humidity_sensor_fail_warning: Humidity sensor failure.
+        heater_fault_warning: Heater fault.
+        background_radiance_warning: High background radiance.
+        ceilometer_board_warning: Ceilometer engine board failure.
+        battery_fail_warning: Battery failure.
+        laser_monitor_fail_warning: Laser monitor failure.
+        receiver_warning: Receiver warning.
+        tilt_angle_warning: Tilt angle > 45 degrees warning.
+        blower_status: Blower is on.
+        blower_heater_status: Blower heater is on.
+        internal_heater_status: Internal heater is on.
+        battery_power_status: Working from battery.
+        standby_status: Standby mode is on.
+        self_test_status: Self test in progress.
+        manual_data_status: Manual data acquisition settings are effective.
+        units_meters: Units are meters if on, else feet.
+        manual_blower_status: Manual blower control.
+        polling_mode_status: Polling mode is on.
+    """
+
+    def __init__(self, status_bits: int):
+        self.transmitter_shutoff_alarm = bool(status_bits & 0x800000000000)
+        self.transmitter_fail_alarm = bool(status_bits & 0x400000000000)
+        self.receiver_fail_alarm = bool(status_bits & 0x200000000000)
+        self.voltage_fail_alarm = bool(status_bits & 0x100000000000)
+        self.memory_error_alarm = bool(status_bits & 0x040000000000)
+        self.light_path_obstruction_alarm = bool(status_bits & 0x020000000000)
+        self.receiver_saturation_alarm = bool(status_bits & 0x010000000000)
+        self.coaxial_cable_fail_alarm = bool(status_bits & 0x000200000000)
+        self.ceilometer_board_fail_alarm = bool(status_bits & 0x000100000000)
+        self.window_contam_warning = bool(status_bits & 0x000080000000)
+        self.battery_low_warning = bool(status_bits & 0x000040000000)
+        self.transmitter_expire_warning = bool(status_bits & 0x000020000000)
+        self.humidity_high_warning = bool(status_bits & 0x000010000000)
+        self.blower_fail_warning = bool(status_bits & 0x000004000000)
+        self.humidity_sensor_fail_warning = bool(status_bits & 0x000001000000)
+        self.heater_fault_warning = bool(status_bits & 0x000000800000)
+        self.background_radiance_warning = bool(status_bits & 0x000000400000)
+        self.ceilometer_board_warning = bool(status_bits & 0x000000200000)
+        self.battery_fail_warning = bool(status_bits & 0x000000100000)
+        self.laser_monitor_fail_warning = bool(status_bits & 0x000000080000)
+        self.receiver_warning = bool(status_bits & 0x000000040000)
+        self.tilt_angle_warning = bool(status_bits & 0x000000020000)
+        self.blower_status = bool(status_bits & 0x000000008000)
+        self.blower_heater_status = bool(status_bits & 0x000000004000)
+        self.internal_heater_status = bool(status_bits & 0x000000002000)
+        self.battery_power_status = bool(status_bits & 0x000000001000)
+        self.standby_status = bool(status_bits & 0x000000000800)
+        self.self_test_status = bool(status_bits & 0x000000000400)
+        self.manual_data_status = bool(status_bits & 0x000000000200)
+        self.units_meters = bool(status_bits & 0x000000000080)
+        self.manual_blower_status = bool(status_bits & 0x000000000040)
+        self.polling_mode_status = bool(status_bits & 0x000000000020)
+
+
+@dataclass
+class ClMessage(Message):
+    """Data message from Vaisala CL31 or CL51.
+
+    Attributes:
+        window_transmission: Window transmission (%).
+        status: Decoded status bits.
+    """
+
+    window_transmission: int
+    status: ClStatus
+
+
+def read_cl_file(
+    filename: str | PathLike,
+) -> tuple[list[datetime.datetime], list[ClMessage]]:
+    """Read Vaisala CL31 or CL51 file."""
+    content = Path(filename).read_bytes()
+    time = []
+    data = []
+    for ts, msg in utils.parse_file(content, FORMAT):
+        try:
+            data.append(read_cl_message(msg))
+            time.append(ts)
+        except (InvalidMessageError, ValueError) as e:
+            logging.debug("Invalid message: %s", e)
+    return time, data
+
+
+def read_cl_message(message: bytes) -> ClMessage:
+    """Read Vaisala CL31 or CL51 data message."""
+    lines = iter(message.splitlines())
+
+    # Line 1
+    line1 = utils.next_line(lines, 8, prefix=b"\x01", suffix=b"\x02")
+    if line1[:2] != b"CL":
+        msg = "Invalid line 1"
+        raise InvalidMessageError(msg)
+    msg_no = line1[6:7]
+    if msg_no not in (b"1", b"2"):
+        msg = f"Invalid message number: {msg_no.decode()}"
+        raise InvalidMessageError(msg)
+    subclass = line1[7:8]
+    if subclass in (b"1", b"2", b"3", b"4"):
+        line3_len = 31  # CL31
+    elif subclass == b"6":
+        line3_len = 40  # CL51
+    else:
+        msg = f"Invalid message subclass: {subclass.decode()}"
+        raise InvalidMessageError(msg)
+    check_content = line1 + b"\x02\r\n"
+
+    # Line 2
+    line2 = utils.next_line(lines, 33)
+    status_bits = int(line2[21:], 16)
+    status = ClStatus(status_bits)
+    check_content += line2 + b"\r\n"
+
+    # Line 3: sky condition
+    if msg_no == b"2":
+        line3 = utils.next_line(lines).rjust(line3_len)
+        check_content += line3 + b"\r\n"
+
+    # Line 3/4
+    line4 = utils.next_line(lines, 47)
+    scale = int(line4[0:5])
+    range_resolution = int(line4[6:8])
+    n_samples = int(line4[9:13])
+    laser_pulse_energy = int(line4[14:17])
+    laser_temperature = int(line4[18:21])
+    window_transmission = int(line4[22:25])
+    tilt_angle = int(line4[26:28])
+    background_light = int(line4[29:33])
+    n_pulses = 1024 * int(line4[35:39])
+    sample_rate = int(line4[41:43])
+    check_content += line4 + b"\r\n"
+
+    # Line 4/5: profile
+    line5 = utils.next_line(lines, 5 * n_samples)
+    raw = utils.read_hex(line5, 5, n_samples)
+    beta = raw * 1e-8 * scale / 100
+    check_content += line5 + b"\r\n\x03"
+
+    # Line 5/6: checksum
+    line6 = utils.next_line(lines, 4, prefix=b"\x03", suffix=b"\x04")
+    expected_checksum = int(line6, 16)
+
+    actual_checksum = utils.crc16(check_content)
+    if expected_checksum != actual_checksum:
+        msg = (
+            "Invalid checksum: "
+            f"expected {expected_checksum:04x}, "
+            f"got {actual_checksum:04x}"
+        )
+        raise InvalidMessageError(msg)
+
+    return ClMessage(
+        range_resolution=range_resolution,
+        laser_pulse_energy=laser_pulse_energy,
+        laser_temperature=laser_temperature,
+        window_transmission=window_transmission,
+        tilt_angle=tilt_angle,
+        background_light=background_light,
+        n_pulses=n_pulses,
+        sample_rate=sample_rate,
+        status=status,
+        beta=beta,
+    )