Add HIRS formats, add copyright headers and replace size fieldname with dim_size

Author: lupemba

src/Instruments/ATOVS/AMSU_A/AMSU_A.jl

@@ -1,2 +1,5 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
 include("AMSU_A_records.jl")
 include("AMSU_A_dimensions.jl")

src/Instruments/ATOVS/AMSU_A/AMSU_A_dimensions.jl

@@ -1,3 +1,6 @@
+# Copyright (c) 2024 EUMETSAT
+# License: MIT
+
 function get_dimensions(T::Type{<:AMSA_XXX_1B})
     return Dict(
         "lat_lon" => 2,

src/Instruments/ATOVS/AMSU_A/AMSU_A_records.jl

@@ -1,3 +1,6 @@
+# Copyright (c) 2024 EUMETSAT
+# License: MIT
+
 const AMSA_XXX_1B_V10_format = @path joinpath(@__DIR__, "csv_formats/AMSA_XXX_1B_V10.csv")
 
 abstract type AMSA_XXX_1B <: ATOVS_1B end

src/Instruments/ATOVS/ATOVS.jl

@@ -1,12 +1,27 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
 abstract type ATOVS_1B <: DataRecord end
 
+const DATA_CAL_NAME = :data_calibration
 const DATA_CAL_NEDT_NAME = :data_calibration_nedt
 const DATA_CAL_QUALITY_NAME = :data_calibration_quality
-const DATA_CAL_NAME = :data_calibration
+
+const ELEMENT_RAD_NAME = :digital_a_data_element_rad
+const ELEMENT_RAD_DATA_NAME = :digital_a_rad
+const ELEMENT_RAD_HEADER_NAME = :digital_a_rad_header
+
+const ELEMENT_FLAG_NAME = :digital_a_data_element_flag
+const ELEMENT_FLAG_DATA_NAME = :digital_a_flag
+const ELEMENT_FLAG_HEADER_NAME = :digital_a_flag_header
+
+const EXTRACTED_VARS = (DATA_CAL_NEDT_NAME, DATA_CAL_QUALITY_NAME, ELEMENT_RAD_DATA_NAME,
+    ELEMENT_RAD_HEADER_NAME, ELEMENT_FLAG_DATA_NAME, ELEMENT_FLAG_HEADER_NAME)
 
 const DATA_CAL_NEDT_DESCRIPTION = "Noise-Equivalent Delta Temperature"
 const DATA_CAL_QUALITY_DESCRIPTION = "Channel Quality Flags"
 
 include("MHS/MHS.jl")
 include("AMSU_A/AMSU_A.jl")
+include("HIRS/HIRS.jl")
 include("VariableExtra.jl")

src/Instruments/ATOVS/HIRS/DiskArrayExtra.jl

@@ -0,0 +1,105 @@
+# Copyright (c) 2024 EUMETSAT
+# License: MIT
+
+# DataElementMetopDiskArray is used to handle the compound types in the HIRS L1B format.
+struct DataElementMetopDiskArray{T, N} <: AbstractMetopDiskArray{T, N}
+    file_pointer::IOStream
+    field_name::Symbol
+    is_header::Bool
+
+    # computed
+    data_element_type::Type
+    field_type::Type
+    offsets_in_file::Vector{Int64}
+    record_type::Type{<:DataRecord}
+    dim_size::NTuple{N, Int64}
+end
+
+function DataElementMetopDiskArray(file_pointer::IOStream,
+        record_layouts::Vector{<:RecordLayout},
+        field_name::Symbol, data_element_field_name::Symbol; auto_convert = true)
+    data_element_type, record_count, offsets_in_file, record_type = layout_info_for_disk_array(
+        record_layouts, data_element_field_name)
+
+    is_header = field_name in (ELEMENT_RAD_HEADER_NAME, ELEMENT_FLAG_HEADER_NAME)
+
+    field_type = if is_header
+        Vector{fieldtypes(eltype(data_element_type))[1]}
+    else
+        Matrix{eltype(fieldtypes(eltype(data_element_type))[2])}
+    end
+
+    T, N = _get_T_and_N(field_type, auto_convert)
+
+    data_element_field_array_size = _get_field_array_size(
+        record_layouts, record_type, data_element_field_name)
+    dim_size = if is_header
+        dim_size = (data_element_field_array_size..., record_count)
+    else
+        dim_size = (20, data_element_field_array_size..., record_count)
+    end
+
+    return DataElementMetopDiskArray{T, N}(
+        file_pointer,
+        field_name,
+        is_header,
+
+        # computed
+        data_element_type,
+        field_type,
+        offsets_in_file,
+        record_type,
+        dim_size)
+end
+
+function _native_read_data_element_header(io::IO, T::Type, byte_to_skip)
+    val = native_read(io, T)
+    skip(io, byte_to_skip)
+    return val
+end
+
+function _native_read_data_element_data(io::IO, T::Type)
+    skip(io, 4)
+    return native_read_array(io, T, (20,))
+end
+
+# Extend get index functions
+function DiskArrays.readblock!(disk_array::DataElementMetopDiskArray{T, N},
+        aout,
+        i::Vararg{OrdinalRange, N}) where {T, N}
+
+    # separate record range.
+    i_record = i[end]
+    i_array = i[1:(end - 1)]
+
+    is_header = disk_array.is_header
+    header_field_type = fieldtypes(eltype(disk_array.data_element_type))[1]
+    data_field_type = fieldtypes(eltype(disk_array.data_element_type))[2]
+
+    byte_to_skip = is_header ? native_sizeof(data_field_type) :
+                   native_sizeof(header_field_type)
+    data_element_per_record = is_header ? disk_array.dim_size[1] : disk_array.dim_size[2]
+
+    field_element_type = eltype(disk_array.field_type)
+
+    for k in eachindex(i_record)
+        record_index = i_record[k]
+        field_start_position = disk_array.offsets_in_file[record_index]
+        seek(disk_array.file_pointer, field_start_position)
+
+        full_field = if is_header
+            [_native_read_data_element_header(
+                 disk_array.file_pointer, field_element_type, byte_to_skip)
+             for i in 1:data_element_per_record]
+        else
+            reduce(hcat,
+                _native_read_data_element_data(
+                    disk_array.file_pointer, Vector{field_element_type})
+                for i in 1:data_element_per_record)
+        end
+
+        aout_rec = selectdim(aout, N, k)
+        aout_rec .= _auto_convert.(T, full_field[i_array...])
+    end
+    return nothing
+end

src/Instruments/ATOVS/HIRS/HIRS.jl

@@ -0,0 +1,6 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+include("HIRS_records.jl")
+include("HIRS_dimensions.jl")
+include("DiskArrayExtra.jl")

src/Instruments/ATOVS/HIRS/HIRS_dimensions.jl

@@ -0,0 +1,53 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+function get_dimensions(T::Type{<:HIRS_XXX_1B})
+    return Dict(
+        "lat_lon" => 2,
+        "channels" => 20,
+        "xtrack" => 56,
+        "transmitter_telemetry" => 9,
+        "roll_pitch_yaw" => 3,
+        "solar_angles" => 4,
+        "flags" => 8,
+        "analog_data" => 16
+    )
+end
+
+function get_field_dimensions(T::Type{<:HIRS_XXX_1B},
+        field_name::Symbol)::Vector{<:AbstractString}
+    if field_name in (DATA_CAL_QUALITY_NAME, DATA_CAL_NEDT_NAME)
+        return ["channels"]
+    elseif field_name == ELEMENT_RAD_DATA_NAME
+        return ["channels", "xtrack"]
+    elseif field_name == ELEMENT_RAD_HEADER_NAME
+        return ["xtrack"]
+    elseif field_name == ELEMENT_FLAG_DATA_NAME
+        return ["channels", "flags"]
+    elseif field_name == ELEMENT_FLAG_HEADER_NAME
+        return ["flags"]
+    end
+
+    if !(fieldtype(T, field_name) <: Array)
+        return String[]
+    end
+
+    array_size = _get_array_size(T, field_name)
+
+    if array_size == (20,)
+        return ["channels"]
+    elseif array_size == (56,)
+        return ["xtrack"]
+    elseif array_size == (2, 56)
+        return ["lat_lon", "xtrack"]
+    else
+        if field_name == :analog_data
+            return ["analog_data"]
+        elseif field_name == :euler_angle
+            return ["roll_pitch_yaw"]
+        elseif field_name == :angular_relation
+            return ["solar_angles", "xtrack"]
+        end
+        error("Dimensions not set for $field_type")
+    end
+end

src/Instruments/ATOVS/HIRS/HIRS_records.jl

@@ -0,0 +1,16 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+const HIRS_XXX_1B_V10_format = @path joinpath(@__DIR__, "csv_formats/HIRS_XXX_1B_V10.csv")
+
+abstract type HIRS_XXX_1B <: ATOVS_1B end
+# create data structure, add description and scale factors
+eval(record_struct_expression(HIRS_XXX_1B_V10_format, HIRS_XXX_1B))
+
+function data_record_type(header::MainProductHeader, product_type::Val{:HIRS_xxx_1B})::Type
+    if header.format_major_version == 10
+        return HIRS_XXX_1B_V10
+    else
+        error("No format found for format major version :$(header.format_major_version)")
+    end
+end

src/Instruments/ATOVS/HIRS/csv_formats/HIRS_XXX_1B_V10.csv

@@ -0,0 +1,39 @@
+FIELD,DESCRIPTION,SF,UNITS,DIM1,DIM2,DIM3,TYPE,TYPE SIZE,FIELD SIZE,OFFSET
+RECORD_HEADER,Generic Record Header,0,,1,1,1,REC_HEAD,20,20,0
+,GENERIC QUALITY INDICATORS,,,,,,,,,20
+DEGRADED_INST_MDR,Quality of MDR has been degraded from nominal due to an instrument degradation,0,NA,1,1,1,boolean,1,1,20
+DEGRADED_PROC_MDR,Quality of MDR has been degraded from nominal due to a processing degradation,0,NA,1,1,1,boolean,1,1,21
+Scan Line Information,,,,,,,,,,22
+LINE_COUNTER,Line counter since last auto-calibration sequence. If calibration enabled then it runs from 0 to 39 else from 0 to 8191 followed by 0 to 8191 and so on,0,NA,1,1,1,u-integer2,2,2,22
+SCAN_TYPE_CODE,"Scan type code (0 = Earth view, 1 = space view, 2 = cold BB view, 3 = warm BB view, 4 = else.",0,NA,1,1,1,u-integer2,2,2,24
+QUALITY_INDICATOR,Quality Indicator Bit Field,0,N/A,1,1,1,bitst(32),4,4,26
+SCAN_LINE_QUALITY,Scan Line Quality Flags,0,N/A,1,1,1,bitst(32),4,4,30
+DATA_CALIBRATION,Noise-Equivalent Delta N and Channel Quality Flags,0,N/A,20,1,1,DATA_CALQUAL,2,40,34
+MEASUREMENT DATA ,,,,,,,,,,74
+DIGITAL_A_DATA_ELEMENT_RAD,Radiances for 56 pixel values of a scan line in the 20 channels. Earth scan pixel values are calibrated to radiances (channel 1 - 19) and reflectance factors (channel 20),4,NA,56,1,1,DATA_ELEM_RAD,84,4704,74
+DIGITAL_A_DATA_ELEMENT_FLAG,Information on the instrument status.,0,NA,8,1,1,DATA_ELEM_FLAG,44,352,4778
+Digital B telemetry,,,,,,,,,,5130
+INSTRUMENT_INVALID_DIGITAL_WORD_FLAG,Invalid Word Bit Flags,0,N/A,1,1,1,bitst(16),2,2,5130
+DIGITAL_B_DATA,Digital B Data,0,N/A,1,1,1,bitst(16),2,2,5132
+Analog telemetry,,,,,,,,,,5134
+INSTRUMENT_INVALID_ANALOG_WORD_FLAG,Invalid Word Bit Flags,0,N/A,1,1,1,bitst(32),4,4,5134
+ANALOG_DATA,Analog Data,0,N/A,16,1,1,u-byte,1,16,5138
+NAVIGATION DATA AT SCAN LINE,,,,,,,,,,5154
+TIME_ATTITUDE,Time Associated with Attitude Angles,0,s,1,1,1,u-integer4,4,4,5154
+EULER_ANGLE,"Euler Angles: Roll, Pitch, Yaw",3,deg,3,1,1,integer2,2,6,5158
+NAVIGATION_STATUS,Navigation Status Bit Field,0,N/A,1,1,1,bitst(32),4,4,5164
+SPACECRAFT_ALTITUDE,Spacecraft Altitude Above Reference Geoid (MSL) ,1,km,1,1,1,u-integer4,4,4,5168
+ANGULAR_RELATION,"Angular relationships: solar zenith angle, satellite zenith angle, solar azimuth angle, satellite azimuth angle (FOV 1 to 56) Note: azimuth angle range is -180 to +180, where minus is west and plus is east.",2,deg,4,56,1,integer2,2,448,5172
+EARTH_LOCATION,"Earth Location: latitude, longitude (FOV 1 to 56)",4,deg,2,56,1,integer4,4,448,5620
+SURFACE_PROPERTY,"Surface property (0 = water, 1 = mixed/coast, 2 = land) ",,,56,1,1,integer2,2,112,6068
+TERRAIN_ELEVATION,Average terrain elevation (FOV 1 to 56),,m,56,1,1,integer2,2,112,6180
+Calibration Coefficient (see Section 3.5.1.12),,,,,,,,,,6292
+PRIMARY_CALIBRATION_SECOND_TERM,Primary Calibration Second Order Term a2 (ch. 1-20) ,12,�mW/m2/sr/cm-1/cnt2 or % alb/cnt2 (ch 20),20,1,1,integer4,4,80,6292
+PRIMARY_CALIBRATION_FIRST_TERM,Primary Calibration First Order Term a1 (ch. 1-20) ,9,�mW/m2/sr/cm-1/cnt or % alb/cnt (ch 20),20,1,1,integer4,4,80,6372
+PRIMARY_CALIBRATION_ZEROTH_TERM,Primary Calibration Zeroth Order Term a0 (ch. 1-20) ,6,�mW/m2/sr/cm-1 or % alb (ch 20) ,20,1,1,integer4,4,80,6452
+SPARE_CALIBRATION_SECOND_TERM,Spare Calibration Second Order Term (ch. 1-20) ,12,�mW/m2/sr/cm-1/cnt2 or % alb/cnt2 (ch 20),20,1,1,integer4,4,80,6532
+SPARE_CALIBRATION_FIRST_TERM,Spare Calibration First Order Term (ch. 1-20) ,9,�mW/m2/sr/cm-1/cnt or % alb/cnt (ch 20),20,1,1,integer4,4,80,6612
+SPARE_CALIBRATION_ZEROTH_TERM,Spare Calibration Zeroth Order Term(ch. 1-20) ,6,�mW/m2/sr/cm-1 or % alb (ch 20),20,1,1,integer4,4,80,6692
+CLOUD COVERAGE INFORMATION,,,,,,,,,,6772
+PERCENTAGE_CLEAR_SKY,Percentage of sky that is clear of clouds,2,%,56,1,1,u-integer2,2,112,6772
+SIZE OF THE MDR,,,,,,,,,,6884

src/Instruments/ATOVS/MHS/MHS.jl

@@ -1,2 +1,5 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
 include("MHS_records.jl")
 include("MHS_dimensions.jl")