Add github actions and documentation page.

.github/github_actions.ABOUT

@@ -0,0 +1,12 @@
+# Component information
+name: Github actions
+description: Github actions generated with PkgTemplates.jl and then modified.
+copyright: 2024 EUMETSAT
+license_spdx: MIT
+home_url: https://github.com/eumetsat/MetopDatasets.jl/
+
+# Package IPR documentation
+license_text_file: ./LICENSE
+
+# File(s) associated to this component - Documentation Templates
+about_resource: ./.github*

.github/workflows/CI.yml

@@ -0,0 +1,74 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+    tags: ['*']
+  pull_request:
+  workflow_dispatch:
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created
+      actions: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 'lts'
+          - '1'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+  docs:
+    name: Documentation
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write # needed to allow julia-actions/cache to proactively delete old caches that it has created
+      contents: write
+      statuses: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+      - uses: julia-actions/cache@v2
+      - name: Configure doc environment
+        shell: julia --project=docs --color=yes {0}
+        run: |
+          using Pkg
+          Pkg.develop(PackageSpec(path=pwd()))
+          Pkg.instantiate()
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-docdeploy@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+      - name: Run doctests
+        shell: julia --project=docs --color=yes {0}
+        run: |
+          using Documenter: DocMeta, doctest
+          using MetopDatasets
+          DocMeta.setdocmeta!(MetopDatasets, :DocTestSetup, :(using MetopDatasets); recursive=true)
+          doctest(MetopDatasets)

.github/workflows/CompatHelper.yml

@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.github/workflows/TagBot.yml

@@ -0,0 +1,31 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+    inputs:
+      lookback:
+        default: "3"
+permissions:
+  actions: read
+  checks: read
+  contents: write
+  deployments: read
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

README.md

@@ -1,7 +1,12 @@
 # Under development 
 This package is still in the early development phase. Note that the package was previously named **MetopNative.jl** and was hosted on the [EUMETSAT GitLab](https://gitlab.eumetsat.int/eumetlab/cross-cutting-tools/MetopNative.jl) 
+
 # MetopDatasets.jl
 
+[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://eumetsat.github.io/MetopDatasets.jl/dev/)
+[![Build Status](https://github.com/eumetsat/MetopDatasets.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/eumetsat/MetopDatasets.jl/actions/workflows/CI.yml?query=branch%3Amain)
+[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
+
 MetopDatasets.jl is a package for reading products from the [METOP satellites](https://www.eumetsat.int/our-satellites/metop-series) using the native binary format specified for each product. The METOP satellites are part of the EUMETSAT-POLAR-SYSTEM (EPS) and produce long series of near real-time weather and climate observation. Learn more and access the products on [EUMETSATs user-portal](https://user.eumetsat.int/dashboard).
 
 MetopDatasets.jl has two APIs for reading native products. The first API is `MetopDataset` that implements the [CommonDataModel.jl](https://github.com/JuliaGeo/CommonDataModel.jl) interface and thus provides data access similar to e.g. [NCDatasets.jl](https://github.com/Alexander-Barth/NCDatasets.jl) and [GRIBDatasets.jl](https://github.com/JuliaGeo/GRIBDatasets.jl). The second API is `MetopProduct` which reads an entire product into a Julia object with a data structure similar to the native file structure. 

docs/make.jl

@@ -9,13 +9,21 @@ DocMeta.setdocmeta!(MetopDatasets, :DocTestSetup, :(using MetopDatasets); recurs
 makedocs(;
     modules = [MetopDatasets],
     authors = "lupemba <simon.koklupemba@eumetsat.int> and contributors",
-    repo = "https://github.com/eumetsat/MetopDatasets.jl/blob/{commit}{path}#{line}",
     sitename = "MetopDatasets.jl",
     format = Documenter.HTML(;
-        prettyurls = get(ENV, "CI", "false") == "true",
         canonical = "https://eumetsat.github.io/MetopDatasets.jl",
         edit_link = "main",
-        assets = String[]),
+        assets = String[]
+    ),
     pages = [
-        "Home" => "index.md"
-    ])
+        "Introduction" => "index.md",
+        "Public API" => "public_api.md",
+        "Internal API" => "internal_api.md"
+    ]
+)
+
+deploydocs(;
+    repo = "github.com/eumetsat/MetopDatasets.jl",
+    devbranch = "main",
+    push_preview = true
+)

docs/src/doc_templates.ABOUT

@@ -1,6 +1,6 @@
 # Component information
-name: Documentation Templates
-description: Markdown templates for auto-generating documentation pages for MetopDatasets.jl
+name: Documentation
+description: Markdown documentation pages for MetopDatasets.jl
 copyright: 2024 EUMETSAT
 license_spdx: MIT
 home_url: https://github.com/eumetsat/MetopDatasets.jl/

docs/src/index.md

@@ -10,17 +10,5 @@ Documentation for [MetopDatasets](https://github.com/eumetsat/MetopDatasets.jl).
 
 This code is licensed under MIT license. See file LICENSE for details on the usage and distribution terms.
 
-## Public functions and types
+## More documentation is under way
 
-```@autodocs
-Modules = [MetopDatasets]
-Private = false
-Order = [:function, :type, :macro]
-```
-
-## Internal functions and types
-```@autodocs
-Modules = [MetopDatasets]
-Public = false
-Order = [:function, :type, :macro]
-```

docs/src/internal_api.md

@@ -0,0 +1,7 @@
+
+## Internal functions and types
+```@autodocs
+Modules = [MetopDatasets]
+Public = false
+Order = [:function, :type, :macro]
+```

docs/src/public_api.md

@@ -0,0 +1,17 @@
+
+
+# MetopDataset
+
+This is the main interface.
+
+```@docs
+MetopDataset
+```
+
+# MetopProduct
+
+This interface might get removed in future versions.
+
+```@docs
+MetopProduct
+```

src/InterfaceDataModel/Dataset.jl

@@ -1,9 +1,6 @@
 # Copyright (c) 2024 EUMETSAT
 # License: MIT
 
-"""
-    MetopDataset{R <: DataRecord} <: CommonDataModel.AbstractDataset
-"""
 struct MetopDataset{R <: DataRecord} <: CDM.AbstractDataset
     file_pointer::IO
     main_product_header::MainProductHeader

src/MetopDatasets.jl

@@ -24,6 +24,6 @@ include("Instruments/IASI/IASI.jl")
 
 const RECORD_DIM_NAME = "atrack"
 
-export Record, DataRecord, MetopProduct, MetopDataset
+export MetopProduct, MetopDataset
 
 end

src/metop_product.jl

@@ -2,11 +2,11 @@
 # License: MIT
 
 """
-MetopProduct{T <: DataRecord} 
-    main_product_header::MainProductHeader
-    internal_pointer_records::Vector{InternalPointerRecord}
-    data_records::Vector{T}
-    dummy_records::Vector{DummyRecord}
+    MetopProduct{T <: DataRecord} 
+        main_product_header::MainProductHeader
+        internal_pointer_records::Vector{InternalPointerRecord}
+        data_records::Vector{T}
+        dummy_records::Vector{DummyRecord}
 
 Generic Metop Product
 """
@@ -21,11 +21,11 @@ struct MetopProduct{T <: DataRecord}
 end
 
 """
-MetopProduct(file_path::AbstractString)
-    main_product_header::MainProductHeader
-    internal_pointer_records::Vector{InternalPointerRecord}
-    data_records::Vector{T}
-    dummy_records::Vector{DummyRecord}
+    MetopProduct(file_path::AbstractString)
+        main_product_header::MainProductHeader
+        internal_pointer_records::Vector{InternalPointerRecord}
+        data_records::Vector{T}
+        dummy_records::Vector{DummyRecord}
 
 Construct a MetopProduct from a file. This allows you to load the entire native file with one function.
 """

test/ASCAT.jl

@@ -22,7 +22,7 @@ end
 
 @testset "ASCAT data records types" begin
     ## ASCA_SZR_1B_V13
-    @test MetopDatasets.ASCA_SZR_1B_V13 <: DataRecord
+    @test MetopDatasets.ASCA_SZR_1B_V13 <: MetopDatasets.DataRecord
     @test MetopDatasets.native_sizeof(MetopDatasets.ASCA_SZR_1B_V13) == 6677
     @test test_dimensions(MetopDatasets.ASCA_SZR_1B_V13)
 
@@ -42,7 +42,8 @@ end
           Dict("num_band" => 3, "xtrack" => 82)
     @test MetopDatasets.get_field_dimensions(MetopDatasets.ASCA_SZR_1B_V13, :sigma0_trip) ==
           ["num_band", "xtrack"]
-    @test MetopDatasets.get_field_dimensions(MetopDatasets.ASCA_SZR_1B_V13, :abs_line_number) ==
+    @test MetopDatasets.get_field_dimensions(
+        MetopDatasets.ASCA_SZR_1B_V13, :abs_line_number) ==
           String[]
     @test MetopDatasets.get_description(MetopDatasets.ASCA_SZR_1B_V13, :sigma0_trip) ==
           "Sigma0 triplet, re-sampled to swath grid, for 3 beams (fore, mid, aft) "
@@ -58,8 +59,10 @@ end
 
     @test MetopDatasets.native_sizeof(MetopDatasets.ASCA_SZF_1B_V13) == 4256
     @test test_dimensions(MetopDatasets.ASCA_SZF_1B_V13)
-    @test MetopDatasets.get_scale_factor(MetopDatasets.ASCA_SZF_1B_V13, :inc_angle_full) == 2
-    @test MetopDatasets.get_dimensions(MetopDatasets.ASCA_SZF_1B_V13) == Dict("xtrack" => 192)
+    @test MetopDatasets.get_scale_factor(MetopDatasets.ASCA_SZF_1B_V13, :inc_angle_full) ==
+          2
+    @test MetopDatasets.get_dimensions(MetopDatasets.ASCA_SZF_1B_V13) ==
+          Dict("xtrack" => 192)
 
     @test MetopDatasets.native_sizeof(MetopDatasets.ASCA_SZF_1B_V12) == 3684
     @test test_dimensions(MetopDatasets.ASCA_SZF_1B_V12)
@@ -83,7 +86,8 @@ end
 
 function get_data_record_type(file)
     data_record_type = open(file, "r") do file_pointer
-        main_header = MetopDatasets.native_read(file_pointer, MetopDatasets.MainProductHeader)
+        main_header = MetopDatasets.native_read(
+            file_pointer, MetopDatasets.MainProductHeader)
         return MetopDatasets.data_record_type(main_header)
     end
     return data_record_type

test/IASI.jl

@@ -5,7 +5,7 @@ using MetopDatasets, Test
 import CommonDataModel as CDM
 
 @testset "IASI data records" begin
-    @test MetopDatasets.IASI_XXX_1C_V11 <: DataRecord
+    @test MetopDatasets.IASI_XXX_1C_V11 <: MetopDatasets.DataRecord
     @test MetopDatasets.native_sizeof(MetopDatasets.IASI_XXX_1C_V11) == 2728908
     @test MetopDatasets.get_scale_factor(MetopDatasets.IASI_XXX_1C_V11, :ggeosondloc) == 6
     field_index = findfirst(fieldnames(MetopDatasets.IASI_XXX_1C_V11) .== :gs1cspect)
@@ -78,9 +78,12 @@ import CommonDataModel as CDM
         @test ds["ggeosondloc"][1, 1, 1, 1]≈156 atol=1 # test first longitude in file
         @test ds["ggeosondloc"][2, 1, 1, 1]≈-52 atol=1 # test first latitude in file
 
-        @test sort(CDM.dimnames(ds)) == ["atrack", "avhrr_channel", "avhrr_image_column", "avhrr_image_line", "band", 
-            "corner_cube_direction", "eigenvalue", "fov_class", "integrated_imager_column", "integrated_imager_line", 
-            "line_column", "lon_lat", "sounder_pixel", "spectral", "subgrid_imager_pixel", "xtrack", "zenith_azimuth"]
+        @test sort(CDM.dimnames(ds)) ==
+              ["atrack", "avhrr_channel", "avhrr_image_column", "avhrr_image_line", "band",
+            "corner_cube_direction", "eigenvalue", "fov_class",
+            "integrated_imager_column", "integrated_imager_line",
+            "line_column", "lon_lat", "sounder_pixel", "spectral",
+            "subgrid_imager_pixel", "xtrack", "zenith_azimuth"]
         @test CDM.dim(ds, :spectral) == 8700
         @test CDM.dim(ds, "atrack") == ds.main_product_header.total_mdr
 

test/auto_generate.jl

@@ -4,13 +4,13 @@
 using MetopDatasets, Test
 
 eval(MetopDatasets.record_struct_expression(
-    joinpath(@__DIR__, "TEST_FORmaT.csv"), DataRecord))
+    joinpath(@__DIR__, "TEST_FORmaT.csv"), MetopDatasets.DataRecord))
 eval(MetopDatasets.record_struct_expression(
-    joinpath(@__DIR__, "TEST_FORMAT2.csv"), DataRecord))
+    joinpath(@__DIR__, "TEST_FORMAT2.csv"), MetopDatasets.DataRecord))
 
 @testset "generate structs" begin
-    @test TEST_FORMAT <: DataRecord
-    @test TEST_FORMAT <: Record
+    @test TEST_FORMAT <: MetopDatasets.DataRecord
+    @test TEST_FORMAT <: MetopDatasets.Record
 
     expected_fields = [:record_header, :degraded_inst_mdr, :degraded_proc_mdr,
         :utc_line_nodes,
@@ -35,7 +35,8 @@ eval(MetopDatasets.record_struct_expression(
 
     # test fallback functions for dimensions
     @test MetopDatasets.get_dimensions(TEST_FORMAT) == Dict("dim_1" => 3, "dim_2" => 82)
-    @test MetopDatasets.get_field_dimensions(TEST_FORMAT, :sigma0_trip) == ["dim_1", "dim_2"]
+    @test MetopDatasets.get_field_dimensions(TEST_FORMAT, :sigma0_trip) ==
+          ["dim_1", "dim_2"]
     @test MetopDatasets.get_field_dimensions(TEST_FORMAT, :latitude) == ["dim_2"]
     @test MetopDatasets.get_field_dimensions(TEST_FORMAT, :utc_line_nodes) == String[]
 

test/generic_functions.jl

@@ -3,7 +3,7 @@
 
 using MetopDatasets, Test
 
-struct TestRecord <: DataRecord
+struct TestRecord <: MetopDatasets.DataRecord
     f1::Array{Int16, 3}
     f2::Array{MetopDatasets.ShortCdsTime, 4}
     f3::Int64