Prepare common data model 0.4

Artifacts.toml

@@ -1,7 +1,7 @@
 [test_data_MetopDatasets]
-git-tree-sha1 = "f3b12f6ed40f19382d48e4661e77d61a6f2dc5e6"
+git-tree-sha1 = "8958418f0be76c385461ed038225f62a8a35de8f"
 lazy = true
 
     [[test_data_MetopDatasets.download]]
-    url = "https://github.com/eumetsat/test-data-MetopDatasets/raw/a237b6eed126f525289d8d8c6aedd0da80218135/reduced_data.tar.gz"
-    sha256 = "6be02adb17179917c3637b6350c77e060479db8eaf6b1c5820a017a212ae98e1"
+    url = "https://github.com/eumetsat/test-data-MetopDatasets/raw/57a1cf647c445e52c1a12a92b047f425983e63f3/reduced_data.tar.gz"
+    sha256 = "b0b1f1ff9b7a30c5035bb5b6e1370b7b8c721e2e15754bded27031be099b33db"

CHANGELOG.md

@@ -2,7 +2,14 @@
 
 ## Unreleased
 
-## v0.2
+## v0.2.1
+- Upgrade to [CommonDataModel v0.4](https://github.com/JuliaGeo/CommonDataModel.jl/releases/tag/v0.4.0) 
+- Add `get_scaled` helper function. The function helps get a field and apply the scale function for a single `Record`. Useful to handle auxiliary records.
+- Add [AMSU-A Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_amsu_a_l1_pfs_c89fa9d9ea.pdf)
+- Add [HIRS Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_97230_eps_hirs4l1_pfs_0ddaefcb74.pdf)
+- Add [MHS Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_97229_eps_mhs_pfs_2069b45efc.pdf)
+
+## v0.2.0
 - Error message for invalid file types
 - (**BREAKING**) Improved padding of flexible IASI L2 variables e.g. "o3_cp_air". The variable dimensions now matches the variable storing the location. See [issue 15](https://github.com/eumetsat/MetopDatasets.jl/issues/15) for more information.
 - Internal changes to only use `FlexibleMetopDiskArray` for the IASI L2 flexible variables that varies in size between each record.

Project.toml

@@ -1,7 +1,7 @@
 name = "MetopDatasets"
 uuid = "0c26954c-4046-4b98-a13c-f9377ca4b9b7"
 authors = ["lupemba <simon.koklupemba@eumetsat.int> and contributors"]
-version = "0.2.0"
+version = "0.2.1"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -16,7 +16,7 @@ RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00"
 [compat]
 Aqua = "0.8.9"
 CSV = "0.10"
-CommonDataModel = "0.3"
+CommonDataModel = "0.4"
 Compat = "4.10"
 Dates = "1"
 DiskArrays = "0.3, 0.4"

docs/make.jl

@@ -27,7 +27,8 @@ makedocs(;
         "Use with Python" => "python.md",
         "Examples" => [
             "ASCAT" => "ASCAT.md",
-            "IASI" => "IASI.md"
+            "IASI" => "IASI.md",
+            "ATOVS (AMSU-A, HIRS, MHS)" => "ATOVS.md"
         ],
         "Full API" => "full_api.md"
     ]

docs/src/ATOVS.md

@@ -0,0 +1,189 @@
+## ATOVS
+Advanced TIROS Operational Sounder (ATOVS) is a sounding instrument package. The ATOVS instruments have both been flown on NOAA satellites and on the METOP satellites. The METOP ATOVS instrument package consists of 3 sensors:
+- Advanced Microwave Sounding Units A (AMSU-A)
+- High Resolution InfraRed Sounder (HIRS)
+- Microwave Humidity Sounder (MHS)
+For more information see [ATOVS Level 1b Product Guide](https://user.eumetsat.int/s3/eup-strapi-media/pdf_atovsl1b_pg_8bbaa8ba48.pdf)
+
+## AMSU-A
+The Advanced Microwave Sounding Unit-A (AMSU-A) is an instrument on the METOP satellites. AMSU-A is a microwave radiometer with 15 channels and is one of the three ATOVS sensors.
+
+The AMSU-A L1B products contain the radiance measured at all 15 channels. See the [AMSU-A Level 1B product pagel](https://data.eumetsat.int/product/EO:EUM:DAT:METOP:AMSUL1) for more information. The frequency of each channel is given in [NOAA KLM User's Guide](https://web.archive.org/web/20060420075941/http://www2.ncdc.noaa.gov/docs/klm/html/c3/sec3-3.htm)
+
+This example is made using the following packages.
+```
+[13f3f980] CairoMakie v0.15.6
+[db073c08] GeoMakie v0.7.15
+```
+
+```julia
+using MetopDatasets
+using CairoMakie, GeoMakie, Statistics
+
+# Open the dataset.
+path = "AMSA_xxx_1B_M03_20250915221320Z_20250915235520Z_N_O_20250915235036Z.nat"
+ds = MetopDataset(path, maskingvalue=NaN);
+
+# read location
+latitude = ds["earth_location"][1,:,:]
+longitude = ds["earth_location"][2,:,:]
+
+# read radiance
+radiance_of_band = ds["scene_radiance"][2,:,:] # Channel 2 is 31.4 GHz
+
+radiance_q05 = quantile(radiance_of_band[:],0.05)
+radiance_q90 = quantile(radiance_of_band[:],0.90)
+
+fig = let
+    # Create figure and axis
+    fig = Figure()
+    ax = GeoAxis(fig[1, 1],
+        title = "AMSU",
+        xlabel = "longitude",
+        ylabel = "latitude")
+
+    # plot data with color map
+        scatter!(ax, longitude[:], latitude[:],
+        color = radiance_of_band[:], colorrange = (radiance_q05,radiance_q90), markersize = 2)
+
+    # Add colorbar
+    Colorbar(fig[1,2], colorrange = (radiance_q05,radiance_q90) , label="mW/m2/sr/cm-1")
+
+    # Add coastlines
+    lines!(ax, GeoMakie.coastlines()) 
+    fig
+end
+```
+![AMSU-A channel 2](AMSU_A_channel_2_plot.png)
+The plot shows the radiance for the AMSU-A channel 2 (31.4 GHz) observed during an orbit.
+
+## HIRS
+The High Resolution Infrared Sounder (HIRS) is an instrument on the METOP satellites. It is a 20 channel infrared sounder and one of the three Advanced TIROS Operational Sounder (ATOVS) sensors. 
+
+The level 1B HIRS files contain the radiance of the 19 channels and the reflectance for channel 20.  See the [HIRS Level 1B product page](https://data.eumetsat.int/product/EO:EUM:DAT:MULT:HIRSL1) for more information.  
+This example shows how we plot the radiance from channel 7 on a map.
+
+This example is made using the following packages.
+```
+[13f3f980] CairoMakie v0.15.6
+[db073c08] GeoMakie v0.7.15
+```
+
+```julia
+using MetopDatasets
+using CairoMakie, GeoMakie, Statistics
+
+# Open the dataset.
+path = "HIRS_xxx_1B_M01_20160720074253Z_20160720092153Z_N_O_20160720083048Z.nat"
+ds = MetopDataset(path, maskingvalue=NaN);
+
+# read location
+latitude = ds["earth_location"][1,:,:]
+longitude = ds["earth_location"][2,:,:]
+
+# read radiance
+radiance_of_band = ds["digital_a_rad"][7,:,:] # Channel 7
+
+radiance_q05 = quantile(radiance_of_band[.!isnan.(radiance_of_band)],0.05)
+radiance_q95 = quantile(radiance_of_band[.!isnan.(radiance_of_band)],0.95)
+
+fig = let
+    # Create figure and axis
+    fig = Figure()
+    ax = GeoAxis(fig[1, 1],
+        title = "HIRS Channel 7",
+        xlabel = "longitude",
+        ylabel = "latitude")
+
+    # plot data with color map
+        scatter!(ax, longitude[:], latitude[:],
+        color = radiance_of_band[:], colorrange = (radiance_q05,radiance_q95), markersize = 2)
+
+    # Add colorbar
+    Colorbar(fig[1,2], colorrange = (radiance_q05,radiance_q95))
+
+    # Add coastlines
+    lines!(ax, GeoMakie.coastlines()) 
+    fig
+end
+```
+![HIRS channel 7](HIRS_plot.png)
+
+## MHS
+The Microwave Humidity Sounder (MHS) is an instrument on the METOP satellites. It measures the earth using 5 microwave channels sensitive to surface temperatures, emissivities, and atmospheric humidity. MHS is one of the three Advanced TIROS Operational Sounder (ATOVS) sensors.
+
+The level 1B MHS files contain the radiance of the 5 channels. See the [MHS Level 1B product page](https://data.eumetsat.int/product/EO:EUM:DAT:METOP:MHSL1) for more information.  
+This example shows how we compute the brightness temperature of channel 3 and plot it on a map. Note that `scene_radiances` can also be plotted directly and this will result in a very similar image but with a different color range. 
+
+This example is made using the following packages.
+```
+[13f3f980] CairoMakie v0.15.6
+[db073c08] GeoMakie v0.7.15
+```
+
+```julia
+
+using MetopDatasets
+using CairoMakie, GeoMakie, Statistics
+
+# Open the dataset.
+path = "MHSx_xxx_1B_M03_20250915084851Z_20250915103051Z_N_O_20250915102514Z.nat"
+ds = MetopDataset(path, maskingvalue=NaN);
+
+# select band to plot: 1,2,3,4 or 5
+band_to_plot = 3
+
+# read location
+latitude = ds["earth_location"][1,:,:]
+longitude = ds["earth_location"][2,:,:]
+
+# read radiance
+radiance_of_band = ds["scene_radiances"][band_to_plot,:,:]
+
+## convert radiance to brightness temperature.
+
+# get wave number of channel
+mhs_giadr = read_first_record(ds, MetopDatasets.GIADR_MHS_RADIANCE)
+wave_number_cm = get_scaled(mhs_giadr, "central_wavenumber_h$(band_to_plot)")
+
+# covert wave number from cm^-1 to m^-1
+wave_number_SI = 10.0^2 * wave_number_cm
+
+# covert radiance from mW/m2/sr/cm-1 to W/m2/sr/m-1
+radiance_SI = radiance_of_band * 10^(-5)
+
+T_brightness_uncorrected = brightness_temperature.(radiance_SI, wave_number_SI)
+
+# Apply linear corrections to the brightness temperature
+a_correction = get_scaled(mhs_giadr, "temperature_h$(band_to_plot)_intercept")
+b_correction = get_scaled(mhs_giadr, "temperature_h$(band_to_plot)_slope")
+T_brightness = a_correction .+ T_brightness_uncorrected .* b_correction
+
+# Plot the correct brightness temperature
+T_brightness_q05 = quantile(T_brightness[:],0.05)
+T_brightness_q95 = quantile(T_brightness[:],0.95)
+
+fig = let
+    # Create figure and axis
+    fig = Figure()
+    ax = GeoAxis(fig[1, 1],
+        title = "MHS band: H$(band_to_plot)",
+        xlabel = "longitude",
+        ylabel = "latitude")
+
+    # plot data with color map
+    scatter!(ax, longitude[:], latitude[:],
+        color = T_brightness[:], colorrange = (T_brightness_q05,T_brightness_q95), markersize = 1)
+
+    # Add colorbar
+    Colorbar(fig[1,2], colorrange = (T_brightness_q05,T_brightness_q95), label="K")
+
+    # Add coastlines
+    lines!(ax, GeoMakie.coastlines()) 
+    fig
+end
+```
+![MHS Brightness temperature](MHS_Tb_plot.png)
+The plot shows the brightness temperature for the MHS H3 channel (183.311 GHz) observed during an orbit.
+
+

docs/src/index.md

@@ -339,24 +339,29 @@ end
 ```
 
 ## Supported formats
+- AMSU-A Level 1B
 - ASCAT Level 1B
 - ASCAT Level 2 Soil Moisture
+- HIRS Level 1B
 - IASI Level 1C
 - IASI Level 2 Combined Sounding
+- MHS Level 1B 
 
 ### Formats not yet supported
-- AMSU-A Level 1B
 - AVHRR Level 1B
 - GOME-2 Level 1B
-- HIRS Level 1B 
-- MHS Level 1B 
+- IASI Level 1C Principal Component Scores
 
 ### Reference documents 
 - [EPS Generic Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_gen_pfs_13e3f0feb7.pdf)
+- [AMSU-A Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_amsu_a_l1_pfs_c89fa9d9ea.pdf)
 - [ASCAT Level 1: Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/ASCAT_Level_1_Product_Format_V12_Annexe_50fe72d349.pdf)
 - [ASCAT Level 2 Soil Moisture: Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_0343_eps_ascatl2_pfs_f509981295.pdf)
+- [HIRS Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_97230_eps_hirs4l1_pfs_0ddaefcb74.pdf)
 - [IASI Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_iasi_level_1_pfs_2105bc9ccf.pdf)
-- [IASI Level 2 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_980760_eps_iasi_l2_f9511c26d2.pdf)
+- [IASI Level 2: Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_980760_eps_iasi_l2_f9511c26d2.pdf)
+- [MHS Level 1 Product Format Specification](https://user.eumetsat.int/s3/eup-strapi-media/pdf_ten_97229_eps_mhs_pfs_2069b45efc.pdf)
+
 
 ## Development status and versioning
 The package was previously named **MetopNative.jl** and was hosted on the [EUMETSAT GitLab](https://gitlab.eumetsat.int/eumetlab/cross-cutting-tools/MetopNative.jl).

src/Instruments/ASCAT/ASCAT_records.jl

@@ -2,8 +2,6 @@
 # License: MIT
 
 ########### SZR ###########
-# Note, there is probably a better way to handle the path. @__DIR__ is a bit of a hack
-#  @__DIR__ ensure that the path is relative to the location of this file.
 const ASCA_SZR_1B_V13_format = @path joinpath(@__DIR__, "csv_formats/ASCA_SZR_1B_V13.csv")
 const ASCA_SZR_1B_V12_format = @path joinpath(@__DIR__, "csv_formats/ASCA_SZR_1B_V12.csv")
 const ASCA_SZR_1B_V11_format = @path joinpath(@__DIR__, "csv_formats/ASCA_SZR_1B_V11.csv")

src/Instruments/ATOVS/AMSU_A/AMSU_A.jl

@@ -0,0 +1,5 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+include("AMSU_A_records.jl")
+include("AMSU_A_dimensions.jl")

src/Instruments/ATOVS/AMSU_A/AMSU_A_dimensions.jl

@@ -0,0 +1,59 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+function get_dimensions(T::Type{<:AMSA_XXX_1B})
+    return Dict(
+        "lat_lon" => 2,
+        "channels" => 15,
+        "xtrack" => 30,
+        "roll_pitch_yaw" => 3,
+        "solar_angles" => 4,
+        "data_calibration" => 16,
+        "calibration_coefficients" => 3,
+        "reflector_readings" => 2,
+        "temperature_dim2" => 2,
+        "temperature_dim3" => 3,
+        "temperature_dim5" => 5,
+        "temperature_dim6" => 6,
+        "temperature_dim7" => 7
+    )
+end
+
+function get_field_dimensions(T::Type{<:AMSA_XXX_1B},
+        field_name::Symbol)::Vector{<:AbstractString}
+    if field_name in (DATA_CAL_QUALITY_NAME, DATA_CAL_NEDT_NAME)
+        return ["data_calibration"]
+    end
+
+    if !(fieldtype(T, field_name) <: Array)
+        return String[]
+    end
+
+    array_size = _get_array_size(T, field_name)
+
+    if array_size == (30,)
+        return ["xtrack"]
+    elseif array_size == (15, 30)
+        return ["channels", "xtrack"]
+    elseif array_size == (3, 15)
+        return ["calibration_coefficients", "channels"]
+    elseif array_size == (2, 30)
+        if field_name == :earth_location
+            return ["lat_lon", "xtrack"]
+        else
+            return ["reflector_readings", "xtrack"]
+        end
+    else
+        if field_name == :euler_angle
+            return ["roll_pitch_yaw"]
+        elseif field_name == :angular_relation
+            return ["solar_angles", "xtrack"]
+        elseif startswith(string(field_name), "reflector")
+            return ["reflector_readings"]
+        elseif only(array_size) in (2, 3, 5, 6, 7)
+            l = only(array_size)
+            return ["temperature_dim$l"]
+        end
+        error("Dimensions not set for $field_type")
+    end
+end

src/Instruments/ATOVS/AMSU_A/AMSU_A_records.jl

@@ -0,0 +1,16 @@
+# Copyright (c) 2025 EUMETSAT
+# License: MIT
+
+const AMSA_XXX_1B_V10_format = @path joinpath(@__DIR__, "csv_formats/AMSA_XXX_1B_V10.csv")
+
+abstract type AMSA_XXX_1B <: ATOVS_1B end
+# create data structure, add description and scale factors
+eval(record_struct_expression(AMSA_XXX_1B_V10_format, AMSA_XXX_1B))
+
+function data_record_type(header::MainProductHeader, product_type::Val{:AMSA_xxx_1B})::Type
+    if header.format_major_version == 10
+        return AMSA_XXX_1B_V10
+    else
+        error("No format found for format major version :$(header.format_major_version)")
+    end
+end