diff --git a/deps/ops/stubs/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc b/deps/ops/stubs/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc index 4a76565c..b7c6b3b7 100644 --- a/deps/ops/stubs/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc +++ b/deps/ops/stubs/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc @@ -313,6 +313,12 @@ SELECT CASE (ObsGroup) StashItem_p,StashItem_p_Surface,StashCode_u10, & StashCode_v10,StashCode_t2,StashCode_rh2,StashCode_pmsl/) + CASE (ObsGroupHIRAS2) + CxFields(1:14) = (/StashItem_theta,StashItem_q,StashItem_qcf,StashItem_SST, & + StashItem_SeaIce,StashItem_orog,StashItem_qcl, & + StashItem_p,StashItem_p_Surface,StashCode_u10, & + StashCode_v10,StashCode_t2,StashCode_rh2,StashCode_pmsl/) + CASE (ObsGroupGIIRSLW, ObsGroupGIIRSMW) CxFields(1:14) = (/StashItem_theta,StashItem_q,StashItem_qcf,StashItem_SST, & StashItem_SeaIce,StashItem_orog,StashItem_qcl, & diff --git a/deps/ops/stubs/OpsMod_Control/Ops_ReadObsGroupSwitches.inc b/deps/ops/stubs/OpsMod_Control/Ops_ReadObsGroupSwitches.inc index 9aad4ab9..fcd305d5 100644 --- a/deps/ops/stubs/OpsMod_Control/Ops_ReadObsGroupSwitches.inc +++ b/deps/ops/stubs/OpsMod_Control/Ops_ReadObsGroupSwitches.inc @@ -102,6 +102,7 @@ Using(ObsGroupGMIlow) = ops_env_is_true ("OPS_GMILOW") Using(ObsGroupGMIhigh) = ops_env_is_true ("OPS_GMIHIGH") Using(ObsGroupHLOSWIND) = ops_env_is_true ("OPS_HLOSWIND") Using(ObsGroupHIRAS) = ops_env_is_true ("OPS_HIRAS") +Using(ObsGroupHIRAS2) = ops_env_is_true ("OPS_HIRAS2") Using(ObsGroupOceanWinds) = ops_env_is_true ("OPS_OCEANWINDS") Using(ObsGroupGIIRSLW) = ops_env_is_true ("OPS_GIIRSLW") Using(ObsGroupGIIRSMW) = ops_env_is_true ("OPS_GIIRSMW") diff --git a/deps/ops/stubs/OpsMod_ODB/OpsFn_ObsToReportType.inc b/deps/ops/stubs/OpsMod_ODB/OpsFn_ObsToReportType.inc index 2b8fcacc..ab04757d 100644 --- a/deps/ops/stubs/OpsMod_ODB/OpsFn_ObsToReportType.inc +++ b/deps/ops/stubs/OpsMod_ODB/OpsFn_ObsToReportType.inc @@ -573,7 +573,7 @@ SELECT CASE (Obs % Header % ObsGroup) IF (Obs % Header % Satid % Present) THEN DO i = ob_start, ob_end IF (Obs % Satid(i) == SatId_FY3D) THEN - report_types(i - ob_start + 1) = rep_type_fy3_d_hiras_rad + report_types(i - ob_start + 1) = rep_type_fy3_d_hiras_rad ELSE found_report_type = .FALSE. DO j = 1, SIZE (map, DIM = 1) @@ -587,6 +587,24 @@ SELECT CASE (Obs % Header % ObsGroup) END IF END DO END IF + CASE (ObsGroupHIRAS2) + IF (Obs % Header % Satid % Present) THEN + DO i = ob_start, ob_end + IF (Obs % Satid(i) == SatId_FY3E) THEN + report_types(i - ob_start + 1) = rep_type_fy3e_hiras2_rad + ELSE + found_report_type = .FALSE. + DO j = 1, SIZE (map, DIM = 1) + IF (map(j,1) == ObsGroupHIRAS2 .AND. map(j,2) == Obs % Satid(i)) THEN + report_types(i - ob_start + 1) = map(j,3) + found_report_type = .TRUE. + EXIT + END IF + END DO + CALL update_unrecognized_satids (Obs % Satid(i)) + END IF + END DO + END IF CASE (ObsGroupGIIRSLW, ObsGroupGIIRSMW) IF (Obs % Header % Satid % Present) THEN DO i = ob_start, ob_end diff --git a/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp.inc b/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp.inc index 8cd30a4f..cbbec080 100644 --- a/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp.inc +++ b/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp.inc @@ -33,7 +33,8 @@ USE OpsMod_ObsGroupInfo, ONLY: & ObsGroupATMS, & ObsGroupCrIS, & ObsGroupHIRAS, & - ObsGroupGIIRSLW, & + ObsGroupHIRAS2, & + ObsGroupGIIRSLW, & ObsGroupGIIRSMW, & ObsGroupOcean, & ObsGroupSatwind, & @@ -407,7 +408,8 @@ ElemDesp2ODBDesp(IndexScanLine) % data_type = odb_integer ElemDesp2ODBDesp(IndexScanLine) % has_missing = c_false IF (ObsGroup == ObsGroupCrIS .or. ObsGroup == ObsGroupHIRAS .or. & - ObsGroup == ObsGroupGIIRSLW .or. ObsGroup == ObsGroupGIIRSMW) THEN + ObsGroup == ObsGroupHIRAS2 .or. ObsGroup == ObsGroupGIIRSLW .or. & + ObsGroup == ObsGroupGIIRSMW) THEN ElemDesp2ODBDesp(IndexPixelNumber) % TableName = 'radiance' ElemDesp2ODBDesp(IndexPixelNumber) % ColName = 'field_of_regard' ElemDesp2ODBDesp(IndexPixelNumber) % data_type = odb_real diff --git a/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp_Body.inc b/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp_Body.inc index 06f72446..61edf7d4 100644 --- a/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp_Body.inc +++ b/deps/ops/stubs/OpsMod_ODB/OpsODB_InitODBElemDesp_Body.inc @@ -27,6 +27,7 @@ USE OpsMod_ObsGroupInfo, ONLY: & ObsGroupCOMSMIClr, & ObsGroupCrIS, & ObsGroupHIRAS, & + ObsGroupHIRAS2, & ObsGroupMWSFY3B, & ObsGroupGeoCloud, & ObsGroupGIIRSLW, & @@ -359,6 +360,24 @@ IF (ObsGroup == ObsGroupHIRAS) THEN ElemDesp2ODBDesp(IndexHIRASBrightnessTemp) % Varno = ops_odb_getval (h, "$rawbt") END IF +IF (ObsGroup == ObsGroupHIRAS2) THEN + ElemDesp2ODBDesp(IndexRadiance) % TableName = 'body' + ElemDesp2ODBDesp(IndexRadiance) % ColName = 'initial_obsvalue' + ElemDesp2ODBDesp(IndexRadiance) % Varno = ops_odb_getval (h, "$rawsca") + ElemDesp2ODBDesp(IndexRadiance) % also_do(1:7) = (/IndexPstar, & + IndexSkinTemperature, & + IndexRH, & + IndexTemp, & + IndexRH2, & + IndexT2, & + IndexHIRAS2BrightnessTemp/) + ElemDesp2ODBDesp(IndexRadiance) % also_do_levels(1:7) = (/1,1,43,43,1,1,431/) + + ElemDesp2ODBDesp(IndexHIRAS2BrightnessTemp) % TableName = 'body' + ElemDesp2ODBDesp(IndexHIRAS2BrightnessTemp) % ColName = 'initial_obsvalue' + ElemDesp2ODBDesp(IndexHIRAS2BrightnessTemp) % Varno = ops_odb_getval (h, "$rawbt") +END IF + IF (ObsGroup == ObsGroupGIIRSLW .or. ObsGroup == ObsGroupGIIRSMW) THEN ElemDesp2ODBDesp(IndexRadiance) % TableName = 'body' ElemDesp2ODBDesp(IndexRadiance) % ColName = 'initial_obsvalue' @@ -1333,7 +1352,7 @@ IF (ObsGroup == ObsGroupGMIlow .OR. ObsGroup == ObsGroupGMIhigh) THEN ElemDesp2ODBDesp(IndexGMIBrightnessTemp) % TableName = 'body' ElemDesp2ODBDesp(IndexGMIBrightnessTemp) % ColName = 'initial_obsvalue' ElemDesp2ODBDesp(IndexGMIBrightnessTemp) % Varno = ops_odb_getval (h, "$rawbt") ! brightness temperature - ElemDesp2ODBDesp(IndexGMIBrightnessTemp) % also_do(1:5) = (/IndexSkinTemperature, & + ElemDesp2ODBDesp(IndexGMIBrightnessTemp) % also_do(1:5) = (/IndexSkinTemperature, & IndexRH, & IndexTemp, & IndexRH2, & diff --git a/deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc b/deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc index d677f265..e6b78c6b 100644 --- a/deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc +++ b/deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc @@ -119,6 +119,12 @@ SELECT CASE (ObsGroup) Varfield_numchans,Varfield_channum,Varfield_britemp, & Varfield_emissivity,Varfield_solzenith, & Varfield_qcinfo/) + CASE (ObsGroupHIRAS2) + Varfields(1:12) = (/Varfield_tskin,Varfield_tcozone,Varfield_satzenith, & + Varfield_surface,Varfield_stratt,Varfield_satid, & + Varfield_numchans,Varfield_channum,Varfield_britemp, & + Varfield_emissivity,Varfield_solzenith, & + Varfield_qcinfo/) IF (.NOT. RTTOV_CloudSwitch) THEN Varfields(13:14) = (/Varfield_cloudtopp, Varfield_cloudfrac/) END IF diff --git a/deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarobsLevDepC.inc b/deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarobsLevDepC.inc index dc01534f..76ec26f3 100644 --- a/deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarobsLevDepC.inc +++ b/deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarobsLevDepC.inc @@ -75,6 +75,7 @@ USE OpsMod_ObsGroupInfo, ONLY: & ObsGroupGMIhigh, & ObsGroupHLOSWIND, & ObsGroupHIRAS, & + ObsGroupHIRAS2, & ObsGroupOceanWinds, & ObsGroupGIIRSLW, & ObsGroupGIIRSMW, & @@ -323,7 +324,7 @@ SELECT CASE (Observations % header % ObsGroup) NumLevelLevs = 1 CASE (ObsGroupHLOSWIND) - + ObsLevelType = 0.0 NumLevelLevs = NumObLev @@ -332,6 +333,10 @@ SELECT CASE (Observations % header % ObsGroup) ObsLevelType = 0.0 NumLevelLevs = 1 + CASE (ObsGroupHIRAS2) + ObsLevelType = 0.0 + NumLevelLevs = 1 + CASE (ObsGroupOceanWinds) ObsLevelType = 0.0 @@ -341,12 +346,12 @@ SELECT CASE (Observations % header % ObsGroup) ObsLevelType = 0.0 NumLevelLevs = 1 - + CASE (ObsGroupSternaMWR) ObsLevelType = 0.0 NumLevelLevs = 1 - + CASE (ObsGroupEPSMWS) ObsLevelType = 0.0 diff --git a/deps/ops/stubs/Ops_Constants/OpsFn_IsSatRadGroup.inc b/deps/ops/stubs/Ops_Constants/OpsFn_IsSatRadGroup.inc index 242f5fcc..843ca577 100644 --- a/deps/ops/stubs/Ops_Constants/OpsFn_IsSatRadGroup.inc +++ b/deps/ops/stubs/Ops_Constants/OpsFn_IsSatRadGroup.inc @@ -28,6 +28,7 @@ IF (obsgroup == ObsGroupABIClr .OR. & obsgroup == ObsGroupFCIClr .OR. & obsgroup == ObsGroupFCIASR .OR. & obsgroup == ObsGroupHIRAS .OR. & + obsgroup == ObsGroupHIRAS2 .OR. & obsgroup == ObsGroupGeoCloud .OR. & obsgroup == ObsGroupGOESImClr .OR. & obsgroup == ObsGroupIASI .OR. & diff --git a/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNameToNum.inc b/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNameToNum.inc index 3e163045..ee5e1134 100644 --- a/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNameToNum.inc +++ b/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNameToNum.inc @@ -50,6 +50,8 @@ SELECT CASE (ObsGroupName) OpsFn_ObsGroupNameToNum = ObsGroupFCIClr CASE ("HIRAS") OpsFn_ObsGroupNameToNum = ObsGroupHIRAS + CASE ("HIRAS2") + OpsFn_ObsGroupNameToNum = ObsGroupHIRAS2 CASE ("MWSFY3B") OpsFn_ObsGroupNameToNum = ObsGroupMWSFY3B CASE ("GeoCloud") diff --git a/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNumToName.inc b/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNumToName.inc index d5e187fa..d29c7362 100644 --- a/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNumToName.inc +++ b/deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNumToName.inc @@ -27,6 +27,8 @@ SELECT CASE (ObsGroup) OpsFn_ObsGroupNumToName = "AIRS" CASE (ObsGroupHIRAS) OpsFn_ObsGroupNumToName = "HIRAS" + CASE (ObsGroupHIRAS2) + OpsFn_ObsGroupNumToName = "HIRAS2" CASE (ObsGroupHLOSwind) OpsFn_ObsGroupNumToName = "HLOSwind" CASE (ObsGroupAltim) diff --git a/deps/ops/stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90 b/deps/ops/stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90 index 06b0851d..97a8b725 100644 --- a/deps/ops/stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90 +++ b/deps/ops/stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90 @@ -68,7 +68,7 @@ MODULE OpsMod_ObsGroupInfo INTEGER, PARAMETER :: ObsGroupGMIhigh = 57 ! => GMI high freq channels INTEGER, PARAMETER :: ObsGroupHLOSwind = 58 ! => ALADIN-AEOLUS HLOS wind INTEGER, PARAMETER :: ObsGroupOceanColour = 59 ! => Ocean Colour Data -INTEGER, PARAMETER :: ObsGroupHIRAS = 60 ! => HIRAS +INTEGER, PARAMETER :: ObsGroupHIRAS = 60 ! => HIRAS INTEGER, PARAMETER :: ObsGroupOceanWinds = 61 ! => Satellite wind speeds INTEGER, PARAMETER :: ObsGroupGIIRSLW = 62 ! => GIIRS LW radiances INTEGER, PARAMETER :: ObsGroupGIIRSMW = 63 ! => GIIRS MW radiances @@ -79,9 +79,10 @@ MODULE OpsMod_ObsGroupInfo INTEGER, PARAMETER :: ObsGroupFCIASR = 68 ! => FCI all-sky radiances INTEGER, PARAMETER :: ObsGroupFCIClr = 69 ! => FCI clear-sky radiances INTEGER, PARAMETER :: ObsGroupSternaMWR = 70 ! => AWS & Sterna microwave radiances -INTEGER, PARAMETER :: ObsGroupEPSMWS = 71 ! => EPS SG MWS +INTEGER, PARAMETER :: ObsGroupEPSMWS = 71 ! => EPS SG MWS INTEGER, PARAMETER :: ObsGroupMWSFY3E = 72 ! => FY3E microwave sounder data -INTEGER, PARAMETER :: max_obs_group_num = 72 ! Number of observation groups +INTEGER, PARAMETER :: ObsGroupHIRAS2 = 74 ! => HIRAS Radiances +INTEGER, PARAMETER :: max_obs_group_num = 74 ! Number of observation groups CONTAINS diff --git a/deps/ops/stubs/Ops_Constants/OpsMod_ObsTypes.f90 b/deps/ops/stubs/Ops_Constants/OpsMod_ObsTypes.f90 index a3814f63..e5dc6d40 100644 --- a/deps/ops/stubs/Ops_Constants/OpsMod_ObsTypes.f90 +++ b/deps/ops/stubs/Ops_Constants/OpsMod_ObsTypes.f90 @@ -285,15 +285,16 @@ MODULE OpsMod_ObsTypes INTEGER, PARAMETER :: ObsTypeGMIlow = 29100 ! => GMI low freq channels INTEGER, PARAMETER :: ObsTypeGMIhigh = 29200 ! => GMI high freq channels -INTEGER, PARAMETER :: ObsTypeHIRAS = 29300 ! => HIRAS -INTEGER, PARAMETER :: ObsTypeHIRASFSR = 29301 ! => HIRAS FSR +INTEGER, PARAMETER :: ObsTypeHIRAS = 29300 ! => HIRAS +INTEGER, PARAMETER :: ObsTypeHIRASFSR = 29301 ! => HIRAS FSR INTEGER, PARAMETER :: ObsTypeHIRASHR = 29302 ! => HIRAS (UK) INTEGER, PARAMETER :: ObsTypeHIRASFSRHR = 29303 ! => HIRAS FSR (UK) +INTEGER, PARAMETER :: ObsTypeHIRAS2 = 29310 ! => HIRAS2 INTEGER, PARAMETER :: ObsTypeGIIRS = 29400 ! => GIIRS INTEGER, PARAMETER :: ObsTypeAWS = 29600 ! => Arctic weather satellite radiances - + INTEGER, PARAMETER :: ObsTypeMWSSG = 29700 ! => EPSSG MWS radiances ! Aircraft data diff --git a/deps/ops/stubs/Ops_Constants/Ops_SubTypeNameToNum.inc b/deps/ops/stubs/Ops_Constants/Ops_SubTypeNameToNum.inc index 932e81ef..9d9705bc 100644 --- a/deps/ops/stubs/Ops_Constants/Ops_SubTypeNameToNum.inc +++ b/deps/ops/stubs/Ops_Constants/Ops_SubTypeNameToNum.inc @@ -116,7 +116,7 @@ SELECT CASE (name) num = ObsTypeESAUWI CASE ("FCIASR") num = ObsTypeFCIEUMASR - CASE ("FCIRADUK") + CASE ("FCIRADUK") num = ObsTypeFCIAUTOUK CASE ("GAUGE") num = ObsTypeGauge diff --git a/deps/ops/stubs/Ops_Constants/Ops_SubTypeNumToName.inc b/deps/ops/stubs/Ops_Constants/Ops_SubTypeNumToName.inc index 1710eb6e..df77282e 100644 --- a/deps/ops/stubs/Ops_Constants/Ops_SubTypeNumToName.inc +++ b/deps/ops/stubs/Ops_Constants/Ops_SubTypeNumToName.inc @@ -97,6 +97,8 @@ SELECT CASE (num) name = "CRIMSS" CASE (ObsTypeHIRASHR, ObsTypeHIRASFSRHR) name = "CRIMSSHR" + CASE (ObsTypeHIRAS2) + name = "CRIMSF" CASE (ObsTypeDropsonde) name = "DROPSOND" CASE (ObsTypeRadwind) diff --git a/etc/global/cx/HIRAS2.nl b/etc/global/cx/HIRAS2.nl new file mode 100644 index 00000000..7e3c5a63 --- /dev/null +++ b/etc/global/cx/HIRAS2.nl @@ -0,0 +1,3 @@ +&CXControlNL +CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222 +/ diff --git a/etc/global/varobs/HIRAS2.nl b/etc/global/varobs/HIRAS2.nl new file mode 100644 index 00000000..e2e9ed74 --- /dev/null +++ b/etc/global/varobs/HIRAS2.nl @@ -0,0 +1,3 @@ +&VarobsControlNL +Varfields=10,11,18,19,21,28,31,35,36,54,55,57,80 +/ diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py index 5f675741..31446377 100644 --- a/test/generate_unittest_netcdfs.py +++ b/test/generate_unittest_netcdfs.py @@ -253,7 +253,7 @@ def output_1d_multi_level_simulated_var_to_netcdf(var_name, file_name): # and the second six correspond to the averaged profiles. var[:] = [1.2, missing_float_nc, 1.4, 1.5, 1.6, 1.7, 1.25, 1.35, 1.45, 1.55, 1.65, 1.75] - + f._ioda_layout = "ObsGroup" f._ioda_layout_version = 0 @@ -553,7 +553,7 @@ def output_simulated_var_profiles_to_netcdf(var_name, file_name): 100300, 90400, 90200, 100400, 80300, 100500, 90500] var = f.createVariable('MetaData/dateTime', np.int64, ('Location')) var.units = iso8601_string - + # The NetCDF4 module doesn't support assigning values to variable-length string variables # using the `var[:] = ...` syntax, so we do it using a loop for i, s in enumerate(["2018-01-01T00:01:01Z", "2018-01-01T00:02:03Z", @@ -824,7 +824,7 @@ def copy_var_to_var(Group, invarname, outvarname, filename): ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], 'testinput/varobs_globalnamelist_atovs.nc4') - # Sterna MWR + # Sterna MWR output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle', 'MetaData/solarZenithAngle'], ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature', 'Emiss/emissivity', @@ -832,15 +832,15 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'], ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], 'testinput/varobs_globalnamelist_sternamwr.nc4') - - # EPSMWS + + # EPSMWS output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle', 'MetaData/solarZenithAngle'], ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature', 'Emiss/emissivity', 'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature', 'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'], ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], - 'testinput/varobs_globalnamelist_epsmws.nc4') + 'testinput/varobs_globalnamelist_epsmws.nc4') # GNSS-RO @@ -981,6 +981,19 @@ def copy_var_to_var(Group, invarname, outvarname, filename): ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier', 'MetaData/observationSubTypeNum'], 'testinput/varobs_globalnamelist_mtgirs.nc4') + # HIRAS2 - this tests the variable_for_quality_control option + output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', + 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle', + 'MetaData/solarZenithAngle', 'OutputToVAR/pressureAtTopOfCloud', 'OneDVar/cloudAmount', + 'MetaData/ozoneTotal'], + ['ObsValue/radiance', 'DerivedObsValue/brightnessTemperature', 'EffectiveError/brightnessTemperature', + 'OneDVar/emissivity', 'BiasCorrObsValue/brightnessTemperature', + 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature', + 'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'], + ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier', 'MetaData/observationSubTypeNum'], + 'testinput/varobs_globalnamelist_hiras2.nc4') + + # Aircraft output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', @@ -1115,22 +1128,22 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels', 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_atovs.nc4') - - # EPSMWS + + # EPSMWS output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'], ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water', 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels', 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_epsmws.nc4') - # SternaMWR + # SternaMWR output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'], ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water', 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels', 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_sternamwr.nc4') - + # SSMIS output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'], @@ -1154,7 +1167,7 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water','air_pressure_levels', 'cloud_volume_fraction_in_atmosphere_layer','liquid_cloud_volume_fraction_in_atmosphere_layer','ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_mwsfy3e.nc4') - + # GNSS-RO output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'], @@ -1270,6 +1283,14 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_mtgirs.nc4') + # HIRAS2 - this tests the variable_for_quality_control option + output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', + 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'], + ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water', + 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels', + 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], + 'testinput/cx_globalnamelist_hiras2.nc4') + # GroundGPS output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m', 'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m'], @@ -1349,4 +1370,3 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'dimensionless_exner_function_levels', 'qrain'], 'testinput/cx_ukvnamelist_radar_reflectivity.nc4') - diff --git a/test/testinput/cx_globalnamelist_hiras2.nc4 b/test/testinput/cx_globalnamelist_hiras2.nc4 new file mode 100644 index 00000000..50828bef Binary files /dev/null and b/test/testinput/cx_globalnamelist_hiras2.nc4 differ diff --git a/test/testinput/cxwriter_globalnamelist_hiras2.yaml b/test/testinput/cxwriter_globalnamelist_hiras2.yaml new file mode 100644 index 00000000..e3e487b4 --- /dev/null +++ b/test/testinput/cxwriter_globalnamelist_hiras2.yaml @@ -0,0 +1,58 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T01:00:00Z + +observations: + - obs space: + name: HIRAS2 + obsdatain: + engine: + type: H5File + obsfile: Data/varobs_globalnamelist_iasi.nc4 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &all_channels 1, 3 + geovals: + filename: Data/cx_globalnamelist_hiras2.nc4 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + ### Assign nominal error (ensure not missing values) ### + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter: 10.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *all_channels + where: + - variable: + name: MetaData/latitude #ensures ob2 is failed + value: is_not_valid + - filter: Cx Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: *all_channels + namelist_directory: ../etc/global/cx + reject_obs_with_all_variables_failing_qc: true + general_mode: debug + IC_PLevels: 5 + - filter: Cx Checker + expected_surface_variables: ["1","2","3","4","5","6","13","16","17"] + expected_upper_air_variables: ["1","5","11","29","30","31","34","35"] + expected_main_table_columns: + - # batch 1 - 9 1dvalues first ; then 2d values in expected order + # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns + - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"] # column 1 - 1st observation + - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"] # column 3 - 3rd observation + - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column 4 - 4th observation + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/varobs_globalnamelist_hiras2.nc4 b/test/testinput/varobs_globalnamelist_hiras2.nc4 new file mode 100644 index 00000000..131860ff Binary files /dev/null and b/test/testinput/varobs_globalnamelist_hiras2.nc4 differ diff --git a/test/testinput/varobswriter_globalnamelist_hiras2.yaml b/test/testinput/varobswriter_globalnamelist_hiras2.yaml new file mode 100644 index 00000000..2e70b9b6 --- /dev/null +++ b/test/testinput/varobswriter_globalnamelist_hiras2.yaml @@ -0,0 +1,68 @@ +time window: + begin: 1999-01-01T00:00:00Z + end: 2032-01-01T02:00:00Z + +observations: + - obs space: + name: HIRAS2 + obsdatain: + engine: + type: H5File + obsfile: Data/varobs_globalnamelist_hiras2.nc4 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &all_channels 1, 3 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + ### Assign nominal error (ensure not missing values) ### + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter: 10.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *all_channels + where: + - variable: + name: MetaData/latitude #ensures ob2 is failed + value: is_not_valid + - filter: VarObs Writer + namelist_directory: ../etc/global/varobs + variables_for_quality_control: + - name: brightnessTemperature + channels: *all_channels + reject_obs_with_all_variables_failing_qc: true + general_mode: verbose + output_varbc_predictors: true + size_of_varobs_array: 4 + fill_obstype_from_ops_subtype: true + - filter: VarObs Checker + expected_main_table_columns: + # Only channels 1 and 3 are simulated and hence written to the VarObs file. + # In the arrays below, rows denote locations and columns are the varobs channels are consecutive. + field: ["10","10","10","10","11","18","19","21","28","31","35","36","54","55","55","57","57","57","57","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","10","10","11","18","19","21","28","31","35","36","54","55","55","57","57","57","57","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","10","10","11","18","19","21","28","31","35","36","54","55","55","57","57","57","57","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80"] + level: ["1","2","3","4","1","1","1","1","1","1","1","1","1","1","2","1","2","3","4","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","3","4","1","1","1","1","1","1","1","1","1","1","2","1","2","3","4","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","3","4","1","1","1","1","1","1","1","1","1","1","2","1","2","3","4","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31"] + ob value: ["41.10000","41.30000","-1073741824.00000","-1073741824.00000","27.10000","77.10000","37.10000","3.00000","13.00000","47.10000","57.10000","67.10000","2.00000","1.00000","2.00000","31.10000","31.30000","-1073741824.00000","-1073741824.00000","0.00000","112.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "43.10000","43.30000","-1073741824.00000","-1073741824.00000","27.30000","77.30000","37.30000","7.00000","17.00000","47.30000","57.30000","67.30000","2.00000","1.00000","2.00000","33.10000","33.30000","-1073741824.00000","-1073741824.00000","0.00000","116.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "44.10000","44.30000","-1073741824.00000","-1073741824.00000","27.40000","77.40000","37.40000","9.00000","19.00000","47.40000","57.40000","67.40000","2.00000","1.00000","2.00000","34.10000","34.30000","-1073741824.00000","-1073741824.00000","0.00000","118.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000"] + lat: ["7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000", + "7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000", + "7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000"] + lon: ["17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000", + "17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000", + "17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000"] + HofX: ObsValue + benchmarkFlag: 1000 + flaggedBenchmark: 0