Skip to content

Commit

Permalink
Dynamic mapping templates depending on iw
Browse files Browse the repository at this point in the history
  • Loading branch information
@RoryPTB
RoryPTB committed Jul 7, 2023
1 parent 10254de commit 7ca718d
Show file tree
Hide file tree
Showing 4 changed files with 209 additions and 38 deletions.
62 changes: 36 additions & 26 deletions synop2bufr/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
'block_no', 'station_no', 'station_id', 'region',
'WMO_station_type',
'lowest_cloud_base', 'visibility', 'cloud_cover',
'time_significance', 'wind_time_period',
'wind_indicator', 'wind_time_period',
'wind_direction', 'wind_speed', 'air_temperature',
'dewpoint_temperature', 'relative_humidity', 'station_pressure',
'isobaric_surface', 'geopotential_height', 'sea_level_pressure',
Expand Down Expand Up @@ -77,17 +77,20 @@
'net_short_wave_radiation_1hr', 'net_short_wave_radiation_24hr',
'direct_solar_radiation_1hr', 'direct_solar_radiation_24hr',
'precipitation_s3', 'ps3_time_period', 'precipitation_24h',
'highest_gust_1', 'highest_gust_2', 'hg2_time_period']
'highest_gust_1', 'highest_gust_2']

# Build the dictionary template
synop_template = dict.fromkeys(_keys)

THISDIR = os.path.dirname(os.path.realpath(__file__))
MAPPINGS = f"{THISDIR}{os.sep}resources{os.sep}synop-mappings.json"
MAPPINGS_307080 = f"{THISDIR}{os.sep}resources{os.sep}synop-mappings-307080.json"
MAPPINGS_307096 = f"{THISDIR}{os.sep}resources{os.sep}synop-mappings-307096.json"

# Load template mappings file, this will be updated for each message.
with open(MAPPINGS) as fh:
_mapping = json.load(fh)
# Load template mappings files, this will be updated for each message.
with open(MAPPINGS_307080) as fh:
_mapping_307080 = json.load(fh)
with open(MAPPINGS_307096) as fh:
_mapping_307096 = json.load(fh)


def parse_synop(message: str, year: int, month: int) -> dict:
Expand Down Expand Up @@ -144,7 +147,13 @@ def parse_synop(message: str, year: int, month: int) -> dict:
else:
output['minute'] = 0

# ! Removed wind instrument indicator as it is not used in 307096
# Translate wind instrument flag from the SYNOP code to the BUFR code
if 'wind_indicator' in decoded:
try:
iw = decoded['wind_indicator']['value']
output['wind_indicator'] = iw
except Exception:
output['wind_indicator'] = None

if 'station_id' in decoded:
try:
Expand Down Expand Up @@ -221,7 +230,6 @@ def parse_synop(message: str, year: int, month: int) -> dict:
# See B/C1.10.5.3
# NOTE: Every time period in the following code shall be a negative number, # noqa
# to indicate measurements have been taken up until the present.
output['time_significance'] = 2
output['wind_time_period'] = -10

try:
Expand Down Expand Up @@ -1030,18 +1038,6 @@ def rad_convert(rad, time):
output['highest_gust_2'] = decoded['highest_gust']['gust_2']['speed']['value'] # noqa
except Exception:
output['highest_gust_2'] = None
# Regulation 6/12.12.2 in the WMO regional guide tells us that the
# 1st max gust speed has fixed time period 10 minutes, and the 2nd has
# time period equal to the length of the observation time in minutes.
# NOTE: All time periods must be negative
if hr % 6 == 0:
output['hg2_time_period'] = -6*60
elif hr % 3 == 0:
output['hg2_time_period'] = -3*60
elif hr % 2 == 0:
output['hg2_time_period'] = -2*60
else:
output['hg2_time_period'] = -60

# ! SECTION 4

Expand Down Expand Up @@ -1230,11 +1226,6 @@ def transform(data: str, metadata: str, year: int,
# create dictionary to store / return result in
result = dict()

# get mapping template, this needs to be reloaded everytime as each
# SYNOP can have a different number of
# replications
mapping = deepcopy(_mapping)

# parse data to dictionary and get number of section 3 and 4
# clouds
try:
Expand All @@ -1246,6 +1237,24 @@ def transform(data: str, metadata: str, year: int,
LOGGER.error(f"Error parsing SYNOP report: {message}. {str(e)}")
continue

# Now determine and load the appropriate mappings
# file depending on the value of the wind indicator.
# This will be updated for each message.
if msg['wind_indicator'] in [1, 4]:
# Use the new template if wind is measured
bufr_template = 307096
# Get mapping template, this needs to be
# reloaded everytime as each SYNOP can have a
# different number of replications
mapping = deepcopy(_mapping_307096)
else:
# Use the old template otherwise
bufr_template = 308070
# Get mapping template, this needs to be
# reloaded everytime as each SYNOP can have a
# different number of replications
mapping = deepcopy(_mapping_307080)

# set WSI
try:
wsi = tsi_mapping[tsi]
Expand Down Expand Up @@ -1343,7 +1352,7 @@ def transform(data: str, metadata: str, year: int,
# At this point we have a dictionary for the data, a
# dictionary of the mappings and the metadata
# The last step is to convert to BUFR.
unexpanded_descriptors = [301150, 307096]
unexpanded_descriptors = [301150, bufr_template]
short_delayed_replications = []
# update replications
delayed_replications = [max(1, num_s3_clouds), max(1, num_s4_clouds)]
Expand Down Expand Up @@ -1429,6 +1438,7 @@ def transform(data: str, metadata: str, year: int,
"data_category": message.get_element("dataCategory")
},
"result": status,
"template": bufr_template,
"csv": csv_string
}

Expand Down
126 changes: 126 additions & 0 deletions synop2bufr/resources/synop-mappings-307080.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
{
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [1,1],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"names_on_row": 1,
"wigos_station_identifier": "data:Station_ID",
"header":[
{"eccodes_key": "edition", "value": "const:4"},
{"eccodes_key": "masterTableNumber", "value": "const:0"},
{"eccodes_key": "bufrHeaderCentre", "value": "const:0"},
{"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"},
{"eccodes_key": "updateSequenceNumber", "value": "const:0"},
{"eccodes_key": "dataCategory", "value": "const:0"},
{"eccodes_key": "internationalDataSubCategory", "value": "const:2"},
{"eccodes_key": "masterTablesVersionNumber", "value": "const:30"},
{"eccodes_key": "numberOfSubsets", "value": "const:1"},
{"eccodes_key": "observedData", "value": "const:1"},
{"eccodes_key": "compressedData", "value": "const:0"},
{"eccodes_key": "typicalYear", "value": "data:year"},
{"eccodes_key": "typicalMonth", "value": "data:month"},
{"eccodes_key": "typicalDay", "value": "data:day"},
{"eccodes_key": "typicalHour", "value": "data:hour"},
{"eccodes_key": "typicalMinute", "value": "data:minute"},
{"eccodes_key": "unexpandedDescriptors", "value":"array:301150, 307080"}
],
"data": [
{"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:_wsi_series"},
{"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:_wsi_issuer"},
{"eccodes_key": "#1#wigosIssueNumber", "value":"data:_wsi_issue_number"},
{"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:_wsi_local"},
{"eccodes_key": "#1#latitude", "value": "data:_latitude"},
{"eccodes_key": "#1#longitude", "value": "data:_longitude"},
{"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:_station_height"},
{"eccodes_key": "#1#blockNumber", "value": "data:block_no"},
{"eccodes_key": "#1#stationNumber", "value": "data:station_no"},
{"eccodes_key": "#1#stationOrSiteName", "value": "data:station_id"},
{"eccodes_key": "#1#stationType", "value": "data:WMO_station_type"},
{"eccodes_key": "#1#year", "value": "data:year"},
{"eccodes_key": "#1#month", "value": "data:month"},
{"eccodes_key": "#1#day", "value": "data:day"},
{"eccodes_key": "#1#hour", "value": "data:hour"},
{"eccodes_key": "#1#minute", "value": "data:minute"},
{"eccodes_key": "#1#nonCoordinatePressure", "value": "data:station_pressure"},
{"eccodes_key": "#1#pressureReducedToMeanSeaLevel", "value": "data:sea_level_pressure"},
{"eccodes_key": "#1#3HourPressureChange", "value": "data:3hr_pressure_change"},
{"eccodes_key": "#1#characteristicOfPressureTendency", "value": "data:pressure_tendency_characteristic"},
{"eccodes_key": "#1#24HourPressureChange", "value": "data:24hr_pressure_change"},
{"eccodes_key": "#1#pressure", "value": "data:isobaric_surface"},
{"eccodes_key": "#1#nonCoordinateGeopotentialHeight", "value": "data:geopotential_height"},
{"eccodes_key": "#1#airTemperature", "value": "data:air_temperature"},
{"eccodes_key": "#1#dewpointTemperature", "value": "data:dewpoint_temperature"},
{"eccodes_key": "#1#relativeHumidity", "value": "data:relative_humidity"},
{"eccodes_key": "#1#horizontalVisibility", "value": "data:visibility"},
{"eccodes_key": "#1#totalPrecipitationPast24Hours", "value": "data:precipitation_24h"},
{"eccodes_key": "#1#cloudCoverTotal", "value": "data:cloud_cover"},
{"eccodes_key": "#1#verticalSignificanceSurfaceObservations", "value": "data:cloud_vs_s1"},
{"eccodes_key": "#1#cloudAmount", "value": "data:cloud_amount_s1"},
{"eccodes_key": "#1#heightOfBaseOfCloud", "value": "data:lowest_cloud_base"},
{"eccodes_key": "#1#cloudType", "value": "data:low_cloud_type"},
{"eccodes_key": "#2#cloudType", "value": "data:middle_cloud_type"},
{"eccodes_key": "#3#cloudType", "value": "data:high_cloud_type"},
{"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7"},
{"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8"},
{"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9"},
{"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:low_cloud_drift_direction"},
{"eccodes_key": "#2#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:middle_cloud_drift_direction"},
{"eccodes_key": "#3#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:high_cloud_drift_direction"},
{"eccodes_key": "#5#verticalSignificanceSurfaceObservations", "value": "const:7"},
{"eccodes_key": "#6#verticalSignificanceSurfaceObservations", "value": "const:8"},
{"eccodes_key": "#7#verticalSignificanceSurfaceObservations", "value": "const:9"},
{"eccodes_key": "#1#stateOfGround", "value": "data:ground_state"},
{"eccodes_key": "#1#totalSnowDepth", "value": "data:snow_depth"},
{"eccodes_key": "#1#groundMinimumTemperaturePast12Hours", "value": "data:ground_temperature"},
{"eccodes_key": "#1#presentWeather", "value": "data:present_weather"},
{"eccodes_key": "#1#pastWeather1", "value": "data:past_weather_1"},
{"eccodes_key": "#1#pastWeather2", "value": "data:past_weather_2"},
{"eccodes_key": "#1#timePeriod", "value": "data:past_weather_time_period"},
{"eccodes_key": "#2#timePeriod", "value": "const:-1"},
{"eccodes_key": "#1#totalSunshine", "value": "data:sunshine_amount_1hr"},
{"eccodes_key": "#3#timePeriod", "value": "const:-24"},
{"eccodes_key": "#2#totalSunshine", "value": "data:sunshine_amount_24hr"},
{"eccodes_key": "#4#timePeriod", "value": "data:ps1_time_period"},
{"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value": "data:precipitation_s1"},
{"eccodes_key": "#5#timePeriod", "value": "data:ps3_time_period"},
{"eccodes_key": "#2#totalPrecipitationOrTotalWaterEquivalent", "value": "data:precipitation_s3"},
{"eccodes_key": "#6#timePeriod", "value": "data:maximum_temperature_period_start"},
{"eccodes_key": "#7#timePeriod", "value": "data:maximum_temperature_period_end"},
{"eccodes_key": "#1#maximumTemperatureAtHeightAndOverPeriodSpecified", "value": "data:maximum_temperature"},
{"eccodes_key": "#8#timePeriod", "value": "data:minimum_temperature_period_start"},
{"eccodes_key": "#9#timePeriod", "value": "data:minimum_temperature_period_end"},
{"eccodes_key": "#1#minimumTemperatureAtHeightAndOverPeriodSpecified", "value": "data:minimum_temperature"},
{"eccodes_key": "#1#instrumentationForWindMeasurement", "value": "data:wind_indicator"},
{"eccodes_key": "#1#timeSignificance", "value": "const:2"},
{"eccodes_key": "#10#timePeriod", "value": "const:-10"},
{"eccodes_key": "#1#windDirection", "value": "data:wind_direction"},
{"eccodes_key": "#1#windSpeed", "value": "data:wind_speed"},
{"eccodes_key": "#11#timePeriod", "value": "const:-10"},
{"eccodes_key": "#1#maximumWindGustSpeed", "value": "data:highest_gust_1"},
{"eccodes_key": "#12#timePeriod", "value": "data:past_weather_time_period"},
{"eccodes_key": "#2#maximumWindGustSpeed", "value": "data:highest_gust_2"},
{"eccodes_key": "#13#timePeriod", "value": "const:-24"},
{"eccodes_key": "#1#typeOfInstrumentationForEvaporationMeasurement", "value": "data:evaporation_instrument"},
{"eccodes_key": "#1#evaporation", "value": "data:evapotranspiration"},
{"eccodes_key": "#4#cloudType", "value": "data:e_cloud_genus"},
{"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:e_cloud_direction"},
{"eccodes_key": "#1#elevation", "value": "data:e_cloud_elevation"},
{"eccodes_key": "#14#timePeriod", "value": "const:-1"},
{"eccodes_key": "#1#netRadiationIntegratedOverPeriodSpecified", "value": "data:net_radiation_1hr"},
{"eccodes_key": "#1#globalSolarRadiationIntegratedOverPeriodSpecified", "value": "data:global_solar_radiation_1hr"},
{"eccodes_key": "#1#diffuseSolarRadiationIntegratedOverPeriodSpecified", "value": "data:diffuse_solar_radiation_1hr"},
{"eccodes_key": "#1#directSolarRadiationIntegratedOverPeriodSpecified", "value": "data:direct_solar_radiation_1hr"},
{"eccodes_key": "#1#longWaveRadiationIntegratedOverPeriodSpecified", "value": "data:long_wave_radiation_1hr"},
{"eccodes_key": "#1#shortWaveRadiationIntegratedOverPeriodSpecified", "value": "data:short_wave_radiation_1hr"},
{"eccodes_key": "#15#timePeriod", "value": "const:-24"},
{"eccodes_key": "#2#netRadiationIntegratedOverPeriodSpecified", "value": "data:net_radiation_24hr"},
{"eccodes_key": "#2#globalSolarRadiationIntegratedOverPeriodSpecified", "value": "data:global_solar_radiation_24hr"},
{"eccodes_key": "#2#diffuseSolarRadiationIntegratedOverPeriodSpecified", "value": "data:diffuse_solar_radiation_24hr"},
{"eccodes_key": "#2#directSolarRadiationIntegratedOverPeriodSpecified", "value": "data:direct_solar_radiation_24hr"},
{"eccodes_key": "#2#longWaveRadiationIntegratedOverPeriodSpecified", "value": "data:long_wave_radiation_24hr"},
{"eccodes_key": "#2#shortWaveRadiationIntegratedOverPeriodSpecified", "value": "data:short_wave_radiation_24hr"},
{"eccodes_key": "#16#timePeriod", "value": "data:past_weather_time_period"},
{"eccodes_key": "#17#timePeriod", "value": "const:0"},
{"eccodes_key": "#1#temperatureChangeOverSpecifiedPeriod", "value": "data:temperature_change"}
]
}
2 changes: 1 addition & 1 deletion synop2bufr/resources/synop-mappings.json → ...bufr/resources/synop-mappings-307096.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
{"eccodes_key": "#1#windSpeed", "value": "data:wind_speed"},
{"eccodes_key": "#7#timePeriod", "value": "const:-10"},
{"eccodes_key": "#1#maximumWindGustSpeed", "value": "data:highest_gust_1"},
{"eccodes_key": "#8#timePeriod", "value": "const:-60"},
{"eccodes_key": "#8#timePeriod", "value": "data:past_weather_time_period"},
{"eccodes_key": "#2#maximumWindGustSpeed", "value": "data:highest_gust_2"},
{"eccodes_key": "#13#timePeriod", "value": "data:maximum_temperature_period_start"},
{"eccodes_key": "#14#timePeriod", "value": "data:maximum_temperature_period_end"},
Expand Down
Loading

0 comments on commit 7ca718d

Please sign in to comment.