Merge remote-tracking branch 'origin/develop' into feature/ECC-1956-a…

…ctivity-experiment-values
ecmwf · Nov 12, 2024 · e52dda5 · e52dda5
2 parents 4681a15 + 8d0f553
commit e52dda5
Show file tree

Hide file tree

Showing 77 changed files with 4,315 additions and 3,969 deletions.
diff --git a/data/bufr/CMakeLists.txt b/data/bufr/CMakeLists.txt
@@ -13,6 +13,7 @@ string(REGEX REPLACE "\n" ";" bufr_refs_to_download "${bufr_refs_to_download}")
 
 # Exceptional case: download bufr files which have to be treated specially
 list(APPEND bufr_files_to_download "vos308014_v3_26.bufr")  # See test ecc-197
+list(APPEND bufr_files_to_download "bad.bufr")  # See ECC-1938
 
 if( ENABLE_EXTRA_TESTS )
     ecbuild_get_test_multidata(

diff --git a/definitions/create_def.pl b/definitions/create_def.pl
@@ -23,7 +23,7 @@
 #my $tarfilesflag = 0;
 
 sub create_cfName {
-    my $p; my %seen; 
+    my $p; my %seen;
     my ($key) = "cfName";
     my $field = "cf.name";
 
@@ -33,6 +33,7 @@ sub create_cfName {
     param,grib_encoding,grib,attribute,centre,units,cf where
     param.hide_def=0 and
     param.retired=0 and
+    grib_encoding.published=1 and
     grib_encoding.id=grib.encoding_id and
     param.id=grib_encoding.param_id and
     attribute.id=grib.attribute_id and
@@ -46,7 +47,7 @@ sub create_cfName {
     my $qh=$dbh->prepare($query);
     $qh->execute();
 
-    # file containing the list of grib api parameters files we want to tar and 
+    # file containing the list of grib api parameters files we want to tar and
     # distribute to users for them to download and update their list of parameter
     # to the latest
     #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
@@ -81,7 +82,7 @@ sub create_cfName {
         }
         $seen{$attribute}=1;
         print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
-        # we need to allow strings in the attribute_value field 
+        # we need to allow strings in the attribute_value field
         # for the moment we apply a patch here
         if ($attribute =~ /stepType/ ) {
             $value="\"accum\"";
@@ -100,7 +101,7 @@ sub create_cfName {
 }
 
 sub create_cfName_legacy {
-    my $p; my %seen; 
+    my $p; my %seen;
     my ($key) = "cfName";
     my $field = "cf.name";
 
@@ -123,7 +124,7 @@ sub create_cfName_legacy {
     my $qh=$dbh->prepare($query);
     $qh->execute();
 
-    # file containing the list of grib api parameters files we want to tar and 
+    # file containing the list of grib api parameters files we want to tar and
     # distribute to users for them to download and update their list of parameter
     # to the latest
     #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
@@ -158,7 +159,7 @@ sub create_cfName_legacy {
         }
         $seen{$attribute}=1;
         print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
-        # we need to allow strings in the attribute_value field 
+        # we need to allow strings in the attribute_value field
         # for the moment we apply a patch here
         if ($attribute =~ /stepType/ ) {
             $value="\"accum\"";
@@ -177,20 +178,21 @@ sub create_cfName_legacy {
 }
 
 sub create_def {
-    my $p; my %seen; 
+    my $p; my %seen;
     my ($key) =@_;
     my $field=$key;
 
-    if ($key =~ /paramId/) { $field="param.id"; } 
-    if ($key =~ /name/) { $field="param.name"; } 
-    if ($key =~ /units/) { $field="units.name"; } 
+    if ($key =~ /paramId/) { $field="param.id"; }
+    if ($key =~ /name/) { $field="param.name"; }
+    if ($key =~ /units/) { $field="units.name"; }
 
     my $query= <<"EOF";
         select $field,force128,edition,
         centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
         from param,grib_encoding,grib,attribute,centre,units where
         param.hide_def=0 and
         param.retired=0 and
+        grib_encoding.published=1 and
         grib_encoding.id=grib.encoding_id and
         param.id=grib_encoding.param_id and
         attribute.id=grib.attribute_id and
@@ -203,7 +205,7 @@ sub create_def {
     my $qh=$dbh->prepare($query);
     $qh->execute();
 
-    # file containing the list of grib api parameters files we want to tar and 
+    # file containing the list of grib api parameters files we want to tar and
     # distribute to users for them to download and update their list of parameter
     # to the latest
     #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
@@ -223,9 +225,9 @@ sub create_def {
                 close $out;
             }
             $filebase="$basedir/grib$edition$conceptDir";
-            mkpath($filebase); 
+            mkpath($filebase);
 
-            #copy("$filebase/$key.def","$filebase/$key.def.bkp") 
+            #copy("$filebase/$key.def","$filebase/$key.def.bkp")
             #  or die ("unable to copy $filebase/$key.def");
 
             print TAR "grib$edition$conceptDir/$key.def\n";
@@ -244,7 +246,7 @@ sub create_def {
         }
         $seen{$attribute}=1;
         print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
-        # we need to allow strings in the attribute_value field 
+        # we need to allow strings in the attribute_value field
         # for the moment we apply a patch here
         if ($attribute =~ /stepType/ ) {
             $value="\"accum\"";
@@ -263,13 +265,13 @@ sub create_def {
 }
 
 sub create_def_legacy {
-    my $p; my %seen; 
+    my $p; my %seen;
     my ($key) =@_;
     my $field=$key;
 
-    if ($key =~ /paramId/) { $field="param.id"; } 
-    if ($key =~ /name/) { $field="param.name"; } 
-    if ($key =~ /units/) { $field="units.name"; } 
+    if ($key =~ /paramId/) { $field="param.id"; }
+    if ($key =~ /name/) { $field="param.name"; }
+    if ($key =~ /units/) { $field="units.name"; }
 
     my $query= <<"EOF";
         select $field,force128,edition,
@@ -289,7 +291,7 @@ sub create_def_legacy {
     my $qh=$dbh->prepare($query);
     $qh->execute();
 
-    # file containing the list of grib api parameters files we want to tar and 
+    # file containing the list of grib api parameters files we want to tar and
     # distribute to users for them to download and update their list of parameter
     # to the latest
     #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
@@ -309,9 +311,9 @@ sub create_def_legacy {
                 close $out;
             }
             $filebase="$basedir/grib$edition$conceptDir";
-            mkpath($filebase); 
+            mkpath($filebase);
 
-            #copy("$filebase/$key.def","$filebase/$key.def.bkp") 
+            #copy("$filebase/$key.def","$filebase/$key.def.bkp")
             #  or die ("unable to copy $filebase/$key.def");
 
             print TAR "grib$edition$conceptDir/$key.legacy.def\n";
@@ -330,7 +332,7 @@ sub create_def_legacy {
         }
         $seen{$attribute}=1;
         print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
-        # we need to allow strings in the attribute_value field 
+        # we need to allow strings in the attribute_value field
         # for the moment we apply a patch here
         if ($attribute =~ /stepType/ ) {
             $value="\"accum\"";
@@ -347,14 +349,101 @@ sub create_def_legacy {
 
     close(TAR);
 }
-
+
+sub create_cfVarName {
+    my $p; my %seen;
+    my ($key) =@_;
+    my $field=$key;
+
+    #if ($key =~ /paramId/) { $field="param.id"; }
+    #if ($key =~ /name/) { $field="param.name"; }
+    #if ($key =~ /units/) { $field="units.name"; }
+    if ($key =~ /cfVarName/) { $field="cfVarName"; }
+
+    my $query= <<"EOF";
+        select $field,force128,edition,
+        centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
+        from param,grib_encoding,grib,attribute,centre,units where
+        param.hide_def=0 and
+        grib_encoding.id=grib.encoding_id and
+        param.id=grib_encoding.param_id and
+        attribute.id=grib.attribute_id and
+        centre.id=grib_encoding.centre_id and
+        units.id=param.units_id
+        and cfVarName IS NOT NULL
+        order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
+EOF
+
+    my $qh=$dbh->prepare($query);
+    $qh->execute();
+
+    # file containing the list of grib api parameters files we want to tar and
+    # distribute to users for them to download and update their list of parameter
+    # to the latest
+    #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
+    #$tarfilesflag=1;
+
+    while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
+    {
+        if ($centre eq "wmo" ) { $conceptDir=""; }
+        else { $conceptDir="/localConcepts/$centre"; }
+        #if ($key =~ /paramId/ && $force128==1 && $keyval >1000) {
+        #  $keyval= $keyval % 1000;
+        #}
+
+        if ($filebase ne "$basedir/grib$edition$conceptDir") {
+            if ($filebase) {
+                print $out "}\n";
+                close $out;
+            }
+            $filebase="$basedir/grib$edition$conceptDir";
+            mkpath($filebase);
+
+            #copy("$filebase/$key.def","$filebase/$key.def.bkp")
+            #  or die ("unable to copy $filebase/$key.def");
+
+            print TAR "grib$edition$conceptDir/$key.def\n";
+            #system("cp -f $filebase/$key.def $filebase/$key.def.orig");
+            open($out,"> $filebase/$key.def")
+                or die "unable to open $filebase/$key.def";
+            print $out "# Automatically generated by $0, do not edit\n";
+            $p=();
+        }
+        if ($p ne $paramId || exists($seen{$attribute}) ) {
+           if ($p) { print $out "\t}\n"; }
+           print $out "#$name\n" ;
+           print $out "\'".$keyval."\' = {\n" ;
+           $p=$paramId;
+           %seen=();
+        }
+        $seen{$attribute}=1;
+        print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
+        # we need to allow strings in the attribute_value field
+        # for the moment we apply a patch here
+        if ($attribute =~ /stepType/ ) {
+            $value="\"accum\"";
+        }
+        if ($value eq '') {
+            $value="missing()";
+        }
+        print $out "\t $attribute = $value ;\n" ;
+    }
+    if ($filebase) {
+       print $out "}\n";
+       close $out;
+    }
+
+    close(TAR);
+}
+
 sub create_paramId_def {
-    my $p; my %seen; 
+    my $p; my %seen;
 
     my $query="select edition,centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
     from param,grib_encoding,grib,attribute,centre where
     param.hide_def=0 and
     param.retired=0 and
+    grib_encoding.published=1 and
     grib_encoding.id=grib.encoding_id and
     param.id=grib_encoding.param_id and
     attribute.id=grib.attribute_id and
@@ -375,9 +464,9 @@ sub create_paramId_def {
                 close $out;
             }
             $filebase="$basedir/grib$edition$conceptDir";
-            mkpath($filebase); 
+            mkpath($filebase);
 
-            copy("$filebase/paramId.def","$filebase/paramId.def.bkp") 
+            copy("$filebase/paramId.def","$filebase/paramId.def.bkp")
                 or die ("unable to copy $filebase/paramId.def");
             open($out,"> $filebase/paramId.def")
                 or die "unable to open $filebase/paramId.def";
@@ -400,7 +489,7 @@ sub create_paramId_def {
        close $out;
     }
 }
- 
+
 sub create_def_old {
     my ($key,$query)=@_;
 
@@ -417,9 +506,9 @@ sub create_def_old {
                 close $out;
             }
             $filebase="$basedir/grib$edition$conceptDir";
-            mkpath($filebase); 
+            mkpath($filebase);
 
-            copy("$filebase/$key.def","$filebase/$key.def.bkp") 
+            copy("$filebase/$key.def","$filebase/$key.def.bkp")
                 or die ("unable to copy $filebase/$key.def");
             open($out,"> $filebase/$key.def")
                 or die "unable to open $filebase/$key.def";
@@ -440,35 +529,36 @@ sub create_def_old {
 create_def_legacy("name");
 create_def("units");
 create_def_legacy("units");
+create_cfVarName("cfVarName");
 create_cfName("cfName");
 create_cfName_legacy("cfName");
 
 # #create_paramId_def();
 
 # $query="select distinct edition,centre.abbreviation,param_id,param.shortName from param,grib_encoding,centre where
-#  param.hide_def=0 and 
-#  param.id=grib_encoding.param_id and 
-#  centre.id=grib_encoding.centre_id and 
+#  param.hide_def=0 and
+#  param.id=grib_encoding.param_id and
+#  centre.id=grib_encoding.centre_id and
 #  shortName!='~' order by abbreviation,edition,param.o,param.id,shortName";
 
 
-# #select distinct edition,centre.abbreviation,param_id,param.shortName 
-# #from param,grib_encoding,grib,centre where param.hide_def=0 and param.id=grib.param_id and 
-# #centre.id=grib_encoding.centre_id and shortName!='~' 
+# #select distinct edition,centre.abbreviation,param_id,param.shortName
+# #from param,grib_encoding,grib,centre where param.hide_def=0 and param.id=grib.param_id and
+# #centre.id=grib_encoding.centre_id and shortName!='~'
 # #order by centre,edition,param.o,param_id";
 
 # #create_def("shortName",$query);
 
-# $query="select distinct edition,centre.abbreviation,param_id,param.name 
-# from param,grib,centre where param.hide_def=0 and param.id=grib.param_id and 
-# centre.id=grib.centre and shortName!='~' 
+# $query="select distinct edition,centre.abbreviation,param_id,param.name
+# from param,grib,centre where param.hide_def=0 and param.id=grib.param_id and
+# centre.id=grib.centre and shortName!='~'
 # order by centre,edition,param.o,param_id";
 
 # #create_def("name",$query);
 
-# $query="select distinct edition,centre.abbreviation,param_id,units.name 
+# $query="select distinct edition,centre.abbreviation,param_id,units.name
 # from param,grib,centre,units where param.hide_def=0 and param.id=grib.param_id and units.id=param.units_id
-# and centre.id=grib.centre and shortName!='~' 
+# and centre.id=grib.centre and shortName!='~'
 # order by centre,edition,param.o,param_id";
 
 # #create_def("units",$query);

diff --git a/definitions/grib1/local.98.16.def b/definitions/grib1/local.98.16.def
@@ -29,7 +29,7 @@ transient hourOfEndOfOverallTimeInterval=23;
 transient minuteOfEndOfOverallTimeInterval=59;
 transient secondOfEndOfOverallTimeInterval=59;
 
-transient indicatorOfUnitForTimeRange=3;
+transient indicatorOfUnitForTimeRange=3; # month
 transient lengthOfTimeRange=1;
 unsigned[1] averagingPeriod : dump ;
 

diff --git a/definitions/grib1/localConcepts/ecmf/cfVarName.def b/definitions/grib1/localConcepts/ecmf/cfVarName.def
@@ -189,6 +189,11 @@
 	 table2Version = 171 ;
 	 indicatorOfParameter = 7 ;
 	}
+#100 metre wind speed anomaly
+'sia100' = {
+	 table2Version = 171 ;
+	 indicatorOfParameter = 8 ;
+	}
 #2 metre specific humidity
 'sh2' = {
 	 table2Version = 174 ;

diff --git a/definitions/grib1/localConcepts/ecmf/name.def b/definitions/grib1/localConcepts/ecmf/name.def
@@ -3774,6 +3774,11 @@
 	 table2Version = 171 ;
 	 indicatorOfParameter = 7 ;
 	}
+#100 metre wind speed anomaly
+'100 metre wind speed anomaly' = {
+	 table2Version = 171 ;
+	 indicatorOfParameter = 8 ;
+	}
 #Lake mix-layer temperature anomaly
 'Lake mix-layer temperature anomaly' = {
 	 table2Version = 171 ;