Merge pull request #64 from ecmwf/feature/metkit-configs

Feature/metkit configs

.gitignore

@@ -8,4 +8,3 @@ doc/latex
 .*.sw*
 .*un~
 *.ccls-cache
-share

CMakeLists.txt

@@ -24,30 +24,6 @@ if(NOT METKIT_CONFIGS_BRANCH)
     set(METKIT_CONFIGS_BRANCH chk)
 endif()
 
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/share")
-    if(IS_SYMLINK "${CMAKE_CURRENT_SOURCE_DIR}/share")
-        if (METKIT_CONFIGS_PATH)
-            #check if it matches with METKIT_CONFIGS_PATH
-            file(REAL_PATH "${CMAKE_CURRENT_SOURCE_DIR}/share" current_metkit_config_path EXPAND_TILDE)
-            file(REAL_PATH "${METKIT_CONFIGS_PATH}" requested_metkit_config_path EXPAND_TILDE)
-            if(NOT (current_metkit_config_path STREQUAL requested_metkit_config_path))
-                message(SEND_ERROR "metkit-configs refers to ${current_metkit_config_path}; requested METKIT_CONFIGS_PATH is ${requested_metkit_config_path}")
-            endif()
-        endif()
-    elseif(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/share")
-        # update the content
-        ecbuild_bundle( PROJECT share    GIT "https://github.com/ecmwf/metkit-configs"    BRANCH "${METKIT_CONFIGS_BRANCH}" UPDATE )
-    else()
-        message(SEND_ERROR "${CMAKE_CURRENT_SOURCE_DIR}/share must point to a valid metkit configuration. Cannot be a regular file")
-    endif()
-else()
-    if(METKIT_CONFIGS_PATH)
-        file( CREATE_LINK "${METKIT_CONFIGS_PATH}" "${CMAKE_CURRENT_SOURCE_DIR}/share" SYMBOLIC )
-    else()
-        ecbuild_bundle( PROJECT share    GIT "https://github.com/ecmwf/metkit-configs"    BRANCH "${METKIT_CONFIGS_BRANCH}" UPDATE )
-    endif()
-endif()
-
 ecbuild_add_option( FEATURE BUILD_TOOLS
                     DEFAULT ON
                     DESCRIPTION "Build the command line tools" )
@@ -84,6 +60,12 @@ ecbuild_add_option( FEATURE ODB
                     DESCRIPTION "Add support for ODB data"
                     REQUIRED_PACKAGES "NAME odc VERSION 1.0" )
 
+# METKIT config files support
+
+ecbuild_add_option( FEATURE METKIT_CONFIG
+                    DEFAULT ON
+                    DESCRIPTION "Install metkit configuration files" )
+
 # Temporary
 ecbuild_add_option( FEATURE FAIL_ON_CCSDS
                     DESCRIPTION "Fail on CCSDS"
@@ -100,6 +82,7 @@ ecbuild_find_package( NAME eckit  VERSION  1.16 REQUIRED )
 include(cmake/compiler_warnings.cmake) # optionally handle compiler specific warnings
 
 add_subdirectory( src )
+add_subdirectory( share )
 add_subdirectory( tests )
 
 ############################################################################################

share/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory( metkit )

share/metkit/.gitignore

@@ -0,0 +1,2 @@
+*.list
+tmp

share/metkit/CMakeLists.txt

@@ -0,0 +1,15 @@
+list( APPEND files language.yaml paramids.yaml params.yaml params-static.yaml param-matching.yaml bufr-subtypes.yaml axis.yaml shortname-context.yaml )
+
+foreach( _file ${files} )
+
+    configure_file( ${_file} ${CMAKE_BINARY_DIR}/share/metkit/${_file} COPYONLY )
+
+    if( HAVE_METKIT_CONFIG )
+        install( FILES  ${CMAKE_BINARY_DIR}/share/metkit/${_file}
+                DESTINATION "share/metkit"
+                PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ )
+    endif()
+
+endforeach()
+
+add_subdirectory( odb )

share/metkit/axis.yaml

@@ -0,0 +1,51 @@
+---
+axes:
+    - class
+    - type
+    - stream
+    - levtype
+    - origin
+    - product
+    - section
+    - method
+    - system
+    - date
+    - refdate
+    - hdate
+    - offsetdate
+    - time
+    - offsettime
+    - anoffset
+    - reference
+    - step
+    - fcmonth
+    - fcperiod
+    - leadtime
+    - opttime
+    - expver
+    - domain
+    - diagnostic
+    - iteration
+    - quantile
+    - number
+    - levelist
+    - latitude
+    - longitude
+    - range
+    - param
+    - ident
+    - obstype
+    - instrument
+    - frequency
+    - direction
+    - channel
+    - reportype
+    - dataset
+    - activity
+    - experiment
+    - generation
+    - model
+    - realization
+    - resolution
+    - year
+    - month

share/metkit/bufr-subtypes.yaml

@@ -0,0 +1,118 @@
+---
+subtypes:
+  - [1, 1]
+  - [2, 1]
+  - [3, 1]
+  - [4, 1]
+  - [7, 1]
+  - [9, 1]
+  - [11, 1]
+  - [12, 1]
+  - [13, 1]
+  - [14, 1]
+  - [19, 1]
+  - [21, 1]
+  - [22, 1]
+  - [23, 1]
+  - [26, 1]
+  - [28, 1]
+  - [31, 8]
+  - [49, 2]
+  - [51, 2]
+  - [53, 2]
+  - [54, 2]
+  - [55, 2]
+  - [57, 2]
+  - [59, 2]
+  - [60, 2]
+  - [61, 2]
+  - [62, 2]
+  - [63, 2]
+  - [65, 2]
+  - [82, 3]
+  - [83, 3]
+  - [84, 3]
+  - [85, 3]
+  - [86, 3]
+  - [87, 3]
+  - [88, 3]
+  - [89, 3]
+  - [91, 4]
+  - [92, 4]
+  - [95, 4]
+  - [96, 4]
+  - [101, 5]
+  - [102, 5]
+  - [103, 5]
+  - [106, 5]
+  - [109, 5]
+  - [110, 1]
+  - [111, 5]
+  - [112, 5]
+  - [113, 5]
+  - [121, 12]
+  - [122, 12]
+  - [123, 12]
+  - [125, 6]
+  - [126, 12]
+  - [127, 12]
+  - [129, 2]
+  - [131, 6]
+  - [132, 6]
+  - [133, 6]
+  - [136, 12]
+  - [137, 12]
+  - [138, 12]
+  - [139, 12]
+  - [140, 1]
+  - [142, 7]
+  - [143, 7]
+  - [144, 7]
+  - [145, 7]
+  - [146, 7]
+  - [147, 1]
+  - [148, 7]
+  - [149, 7]
+  - [150, 7]
+  - [151, 7]
+  - [153, 12]
+  - [154, 2]
+  - [155, 2]
+  - [156, 2]
+  - [164, 10]
+  - [165, 1]
+  - [170, 1]
+  - [172, 1]
+  - [176, 1]
+  - [178, 1]
+  - [180, 1]
+  - [181, 1]
+  - [182, 1]
+  - [189, 3]
+  - [190, 3]
+  - [201, 12]
+  - [202, 12]
+  - [203, 30]
+  - [204, 30]
+  - [206, 2]
+  - [207, 2]
+  - [208, 2]
+  - [209, 2]
+  - [210, 12]
+  - [211, 2]
+  - [212, 3]
+  - [213, 12]
+  - [214, 12]
+  - [216, 2]
+  - [217, 12]
+  - [218, 12]
+  - [224, 12]
+  - [220, 12]
+  - [230, 5]
+  - [231, 5]
+  - [233, 12]
+  - [237, 1]
+  - [240, 2]
+  - [250, 2]
+  - [251, 2]
+  - [255, 2]

share/metkit/check-params-yaml.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+import MySQLdb
+import yaml
+import re
+import os
+
+
+db = MySQLdb.connect("grib-param-db-prod.ecmwf.int", "ecmwf_ro", "ecmwf_ro", "param")
+
+
+cursor = db.cursor()
+
+
+with open("params.yaml") as f:
+    PARAMS = yaml.load(f.read())
+
+for entry in PARAMS:
+    when, parms = entry
+    for p in parms:
+        cursor.execute("select count(*) from param where id=%s" % p)
+        for data in cursor.fetchall():
+            if data[0] == 0:
+                print("%s: %s is not in param DB" % (when, p))

share/metkit/compare.py

@@ -0,0 +1,35 @@
+import yaml
+import sys
+
+
+def read(path):
+    r = {}
+    with open(path) as f:
+        for k, v in yaml.safe_load(f):
+            u = (k.get("stream"), k.get("type"), k.get("levtype", "sfc"))
+            r[u] = set(v)
+    return r
+
+
+r = read("fieldsdb.yaml")
+s = read("params.yaml")
+
+
+for k, v in sorted(r.items()):
+    if k not in s:
+        print("Only in db", k)
+
+print()
+for k, v in sorted(s.items()):
+    if k not in r:
+        print("Only in params", k)
+
+print()
+for k in sorted(set(r.keys()) & set(s.keys())):
+    print(k)
+    a = r[k] - s[k]
+    b = s[k] - r[k]
+    if a:
+        print("    In db and not in params", a)
+    if b:
+        print("    In params and not in db", b)