diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 101149f..feb0d42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,10 +4,10 @@ on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - - 'master' - - 'develop' + - "master" + - "develop" tags-ignore: - - '**' + - "**" # Trigger the workflow on pull request pull_request: ~ @@ -27,6 +27,7 @@ jobs: uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main with: eccodes-python: ecmwf/eccodes-python@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true secrets: inherit # Build downstream packages on HPC diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eea584f..3aa9462 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog for eccodes-python ============================ +1.7.0 (2024-02-26) +-------------------- +- ECC-1761: Add function to extract message offsets and sizes +- ECC-1742: Add function to clone only the meta-data of a message + 1.6.1 (2023-10-02) -------------------- diff --git a/README.rst b/README.rst index 31d11b0..ee53913 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ Features: - reads and writes GRIB 1 and 2 files, - reads and writes BUFR 3 and 4 files, -- supports all modern versions of Python 3.11, 3.10, 3.9, 3.8 and PyPy3, +- supports all modern versions of Python and PyPy3, - works on most *Linux* distributions and *MacOS*, the *ecCodes* C-library is the only system dependency, - PyPI package can be installed without compiling, diff --git a/docs/index.rst b/docs/index.rst index 4ce5848..5d18fb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ Features: - reads and writes GRIB 1 and 2 files, - reads and writes BUFR 3 and 4 files, -- supports all modern versions of Python 3.11, 3.10, 3.9, 3.8 and PyPy3, +- supports all modern versions of Python and PyPy3, - works on most *Linux* distributions and *MacOS*, the *ecCodes* C-library is the only system dependency, - PyPI package can be installed without compiling, at the cost of being twice as slow as the original *ecCodes* module, diff --git a/eccodes/eccodes.py b/eccodes/eccodes.py index e8026fd..ef8640e 100644 --- a/eccodes/eccodes.py +++ b/eccodes/eccodes.py @@ -43,6 +43,7 @@ codes_definition_path, codes_dump, codes_extract_offsets, + codes_extract_offsets_sizes, codes_get_gaussian_latitudes, codes_get_library_path, codes_get_version_info, @@ -245,6 +246,7 @@ "codes_count_in_file", "codes_definition_path", "codes_extract_offsets", + "codes_extract_offsets_sizes", "codes_get_api_version", "codes_get_array", "codes_get_double_array", diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 0d1f98e..32fdc34 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -21,7 +21,7 @@ import cffi -__version__ = "1.6.1" +__version__ = "1.7.0" LOG = logging.getLogger(__name__) diff --git a/gribapi/eccodes.h b/gribapi/eccodes.h index fea2441..9304a6c 100644 --- a/gribapi/eccodes.h +++ b/gribapi/eccodes.h @@ -22,6 +22,7 @@ void codes_bufr_multi_element_constant_arrays_on(codes_context* c); void codes_bufr_multi_element_constant_arrays_off(codes_context* c); int codes_bufr_extract_headers_malloc(codes_context* c, const char* filename, codes_bufr_header** result, int* num_messages, int strict_mode); int codes_extract_offsets_malloc(codes_context* c, const char* filename, ProductKind product, long int** offsets, int* num_messages, int strict_mode); +int codes_extract_offsets_sizes_malloc(codes_context* c, const char* filename, ProductKind product, long int** offsets, size_t** sizes, int* num_messages, int strict_mode); int codes_bufr_key_is_header(const codes_handle* h, const char* key, int* err); int codes_bufr_key_is_coordinate(const codes_handle* h, const char* key, int* err); diff --git a/gribapi/errors.py b/gribapi/errors.py index 28e16fe..520d223 100644 --- a/gribapi/errors.py +++ b/gribapi/errors.py @@ -192,7 +192,7 @@ class InvalidIndexError(GribInternalError): class InvalidGribError(GribInternalError): - """Invalid grib id.""" + """Invalid GRIB id.""" class InvalidFileError(GribInternalError): diff --git a/gribapi/grib_api.h b/gribapi/grib_api.h index 947788d..e0df7b9 100644 --- a/gribapi/grib_api.h +++ b/gribapi/grib_api.h @@ -38,7 +38,7 @@ struct grib_values { int has_value; int equal; grib_values* next; -} ; +}; typedef struct grib_handle grib_handle; typedef struct grib_multi_handle grib_multi_handle; @@ -74,7 +74,8 @@ int grib_count_in_file(grib_context* c, FILE* f,int* n); grib_handle* grib_handle_new_from_file(grib_context* c, FILE* f, int* error); grib_handle* grib_handle_new_from_message_copy(grib_context* c, const void* data, size_t data_len); grib_handle* grib_handle_new_from_samples (grib_context* c, const char* sample_name); -grib_handle* grib_handle_clone(const grib_handle* h) ; +grib_handle* grib_handle_clone(const grib_handle* h); +grib_handle* grib_handle_clone_headers_only(const grib_handle* h); int grib_handle_delete(grib_handle* h); grib_multi_handle* grib_multi_handle_new(grib_context* c); int grib_multi_handle_append(grib_handle* h,int start_section,grib_multi_handle* mh); @@ -125,7 +126,7 @@ void grib_dump_content(const grib_handle* h, FILE* out, const char* mode, unsign grib_context* grib_context_get_default(void); void grib_context_delete(grib_context* c); -void grib_gts_header_on(grib_context* c) ; +void grib_gts_header_on(grib_context* c); void grib_gts_header_off(grib_context* c); void grib_gribex_mode_on(grib_context* c); void grib_gribex_mode_off(grib_context* c); diff --git a/gribapi/gribapi.py b/gribapi/gribapi.py index 7c21f56..2097e3f 100644 --- a/gribapi/gribapi.py +++ b/gribapi/gribapi.py @@ -162,13 +162,13 @@ def wrapper(*args): def get_handle(msgid): h = ffi.cast("grib_handle*", msgid) if h == ffi.NULL: - raise errors.InvalidGribError(f"get_handle: Bad message ID {msgid}") + raise errors.NullHandleError(f"get_handle: Bad message ID {msgid}") return h def put_handle(handle): if handle == ffi.NULL: - raise errors.InvalidGribError(f"put_handle: Bad message ID {handle}") + raise errors.NullHandleError("put_handle: Bad message ID (handle is NULL)") return int(ffi.cast("size_t", handle)) @@ -752,12 +752,10 @@ def grib_iterator_next(iterid): lat_p = ffi.new("double*") lon_p = ffi.new("double*") value_p = ffi.new("double*") - err = lib.grib_iterator_next(iterh, lat_p, lon_p, value_p) - if err == 0: + retval = lib.grib_iterator_next(iterh, lat_p, lon_p, value_p) + if retval == 0: + # No more data available. End of iteration return [] - elif err < 0: - GRIB_CHECK(err) - return None else: return (lat_p[0], lon_p[0], value_p[0]) @@ -803,8 +801,7 @@ def grib_keys_iterator_next(iterid): """ kih = get_grib_keys_iterator(iterid) res = lib.grib_keys_iterator_next(kih) - if res < 0: - GRIB_CHECK(res) + # res is 0 or 1 return res @@ -887,8 +884,7 @@ def codes_bufr_keys_iterator_next(iterid): """ bki = get_bufr_keys_iterator(iterid) res = lib.codes_bufr_keys_iterator_next(bki) - if res < 0: - GRIB_CHECK(res) + # res is 0 or 1 return res @@ -1121,23 +1117,29 @@ def codes_bufr_copy_data(msgid_src, msgid_dst): @require(msgid_src=int) -def grib_clone(msgid_src): +def grib_clone(msgid_src, headers_only=False): r""" @brief Create a copy of a message. Create a copy of a given message (\em msgid_src) resulting in a new message in memory (\em msgid_dest) identical to the original one. + If the headers_only option is enabled, the clone will not contain + the Bitmap and Data sections \b Examples: \ref grib_clone.py "grib_clone.py" - @param msgid_src id of message to be cloned - @return id of clone + @param msgid_src id of message to be cloned + @param headers_only whether or not to clone the message with the headers only + @return id of clone @exception CodesInternalError """ h_src = get_handle(msgid_src) - h_dest = lib.grib_handle_clone(h_src) + if headers_only: + h_dest = lib.grib_handle_clone_headers_only(h_src) + else: + h_dest = lib.grib_handle_clone(h_src) if h_dest == ffi.NULL: - raise errors.InvalidGribError("clone failed") + raise errors.MessageInvalidError("clone failed") return put_handle(h_dest) @@ -2381,7 +2383,7 @@ def grib_new_from_message(message): message = message.encode(ENC) h = lib.grib_handle_new_from_message_copy(ffi.NULL, message, len(message)) if h == ffi.NULL: - raise errors.InvalidGribError("new_from_message failed") + raise errors.MessageInvalidError("new_from_message failed") return put_handle(h) @@ -2555,10 +2557,10 @@ def codes_extract_offsets(filepath, product_kind, is_strict=True): """ @brief Message offset extraction - @param filepath path of input file - @product_kind one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS - @param is_strict if True, fail as soon as any invalid message is encountered - @return a generator that yields offsets (each offset is an integer) + @param filepath path of input file + @param product_kind one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS + @param is_strict if True, fail as soon as any invalid message is encountered + @return a generator that yields offsets (as integers) @exception CodesInternalError """ context = lib.grib_context_get_default() @@ -2579,6 +2581,42 @@ def codes_extract_offsets(filepath, product_kind, is_strict=True): i += 1 +def codes_extract_offsets_sizes(filepath, product_kind, is_strict=True): + """ + @brief Message offset and size extraction + + @param filepath path of input file + @param product_kind one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS + @param is_strict if True, fail as soon as any invalid message is encountered + @return a generator that yields lists of pairs of offsets and sizes (as integers) + @exception CodesInternalError + """ + context = lib.grib_context_get_default() + offsets_p = ffi.new("long int**") + sizes_p = ffi.new("size_t**") + num_message_p = ffi.new("int*") + + err = lib.codes_extract_offsets_sizes_malloc( + context, + filepath.encode(ENC), + product_kind, + offsets_p, + sizes_p, + num_message_p, + is_strict, + ) + GRIB_CHECK(err) + + num_messages = num_message_p[0] + offsets = offsets_p[0] + sizes = sizes_p[0] + + i = 0 + while i < num_messages: + yield (offsets[i], sizes[i]) + i += 1 + + # ------------------------------- # EXPERIMENTAL FEATURES # ------------------------------- diff --git a/tests/test_eccodes.py b/tests/test_eccodes.py index 2230b11..4c10f5d 100644 --- a/tests/test_eccodes.py +++ b/tests/test_eccodes.py @@ -49,7 +49,7 @@ def test_codes_samples_path(): def test_codes_set_definitions_path(): - eccodes.eccodes.codes_set_definitions_path(eccodes.codes_definition_path()) + eccodes.codes_set_definitions_path(eccodes.codes_definition_path()) def test_codes_set_samples_path(): @@ -69,8 +69,8 @@ def test_api_version(): def test_version_info(): vinfo = eccodes.codes_get_version_info() + print("ecCodes version information: ", vinfo) assert len(vinfo) == 2 - print(vinfo) def test_codes_is_defined(): @@ -84,8 +84,8 @@ def test_codes_get_native_type(): assert eccodes.codes_get_native_type(gid, "referenceValue") is float assert eccodes.codes_get_native_type(gid, "stepType") is str assert eccodes.codes_get_native_type(gid, "section_1") is None - with pytest.raises(eccodes.InvalidGribError): - eccodes.codes_get_native_type(0, "aKey") + with pytest.raises(eccodes.NullHandleError): + eccodes.codes_get_native_type(0, "aKey") # NULL handle def test_new_from_file(): @@ -174,6 +174,26 @@ def test_extract_offsets(): assert offsets_list == expected +def test_extract_offsets_sizes(): + if eccodes.codes_get_api_version(int) < 23400: + pytest.skip("ecCodes version too old") + + offsets_sizes = eccodes.codes_extract_offsets_sizes( + TEST_GRIB_TIGGE_DATA, eccodes.CODES_PRODUCT_GRIB, is_strict=True + ) + result = list(offsets_sizes) + expected = [ + (0, 432), + (432, 432), + (864, 432), + (1296, 432), + (1728, 432), + (2160, 456), + (2616, 456), + ] + assert result == expected + + def test_any_new_from_samples(): msgid = eccodes.codes_new_from_samples( "reduced_gg_ml_grib2", eccodes.CODES_PRODUCT_ANY @@ -239,7 +259,7 @@ def test_grib_codes_set_missing(): eccodes.codes_set(gid, "typeOfFirstFixedSurface", "sfc") eccodes.codes_set_missing(gid, "scaleFactorOfFirstFixedSurface") eccodes.codes_set_missing(gid, "scaledValueOfFirstFixedSurface") - assert eccodes.eccodes.codes_is_missing(gid, "scaleFactorOfFirstFixedSurface") + assert eccodes.codes_is_missing(gid, "scaleFactorOfFirstFixedSurface") def test_grib_set_key_vals(): @@ -282,8 +302,7 @@ def test_grib_get_array(): def test_grib_get_array_single_precision(): if eccodes.codes_get_api_version(int) < 23100: - print("Test skipped (ecCodes version too old)") - return + pytest.skip("ecCodes version too old") gid = eccodes.codes_grib_new_from_samples("reduced_gg_pl_160_grib2") @@ -350,9 +369,27 @@ def test_grib_clone(): eccodes.codes_release(clone) +def test_grib_clone_headers_only(): + if eccodes.codes_get_api_version(int) < 23400: + pytest.skip("ecCodes version too old") + + with open(TEST_GRIB_ERA5_DATA, "rb") as f: + msgid1 = eccodes.codes_grib_new_from_file(f) + msgid2 = eccodes.codes_clone(msgid1, headers_only=True) + msg1_size = eccodes.codes_get_message_size(msgid1) + msg2_size = eccodes.codes_get_message_size(msgid2) + assert msg1_size > msg2_size + assert eccodes.codes_get(msgid1, "totalLength") == 14752 + assert eccodes.codes_get(msgid2, "totalLength") == 112 + assert eccodes.codes_get(msgid1, "bitsPerValue") == 16 + assert eccodes.codes_get(msgid2, "bitsPerValue") == 0 + eccodes.codes_release(msgid1) + eccodes.codes_release(msgid2) + + def test_grib_keys_iterator(): gid = eccodes.codes_grib_new_from_samples("reduced_gg_pl_1280_grib1") - iterid = eccodes.eccodes.codes_keys_iterator_new(gid, "ls") + iterid = eccodes.codes_keys_iterator_new(gid, "ls") count = 0 while eccodes.codes_keys_iterator_next(iterid): keyname = eccodes.codes_keys_iterator_get_name(iterid) @@ -361,7 +398,7 @@ def test_grib_keys_iterator(): count += 1 assert count == 10 eccodes.codes_keys_iterator_rewind(iterid) - eccodes.eccodes.codes_keys_iterator_delete(iterid) + eccodes.codes_keys_iterator_delete(iterid) eccodes.codes_release(gid) @@ -429,6 +466,13 @@ def test_grib_get_double_elements(): assert elems == elems2 +def test_grib_get_values(): + gid = eccodes.codes_grib_new_from_samples("gg_sfc_grib2") + with pytest.raises(TypeError): + eccodes.codes_get_values(gid, str) + eccodes.codes_release(gid) + + def test_grib_geoiterator(): gid = eccodes.codes_grib_new_from_samples("reduced_gg_pl_256_grib2") iterid = eccodes.codes_grib_iterator_new(gid, 0) @@ -482,6 +526,9 @@ def test_grib_nearest_multiple(): eccodes.codes_release(gid) assert nearest[0].index == 1770 assert nearest[1].index == 2500 + # Error condition + with pytest.raises(ValueError): + eccodes.codes_grib_find_nearest_multiple(gid, is_lsm, (1, 2), (1, 2, 3)) def test_grib_ecc_1042(): @@ -520,8 +567,8 @@ def test_grib_ecc_1007(): values = np.zeros((numvals,)) values[0] = 12 # Make sure it's not a constant field eccodes.codes_set_values(gid, values) - maxv = eccodes.eccodes.codes_get(gid, "max") - minv = eccodes.eccodes.codes_get(gid, "min") + maxv = eccodes.codes_get(gid, "max") + minv = eccodes.codes_get(gid, "min") assert minv == 0 assert maxv == 12 eccodes.codes_release(gid) @@ -609,7 +656,7 @@ def test_grib_index_new_from_file(tmpdir): eccodes.codes_index_write(iid, index_file) key = "level" - assert eccodes.eccodes.codes_index_get_size(iid, key) == 1 + assert eccodes.codes_index_get_size(iid, key) == 1 # Cannot get the native type of a key from an index # so right now the default is str. @@ -701,6 +748,12 @@ def test_grib_dump(tmp_path): eccodes.codes_release(gid) +def test_grib_copy_namespace(): + gid1 = eccodes.codes_grib_new_from_samples("GRIB2") + gid2 = eccodes.codes_grib_new_from_samples("reduced_gg_pl_32_grib2") + eccodes.codes_copy_namespace(gid1, "ls", gid2) + + # --------------------------------------------- # PRODUCT BUFR # --------------------------------------------- @@ -795,7 +848,7 @@ def test_bufr_keys_iterator(): def test_bufr_codes_is_missing(): - bid = eccodes.eccodes.codes_bufr_new_from_samples("BUFR4_local") + bid = eccodes.codes_bufr_new_from_samples("BUFR4_local") eccodes.codes_set(bid, "unpack", 1) assert eccodes.codes_is_missing(bid, "heightOfBarometerAboveMeanSeaLevel") == 1 assert eccodes.codes_is_missing(bid, "blockNumber") == 1 @@ -859,8 +912,7 @@ def test_codes_bufr_key_is_header(): def test_codes_bufr_key_is_coordinate(): if eccodes.codes_get_api_version(int) < 23100: - print("Test skipped (ecCodes version too old)") - return + pytest.skip("ecCodes version too old") bid = eccodes.codes_bufr_new_from_samples("BUFR4") assert not eccodes.codes_bufr_key_is_coordinate(bid, "edition") @@ -901,6 +953,13 @@ def test_bufr_dump(tmp_path): eccodes.codes_release(bid) +def test_bufr_copy_data(): + bid1 = eccodes.codes_bufr_new_from_samples("BUFR4_local") + bid2 = eccodes.codes_bufr_new_from_samples("BUFR4") + bid3 = eccodes.codes_bufr_copy_data(bid1, bid2) + assert bid3 + + # --------------------------------------------- # Experimental features # --------------------------------------------- @@ -924,5 +983,12 @@ def test_grib_nearest2(): assert sorted(expected_indexes) == sorted(returned_indexes) assert math.isclose(nearest[0].value, 295.22085, abs_tol=0.0001) assert math.isclose(nearest[2].distance, 24.16520, abs_tol=0.0001) + + # Error conditions + with pytest.raises(eccodes.FunctionNotImplementedError): + eccodes.codes_grib_nearest_find(nid, gid, lat, lon, flags, is_lsm=True) + with pytest.raises(eccodes.FunctionNotImplementedError): + eccodes.codes_grib_nearest_find(nid, gid, lat, lon, flags, False, npoints=5) + eccodes.codes_release(gid) eccodes.codes_grib_nearest_delete(nid)