From c67da0ab5b5a687c9b360b24217196110aab5edb Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 08:49:29 +0800 Subject: [PATCH 01/24] fix readthedocs: https://blog.readthedocs.com/migrate-configuration-v2/ --- .readthedocs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index a4f6d73..bf26db7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,5 +1,9 @@ version: 2 formats: all +build: + os: ubuntu-22.04 + tools: + python: "3.10" mkdocs: fail_on_warning: false python: From 7315d238205a164b75af0523be378dda1a0381ac Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 09:35:39 +0800 Subject: [PATCH 02/24] weird --- src/pybind11_geojson.cpp | 3 +-- tests/test_geobuf.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index 09a3681..e53bfc4 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -352,7 +352,7 @@ void bind_geojson(py::module &geojson) .def("__setitem__", [](mapbox::geojson::geometry &self, const std::string &key, const py::object &value) { - if (key == "type" || key == "coordinates") { + if (key == "type" || key == "coordinates" || key == "geometries") { throw pybind11::key_error(key); } self.custom_properties[key] = to_geojson_value(value); @@ -360,7 +360,6 @@ void bind_geojson(py::module &geojson) }) .def("__len__", [](mapbox::geojson::geometry &self) { return __len__(self); }) - // TODO, remove push_backs .def( "push_back", [](mapbox::geojson::geometry &self, diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index a231406..01088bb 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -449,6 +449,9 @@ def test_geojson_point(): g1.round(lon=-1, lat=-1) # normally you don't do it assert np.all(g1.as_numpy() == [120, 460, 0]) assert not g1.deduplicate_xyz() + assert g1 != geojson.Point() + g1.clear() + assert g1 == geojson.Point() def test_geojson_point2(): @@ -517,7 +520,6 @@ def test_geojson_multi_point(): g1[0] = geojson.Point([1, 2]) assert g1() == [[1, 2, 0], [7, 8, 9]] for idx, pt in enumerate(g1): - print(idx, pt) assert isinstance(pt, geojson.Point) if idx == 0: assert pt() == [1, 2, 0] @@ -574,6 +576,13 @@ def test_geojson_multi_point(): assert not g.deduplicate_xyz() assert len(g) == 3 + g = geojson.Geometry(g) + assert len(g) == 3 + g.push_back(geojson.Point(6, 7)) + assert len(g) == 4 + g.clear() + assert len(g) == 0 + def test_geojson_line_string(): g1 = geojson.LineString() @@ -685,6 +694,13 @@ def test_geojson_multi_line_string(): g1[0] = [[7, 8], [2, 3]] assert g1() == [[[7, 8, 0], [2, 3, 0]]] + gg = geojson.Geometry(g1) + g1.push_back(geojson.LineString([[2,4,6], [7,8,9]])) + assert g1() == [[[7, 8, 0], [2, 3, 0]], [[2,4,6], [7,8,9]]] + # support by (self: _pybind11_geobuf.geojson.Geometry, arg0: _pybind11_geobuf.geojson.Geometry) + gg.push_back(geojson.LineString([[2,4,6], [7,8,9]])) + assert gg()['coordinates'] == [[[7, 8, 0], [2, 3, 0]], [[2,4,6], [7,8,9]]] + g1.clear() assert len(g1) == 0 @@ -1498,6 +1514,9 @@ def test_geojson_feature(): == 3.141592653 ) # noqa + feature = geojson.Feature(sample_geojson()) + print() # shit + def test_geojson_load_dump(): dirname = os.path.abspath(f"{__pwd}/../data") @@ -1778,6 +1797,11 @@ def test_query(): if __name__ == "__main__": - np.set_printoptions(suppress=True) - pwd = os.path.abspath(os.path.dirname(__file__)) - pytest_main(pwd, test_file=os.path.basename(__file__)) + # test_geojson_geometry_collection() + test_geojson_multi_line_string() + test_geojson_multi_point() + test_geojson_geometry() + test_geojson_feature() + # np.set_printoptions(suppress=True) + # pwd = os.path.abspath(os.path.dirname(__file__)) + # pytest_main(pwd, test_file=os.path.basename(__file__)) From 0a8ab5e6c4b6d90eaea28d4d7dfa6cff16fa9c0d Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 09:40:05 +0800 Subject: [PATCH 03/24] avoids copy --- src/pybind11_geojson.cpp | 15 +++++++++++++++ tests/test_geobuf.py | 11 +++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index e53bfc4..03da291 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -406,6 +406,21 @@ void bind_geojson(py::module &geojson) -> mapbox::geojson::geometry & { if (self.is()) { self.get().push_back(geom); + } else { + std::cerr << "can only push_back Polygon to MultiPolygon, current type: " << geometry_type(self) << std::endl; + } + return self; + }, + rvp::reference_internal) + .def( + "push_back", + [](mapbox::geojson::geometry &self, + const mapbox::geojson::line_string &geom) + -> mapbox::geojson::geometry & { + if (self.is()) { + self.get().push_back(geom); + } else { + std::cerr << "can only push_back LineString to MultiLineString, current type: " << geometry_type(self) << std::endl; } return self; }, diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 01088bb..0f96772 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -695,11 +695,10 @@ def test_geojson_multi_line_string(): assert g1() == [[[7, 8, 0], [2, 3, 0]]] gg = geojson.Geometry(g1) - g1.push_back(geojson.LineString([[2,4,6], [7,8,9]])) - assert g1() == [[[7, 8, 0], [2, 3, 0]], [[2,4,6], [7,8,9]]] - # support by (self: _pybind11_geobuf.geojson.Geometry, arg0: _pybind11_geobuf.geojson.Geometry) - gg.push_back(geojson.LineString([[2,4,6], [7,8,9]])) - assert gg()['coordinates'] == [[[7, 8, 0], [2, 3, 0]], [[2,4,6], [7,8,9]]] + g1.push_back(geojson.LineString([[2, 4, 6], [7, 8, 9]])) + assert g1() == [[[7, 8, 0], [2, 3, 0]], [[2, 4, 6], [7, 8, 9]]] + gg.push_back(geojson.LineString([[2, 4, 6], [7, 8, 9]])) + assert gg()["coordinates"] == [[[7, 8, 0], [2, 3, 0]], [[2, 4, 6], [7, 8, 9]]] g1.clear() assert len(g1) == 0 @@ -1515,7 +1514,7 @@ def test_geojson_feature(): ) # noqa feature = geojson.Feature(sample_geojson()) - print() # shit + print() # shit def test_geojson_load_dump(): From 0186b1b649bea6c0dc978dd400f27b20f3e87f79 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 09:45:32 +0800 Subject: [PATCH 04/24] clean code --- src/pybind11_geojson.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index 03da291..0da0ce8 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -478,30 +478,6 @@ void bind_geojson(py::module &geojson) BIND_PY_FLUENT_ATTRIBUTE(mapbox::geojson::geometry, // PropertyMap, // custom_properties) // - .def("__getitem__", - [](mapbox::geojson::geometry &self, - const std::string &key) -> mapbox::geojson::value & { - return self.custom_properties.at(key); - }, - rvp::reference_internal) // - .def( - "get", - [](mapbox::geojson::geometry &self, - const std::string &key) -> mapbox::geojson::value * { - auto &obj = self.custom_properties; - auto itr = obj.find(key); - if (itr == obj.end()) { - return nullptr; - } - return &itr->second; - }, - "key"_a, rvp::reference_internal) - .def("__setitem__", - [](mapbox::geojson::geometry &self, const std::string &key, const py::object &value) { - auto &obj = self.custom_properties; - obj[key] = to_geojson_value(value); - return value; - }) .def("keys", [](mapbox::geojson::geometry &self) { return py::make_key_iterator(self.custom_properties); From 45600c7c7e0956530597a85078fff32b60aea164 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 10:05:57 +0800 Subject: [PATCH 05/24] better logging --- src/geobuf/geojson_helpers.hpp | 37 ++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/geobuf/geojson_helpers.hpp b/src/geobuf/geojson_helpers.hpp index 6590ae3..37f1c85 100644 --- a/src/geobuf/geojson_helpers.hpp +++ b/src/geobuf/geojson_helpers.hpp @@ -291,19 +291,20 @@ inline std::string get_type(const mapbox::geojson::value &self) inline void geometry_push_back(mapbox::geojson::geometry &self, const mapbox::geojson::point &point) { - self.match([&](mapbox::geojson::multi_point &g) { g.push_back(point); }, - [&](mapbox::geojson::line_string &g) { g.push_back(point); }, - [&](mapbox::geojson::multi_line_string &g) { - g.back().push_back(point); - }, - [&](mapbox::geojson::polygon &g) { g.back().push_back(point); }, - [&](mapbox::geojson::multi_polygon &g) { - g.back().back().push_back(point); - }, - [&](auto &g) { - std::cerr << "geometry_push_back not handled for this type: " - << geometry_type(g) << std::endl; - }); + self.match( + [&](mapbox::geojson::multi_point &g) { g.push_back(point); }, + [&](mapbox::geojson::line_string &g) { g.push_back(point); }, + [&](mapbox::geojson::multi_line_string &g) { + g.back().push_back(point); + }, + [&](mapbox::geojson::polygon &g) { g.back().push_back(point); }, + [&](mapbox::geojson::multi_polygon &g) { + g.back().back().push_back(point); + }, + [&](auto &g) { + std::cerr << "geometry_push_back(point) not handled for this type: " + << geometry_type(g) << std::endl; + }); } inline void geometry_push_back(mapbox::geojson::geometry &self, @@ -325,8 +326,9 @@ inline void geometry_push_back(mapbox::geojson::geometry &self, eigen2geom(points, g.back()); }, [&](auto &g) { - std::cerr << "geometry_push_back not handled for this type: " - << geometry_type(g) << std::endl; + std::cerr + << "geometry_push_back(ndarray) not handled for this type: " + << geometry_type(g) << std::endl; }); } @@ -336,8 +338,9 @@ inline void geometry_push_back(mapbox::geojson::geometry &self, self.match( [&](mapbox::geojson::geometry_collection &g) { g.push_back(geom); }, [&](auto &g) { - std::cerr << "geometry_push_back(geom) not handled for this type: " - << geometry_type(g) << std::endl; + std::cerr << "geometry_push_back(geom:" << geometry_type(geom) + << ") not handled for this type: " << geometry_type(g) + << std::endl; }); } From 9c221d4d409b153a1154935790943ba425704b24 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 10:14:53 +0800 Subject: [PATCH 06/24] fix segment fault --- src/geobuf/geojson_helpers.hpp | 17 ++++++++++++++++- tests/test_geobuf.py | 5 +++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/geobuf/geojson_helpers.hpp b/src/geobuf/geojson_helpers.hpp index 37f1c85..79f80d0 100644 --- a/src/geobuf/geojson_helpers.hpp +++ b/src/geobuf/geojson_helpers.hpp @@ -295,10 +295,25 @@ inline void geometry_push_back(mapbox::geojson::geometry &self, [&](mapbox::geojson::multi_point &g) { g.push_back(point); }, [&](mapbox::geojson::line_string &g) { g.push_back(point); }, [&](mapbox::geojson::multi_line_string &g) { + if (g.empty()) { + throw std::invalid_argument( + "can't push_back Point to empty MultiLineString, may " + "resize first"); + } + g.back().push_back(point); + }, + [&](mapbox::geojson::polygon &g) { + if (g.empty()) { + throw std::invalid_argument( + "can't push_back Point to empty Polygon, may resize first"); + } g.back().push_back(point); }, - [&](mapbox::geojson::polygon &g) { g.back().push_back(point); }, [&](mapbox::geojson::multi_polygon &g) { + if (g.empty() || g.back().empty()) { + throw std::invalid_argument("can't push_back Point to invalid " + "MultiPolygon, may resize first"); + } g.back().back().push_back(point); }, [&](auto &g) { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 0f96772..40387a3 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1177,8 +1177,9 @@ def test_geojson_geometry(): assert g5() == g5.push_back(g4)() # push_back geometry silent ignore g5 = geojson.Geometry(geojson.MultiLineString()) - # g5.push_back([70, 80, 90]), don't do this, will segment fault - # TODO, raise Exception for push_back + with pytest.raises(ValueError) as excinfo: + g5.push_back([70, 80, 90]) + assert "can't push_back Point to empty MultiLineString" in str(excinfo) g6 = geojson.Geometry(geojson.Polygon([[1, 2, 3], [4, 5, 6]])) assert np.array(g6()["coordinates"]).shape == (1, 2, 3) From 08cddfe35240a9398bf6db8a612c984229de4a9c Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 10:23:53 +0800 Subject: [PATCH 07/24] resize --- src/geobuf/geojson_helpers.hpp | 14 ++++++++++++++ src/pybind11_geojson.cpp | 7 +++++++ tests/test_geobuf.py | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/src/geobuf/geojson_helpers.hpp b/src/geobuf/geojson_helpers.hpp index 79f80d0..5165b6d 100644 --- a/src/geobuf/geojson_helpers.hpp +++ b/src/geobuf/geojson_helpers.hpp @@ -405,6 +405,20 @@ inline void geometry_clear(mapbox::geojson::geometry &self) }); } +inline void geometry_resize(mapbox::geojson::geometry &self, int size) +{ + self.match([&](mapbox::geojson::multi_point &g) { g.resize(size); }, + [&](mapbox::geojson::line_string &g) { g.resize(size); }, + [&](mapbox::geojson::multi_line_string &g) { g.resize(size); }, + [&](mapbox::geojson::polygon &g) { g.resize(size); }, + [&](mapbox::geojson::multi_polygon &g) { g.resize(size); }, + [&](mapbox::geojson::geometry_collection &g) { g.resize(size); }, + [&](auto &g) { + std::cerr << "geometry_resize() not handled for this type: " + << geometry_type(g) << std::endl; + }); +} + inline void geojson_value_clear(mapbox::geojson::value &self) { self.match([](mapbox::geojson::value::array_type &arr) { arr.clear(); }, diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index 0da0ce8..5390f19 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -432,6 +432,13 @@ void bind_geojson(py::module &geojson) return self; }, rvp::reference_internal) + .def( + "resize", + [](mapbox::geojson::geometry &self, int size) -> mapbox::geojson::geometry & { + geometry_resize(self, size); + return self; + }, + rvp::reference_internal) .def( "clear", [](mapbox::geojson::geometry &self) -> mapbox::geojson::geometry & { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 40387a3..02e2083 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1180,6 +1180,10 @@ def test_geojson_geometry(): with pytest.raises(ValueError) as excinfo: g5.push_back([70, 80, 90]) assert "can't push_back Point to empty MultiLineString" in str(excinfo) + g5.resize(1) + assert g5() == {'type': 'MultiLineString', 'coordinates': [[]]} + g5.push_back([70, 80, 90]) + assert g5() == {'type': 'MultiLineString', 'coordinates': [[[70.0, 80.0, 90.0]]]} g6 = geojson.Geometry(geojson.Polygon([[1, 2, 3], [4, 5, 6]])) assert np.array(g6()["coordinates"]).shape == (1, 2, 3) From 7574c64f632c5feb7383aeab4f231b1ac75d39a5 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 10:34:57 +0800 Subject: [PATCH 08/24] resize --- src/pybind11_geojson.cpp | 6 ++++++ tests/test_geobuf.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index 5390f19..b7c650c 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -806,6 +806,12 @@ void bind_geojson(py::module &geojson) eigen2geom(points, self); \ return self; \ })) \ + .def("resize", \ + [](mapbox::geojson::geom_type &self, \ + int size) -> mapbox::geojson::geom_type & { \ + self.resize(size); \ + return self; \ + }) \ .def(py::pickle( \ [](const mapbox::geojson::geom_type &self) { \ return to_python(mapbox::geojson::geometry{self}); \ diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 02e2083..ee5cbc2 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -508,6 +508,13 @@ def test_geojson_multi_point(): g2 = geojson.MultiPoint([g1[0], g1[1], g1[0], g1[1]]) assert g2() == [*g1(), *g1()] + assert len(g2) == 4 + del g2[1] + assert len(g2) == 3 + assert g2() == [[1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + assert g2.resize(2)() == [g1[0](), g1[0]()] + assert len(g2) == 2 + g3 = geojson.MultiPoint([[1, 2], [3, 4]]) assert np.all(g3.as_numpy() == [[1, 2, 0], [3, 4, 0]]) @@ -1181,9 +1188,9 @@ def test_geojson_geometry(): g5.push_back([70, 80, 90]) assert "can't push_back Point to empty MultiLineString" in str(excinfo) g5.resize(1) - assert g5() == {'type': 'MultiLineString', 'coordinates': [[]]} + assert g5() == {"type": "MultiLineString", "coordinates": [[]]} g5.push_back([70, 80, 90]) - assert g5() == {'type': 'MultiLineString', 'coordinates': [[[70.0, 80.0, 90.0]]]} + assert g5() == {"type": "MultiLineString", "coordinates": [[[70.0, 80.0, 90.0]]]} g6 = geojson.Geometry(geojson.Polygon([[1, 2, 3], [4, 5, 6]])) assert np.array(g6()["coordinates"]).shape == (1, 2, 3) From a3df855a0f0161d8bbb964ab217cee774f96a8d5 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 10:45:24 +0800 Subject: [PATCH 09/24] clear --- src/pybind11_geojson.cpp | 5 +++++ tests/test_geobuf.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/pybind11_geojson.cpp b/src/pybind11_geojson.cpp index b7c650c..e5833d6 100644 --- a/src/pybind11_geojson.cpp +++ b/src/pybind11_geojson.cpp @@ -443,6 +443,7 @@ void bind_geojson(py::module &geojson) "clear", [](mapbox::geojson::geometry &self) -> mapbox::geojson::geometry & { geometry_clear(self); + self.custom_properties.clear(); return self; }, rvp::reference_internal) @@ -1777,6 +1778,10 @@ void bind_geojson(py::module &geojson) .def( "clear", [](mapbox::geojson::feature &self) -> mapbox::geojson::feature & { + geometry_clear(self.geometry); + self.geometry.custom_properties.clear(); + self.properties.clear(); + self.id = mapbox::geojson::null_value_t(); self.custom_properties.clear(); return self; }, diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index ee5cbc2..9cde832 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1526,6 +1526,8 @@ def test_geojson_feature(): ) # noqa feature = geojson.Feature(sample_geojson()) + assert len(feature.properties()) == 0 + assert len(feature.custom_properties()) == 0 print() # shit @@ -1813,6 +1815,7 @@ def test_query(): test_geojson_multi_point() test_geojson_geometry() test_geojson_feature() + test_rapidjson_obj() # np.set_printoptions(suppress=True) # pwd = os.path.abspath(os.path.dirname(__file__)) # pytest_main(pwd, test_file=os.path.basename(__file__)) From 8792419bed94ac5e856d4949f4669a8c97c96dad Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 11:01:02 +0800 Subject: [PATCH 10/24] should cast uint64_t --- tests/test_geobuf.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 9cde832..6c9d6be 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1525,10 +1525,24 @@ def test_geojson_feature(): == 3.141592653 ) # noqa - feature = geojson.Feature(sample_geojson()) + feature = geojson.Feature({**sample_geojson(), 'id': 666}) + assert feature.id() == 666 + feature.clear() assert len(feature.properties()) == 0 assert len(feature.custom_properties()) == 0 - print() # shit + assert feature() == {'type': 'Feature', 'geometry': {'type': 'LineString', 'coordinates': []}, 'properties': {}} + assert feature.id() is None + # using identifier = mapbox::util::variant; + assert feature.id(42).id() == 42 + assert feature.id(-1024).id() == -1024 + assert feature.id(3.14).id() == 3.14 + assert feature.id('string').id() == 'string' + assert feature.id(None).id() is None + assert feature.id(-2**63).id() == -9223372036854775808 + assert feature.id(2**64-1).id() == 18446744073709551615 + with pytest.raises(RuntimeError) as excinfo: + feature.id(2**63) # out of range of uint64_t + print() # shit def test_geojson_load_dump(): @@ -1810,12 +1824,7 @@ def test_query(): if __name__ == "__main__": - # test_geojson_geometry_collection() - test_geojson_multi_line_string() - test_geojson_multi_point() - test_geojson_geometry() test_geojson_feature() - test_rapidjson_obj() # np.set_printoptions(suppress=True) # pwd = os.path.abspath(os.path.dirname(__file__)) # pytest_main(pwd, test_file=os.path.basename(__file__)) From 965113b91404414621f250e33e085aab4bcad7fa Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 11:13:52 +0800 Subject: [PATCH 11/24] fix id --- src/geobuf/pybind11_helpers.hpp | 21 ++++++++++++++++----- tests/test_geobuf.py | 18 +++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/geobuf/pybind11_helpers.hpp b/src/geobuf/pybind11_helpers.hpp index da940c4..359a3aa 100644 --- a/src/geobuf/pybind11_helpers.hpp +++ b/src/geobuf/pybind11_helpers.hpp @@ -319,12 +319,23 @@ inline mapbox::geojson::identifier to_feature_id(const py::object &obj) return {}; } if (py::isinstance(obj)) { - auto val = obj.cast(); - if (val < 0) { - return int64_t(val); - } else { - return uint64_t(val); + // __py_int_to_rapidjson + try { + auto num = obj.cast(); + if (py::int_(num).equal(obj)) { + return num; + } + } catch (...) { + } + try { + auto num = obj.cast(); + if (py::int_(num).equal(obj)) { + return num; + } + } catch (...) { } + throw std::runtime_error("integer out of range of int64_t/uint64_t: " + + py::repr(obj).cast()); } else if (py::isinstance(obj)) { return obj.cast(); } else if (py::isinstance(obj)) { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 6c9d6be..ca6fae8 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1525,24 +1525,28 @@ def test_geojson_feature(): == 3.141592653 ) # noqa - feature = geojson.Feature({**sample_geojson(), 'id': 666}) + feature = geojson.Feature({**sample_geojson(), "id": 666}) assert feature.id() == 666 feature.clear() assert len(feature.properties()) == 0 assert len(feature.custom_properties()) == 0 - assert feature() == {'type': 'Feature', 'geometry': {'type': 'LineString', 'coordinates': []}, 'properties': {}} + assert feature() == { + "type": "Feature", + "geometry": {"type": "LineString", "coordinates": []}, + "properties": {}, + } assert feature.id() is None # using identifier = mapbox::util::variant; assert feature.id(42).id() == 42 assert feature.id(-1024).id() == -1024 assert feature.id(3.14).id() == 3.14 - assert feature.id('string').id() == 'string' + assert feature.id("string").id() == "string" assert feature.id(None).id() is None - assert feature.id(-2**63).id() == -9223372036854775808 - assert feature.id(2**64-1).id() == 18446744073709551615 + assert feature.id(-(2**63)).id() == -9223372036854775808 + assert feature.id(2**64 - 1).id() == 18446744073709551615 with pytest.raises(RuntimeError) as excinfo: - feature.id(2**63) # out of range of uint64_t - print() # shit + feature.id(2**63) # out of range of uint64_t + print() # shit def test_geojson_load_dump(): From 5178020a37d726e1eeb5a05a2c7ebec1971a97cd Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 11:25:06 +0800 Subject: [PATCH 12/24] not ready --- src/geobuf/pybind11_helpers.hpp | 36 +++++++++++++++++++++++++-------- tests/test_geobuf.py | 7 +++++-- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/geobuf/pybind11_helpers.hpp b/src/geobuf/pybind11_helpers.hpp index 359a3aa..20a4473 100644 --- a/src/geobuf/pybind11_helpers.hpp +++ b/src/geobuf/pybind11_helpers.hpp @@ -31,6 +31,7 @@ using geojson_value = mapbox::geojson::value; inline RapidjsonValue __py_int_to_rapidjson(const py::handle &obj) { + // https://github.com/pybind/pybind11_json/blob/master/include/pybind11_json/pybind11_json.hpp#L84-L104 try { auto num = obj.cast(); if (py::int_(num).equal(obj)) { @@ -319,7 +320,7 @@ inline mapbox::geojson::identifier to_feature_id(const py::object &obj) return {}; } if (py::isinstance(obj)) { - // __py_int_to_rapidjson + // similar to __py_int_to_rapidjson try { auto num = obj.cast(); if (py::int_(num).equal(obj)) { @@ -397,26 +398,37 @@ inline py::object to_python(const mapbox::geojson::feature_collection &features) inline geojson_value to_geojson_value(const py::handle &obj) { if (obj.ptr() == nullptr || obj.is_none()) { - return nullptr; + return {}; } if (py::isinstance(obj)) { return obj.cast(); } if (py::isinstance(obj)) { - auto val = obj.cast(); - if (val < 0) { - return int64_t(val); - } else { - return uint64_t(val); + // similar to __py_int_to_rapidjson + try { + auto num = obj.cast(); + if (py::int_(num).equal(obj)) { + return num; + } + } catch (...) { } + try { + auto num = obj.cast(); + if (py::int_(num).equal(obj)) { + return num; + } + } catch (...) { + } + throw std::runtime_error("integer out of range of int64_t/uint64_t: " + + py::repr(obj).cast()); } + if (py::isinstance(obj)) { return obj.cast(); } if (py::isinstance(obj)) { return obj.cast(); } - // TODO, bytes if (py::isinstance(obj) || py::isinstance(obj)) { geojson_value::array_type ret; for (const py::handle &e : obj) { @@ -431,6 +443,14 @@ inline geojson_value to_geojson_value(const py::handle &obj) } return ret; } + if (py::isinstance(obj)) { + // https://github.com/pybind/pybind11_json/blob/master/include/pybind11_json/pybind11_json.hpp#L112 + py::module base64 = py::module::import("base64"); + auto str = base64.attr("b64encode")(obj) + .attr("decode")("utf-8") + .cast(); + return str; + } throw std::runtime_error( "to_geojson_value not implemented for this type of object: " + py::repr(obj).cast()); diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index ca6fae8..59b1a70 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1545,8 +1545,11 @@ def test_geojson_feature(): assert feature.id(-(2**63)).id() == -9223372036854775808 assert feature.id(2**64 - 1).id() == 18446744073709551615 with pytest.raises(RuntimeError) as excinfo: - feature.id(2**63) # out of range of uint64_t - print() # shit + feature.id(2**64) # out of range of uint64_t + assert "integer out of range of int64_t/uint64_t: 18446744073709551616" in str( + excinfo + ) + print() def test_geojson_load_dump(): From dfa6fdd07392a7266181315f7768205db6ddfacd Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 11:31:07 +0800 Subject: [PATCH 13/24] fix parsing --- tests/test_geobuf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 59b1a70..d21590e 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1549,7 +1549,15 @@ def test_geojson_feature(): assert "integer out of range of int64_t/uint64_t: 18446744073709551616" in str( excinfo ) - print() + feature["num"] = -(2**63) + assert feature["num"]() == -9223372036854775808 + feature["num"] = 2**64 - 1 + assert feature["num"]() == 18446744073709551615 + with pytest.raises(RuntimeError) as excinfo: + feature["num"] = 2**64 + assert "integer out of range of int64_t/uint64_t: 18446744073709551616" in str( + excinfo + ) def test_geojson_load_dump(): From a4f7683f3d5909963aa48788e5da9543e6341ed2 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 12:10:10 +0800 Subject: [PATCH 14/24] id not working --- src/geobuf/geobuf.cpp | 18 ++++++++++++++---- tests/test_geobuf.py | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index 2f11a32..dbbaa64 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -411,17 +411,19 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) // int64_t, double, std::string>; feature.id.match( [&](uint64_t uid) { - int id = static_cast(uid); - if (id >= 0) { - pbf.add_int64(12, id); + if (uid <= static_cast( + std::numeric_limits::max())) { + pbf.add_int64(12, static_cast(uid)); } else { pbf.add_string(11, std::to_string(uid)); } }, [&](int64_t id) { pbf.add_int64(12, id); }, + [&](double d) { pbf.add_string(11, std::to_string(d)); }, [&](const std::string &id) { pbf.add_string(11, id); }, [&](const auto &) { - pbf.add_string(11, dump(to_json(feature.id))); + throw std::invalid_argument("invalid id: " + + dump(to_json(feature.id))); }); } if (!feature.properties.empty()) { @@ -702,7 +704,15 @@ mapbox::geojson::feature Decoder::readFeature(Pbf &pbf) protozero::pbf_reader pbf_g = pbf.get_message(); f.geometry = readGeometry(pbf_g); } else if (tag == 11) { + // shit f.id = pbf.get_string(); // TODO, restore to mapbox::geojson id + // https://github.com/mapbox/geobuf/blob/daad5e039f842f4d4f24ed7d59f31586563b71b8/geobuf.proto#L18-L21 + // oneof id_type { + // string id = 11; + // sint64 int_id = 12; + // } + // using identifier = mapbox::util::variant; } else if (tag == 12) { f.id = pbf.get_int64(); } else if (tag == 13) { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index d21590e..df22942 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1559,6 +1559,30 @@ def test_geojson_feature(): excinfo ) + feature.clear() + + feature.id(2**63 - 1) + assert feature.id() == 9223372036854775807 + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + assert "12: 9223372036854775807" in text + + feature.id(2**63) + assert feature.id() == 9223372036854775808 + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + assert '11: "9223372036854775808"' in text + + feature.id(3.14) + assert feature.id() == 3.14 + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + assert '11: "3.14"' in text + + print() + + # shit + def test_geojson_load_dump(): dirname = os.path.abspath(f"{__pwd}/../data") From fe79b2a87e373cdf0a0865fc698f56faac0a227b Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 12:24:26 +0800 Subject: [PATCH 15/24] fix --- src/geobuf/geobuf.cpp | 12 +++++++----- tests/test_geobuf.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index dbbaa64..31e6143 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -21,12 +21,14 @@ // https://github.com/mapbox/geobuf/blob/master/encode.js // https://github.com/mapbox/geobuf/blob/master/decode.js -#ifdef NDEBUG -#define dbg(x) x -#else -#define DBG_MACRO_NO_WARNING #include "dbg.h" -#endif + +// #ifdef NDEBUG +// #define dbg(x) x +// #else +// #define DBG_MACRO_NO_WARNING +// #include "dbg.h" +// #endif constexpr const auto RJFLAGS = rapidjson::kParseDefaultFlags | // rapidjson::kParseCommentsFlag | // diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index df22942..975b2ce 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1573,14 +1573,17 @@ def test_geojson_feature(): text = pbf_decode(pbf) assert '11: "9223372036854775808"' in text + feature.id("text") + assert feature.id() == "text" + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + assert '11: "text"' in text + feature.id(3.14) assert feature.id() == 3.14 pbf = feature.to_geobuf() text = pbf_decode(pbf) - assert '11: "3.14"' in text - - print() - + # assert '11: "3.14"' in text # shit @@ -1863,7 +1866,6 @@ def test_query(): if __name__ == "__main__": - test_geojson_feature() - # np.set_printoptions(suppress=True) - # pwd = os.path.abspath(os.path.dirname(__file__)) - # pytest_main(pwd, test_file=os.path.basename(__file__)) + np.set_printoptions(suppress=True) + pwd = os.path.abspath(os.path.dirname(__file__)) + pytest_main(pwd, test_file=os.path.basename(__file__)) From 6f43a28f148b6569ccd8c882385b3b1573a002e8 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 12:40:41 +0800 Subject: [PATCH 16/24] parse --- src/geobuf/geobuf.cpp | 29 ++++++++++++++++++++--------- tests/test_geobuf.py | 2 ++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index 31e6143..5621ead 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -412,6 +412,7 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) // using identifier = mapbox::util::variant; feature.id.match( + // shit [&](uint64_t uid) { if (uid <= static_cast( std::numeric_limits::max())) { @@ -706,15 +707,25 @@ mapbox::geojson::feature Decoder::readFeature(Pbf &pbf) protozero::pbf_reader pbf_g = pbf.get_message(); f.geometry = readGeometry(pbf_g); } else if (tag == 11) { - // shit - f.id = pbf.get_string(); // TODO, restore to mapbox::geojson id - // https://github.com/mapbox/geobuf/blob/daad5e039f842f4d4f24ed7d59f31586563b71b8/geobuf.proto#L18-L21 - // oneof id_type { - // string id = 11; - // sint64 int_id = 12; - // } - // using identifier = mapbox::util::variant; + // see "feature.id.match(" in this source file + // protobuf: oneof id_type { + // string id = 11; + // sint64 int_id = 12; + // } + // geojson: id := + auto text = pbf.get_string(); + auto json = parse(text); + if (json.IsNumber()) { + if (json.IsUint64()) { + f.id = json.GetUint64(); + } else if (json.IsInt64()) { + f.id = json.GetInt64(); + } else { + f.id = json.GetDouble(); + } + } else { + f.id = text; + } } else if (tag == 12) { f.id = pbf.get_int64(); } else if (tag == 13) { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 975b2ce..4dfeb45 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1866,6 +1866,8 @@ def test_query(): if __name__ == "__main__": + test_geojson_feature() + np.set_printoptions(suppress=True) pwd = os.path.abspath(os.path.dirname(__file__)) pytest_main(pwd, test_file=os.path.basename(__file__)) From 50673bf6fc3b37164f2eb4094409d2b08afc03e5 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 12:49:48 +0800 Subject: [PATCH 17/24] something went wrong? --- src/geobuf/geobuf.cpp | 9 ++++++++- tests/test_geobuf.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index 5621ead..969f93d 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -6,6 +6,8 @@ #include #include +#include "spdlog/spdlog.h" + #include "rapidjson/error/en.h" #include "rapidjson/filereadstream.h" #include "rapidjson/filewritestream.h" @@ -422,7 +424,12 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) } }, [&](int64_t id) { pbf.add_int64(12, id); }, - [&](double d) { pbf.add_string(11, std::to_string(d)); }, + [&](double d) { + pbf.add_string(11, "what"); + // pbf.add_string(11, dbg(fmt::format("{}", d))); + // pbf.add_string(11, std::to_string(d)); + // + }, [&](const std::string &id) { pbf.add_string(11, id); }, [&](const auto &) { throw std::invalid_argument("invalid id: " + diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 4dfeb45..e5b1a91 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1573,6 +1573,12 @@ def test_geojson_feature(): text = pbf_decode(pbf) assert '11: "9223372036854775808"' in text + feature.id(2**64 - 1) + assert feature.id() == 18446744073709551615 + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + assert '11: "18446744073709551615"' in text + feature.id("text") assert feature.id() == "text" pbf = feature.to_geobuf() @@ -1583,6 +1589,7 @@ def test_geojson_feature(): assert feature.id() == 3.14 pbf = feature.to_geobuf() text = pbf_decode(pbf) + print() # assert '11: "3.14"' in text # shit From b35765ae696a8b2b52249ab9588a00ef4bf085e4 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 16:44:52 +0800 Subject: [PATCH 18/24] update --- src/geobuf/geobuf.cpp | 6 +----- src/geobuf/pbf_decoder.cpp | 6 +++--- tests/test_geobuf.py | 9 +++++++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index 969f93d..be505eb 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -414,7 +414,6 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) // using identifier = mapbox::util::variant; feature.id.match( - // shit [&](uint64_t uid) { if (uid <= static_cast( std::numeric_limits::max())) { @@ -425,10 +424,7 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) }, [&](int64_t id) { pbf.add_int64(12, id); }, [&](double d) { - pbf.add_string(11, "what"); - // pbf.add_string(11, dbg(fmt::format("{}", d))); - // pbf.add_string(11, std::to_string(d)); - // + pbf.add_string(11, std::to_string(d)); }, [&](const std::string &id) { pbf.add_string(11, id); }, [&](const auto &) { diff --git a/src/geobuf/pbf_decoder.cpp b/src/geobuf/pbf_decoder.cpp index bc615ed..f66eb80 100644 --- a/src/geobuf/pbf_decoder.cpp +++ b/src/geobuf/pbf_decoder.cpp @@ -61,11 +61,11 @@ bool decode_printable_string(std::stringstream &out, static constexpr const std::size_t max_string_length = 60; const std::string str{view.data(), view.size()}; - if (str.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUV" - "WXYZ0123456789_:-") != std::string::npos) { + if (!std::all_of(str.begin(), str.end(), [](char c) { + return std::isprint(static_cast(c)); + })) { return false; } - if (str.size() > max_string_length) { out << '"' << str.substr(0, max_string_length) << "\"...\n"; } else { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index e5b1a91..062147b 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1561,6 +1561,12 @@ def test_geojson_feature(): feature.clear() + feature.id(3.14) + assert feature.id() == 3.14 + pbf = feature.to_geobuf() + text = pbf_decode(pbf) + print(text) + feature.id(2**63 - 1) assert feature.id() == 9223372036854775807 pbf = feature.to_geobuf() @@ -1588,6 +1594,9 @@ def test_geojson_feature(): feature.id(3.14) assert feature.id() == 3.14 pbf = feature.to_geobuf() + + f2 = geojson.Feature().from_geobuf(pbf) + assert f2.id() == 3.14 text = pbf_decode(pbf) print() # assert '11: "3.14"' in text From 0514a31401c978fece1115f37327416005f2e7ce Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 16:51:07 +0800 Subject: [PATCH 19/24] std::to_string has many trailing zeros --- src/geobuf/geobuf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index be505eb..ce42b0a 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -424,7 +424,7 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) }, [&](int64_t id) { pbf.add_int64(12, id); }, [&](double d) { - pbf.add_string(11, std::to_string(d)); + pbf.add_string(11, dbg(fmt::format("{}", d))); }, [&](const std::string &id) { pbf.add_string(11, id); }, [&](const auto &) { From f0a5a0bece9927a1d99b6ac650047285f00ff496 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 17:01:23 +0800 Subject: [PATCH 20/24] geobuf id --- src/geobuf/geobuf.cpp | 19 +++++++++---------- tests/test_geobuf.py | 35 +++++++++++++++++------------------ 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/geobuf/geobuf.cpp b/src/geobuf/geobuf.cpp index ce42b0a..e135a3e 100644 --- a/src/geobuf/geobuf.cpp +++ b/src/geobuf/geobuf.cpp @@ -6,8 +6,6 @@ #include #include -#include "spdlog/spdlog.h" - #include "rapidjson/error/en.h" #include "rapidjson/filereadstream.h" #include "rapidjson/filewritestream.h" @@ -23,14 +21,12 @@ // https://github.com/mapbox/geobuf/blob/master/encode.js // https://github.com/mapbox/geobuf/blob/master/decode.js +#ifdef NDEBUG +#define dbg(x) x +#else +#define DBG_MACRO_NO_WARNING #include "dbg.h" - -// #ifdef NDEBUG -// #define dbg(x) x -// #else -// #define DBG_MACRO_NO_WARNING -// #include "dbg.h" -// #endif +#endif constexpr const auto RJFLAGS = rapidjson::kParseDefaultFlags | // rapidjson::kParseCommentsFlag | // @@ -424,7 +420,10 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf) }, [&](int64_t id) { pbf.add_int64(12, id); }, [&](double d) { - pbf.add_string(11, dbg(fmt::format("{}", d))); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.Double(d); + pbf.add_string(11, s.GetString()); }, [&](const std::string &id) { pbf.add_string(11, id); }, [&](const auto &) { diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 062147b..09b81b4 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1561,46 +1561,47 @@ def test_geojson_feature(): feature.clear() - feature.id(3.14) - assert feature.id() == 3.14 - pbf = feature.to_geobuf() - text = pbf_decode(pbf) - print(text) - feature.id(2**63 - 1) - assert feature.id() == 9223372036854775807 pbf = feature.to_geobuf() + assert feature.id() == 9223372036854775807 + assert geojson.Feature().from_geobuf(pbf).id() == 9223372036854775807 text = pbf_decode(pbf) assert "12: 9223372036854775807" in text feature.id(2**63) - assert feature.id() == 9223372036854775808 pbf = feature.to_geobuf() + assert feature.id() == 9223372036854775808 + assert geojson.Feature().from_geobuf(pbf).id() == 9223372036854775808 text = pbf_decode(pbf) assert '11: "9223372036854775808"' in text feature.id(2**64 - 1) - assert feature.id() == 18446744073709551615 pbf = feature.to_geobuf() + assert feature.id() == 18446744073709551615 + assert geojson.Feature().from_geobuf(pbf).id() == 18446744073709551615 text = pbf_decode(pbf) assert '11: "18446744073709551615"' in text feature.id("text") - assert feature.id() == "text" pbf = feature.to_geobuf() + assert feature.id() == "text" + assert geojson.Feature().from_geobuf(pbf).id() == "text" text = pbf_decode(pbf) assert '11: "text"' in text feature.id(3.14) - assert feature.id() == 3.14 pbf = feature.to_geobuf() + assert feature.id() == 3.14 + assert geojson.Feature().from_geobuf(pbf).id() == 3.14 + text = pbf_decode(pbf) + assert '11: "3.14"' in text - f2 = geojson.Feature().from_geobuf(pbf) - assert f2.id() == 3.14 + feature.id("3.14") + pbf = feature.to_geobuf() + assert feature.id() == "3.14" + assert geojson.Feature().from_geobuf(pbf).id() == 3.14 # note that not "3.14" text = pbf_decode(pbf) - print() - # assert '11: "3.14"' in text - # shit + assert '11: "3.14"' in text def test_geojson_load_dump(): @@ -1882,8 +1883,6 @@ def test_query(): if __name__ == "__main__": - test_geojson_feature() - np.set_printoptions(suppress=True) pwd = os.path.abspath(os.path.dirname(__file__)) pytest_main(pwd, test_file=os.path.basename(__file__)) From 8f97a1208a7658ad43aebe74171072378c1f3f75 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 17:20:44 +0800 Subject: [PATCH 21/24] query --- src/geobuf/planet.hpp | 45 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/geobuf/planet.hpp b/src/geobuf/planet.hpp index 277b264..76c285f 100644 --- a/src/geobuf/planet.hpp +++ b/src/geobuf/planet.hpp @@ -33,12 +33,12 @@ struct Planet const Eigen::Vector2d &max) const { auto &tree = this->rtree(); - auto hits = tree.search(min[0], min[1], max[0], max[1]); - Eigen::VectorXi index(hits.size()); - for (int i = 0, N = hits.size(); i < N; ++i) { - index[i] = hits[i].offset; + std::set hits; + for (auto h : tree.search(min[0], min[1], max[0], max[1])) { + hits.insert(h.offset); } - return index; + std::vector index(hits.begin(), hits.end()); + return Eigen::VectorXi::Map(index.data(), index.size()); } FeatureCollection copy(const Eigen::VectorXi &index) const { @@ -169,11 +169,42 @@ struct Planet if (rtree_) { return *rtree_; } + size_t N = 0; + for (auto &feature : features_) { + if (!feature.geometry.is()) { + ++N; + continue; + } + auto &ls = feature.geometry.get(); + N += std::max(ls.size() - 1, size_t{1}); + } auto nodes = std::vector{}; - nodes.reserve(features_.size()); + nodes.reserve(N); uint64_t index{0}; for (auto &feature : features_) { - nodes.emplace_back(envelope_2d(feature.geometry, index++)); + if (!feature.geometry.is()) { + nodes.emplace_back(envelope_2d(feature.geometry, index++)); + continue; + } + auto &ls = feature.geometry.get(); + if (ls.size() < 2) { + nodes.emplace_back(envelope_2d(feature.geometry, index++)); + continue; + } + for (size_t i = 0, I = ls.size() - 1; i < I; ++i) { + double xmin = ls[i].x; + double xmax = ls[i + 1].x; + double ymin = ls[i].y; + double ymax = ls[i + 1].y; + if (xmin > xmax) { + std::swap(xmin, xmax); + } + if (ymin > ymax) { + std::swap(ymin, ymax); + } + nodes.emplace_back(xmin, ymin, xmax, ymax, index); + } + ++index; } auto extent = calcExtent(nodes); hilbertSort(nodes, extent); From 15f1663d4b989cc8cdc7ca00e9000ea6a4c44c47 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 17:37:39 +0800 Subject: [PATCH 22/24] not ready --- .gitignore | 1 + src/geobuf/planet.hpp | 2 +- tests/test_geobuf.py | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 55226fe..59689bb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ benchmarks/*.json benchmarks/*.pbf* # https://github.com/cubao/nano-fmm/blob/master/data/Makefile data/suzhoubeizhan.json +tests/*.json diff --git a/src/geobuf/planet.hpp b/src/geobuf/planet.hpp index 76c285f..5768a6b 100644 --- a/src/geobuf/planet.hpp +++ b/src/geobuf/planet.hpp @@ -202,7 +202,7 @@ struct Planet if (ymin > ymax) { std::swap(ymin, ymax); } - nodes.emplace_back(xmin, ymin, xmax, ymax, index); + nodes.push_back({xmin, ymin, xmax, ymax, index}); } ++index; } diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index 09b81b4..f788720 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -1867,6 +1867,7 @@ def test_query(): planet = Planet(fc) hits = planet.query([120.64094, 31.41515], [120.64137, 31.41534]) assert len(hits) == 4 + assert hits.tolist() == [529, 530, 536, 659] path = f"{__pwd}/../data/suzhoubeizhan_crossover.json" polygon = geojson.Feature().load(path).to_numpy() @@ -1878,8 +1879,9 @@ def test_query(): assert len(cropped1) == len(cropped2) == 54 assert len(list(cropped1[0].properties().keys())) == 6 assert list(cropped2[0].properties().keys()) == ["index"] - assert cropped2[0].properties()["index"]() == 438 - assert fc[438] == cropped1[0] + assert cropped2[-1].properties()["index"]() == 977 + hits = [f.properties()["index"]() for f in cropped2] + # assert fc[977] == cropped1[-1] if __name__ == "__main__": From 6625dde9708e2a7e7534c10d2cb7520185045c41 Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 18:04:11 +0800 Subject: [PATCH 23/24] add release notes --- docs/about/release-notes.md | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md index ba82d3e..f974019 100644 --- a/docs/about/release-notes.md +++ b/docs/about/release-notes.md @@ -10,6 +10,13 @@ To upgrade `pybind11-geobuf` to the latest version, use pip: pip install -U pybind11-geobuf ``` +## Version 0.1.8 (2023-11-11) + +* Fix readthedocs +* Full support of geobuf Feature id +* Fix geojson value uint64_t/int64_t +* Misc updates + ## Version 0.1.7 (2023-11-11) * Integrate PackedRTree (rbush) diff --git a/setup.py b/setup.py index a9c3d58..f2db1f0 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ def build_extension(self, ext): # logic and declaration, and simpler if you include description/version in a file. setup( name="pybind11_geobuf", - version="0.1.7", + version="0.1.8", author="tzx", author_email="dvorak4tzx@gmail.com", url="https://geobuf-cpp.readthedocs.io", From 42f280df4e02e0e5b3925c980616e635f29a369f Mon Sep 17 00:00:00 2001 From: TANG ZHIXIONG Date: Sat, 11 Nov 2023 18:09:33 +0800 Subject: [PATCH 24/24] fix lint --- tests/test_geobuf.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_geobuf.py b/tests/test_geobuf.py index f788720..0486d8d 100644 --- a/tests/test_geobuf.py +++ b/tests/test_geobuf.py @@ -705,7 +705,10 @@ def test_geojson_multi_line_string(): g1.push_back(geojson.LineString([[2, 4, 6], [7, 8, 9]])) assert g1() == [[[7, 8, 0], [2, 3, 0]], [[2, 4, 6], [7, 8, 9]]] gg.push_back(geojson.LineString([[2, 4, 6], [7, 8, 9]])) - assert gg()["coordinates"] == [[[7, 8, 0], [2, 3, 0]], [[2, 4, 6], [7, 8, 9]]] + assert gg()["coordinates"] == [ + [[7, 8, 0], [2, 3, 0]], + [[2, 4, 6], [7, 8, 9]], + ] g1.clear() assert len(g1) == 0 @@ -1190,7 +1193,10 @@ def test_geojson_geometry(): g5.resize(1) assert g5() == {"type": "MultiLineString", "coordinates": [[]]} g5.push_back([70, 80, 90]) - assert g5() == {"type": "MultiLineString", "coordinates": [[[70.0, 80.0, 90.0]]]} + assert g5() == { + "type": "MultiLineString", + "coordinates": [[[70.0, 80.0, 90.0]]], + } g6 = geojson.Geometry(geojson.Polygon([[1, 2, 3], [4, 5, 6]])) assert np.array(g6()["coordinates"]).shape == (1, 2, 3) @@ -1536,7 +1542,7 @@ def test_geojson_feature(): "properties": {}, } assert feature.id() is None - # using identifier = mapbox::util::variant; + # id := variant assert feature.id(42).id() == 42 assert feature.id(-1024).id() == -1024 assert feature.id(3.14).id() == 3.14 @@ -1546,18 +1552,15 @@ def test_geojson_feature(): assert feature.id(2**64 - 1).id() == 18446744073709551615 with pytest.raises(RuntimeError) as excinfo: feature.id(2**64) # out of range of uint64_t - assert "integer out of range of int64_t/uint64_t: 18446744073709551616" in str( - excinfo - ) + prefix = "integer out of range of int64_t/uint64_t" + assert f"{prefix}: 18446744073709551616" in str(excinfo) feature["num"] = -(2**63) assert feature["num"]() == -9223372036854775808 feature["num"] = 2**64 - 1 assert feature["num"]() == 18446744073709551615 with pytest.raises(RuntimeError) as excinfo: feature["num"] = 2**64 - assert "integer out of range of int64_t/uint64_t: 18446744073709551616" in str( - excinfo - ) + assert f"{prefix}: 18446744073709551616" in str(excinfo) feature.clear() @@ -1599,7 +1602,8 @@ def test_geojson_feature(): feature.id("3.14") pbf = feature.to_geobuf() assert feature.id() == "3.14" - assert geojson.Feature().from_geobuf(pbf).id() == 3.14 # note that not "3.14" + # note that not "3.14" + assert geojson.Feature().from_geobuf(pbf).id() == 3.14 text = pbf_decode(pbf) assert '11: "3.14"' in text