full support of geobuf Feature id, fix geojson uint64_t/int64_t, other misc updates

.gitignore

@@ -8,3 +8,4 @@ benchmarks/*.json
 benchmarks/*.pbf*
 # https://github.com/cubao/nano-fmm/blob/master/data/Makefile
 data/suzhoubeizhan.json
+tests/*.json

.readthedocs.yml

@@ -1,5 +1,9 @@
 version: 2
 formats: all
+build:
+    os: ubuntu-22.04
+    tools:
+        python: "3.10"
 mkdocs:
   fail_on_warning: false
 python:

docs/about/release-notes.md

@@ -10,6 +10,13 @@ To upgrade `pybind11-geobuf` to the latest version, use pip:
 pip install -U pybind11-geobuf
 ```
 
+## Version 0.1.8 (2023-11-11)
+
+*   Fix readthedocs
+*   Full support of geobuf Feature id
+*   Fix geojson value uint64_t/int64_t
+*   Misc updates
+
 ## Version 0.1.7 (2023-11-11)
 
 *   Integrate PackedRTree (rbush)

setup.py

@@ -127,7 +127,7 @@ def build_extension(self, ext):
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="pybind11_geobuf",
-    version="0.1.7",
+    version="0.1.8",
     author="tzx",
     author_email="dvorak4tzx@gmail.com",
     url="https://geobuf-cpp.readthedocs.io",

src/geobuf/geobuf.cpp

@@ -411,17 +411,24 @@ void Encoder::writeFeature(const mapbox::geojson::feature &feature, Pbf &pbf)
         // int64_t, double, std::string>;
         feature.id.match(
             [&](uint64_t uid) {
-                int id = static_cast<int64_t>(uid);
-                if (id >= 0) {
-                    pbf.add_int64(12, id);
+                if (uid <= static_cast<uint64_t>(
+                               std::numeric_limits<int64_t>::max())) {
+                    pbf.add_int64(12, static_cast<int64_t>(uid));
                 } else {
                     pbf.add_string(11, std::to_string(uid));
                 }
             },
             [&](int64_t id) { pbf.add_int64(12, id); },
+            [&](double d) {
+                rapidjson::StringBuffer s;
+                rapidjson::Writer<rapidjson::StringBuffer> writer(s);
+                writer.Double(d);
+                pbf.add_string(11, s.GetString());
+            },
             [&](const std::string &id) { pbf.add_string(11, id); },
             [&](const auto &) {
-                pbf.add_string(11, dump(to_json(feature.id)));
+                throw std::invalid_argument("invalid id: " +
+                                            dump(to_json(feature.id)));
             });
     }
     if (!feature.properties.empty()) {
@@ -702,7 +709,25 @@ mapbox::geojson::feature Decoder::readFeature(Pbf &pbf)
             protozero::pbf_reader pbf_g = pbf.get_message();
             f.geometry = readGeometry(pbf_g);
         } else if (tag == 11) {
-            f.id = pbf.get_string(); // TODO, restore to mapbox::geojson id
+            // see "feature.id.match(" in this source file
+            // protobuf:    oneof id_type {
+            //                  string id = 11;
+            //                  sint64 int_id = 12;
+            //              }
+            // geojson:     id := <null, uint64_t, int64_t, double, string>
+            auto text = pbf.get_string();
+            auto json = parse(text);
+            if (json.IsNumber()) {
+                if (json.IsUint64()) {
+                    f.id = json.GetUint64();
+                } else if (json.IsInt64()) {
+                    f.id = json.GetInt64();
+                } else {
+                    f.id = json.GetDouble();
+                }
+            } else {
+                f.id = text;
+            }
         } else if (tag == 12) {
             f.id = pbf.get_int64();
         } else if (tag == 13) {

src/geobuf/geojson_helpers.hpp

@@ -291,19 +291,35 @@ inline std::string get_type(const mapbox::geojson::value &self)
 inline void geometry_push_back(mapbox::geojson::geometry &self,
                                const mapbox::geojson::point &point)
 {
-    self.match([&](mapbox::geojson::multi_point &g) { g.push_back(point); },
-               [&](mapbox::geojson::line_string &g) { g.push_back(point); },
-               [&](mapbox::geojson::multi_line_string &g) {
-                   g.back().push_back(point);
-               },
-               [&](mapbox::geojson::polygon &g) { g.back().push_back(point); },
-               [&](mapbox::geojson::multi_polygon &g) {
-                   g.back().back().push_back(point);
-               },
-               [&](auto &g) {
-                   std::cerr << "geometry_push_back not handled for this type: "
-                             << geometry_type(g) << std::endl;
-               });
+    self.match(
+        [&](mapbox::geojson::multi_point &g) { g.push_back(point); },
+        [&](mapbox::geojson::line_string &g) { g.push_back(point); },
+        [&](mapbox::geojson::multi_line_string &g) {
+            if (g.empty()) {
+                throw std::invalid_argument(
+                    "can't push_back Point to empty MultiLineString, may "
+                    "resize first");
+            }
+            g.back().push_back(point);
+        },
+        [&](mapbox::geojson::polygon &g) {
+            if (g.empty()) {
+                throw std::invalid_argument(
+                    "can't push_back Point to empty Polygon, may resize first");
+            }
+            g.back().push_back(point);
+        },
+        [&](mapbox::geojson::multi_polygon &g) {
+            if (g.empty() || g.back().empty()) {
+                throw std::invalid_argument("can't push_back Point to invalid "
+                                            "MultiPolygon, may resize first");
+            }
+            g.back().back().push_back(point);
+        },
+        [&](auto &g) {
+            std::cerr << "geometry_push_back(point) not handled for this type: "
+                      << geometry_type(g) << std::endl;
+        });
 }
 
 inline void geometry_push_back(mapbox::geojson::geometry &self,
@@ -325,8 +341,9 @@ inline void geometry_push_back(mapbox::geojson::geometry &self,
             eigen2geom(points, g.back());
         },
         [&](auto &g) {
-            std::cerr << "geometry_push_back not handled for this type: "
-                      << geometry_type(g) << std::endl;
+            std::cerr
+                << "geometry_push_back(ndarray) not handled for this type: "
+                << geometry_type(g) << std::endl;
         });
 }
 
@@ -336,8 +353,9 @@ inline void geometry_push_back(mapbox::geojson::geometry &self,
     self.match(
         [&](mapbox::geojson::geometry_collection &g) { g.push_back(geom); },
         [&](auto &g) {
-            std::cerr << "geometry_push_back(geom) not handled for this type: "
-                      << geometry_type(g) << std::endl;
+            std::cerr << "geometry_push_back(geom:" << geometry_type(geom)
+                      << ") not handled for this type: " << geometry_type(g)
+                      << std::endl;
         });
 }
 
@@ -387,6 +405,20 @@ inline void geometry_clear(mapbox::geojson::geometry &self)
                });
 }
 
+inline void geometry_resize(mapbox::geojson::geometry &self, int size)
+{
+    self.match([&](mapbox::geojson::multi_point &g) { g.resize(size); },
+               [&](mapbox::geojson::line_string &g) { g.resize(size); },
+               [&](mapbox::geojson::multi_line_string &g) { g.resize(size); },
+               [&](mapbox::geojson::polygon &g) { g.resize(size); },
+               [&](mapbox::geojson::multi_polygon &g) { g.resize(size); },
+               [&](mapbox::geojson::geometry_collection &g) { g.resize(size); },
+               [&](auto &g) {
+                   std::cerr << "geometry_resize() not handled for this type: "
+                             << geometry_type(g) << std::endl;
+               });
+}
+
 inline void geojson_value_clear(mapbox::geojson::value &self)
 {
     self.match([](mapbox::geojson::value::array_type &arr) { arr.clear(); },

src/geobuf/pbf_decoder.cpp

@@ -61,11 +61,11 @@ bool decode_printable_string(std::stringstream &out,
     static constexpr const std::size_t max_string_length = 60;
 
     const std::string str{view.data(), view.size()};
-    if (str.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUV"
-                              "WXYZ0123456789_:-") != std::string::npos) {
+    if (!std::all_of(str.begin(), str.end(), [](char c) {
+            return std::isprint(static_cast<unsigned char>(c));
+        })) {
         return false;
     }
-
     if (str.size() > max_string_length) {
         out << '"' << str.substr(0, max_string_length) << "\"...\n";
     } else {

src/geobuf/planet.hpp

@@ -33,12 +33,12 @@ struct Planet
                           const Eigen::Vector2d &max) const
     {
         auto &tree = this->rtree();
-        auto hits = tree.search(min[0], min[1], max[0], max[1]);
-        Eigen::VectorXi index(hits.size());
-        for (int i = 0, N = hits.size(); i < N; ++i) {
-            index[i] = hits[i].offset;
+        std::set<int> hits;
+        for (auto h : tree.search(min[0], min[1], max[0], max[1])) {
+            hits.insert(h.offset);
         }
-        return index;
+        std::vector<int> index(hits.begin(), hits.end());
+        return Eigen::VectorXi::Map(index.data(), index.size());
     }
     FeatureCollection copy(const Eigen::VectorXi &index) const
     {
@@ -169,11 +169,42 @@ struct Planet
         if (rtree_) {
             return *rtree_;
         }
+        size_t N = 0;
+        for (auto &feature : features_) {
+            if (!feature.geometry.is<mapbox::geojson::line_string>()) {
+                ++N;
+                continue;
+            }
+            auto &ls = feature.geometry.get<mapbox::geojson::line_string>();
+            N += std::max(ls.size() - 1, size_t{1});
+        }
         auto nodes = std::vector<FlatGeobuf::NodeItem>{};
-        nodes.reserve(features_.size());
+        nodes.reserve(N);
         uint64_t index{0};
         for (auto &feature : features_) {
-            nodes.emplace_back(envelope_2d(feature.geometry, index++));
+            if (!feature.geometry.is<mapbox::geojson::line_string>()) {
+                nodes.emplace_back(envelope_2d(feature.geometry, index++));
+                continue;
+            }
+            auto &ls = feature.geometry.get<mapbox::geojson::line_string>();
+            if (ls.size() < 2) {
+                nodes.emplace_back(envelope_2d(feature.geometry, index++));
+                continue;
+            }
+            for (size_t i = 0, I = ls.size() - 1; i < I; ++i) {
+                double xmin = ls[i].x;
+                double xmax = ls[i + 1].x;
+                double ymin = ls[i].y;
+                double ymax = ls[i + 1].y;
+                if (xmin > xmax) {
+                    std::swap(xmin, xmax);
+                }
+                if (ymin > ymax) {
+                    std::swap(ymin, ymax);
+                }
+                nodes.push_back({xmin, ymin, xmax, ymax, index});
+            }
+            ++index;
         }
         auto extent = calcExtent(nodes);
         hilbertSort(nodes, extent);

src/geobuf/pybind11_helpers.hpp

@@ -31,6 +31,7 @@ using geojson_value = mapbox::geojson::value;
 
 inline RapidjsonValue __py_int_to_rapidjson(const py::handle &obj)
 {
+    // https://github.com/pybind/pybind11_json/blob/master/include/pybind11_json/pybind11_json.hpp#L84-L104
     try {
         auto num = obj.cast<int64_t>();
         if (py::int_(num).equal(obj)) {
@@ -319,12 +320,23 @@ inline mapbox::geojson::identifier to_feature_id(const py::object &obj)
         return {};
     }
     if (py::isinstance<py::int_>(obj)) {
-        auto val = obj.cast<long long>();
-        if (val < 0) {
-            return int64_t(val);
-        } else {
-            return uint64_t(val);
+        // similar to __py_int_to_rapidjson
+        try {
+            auto num = obj.cast<int64_t>();
+            if (py::int_(num).equal(obj)) {
+                return num;
+            }
+        } catch (...) {
         }
+        try {
+            auto num = obj.cast<uint64_t>();
+            if (py::int_(num).equal(obj)) {
+                return num;
+            }
+        } catch (...) {
+        }
+        throw std::runtime_error("integer out of range of int64_t/uint64_t: " +
+                                 py::repr(obj).cast<std::string>());
     } else if (py::isinstance<py::float_>(obj)) {
         return obj.cast<double>();
     } else if (py::isinstance<py::str>(obj)) {
@@ -386,26 +398,37 @@ inline py::object to_python(const mapbox::geojson::feature_collection &features)
 inline geojson_value to_geojson_value(const py::handle &obj)
 {
     if (obj.ptr() == nullptr || obj.is_none()) {
-        return nullptr;
+        return {};
     }
     if (py::isinstance<py::bool_>(obj)) {
         return obj.cast<bool>();
     }
     if (py::isinstance<py::int_>(obj)) {
-        auto val = obj.cast<long long>();
-        if (val < 0) {
-            return int64_t(val);
-        } else {
-            return uint64_t(val);
+        // similar to __py_int_to_rapidjson
+        try {
+            auto num = obj.cast<int64_t>();
+            if (py::int_(num).equal(obj)) {
+                return num;
+            }
+        } catch (...) {
+        }
+        try {
+            auto num = obj.cast<uint64_t>();
+            if (py::int_(num).equal(obj)) {
+                return num;
+            }
+        } catch (...) {
         }
+        throw std::runtime_error("integer out of range of int64_t/uint64_t: " +
+                                 py::repr(obj).cast<std::string>());
     }
+
     if (py::isinstance<py::float_>(obj)) {
         return obj.cast<double>();
     }
     if (py::isinstance<py::str>(obj)) {
         return obj.cast<std::string>();
     }
-    // TODO, bytes
     if (py::isinstance<py::tuple>(obj) || py::isinstance<py::list>(obj)) {
         geojson_value::array_type ret;
         for (const py::handle &e : obj) {
@@ -420,6 +443,14 @@ inline geojson_value to_geojson_value(const py::handle &obj)
         }
         return ret;
     }
+    if (py::isinstance<py::bytes>(obj)) {
+        // https://github.com/pybind/pybind11_json/blob/master/include/pybind11_json/pybind11_json.hpp#L112
+        py::module base64 = py::module::import("base64");
+        auto str = base64.attr("b64encode")(obj)
+                       .attr("decode")("utf-8")
+                       .cast<std::string>();
+        return str;
+    }
     throw std::runtime_error(
         "to_geojson_value not implemented for this type of object: " +
         py::repr(obj).cast<std::string>());