diff --git a/geobuf.cpp b/geobuf.cpp index b754eac2..02d8ded3 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -270,14 +270,14 @@ std::vector readGeometry(protozero::pbf_reader &pbf, size_t dim, d return ret; } -void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector &keys, struct serialization_state *sst, int layer, std::string layername) { +void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector &keys, struct serialization_state *sst, int layer, std::string layername, key_pool &key_pool) { std::vector dv; long long id = 0; bool has_id = false; std::vector values; std::map other; - std::vector full_keys; + std::vector> full_keys; std::vector full_values; while (pbf.next()) { @@ -338,7 +338,7 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectorstart; i < qra->end; i++) { struct queued_feature &qf = feature_queue[i]; - readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername); + readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername, key_pool); } return NULL; diff --git a/geocsv.cpp b/geocsv.cpp index 69f7d507..2c174999 100644 --- a/geocsv.cpp +++ b/geocsv.cpp @@ -58,6 +58,7 @@ void parse_geocsv(std::vector &sst, std::string fnam } size_t seq = 0; + key_pool key_pool; while ((s = csv_getline(f)).size() > 0) { std::string err = check_utf8(s); if (err != "") { @@ -89,7 +90,7 @@ void parse_geocsv(std::vector &sst, std::string fnam drawvec dv; dv.push_back(draw(VT_MOVETO, x, y)); - std::vector full_keys; + std::vector> full_keys; std::vector full_values; for (size_t i = 0; i < line.size(); i++) { @@ -107,7 +108,7 @@ void parse_geocsv(std::vector &sst, std::string fnam } sv.s = line[i]; - full_keys.push_back(header[i]); + full_keys.push_back(key_pool.pool(header[i])); full_values.push_back(sv); } } diff --git a/geojson.cpp b/geojson.cpp index f724e0bd..dae22ec3 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -182,17 +182,18 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom nprop = properties->value.object.length; } - std::vector keys; + std::vector> full_keys; std::vector values; - keys.reserve(nprop); + full_keys.reserve(nprop); values.reserve(nprop); + key_pool key_pool; for (size_t i = 0; i < nprop; i++) { if (properties->value.object.keys[i]->type == JSON_STRING) { serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); - keys.emplace_back(properties->value.object.keys[i]->value.string.string); + full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); values.push_back(std::move(sv)); } } @@ -211,7 +212,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom sf.geometry = dv; sf.feature_minzoom = 0; // Will be filled in during index merging sf.seq = *(sst->layer_seq); - sf.full_keys = std::move(keys); + sf.full_keys = std::move(full_keys); sf.full_values = std::move(values); return serialize_feature(sst, sf, tippecanoe_layername); diff --git a/plugin.cpp b/plugin.cpp index e5d75809..2b06e335 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -145,7 +145,7 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter) { +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { serial_feature sf; while (1) { @@ -354,7 +354,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: // would have already run before prefiltering if (v.type != mvt_null) { - sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string)); + sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string))); sf.full_values.push_back(v); if (!postfilter) { diff --git a/plugin.hpp b/plugin.hpp index 595e7f13..fc901daf 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,3 +1,4 @@ +struct key_pool; std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters); +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); diff --git a/serial.cpp b/serial.cpp index 448fcea6..5afc2ed2 100644 --- a/serial.cpp +++ b/serial.cpp @@ -413,8 +413,9 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g) } // called from frontends -int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername, key_pool &key_pool) { +int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername) { struct reader *r = &(*sst->readers)[sst->segment]; + key_pool key_pool; sf.bbox[0] = LLONG_MAX; sf.bbox[1] = LLONG_MAX; diff --git a/tile.cpp b/tile.cpp index 74ec407b..4387537b 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1063,6 +1063,7 @@ struct next_feature_state { // geometry type set to -2. static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate); + while (1) { serial_feature sf; long long len; @@ -1859,7 +1860,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (prefilter == NULL) { sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate); } else { - sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); + sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL, key_pool); } if (sf.t < 0) {