Skip to content

Commit

Permalink
More plumbing for key_pool
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-n-f
e-n-f committed Nov 4, 2024
1 parent ab95915 commit 7670174
Show file tree
Hide file tree
Showing 7 changed files with 21 additions and 15 deletions.
9 changes: 5 additions & 4 deletions geobuf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,14 +270,14 @@ std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, d
return ret;
}

void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) {
void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername, key_pool &key_pool) {
std::vector<drawvec_type> dv;
long long id = 0;
bool has_id = false;
std::vector<serial_val> values;
std::map<std::string, serial_val> other;

std::vector<std::string> full_keys;
std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> full_values;

while (pbf.next()) {
Expand Down Expand Up @@ -338,7 +338,7 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<s
exit(EXIT_IMPOSSIBLE);
}

full_keys.push_back(keys[properties[i]]);
full_keys.push_back(key_pool.pool(keys[properties[i]]));
full_values.push_back(values[properties[i + 1]]);
}

Expand Down Expand Up @@ -434,10 +434,11 @@ struct queue_run_arg {

void *run_parse_feature(void *v) {
struct queue_run_arg *qra = (struct queue_run_arg *) v;
key_pool key_pool;

for (size_t i = qra->start; i < qra->end; i++) {
struct queued_feature &qf = feature_queue[i];
readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername);
readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername, key_pool);
}

return NULL;
Expand Down
5 changes: 3 additions & 2 deletions geocsv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
}

size_t seq = 0;
key_pool key_pool;
while ((s = csv_getline(f)).size() > 0) {
std::string err = check_utf8(s);
if (err != "") {
Expand Down Expand Up @@ -89,7 +90,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
drawvec dv;
dv.push_back(draw(VT_MOVETO, x, y));

std::vector<std::string> full_keys;
std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> full_values;

for (size_t i = 0; i < line.size(); i++) {
Expand All @@ -107,7 +108,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
}
sv.s = line[i];

full_keys.push_back(header[i]);
full_keys.push_back(key_pool.pool(header[i]));
full_values.push_back(sv);
}
}
Expand Down
9 changes: 5 additions & 4 deletions geojson.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,17 +182,18 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
nprop = properties->value.object.length;
}

std::vector<std::string> keys;
std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> values;

keys.reserve(nprop);
full_keys.reserve(nprop);
values.reserve(nprop);
key_pool key_pool;

for (size_t i = 0; i < nprop; i++) {
if (properties->value.object.keys[i]->type == JSON_STRING) {
serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature);

keys.emplace_back(properties->value.object.keys[i]->value.string.string);
full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string));
values.push_back(std::move(sv));
}
}
Expand All @@ -211,7 +212,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
sf.geometry = dv;
sf.feature_minzoom = 0; // Will be filled in during index merging
sf.seq = *(sst->layer_seq);
sf.full_keys = std::move(keys);
sf.full_keys = std::move(full_keys);
sf.full_values = std::move(values);

return serialize_feature(sst, sf, tippecanoe_layername);
Expand Down
4 changes: 2 additions & 2 deletions plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
}

// Reads from the prefilter
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter) {
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter, key_pool &key_pool) {
serial_feature sf;

while (1) {
Expand Down Expand Up @@ -354,7 +354,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
// would have already run before prefiltering

if (v.type != mvt_null) {
sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string));
sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string)));
sf.full_values.push_back(v);

if (!postfilter) {
Expand Down
3 changes: 2 additions & 1 deletion plugin.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
struct key_pool;
std::vector<mvt_layer> filter_layers(const char *filter, std::vector<mvt_layer> &layer, unsigned z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, int extent);
void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y);
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters);
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters, key_pool &key_pool);
3 changes: 2 additions & 1 deletion serial.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,8 +413,9 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g)
}

// called from frontends
int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername, key_pool &key_pool) {
int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername) {
struct reader *r = &(*sst->readers)[sst->segment];
key_pool key_pool;

sf.bbox[0] = LLONG_MAX;
sf.bbox[1] = LLONG_MAX;
Expand Down
3 changes: 2 additions & 1 deletion tile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1063,6 +1063,7 @@ struct next_feature_state {
// geometry type set to -2.
static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, std::atomic<bool> *within, compressor **geomfile, std::atomic<long long> *geompos, long long start_geompos[], std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data, next_feature_state &next_feature_state, double droprate) {
double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate);

while (1) {
serial_feature sf;
long long len;
Expand Down Expand Up @@ -1859,7 +1860,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate);
} else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL, key_pool);
}

if (sf.t < 0) {
Expand Down

0 comments on commit 7670174

Please sign in to comment.