Store objects sequentially (breaks include_ids)

include/geom.h

@@ -25,13 +25,6 @@ using uint = unsigned int;
 #include <boost/interprocess/managed_external_buffer.hpp>
 #include <boost/interprocess/allocators/node_allocator.hpp>
 
-#define OSMID_TYPE_OFFSET 40
-#define OSMID_MASK      ((1L<<OSMID_TYPE_OFFSET)-1)
-#define OSMID_SHAPE     (0L<<OSMID_TYPE_OFFSET)
-#define OSMID_NODE      (1L<<OSMID_TYPE_OFFSET)
-#define OSMID_WAY       (2L<<OSMID_TYPE_OFFSET)
-#define OSMID_RELATION  (3L<<OSMID_TYPE_OFFSET)
-
 namespace bi = boost::interprocess;
 
 typedef boost::geometry::model::d2::point_xy<double> Point; 

include/osm_lua_processing.h

@@ -222,7 +222,6 @@ class OsmLuaProcessing {
 	class OsmMemTiles &osmMemTiles;
 	AttributeStore &attributeStore;			// key/value store
 
-	uint64_t osmID;							///< ID of OSM object (relations have decrementing way IDs)
 	int64_t originalOsmID;					///< Original OSM object ID
 	bool isWay, isRelation, isClosed;		///< Way, node, relation?
 

include/output_object.h

@@ -23,13 +23,6 @@ typedef int8_t ZOrder;
 
 enum OutputGeometryType : unsigned int { POINT_, LINESTRING_, MULTILINESTRING_, POLYGON_ };
 
-#define OSMID_TYPE_OFFSET	40
-#define OSMID_MASK 		((1ULL<<OSMID_TYPE_OFFSET)-1)
-#define OSMID_SHAPE 	(0ULL<<OSMID_TYPE_OFFSET)
-#define OSMID_NODE 		(1ULL<<OSMID_TYPE_OFFSET)
-#define OSMID_WAY 		(2ULL<<OSMID_TYPE_OFFSET)
-#define OSMID_RELATION 	(3ULL<<OSMID_TYPE_OFFSET)
-
 //\brief Display the geometry type
 std::ostream& operator<<(std::ostream& os, OutputGeometryType geomType);
 
@@ -50,7 +43,7 @@ class OutputObject {
 
 
 public:
-	NodeID objectID 			: 42;					// id of way (linestring/polygon) or node (point)
+	NodeID objectID 			: 42;					// id of point/linestring/polygon
 	OutputGeometryType geomType : 2;					// point, linestring, polygon
 	unsigned minZoom 			: 4;					// minimum zoom level in which object is written
 	uint_least8_t layer 		: 8;					// what layer is it in?

include/tile_data.h

@@ -29,20 +29,20 @@ class TileDataSource {
 	unsigned int baseZoom;
 
 	// Store for generated geometries
-	using point_store_t = std::deque<std::pair<NodeID, Point>>;
+	using point_store_t = std::vector<Point>;
 
 	using linestring_t = boost::geometry::model::linestring<Point, std::vector, mmap_allocator>;
-	using linestring_store_t = std::deque<std::pair<NodeID, linestring_t>>;
+	using linestring_store_t = std::vector<linestring_t>;
 
 	using multi_linestring_t = boost::geometry::model::multi_linestring<linestring_t, std::vector, mmap_allocator>;
-	using multi_linestring_store_t = std::deque<std::pair<NodeID, multi_linestring_t>>;
+	using multi_linestring_store_t = std::vector<multi_linestring_t>;
 
 	using polygon_t = boost::geometry::model::polygon<Point, true, true, std::vector, std::vector, mmap_allocator, mmap_allocator>;
 	using multi_polygon_t = boost::geometry::model::multi_polygon<polygon_t, std::vector, mmap_allocator>;
-	using multi_polygon_store_t = std::deque<std::pair<NodeID, multi_polygon_t>>;
+	using multi_polygon_store_t = std::vector<multi_polygon_t>;
 
-	std::mutex points_store_mutex;
-	std::unique_ptr<point_store_t> points_store;
+	std::mutex point_store_mutex;
+	std::unique_ptr<point_store_t> point_store;
 
 	std::mutex linestring_store_mutex;
 	std::unique_ptr<linestring_store_t> linestring_store;
@@ -92,16 +92,14 @@ class TileDataSource {
 
 	void MergeLargeObjects(TileCoordinates dstIndex, uint zoom, std::vector<OutputObjectRef> &dstTile);
 
-	void SortGeometries(unsigned int threadNum);
-
 	std::vector<OutputObjectRef> getTileData(std::vector<bool> const &sortOrders, 
 	                                         TileCoordinates coordinates, unsigned int zoom);
 
 	Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox) const;
 	LatpLon buildNodeGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox) const;
 
 	void open() {
-		points_store = std::make_unique<point_store_t>();
+		point_store = std::make_unique<point_store_t>();
 		linestring_store = std::make_unique<linestring_store_t>();
 		multi_polygon_store = std::make_unique<multi_polygon_store_t>();
 		multi_linestring_store = std::make_unique<multi_linestring_store_t>();
@@ -112,58 +110,52 @@ class TileDataSource {
 	using handle_t = void *;
 
 	template<typename T>
-	void store_point(NodeID id, T const &input) {	
-		std::lock_guard<std::mutex> lock(points_store_mutex);
-		points_store->emplace_back(id, input);		   	
+	NodeID store_point(T const &input) {
+		std::lock_guard<std::mutex> lock(point_store_mutex);
+		NodeID id = point_store->size();
+		point_store->emplace_back(input);
+		return id;
 	}
 
 	Point const &retrieve_point(NodeID id) const {
-		auto iter = std::lower_bound(points_store->begin(), points_store->end(), id, [](auto const &e, auto id) { 
-			return e.first < id; 
-		});
-		if(iter == points_store->end() || iter->first != id)
-			throw std::out_of_range("Could not find generated node with id " + std::to_string(id));
-		return iter->second;
+		if (id > point_store->size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id));
+		return point_store->at(id);
 	}
 
 	template<typename Input>
-	void store_linestring(NodeID id, Input const &src) {
+	NodeID store_linestring(Input const &src) {
 		linestring_t dst(src.begin(), src.end());
 		std::lock_guard<std::mutex> lock(linestring_store_mutex);
-		linestring_store->emplace_back(id, std::move(dst));
+		NodeID id = linestring_store->size();
+		linestring_store->emplace_back(std::move(dst));
+		return id;
 	}
 
 	linestring_t const &retrieve_linestring(NodeID id) const {
-		auto iter = std::lower_bound(linestring_store->begin(), linestring_store->end(), id, [](auto const &e, auto id) { 
-			return e.first < id; 
-		});
-		if(iter == linestring_store->end() || iter->first != id)
-			throw std::out_of_range("Could not find generated linestring with id " + std::to_string(id));
-		return iter->second;
+		if (id > linestring_store->size()) throw std::out_of_range("Could not find generated linestring with id " + std::to_string(id));
+		return linestring_store->at(id);
 	}
 
 	template<typename Input>
-	void store_multi_linestring(NodeID id, Input const &src) {
+	NodeID store_multi_linestring(Input const &src) {
 		multi_linestring_t dst;
 		dst.resize(src.size());
 		for (std::size_t i=0; i<src.size(); ++i) {
 			boost::geometry::assign(dst[i], src[i]);
 		}
 		std::lock_guard<std::mutex> lock(multi_linestring_store_mutex);
-		multi_linestring_store->emplace_back(id, std::move(dst));
+		NodeID id = multi_linestring_store->size();
+		multi_linestring_store->emplace_back(std::move(dst));
+		return id;
 	}
 
 	multi_linestring_t const &retrieve_multi_linestring(NodeID id) const {
-		auto iter = std::lower_bound(multi_linestring_store->begin(), multi_linestring_store->end(), id, [](auto const &e, auto id) { 
-			return e.first < id; 
-		});
-		if(iter == multi_linestring_store->end() || iter->first != id)
-			throw std::out_of_range("Could not find generated multi-linestring with id " + std::to_string(id));
-		return iter->second;
+		if (id > multi_linestring_store->size()) throw std::out_of_range("Could not find generated multi-linestring with id " + std::to_string(id));
+		return multi_linestring_store->at(id);
 	}
 
 	template<typename Input>
-	void store_multi_polygon(NodeID id, Input const &src) {
+	NodeID store_multi_polygon(Input const &src) {
 		multi_polygon_t dst;
 		dst.resize(src.size());
 		for(std::size_t i = 0; i < src.size(); ++i) {
@@ -177,16 +169,14 @@ class TileDataSource {
 			}
 		}
 		std::lock_guard<std::mutex> lock(multi_polygon_store_mutex);
-		multi_polygon_store->emplace_back(id, std::move(dst));
+		NodeID id = multi_polygon_store->size();
+		multi_polygon_store->emplace_back(std::move(dst));
+		return id;
 	}
 
 	multi_polygon_t const &retrieve_multi_polygon(NodeID id) const {
-		auto iter = std::lower_bound(multi_polygon_store->begin(), multi_polygon_store->end(), id, [](auto const &e, auto id) { 
-			return e.first < id; 
-		});
-		if(iter == multi_polygon_store->end() || iter->first != id)
-			throw std::out_of_range("Could not find generated multi polygon with id " + std::to_string(id));
-		return iter->second;
+		if (id > multi_polygon_store->size()) throw std::out_of_range("Could not find generated multi-polygon with id " + std::to_string(id));
+		return multi_polygon_store->at(id);
 	}
 
 

src/osm_lua_processing.cpp

@@ -331,9 +331,9 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 
             if(!CorrectGeometry(p)) return;
 
-			osmMemTiles.store_point(osmID, p);
+			NodeID id = osmMemTiles.store_point(p);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectPoint(geomType, 
-							layers.layerMap[layerName], osmID, 0, layerMinZoom));
+							layers.layerMap[layerName], id, 0, layerMinZoom));
 			outputs.push_back(std::make_pair(oo, attributes));
             return;
 		}
@@ -361,9 +361,9 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 
             if(!CorrectGeometry(mp)) return;
 
-			osmMemTiles.store_multi_polygon(osmID, mp);
+			NodeID id = osmMemTiles.store_multi_polygon(mp);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectMultiPolygon(geomType, 
-							layers.layerMap[layerName], osmID, 0, layerMinZoom));
+							layers.layerMap[layerName], id, 0, layerMinZoom));
 			outputs.push_back(std::make_pair(oo, attributes));
 		}
 		else if (geomType==MULTILINESTRING_) {
@@ -377,9 +377,9 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 			}
 			if (!CorrectGeometry(mls)) return;
 
-			osmMemTiles.store_multi_linestring(osmID, mls);
+			NodeID id = osmMemTiles.store_multi_linestring(mls);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectMultiLinestring(geomType, 
-							layers.layerMap[layerName], osmID, 0, layerMinZoom));
+							layers.layerMap[layerName], id, 0, layerMinZoom));
 			outputs.push_back(std::make_pair(oo, attributes));
 		}
 		else if (geomType==LINESTRING_) {
@@ -388,9 +388,9 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 
             if(!CorrectGeometry(ls)) return;
 
-			osmMemTiles.store_linestring(osmID, ls);
+			NodeID id = osmMemTiles.store_linestring(ls);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectLinestring(geomType, 
-						layers.layerMap[layerName], osmID, 0, layerMinZoom));
+						layers.layerMap[layerName], id, 0, layerMinZoom));
 			outputs.push_back(std::make_pair(oo, attributes));
 		}
 	} catch (std::invalid_argument &err) {
@@ -424,9 +424,9 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName) {
 		return;
 	}
 
-	osmMemTiles.store_point(osmID, geomp);
+	NodeID id = osmMemTiles.store_point(geomp);
 	OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectPoint(POINT_,
-					layers.layerMap[layerName], osmID, 0, layerMinZoom));
+					layers.layerMap[layerName], id, 0, layerMinZoom));
 	outputs.push_back(std::make_pair(oo, attributes));
 }
 
@@ -559,7 +559,6 @@ bool OsmLuaProcessing::scanRelation(WayID id, const tag_map_t &tags) {
 void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) {
 
 	reset();
-	osmID = (id & OSMID_MASK) | OSMID_NODE;
 	originalOsmID = id;
 	isWay = false;
 	isRelation = false;
@@ -588,7 +587,6 @@ void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) {
 // We are now processing a way
 void OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags) {
 	reset();
-	osmID = (wayId & OSMID_MASK) | OSMID_WAY;
 	originalOsmID = wayId;
 	isWay = true;
 	isRelation = false;
@@ -638,7 +636,6 @@ void OsmLuaProcessing::setRelation(int64_t relationId, WayVec const &outerWayVec
                                    bool isNativeMP,      // only OSM type=multipolygon
                                    bool isInnerOuter) {  // any OSM relation with "inner" and "outer" roles (e.g. type=multipolygon|boundary)
 	reset();
-	osmID = (relationId & OSMID_MASK) | OSMID_RELATION;
 	originalOsmID = relationId;
 	isWay = true;
 	isRelation = true;

src/read_pbf.cpp

@@ -75,7 +75,7 @@ bool PbfReader::ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, Primitive
 		for (int j=0; j<pg.ways_size(); j++) {
 			pbfWay = pg.ways(j);
 			WayID wayId = static_cast<WayID>(pbfWay.id());
-			if (wayId > OSMID_MASK) throw std::runtime_error("Way ID negative or too large: "+std::to_string(wayId));
+			if (wayId >= pow(2,42)) throw std::runtime_error("Way ID negative or too large: "+std::to_string(wayId));
 
 			// Assemble nodelist
 			LatpLonVec llVec;
@@ -146,7 +146,7 @@ bool PbfReader::ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, Prim
 		for (int n=0; n < pbfRelation.memids_size(); n++) {
 			lastID += pbfRelation.memids(n);
 			if (pbfRelation.types(n) != Relation_MemberType_WAY) { continue; }
-			if (lastID > OSMID_MASK) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID));
+			if (lastID >= pow(2,42)) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID));
 			osmStore.mark_way_used(static_cast<WayID>(lastID));
 			if (isAccepted) { osmStore.relation_contains_way(relid, lastID); }
 		}

src/shp_mem_tiles.cpp

@@ -75,8 +75,8 @@ OutputObjectRef ShpMemTiles::StoreShapefileGeometry(uint_least8_t layerNum,
 			if (p != nullptr) {
 
 				Point sp(p->x()*10000000.0, p->y()*10000000.0);
-				store_point(id, sp);
-				oo = CreateObject(OutputObjectPoint(geomType, layerNum, id, attrIdx, minzoom));
+				NodeID oid = store_point(sp);
+				oo = CreateObject(OutputObjectPoint(geomType, layerNum, oid, attrIdx, minzoom));
 				if (isIndexed) indexedGeometries.push_back(oo);
 
 				tilex =  lon2tilex(p->x(), baseZoom);
@@ -87,8 +87,8 @@ OutputObjectRef ShpMemTiles::StoreShapefileGeometry(uint_least8_t layerNum,
 
 		case LINESTRING_:
 		{
-			store_linestring(id, boost::get<Linestring>(geometry));
-			oo = CreateObject(OutputObjectLinestring(geomType, layerNum, id, attrIdx, minzoom));
+			NodeID oid = store_linestring(boost::get<Linestring>(geometry));
+			oo = CreateObject(OutputObjectLinestring(geomType, layerNum, oid, attrIdx, minzoom));
 			if (isIndexed) indexedGeometries.push_back(oo);
 
 			OutputRefsWithAttributes oolist { std::make_pair(oo, AttributeSet()) };
@@ -98,8 +98,8 @@ OutputObjectRef ShpMemTiles::StoreShapefileGeometry(uint_least8_t layerNum,
 
 		case POLYGON_:
 		{
-			store_multi_polygon(id, boost::get<MultiPolygon>(geometry));
-			oo = CreateObject(OutputObjectMultiPolygon(geomType, layerNum, id, attrIdx, minzoom));
+			NodeID oid = store_multi_polygon(boost::get<MultiPolygon>(geometry));
+			oo = CreateObject(OutputObjectMultiPolygon(geomType, layerNum, oid, attrIdx, minzoom));
 			if (isIndexed) indexedGeometries.push_back(oo);
 
 			OutputRefsWithAttributes oolist { std::make_pair(oo, AttributeSet()) };

src/tile_data.cpp

@@ -88,34 +88,6 @@ void TileDataSource::MergeLargeObjects(TileCoordinates dstIndex, uint zoom, std:
 		dstTile.push_back(result.second);
 }
 
-// Sort all generated geometries
-void TileDataSource::SortGeometries(unsigned int threadNum) {
-	std::cout << "Sorting generated geometries" << std::endl;
-
-	std::lock_guard<std::mutex> lock_points(points_store_mutex);
-	boost::sort::block_indirect_sort(
-		points_store->begin(), points_store->end(), 
-		[](auto const &a, auto const &b) { return a.first < b.first; }, threadNum);
-
-	std::lock_guard<std::mutex> lock_linestring(linestring_store_mutex);
-	boost::sort::block_indirect_sort(
-		linestring_store->begin(), linestring_store->end(), 
-		[](auto const &a, auto const &b) { return a.first < b.first; }, 
-		threadNum);
-
-	std::lock_guard<std::mutex> lock_multi_linestring(multi_linestring_store_mutex);
-	boost::sort::block_indirect_sort(
-		multi_linestring_store->begin(), multi_linestring_store->end(), 
-		[](auto const &a, auto const &b) { return a.first < b.first; }, 
-		threadNum);
-
-	std::lock_guard<std::mutex> lock_multi_polygon(multi_polygon_store_mutex);
-	boost::sort::block_indirect_sort(
-		multi_polygon_store->begin(), multi_polygon_store->end(), 
-		[](auto const &a, auto const &b) { return a.first < b.first; }, 
-		threadNum);
-}
-
 // Build node and way geometries
 Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, 
                                           NodeID const objectID, const TileBbox &bbox) const {
@@ -244,7 +216,7 @@ LatpLon TileDataSource::buildNodeGeometry(OutputGeometryType const geomType,
 
 // Report number of stored geometries
 void TileDataSource::reportSize() const {
-	std::cout << "Generated points: " << points_store->size() << ", lines: " << (linestring_store->size() + multi_linestring_store->size()) << ", polygons: " << multi_polygon_store->size() << std::endl;
+	std::cout << "Generated points: " << point_store->size() << ", lines: " << (linestring_store->size() + multi_linestring_store->size()) << ", polygons: " << multi_polygon_store->size() << std::endl;
 }
 
 TileCoordinatesSet GetTileCoordinates(std::vector<class TileDataSource *> const &sources, unsigned int zoom) {

src/tile_worker.cpp

@@ -153,7 +153,8 @@ void ProcessObjects(TileDataSource const *source, AttributeStore const &attribut
 			featurePtr->set_type(vector_tile::Tile_GeomType_POINT);
 
 			oo->writeAttributes(&keyList, &valueList, attributeStore, featurePtr, zoom);
-			if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
+			// not currently supported:
+			// if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
 		} else {
 			Geometry g;
 			try {
@@ -185,7 +186,8 @@ void ProcessObjects(TileDataSource const *source, AttributeStore const &attribut
 			boost::apply_visitor(w, g);
 			if (featurePtr->geometry_size()==0) { vtLayer->mutable_features()->RemoveLast(); continue; }
 			oo->writeAttributes(&keyList, &valueList, attributeStore, featurePtr, zoom);
-			if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
+			// not currently supported:
+			// if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
 
 		}
 	}