Initial working demo

include/attribute_store.h

@@ -55,9 +55,9 @@ struct AttributeStore
 		switch(lhs_id) {
 			case Index::BOOL:
 				return lhs.bool_value() < rhs.bool_value();
-			case Index::FLOAT:	
+			case Index::FLOAT:
 				return lhs.float_value() < rhs.float_value();
-			case Index::STRING:	
+			case Index::STRING:
 				return lhs.string_value() < rhs.string_value();
 		}
 
@@ -111,7 +111,7 @@ struct AttributeStore
 
 	key_value_set_ref_t empty_set() const { return new key_value_set(); }
 
-    key_value_set_ref_t store_set(key_value_set_ref_t attributes) {
+	key_value_set_ref_t store_set(key_value_set_ref_t attributes) {
 		auto idx = attributes->values.size();
 		for(auto const &i: attributes->values) {
 			boost::hash_combine(idx, i.minzoom);

include/osm_lua_processing.h

@@ -157,6 +157,7 @@ class OsmLuaProcessing {
 	void AttributeWithMinZoom(const std::string &key, const std::string &val, const char minzoom);
 	void AttributeNumeric(const std::string &key, const float val);
 	void AttributeNumericWithMinZoom(const std::string &key, const float val, const char minzoom);
+	void SetRank(const float val);
 	void AttributeBoolean(const std::string &key, const bool val);
 	void AttributeBooleanWithMinZoom(const std::string &key, const bool val, const char minzoom);
 	void MinZoom(const double z);

include/output_object.h

@@ -57,6 +57,7 @@ class OutputObject {
 	unsigned minZoom 			: 4;
 
 	AttributeStoreRef attributes;
+	float_t rankValue = std::numeric_limits<float_t>::infinity();
 
 	void setZOrder(const ZOrder z) {
 #ifndef FLOAT_Z_ORDER
@@ -75,9 +76,12 @@ class OutputObject {
 		this->attributes = attributes;
 	}
 
+	void setRankValue(const float_t value);
+
 	//\brief Write attribute key/value pairs (dictionary-encoded)
 	void writeAttributes(std::vector<std::string> *keyList, 
-		std::vector<vector_tile::Tile_Value> *valueList, vector_tile::Tile_Feature *featurePtr, char zoom) const;
+		std::vector<vector_tile::Tile_Value> *valueList, vector_tile::Tile_Feature *featurePtr, 
+		AttributeStoreRef extraAttributes, char zoom) const;
 
 	/**
 	 * \brief Find a value in the value dictionary

include/shared_data.h

@@ -31,6 +31,7 @@ struct LayerDef {
 	bool indexed;
 	std::string indexName;
 	std::map<std::string, uint> attributeMap;
+	uint64_t rankMax;
 	bool writeTo;
 };
 
@@ -51,6 +52,7 @@ class LayerDefinition {
 			bool allSourceColumns,
 			bool indexed,
 			const std::string &indexName,
+			const uint64_t rankMax,
 			const std::string &writeTo);
 	std::vector<bool> getSortOrders();
 	rapidjson::Value serialiseToJSONValue(rapidjson::Document::AllocatorType &allocator) const;

resources/config-openmaptiles.json

@@ -1,10 +1,10 @@
 {
 	"layers": {
-		"place":            { "minzoom":  0, "maxzoom": 14 },
+		"place":            { "minzoom":  0, "maxzoom": 14, "rank_max": 32 },
 		"boundary":         { "minzoom":  0, "maxzoom": 14, "simplify_below": 12, "simplify_level": 0.0003, "simplify_ratio": 2 },
 
-		"poi":              { "minzoom": 12, "maxzoom": 14 },
-		"poi_detail":       { "minzoom": 14, "maxzoom": 14, "write_to": "poi"},
+		"poi":              { "minzoom": 12, "maxzoom": 14, "rank_max": 64 },
+		"poi_detail":       { "minzoom": 14, "maxzoom": 14, "rank_max": 64, "write_to": "poi"},
 
 		"housenumber":      { "minzoom": 14, "maxzoom": 14 },
 

resources/process-openmaptiles.lua

@@ -76,15 +76,14 @@ function node_function(node)
 	--   we could potentially approximate it for cities based on the population tag
 	local place = node:Find("place")
 	if place ~= "" then
-		local rank = nil
 		local mz = 13
 		local pop = tonumber(node:Find("population")) or 0
 
 		if     place == "continent"     then mz=0
 		elseif place == "country"       then
-			if     pop>50000000 then rank=1; mz=1
-			elseif pop>20000000 then rank=2; mz=2
-			else                     rank=3; mz=3 end
+			if     pop>50000000 then           mz=1
+			elseif pop>20000000 then           mz=2
+			else                               mz=3 end
 		elseif place == "state"         then mz=4
 		elseif place == "city"          then mz=5
 		elseif place == "town" and pop>8000 then mz=7
@@ -100,7 +99,7 @@ function node_function(node)
 		node:Layer("place", false)
 		node:Attribute("class", place)
 		node:MinZoom(mz)
-		if rank then node:AttributeNumeric("rank", rank) end
+		node:SetRank(pop)
 		if place=="country" then node:Attribute("iso_a2", node:Find("ISO3166-1:alpha2")) end
 		SetNameAttributes(node)
 		return
@@ -579,7 +578,7 @@ function WritePOI(obj,class,subclass,rank)
 	if rank>4 then layer="poi_detail" end
 	obj:LayerAsCentroid(layer)
 	SetNameAttributes(obj)
-	obj:AttributeNumeric("rank", rank)
+	obj:SetRank(rank)
 	obj:Attribute("class", class)
 	obj:Attribute("subclass", subclass)
 end

src/osm_lua_processing.cpp

@@ -53,6 +53,7 @@ OsmLuaProcessing::OsmLuaProcessing(
 		.addFunction("LayerAsCentroid", &OsmLuaProcessing::LayerAsCentroid)
 		.addOverloadedFunctions("Attribute", &OsmLuaProcessing::Attribute, &OsmLuaProcessing::AttributeWithMinZoom)
 		.addOverloadedFunctions("AttributeNumeric", &OsmLuaProcessing::AttributeNumeric, &OsmLuaProcessing::AttributeNumericWithMinZoom)
+		.addFunction("SetRank", &OsmLuaProcessing::SetRank)
 		.addOverloadedFunctions("AttributeBoolean", &OsmLuaProcessing::AttributeBoolean, &OsmLuaProcessing::AttributeBooleanWithMinZoom)
 		.addFunction("MinZoom", &OsmLuaProcessing::MinZoom)
 		.addOverloadedFunctions("ZOrder", &OsmLuaProcessing::ZOrder, &OsmLuaProcessing::ZOrderWithScale)
@@ -328,13 +329,13 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 		if (geomType==POINT_) {
 			Point p = Point(lon, latp);
 
-            if(!CorrectGeometry(p)) return;
+			if(!CorrectGeometry(p)) return;
 
 			osmStore.store_point(osmStore.osm(), osmID, p);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectOsmStorePoint(geomType, 
 							layers.layerMap[layerName], osmID, attributeStore.empty_set(), layerMinZoom));
 			outputs.push_back(std::make_pair(oo, attributeStore.empty_set()));
-            return;
+			return;
 		}
 		else if (geomType==POLYGON_) {
 			// polygon
@@ -358,7 +359,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) {
 				mp.push_back(p);
 			}
 
-            if(!CorrectGeometry(mp)) return;
+			if(!CorrectGeometry(mp)) return;
 
 			osmStore.store_multi_polygon(osmStore.osm(), osmID, mp);
 			OutputObjectRef oo = osmMemTiles.CreateObject(OutputObjectOsmStoreMultiPolygon(geomType, 
@@ -476,6 +477,16 @@ void OsmLuaProcessing::AttributeNumericWithMinZoom(const string &key, const floa
 	setVectorLayerMetadata(outputs.back().first->layer, key, 1);
 }
 
+void OsmLuaProcessing::SetRank(const float val) {
+	if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; }
+
+	// First, set the attribute so it will be registered in the metadata
+	setVectorLayerMetadata(outputs.back().first->layer, "rank", 1);
+
+	// Set the ranked attribute, used for relative processing during output.
+	outputs.back().first->setRankValue(val);
+}
+
 void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val) { AttributeBooleanWithMinZoom(key,val,0); }
 void OsmLuaProcessing::AttributeBooleanWithMinZoom(const string &key, const bool val, const char minzoom) {
 	if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; }

src/output_object.cpp

@@ -30,42 +30,49 @@ std::ostream& operator<<(std::ostream& os, OutputGeometryType geomType)
 	return os;
 }
 
+void OutputObject::setRankValue(const float_t value) {
+	rankValue = value;
+}
 
 // Write attribute key/value pairs (dictionary-encoded)
 void OutputObject::writeAttributes(
 	vector<string> *keyList, 
 	vector<vector_tile::Tile_Value> *valueList, 
 	vector_tile::Tile_Feature *featurePtr,
+	AttributeStoreRef extraAttributes,
 	char zoom) const {
 
-	for(auto const &it: attributes->values) {
-		if (it.minzoom > zoom) continue;
-
-		// Look for key
-		std::string const &key = it.key;
-		auto kt = find(keyList->begin(), keyList->end(), key);
-		if (kt != keyList->end()) {
-			uint32_t subscript = kt - keyList->begin();
-			featurePtr->add_tags(subscript);
-		} else {
-			uint32_t subscript = keyList->size();
-			keyList->push_back(key);
-			featurePtr->add_tags(subscript);
-		}
-
-		// Look for value
-		vector_tile::Tile_Value const &value = it.value;
-		int subscript = findValue(valueList, value);
-		if (subscript>-1) {
-			featurePtr->add_tags(subscript);
-		} else {
-			uint32_t subscript = valueList->size();
-			valueList->push_back(value);
-			featurePtr->add_tags(subscript);
-		}
+	auto attributeList = { attributes, extraAttributes };
+	for (auto const attributes: attributeList) {
+		for(auto const &it: attributes->values) {
+			if (it.minzoom > zoom) continue;
+
+			// Look for key
+			std::string const &key = it.key;
+			auto kt = find(keyList->begin(), keyList->end(), key);
+			if (kt != keyList->end()) {
+				uint32_t subscript = kt - keyList->begin();
+				featurePtr->add_tags(subscript);
+			} else {
+				uint32_t subscript = keyList->size();
+				keyList->push_back(key);
+				featurePtr->add_tags(subscript);
+			}
+
+			// Look for value
+			vector_tile::Tile_Value const &value = it.value;
+			int subscript = findValue(valueList, value);
+			if (subscript>-1) {
+				featurePtr->add_tags(subscript);
+			} else {
+				uint32_t subscript = valueList->size();
+				valueList->push_back(value);
+				featurePtr->add_tags(subscript);
+			}
 
-		//if(value.has_string_value())
-		//	std::cout << "Write attr: " << key << " " << value.string_value() << std::endl;	
+			//if(value.has_string_value())
+			//	std::cout << "Write attr: " << key << " " << value.string_value() << std::endl;	
+		}
 	}
 }
 

src/shared_data.cpp

@@ -25,13 +25,14 @@ uint LayerDefinition::addLayer(string name, uint minzoom, uint maxzoom,
 		bool allSourceColumns,
 		bool indexed,
 		const std::string &indexName,
+		const uint64_t rankMax,
 		const std::string &writeTo)  {
 
 	bool isWriteTo = !writeTo.empty();
 	LayerDef layer = { name, minzoom, maxzoom, simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio, 
 		filterBelow, filterArea, combinePolygonsBelow, sortZOrderAscending,
 		source, sourceColumns, allSourceColumns, indexed, indexName,
-		std::map<std::string,uint>(), isWriteTo };
+		std::map<std::string,uint>(), rankMax, isWriteTo };
 	layers.push_back(layer);
 	uint layerNum = layers.size()-1;
 	layerMap[name] = layerNum;
@@ -221,11 +222,17 @@ void Config::readConfig(rapidjson::Document &jsonConfig, bool &hasClippingBox, B
 		}
 		string indexName = it->value.HasMember("index_column") ? it->value["index_column"].GetString() : "";
 
+		uint64_t rankMax = std::numeric_limits<uint64_t>::max();
+		if (it->value.HasMember("rank_max")) {
+			rankMax = it->value["rank_max"].GetUint64();
+			if (rankMax == 0) rankMax = std::numeric_limits<uint64_t>::max();
+		}
+
 		layers.addLayer(layerName, minZoom, maxZoom,
 				simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio, 
 				filterBelow, filterArea, combinePolyBelow, sortZOrderAscending,
 				source, sourceColumns, allSourceColumns, indexed, indexName,
-				writeTo);
+				rankMax, writeTo);
 
 		cout << "Layer " << layerName << " (z" << minZoom << "-" << maxZoom << ")";
 		if (it->value.HasMember("write_to")) { cout << " -> " << it->value["write_to"].GetString(); }

src/tile_worker.cpp

@@ -1,6 +1,7 @@
 /*! \file */ 
 #include "tile_worker.h"
 #include <fstream>
+#include <numeric>
 #include <boost/filesystem.hpp>
 #include <signal.h>
 #include "helpers.h"
@@ -134,14 +135,58 @@ void RemoveInnersBelowSize(MultiPolygon &g, double filterArea) {
 	}
 }
 
+void ProcessTileRelativeObjects(OutputObjectsConstIt ooSameLayerBegin, OutputObjectsConstIt ooSameLayerEnd, 
+	std::map<NodeID, uint64_t> *ooRankMap) {
+
+	std::vector<OutputObjectRef> tileRelativeObjectsForSort;
+	std::copy_if(ooSameLayerBegin, ooSameLayerEnd, std::back_inserter(tileRelativeObjectsForSort), 
+		[](OutputObjectRef const &oo) -> bool {
+			// TODO: support multiple ranked attributes
+			return oo->rankValue < std::numeric_limits<float_t>::infinity();
+		});
+
+	uint64_t currentSortIdx = 0;
+	std::sort(tileRelativeObjectsForSort.begin(), tileRelativeObjectsForSort.end(), 
+		[](const OutputObjectRef &lhs, const OutputObjectRef &rhs) -> bool {
+			return lhs->rankValue < rhs->rankValue;
+		});
+
+	for (auto jt = tileRelativeObjectsForSort.begin(); jt != tileRelativeObjectsForSort.end(); ++jt) {
+		// rank is a 1-indexed value
+		uint64_t index = distance(tileRelativeObjectsForSort.begin(), jt) + 1;
+		OutputObjectRef oo = *jt;
+		ooRankMap->emplace(oo->objectID & OSMID_MASK, index);
+	}
+}
+
 void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, OutputObjectsConstIt ooSameLayerEnd, 
-	class SharedData &sharedData, double simplifyLevel, double filterArea, bool combinePolygons, unsigned zoom, const TileBbox &bbox,
+	class SharedData &sharedData, double simplifyLevel, double filterArea, bool combinePolygons, unsigned zoom, 
+	uint64_t rankMax, const TileBbox &bbox,
 	vector_tile::Tile_Layer *vtLayer, vector<string> &keyList, vector<vector_tile::Tile_Value> &valueList) {
 
+	std::map<NodeID, uint64_t> ooRankMap = std::map<NodeID, uint64_t>();
+	ProcessTileRelativeObjects(ooSameLayerBegin, ooSameLayerEnd, &ooRankMap);
+
 	for (auto jt = ooSameLayerBegin; jt != ooSameLayerEnd; ++jt) {
 		OutputObjectRef oo = *jt;
 		if (zoom < oo->minZoom) { continue; }
 
+		AttributeStoreRef extraAttributes = new AttributeStore::key_value_set();
+
+		auto ooRank = ooRankMap.find(oo->objectID & OSMID_MASK);
+		if (ooRank != ooRankMap.end()) {
+			uint64_t rankInt = ooRank->second;
+
+			if (rankInt > rankMax) {
+				// Do not write the object to the tile if it is greater than the maximum allowed rank in the layer config
+				continue;
+			}
+
+			vector_tile::Tile_Value v;
+			v.set_uint_value(rankInt);
+			extraAttributes->values.emplace("rank", v, 0);
+		}
+
 		if (oo->geomType == POINT_) {
 			vector_tile::Tile_Feature *featurePtr = vtLayer->add_features();
 			LatpLon pos = buildNodeGeometry(osmStore, *oo, bbox);
@@ -151,7 +196,7 @@ void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, O
 			featurePtr->add_geometry((xy.second << 1) ^ (xy.second >> 31));
 			featurePtr->set_type(vector_tile::Tile_GeomType_POINT);
 
-			oo->writeAttributes(&keyList, &valueList, featurePtr, zoom);
+			oo->writeAttributes(&keyList, &valueList, featurePtr, extraAttributes, zoom);
 			if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
 		} else {
 			Geometry g;
@@ -183,9 +228,8 @@ void ProcessObjects(OSMStore &osmStore, OutputObjectsConstIt ooSameLayerBegin, O
 			WriteGeometryVisitor w(&bbox, featurePtr, simplifyLevel);
 			boost::apply_visitor(w, g);
 			if (featurePtr->geometry_size()==0) { vtLayer->mutable_features()->RemoveLast(); continue; }
-			oo->writeAttributes(&keyList, &valueList, featurePtr, zoom);
+			oo->writeAttributes(&keyList, &valueList, featurePtr, extraAttributes, zoom);
 			if (sharedData.config.includeID) { featurePtr->set_id(oo->objectID & OSMID_MASK); }
-
 		}
 	}
 }
@@ -237,9 +281,10 @@ void ProcessLayer(OSMStore &osmStore,
 		}
 
 		auto ooListSameLayer = GetObjectsAtSubLayer(data, layerNum);
+
 		// Loop through output objects
 		ProcessObjects(osmStore, ooListSameLayer.first, ooListSameLayer.second, sharedData, 
-			simplifyLevel, filterArea, zoom < ld.combinePolygonsBelow, zoom, bbox, vtLayer, keyList, valueList);
+			simplifyLevel, filterArea, zoom < ld.combinePolygonsBelow, zoom, ld.rankMax, bbox, vtLayer, keyList, valueList);
 	}
 	if (verbose && std::time(0)-start>3) {
 		std::cout << "Layer " << layerName << " at " << zoom << "/" << index.x << "/" << index.y << " took " << (std::time(0)-start) << " seconds" << std::endl;