lua: use non-member functions for interop

CMakeLists.txt

@@ -102,6 +102,7 @@ file(GLOB tilemaker_src_files
 	src/pbf_blocks.cpp
 	src/read_shp.cpp
 	src/shp_mem_tiles.cpp
+	src/tag_map.cpp
 	src/tilemaker.cpp
 	src/write_geometry.cpp
   )

Makefile

@@ -91,7 +91,7 @@ INC := -I$(PLATFORM_PATH)/include -isystem ./include -I./src $(LUA_CFLAGS)
 
 all: tilemaker
 
-tilemaker: include/osmformat.pb.o include/vector_tile.pb.o src/mbtiles.o src/pbf_blocks.o src/coordinates.o src/osm_store.o src/helpers.o src/output_object.o src/read_shp.o src/read_pbf.o src/osm_lua_processing.o src/write_geometry.o src/shared_data.o src/tile_worker.o src/tile_data.o src/osm_mem_tiles.o src/shp_mem_tiles.o src/attribute_store.o src/tilemaker.o src/geom.o
+tilemaker: include/osmformat.pb.o include/vector_tile.pb.o src/mbtiles.o src/pbf_blocks.o src/coordinates.o src/osm_store.o src/helpers.o src/output_object.o src/read_shp.o src/read_pbf.o src/osm_lua_processing.o src/write_geometry.o src/shared_data.o src/tile_worker.o src/tile_data.o src/osm_mem_tiles.o src/shp_mem_tiles.o src/attribute_store.o src/tilemaker.o src/geom.o src/tag_map.o
 	$(CXX) $(CXXFLAGS) -o tilemaker $^ $(INC) $(LIB) $(LDFLAGS)
 
 %.o: %.cpp

docs/CONFIGURATION.md

@@ -107,13 +107,16 @@ For example:
 
 ### Lua processing
 
-Your Lua file needs to supply 5 things:
+Your Lua file needs to supply a few things:
 
 1. `node_keys`, a list of those OSM keys which indicate that a node should be processed
-2. `init_function(name)` (optional), a function to initialize Lua logic
-2. `node_function(node)`, a function to process an OSM node and add it to layers
-3. `way_function(way)`, a function to process an OSM way and add it to layers
-3. `exit_function` (optional), a function to finalize Lua logic (useful to show statistics)
+2. `node_function()`, a function to process an OSM node and add it to layers
+3. `way_function()`, a function to process an OSM way and add it to layers
+4. (optional) `init_function(name)`, a function to initialize Lua logic
+5. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics)
+6. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation 
+7. (optional) `relation_function`, a function to process an OSM relation and add it to layers
+8. (optional) `attribute_function`, a function to remap attributes from shapefiles
 
 `node_keys` is a simple list (or in Lua parlance, a 'table') of OSM tag keys. If a node has one of those keys, it will be processed by `node_function`; if not, it'll be skipped. For example, if you wanted to show highway crossings and railway stations, it should be `{ "highway", "railway" }`. (This avoids the need to process the vast majority of nodes which contain no important tags at all.)
 
@@ -127,28 +130,30 @@ Note the order: you write to a layer first, then set attributes after.
 
 To do that, you use these methods:
 
-* `node:Find(key)` or `way:Find(key)`: get the value for a tag, or the empty string if not present. For example, `way:Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
-* `node:Holds(key)` or `way:Holds(key)`: returns true if that key exists, false otherwise.
-* `node:Layer("layer_name", false)` or `way:Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
-* `way:LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
-* `node:Attribute(key,value,minzoom)` or `node:Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
-* `node:AttributeNumeric(key,value,minzoom)`, `node:AttributeBoolean(key,value,minzoom)` (and `way:`...): for numeric/boolean columns.
-* `node:Id()` or `way:Id()`: get the OSM ID of the current object.
-* `node:ZOrder(number)` or `way:ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
-* `node:MinZoom(zoom)` or `way:MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
-* `way:Length()` and `way:Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
-* `way:Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
+* `Find(key)`: get the value for a tag, or the empty string if not present. For example, `Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
+* `Holds(key)`: returns true if that key exists, false otherwise.
+* `Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
+* `LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
+* `Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
+* `AttributeNumeric(key,value,minzoom)`, `AttributeBoolean(key,value,minzoom)`: for numeric/boolean columns.
+* `Id()`: get the OSM ID of the current object.
+* `ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
+* `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
+* `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
+* `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
 
 The simplest possible function, to include roads/paths and nothing else, might look like this:
 
-    function way_function(way)
-      local highway = way:Find("highway")
+```lua
+    function way_function()
+      local highway = Find("highway")
       if highway~="" then
-        way:Layer("roads", false)
-        way:Attribute("name", way:Find("name"))
-        way:Attribute("type", highway)
+        Layer("roads", false)
+        Attribute("name", Find("name"))
+        Attribute("type", highway)
       end
     end
+```
 
 Take a look at the supplied process.lua for a simple example, or the more complex OpenMapTiles-compatible script in `resources/`. You can specify another filename with the `--process` option.
 
@@ -197,11 +202,11 @@ When processing OSM objects with your Lua script, you can perform simple spatial
 
 You can then find out whether a node is within one of these polygons using the `Intersects` method:
 
-    if node:Intersects("countries") then print("Looks like it's on land"); end
+    if Intersects("countries") then print("Looks like it's on land"); end
 
 Or you can find out what country(/ies) the node is within using `FindIntersecting`, which returns a table:
 
-    names = node:FindIntersecting("countries")
+    names = FindIntersecting("countries")
     print(table.concat(name,","))
 
 To enable these functions, set `index` to true in your shapefile layer definition. `index_column` is not needed for `Intersects` but required for `FindIntersecting`.

docs/RELATIONS.md

@@ -22,26 +22,30 @@ This is a two-stage process: first, when reading relations, indicate that these
 
 To define which relations should be accepted, add a `relation_scan_function`:
 
-    function relation_scan_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        local network = relation:Find("network")
-        if network=="ncn" then relation:Accept() end
+```lua
+    function relation_scan_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        local network = Find("network")
+        if network=="ncn" then Accept() end
       end
     end
+```
 
-This function takes the relation as its sole argument. Examine the tags using `relation:Find(key)` as normal. (You can also use `relation:Holds(key)` and `relation:Id()`.) If you want to use this relation, call `relation:Accept()`.
+Examine the tags using `Find(key)` as normal. (You can also use `Holds(key)` and `Id()`.) If you want to use this relation, call `Accept()`.
 
 #### Stage 2: accessing relations from ways
 
-Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`way:NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `way:FindInRelation(key)`. For example:
+Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `FindInRelation(key)`. For example:
 
+```lua
     while true do
-      local rel = way:NextRelation()
+      local rel = NextRelation()
       if not rel then break end
-      print ("Part of route "..way:FindInRelation("ref"))
+      print ("Part of route "..FindInRelation("ref"))
     end
+```
 
-(Should you need to re-read the relations, you can reset the iterator with `way:RestartRelations()`.)
+(Should you need to re-read the relations, you can reset the iterator with `RestartRelations()`.)
 
 
 ### Writing relation geometries
@@ -52,13 +56,15 @@ First, make sure that you have accepted the relations using `relation_scan_funct
 
 Then write a `relation_function`, which works in the same way as `way_function` would:
 
-    function relation_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        relation:Layer("bike_routes", false)
-        relation:Attribute("class", relation:Find("network"))
-        relation:Attribute("ref", relation:Find("ref"))
+```lua
+    function relation_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        Layer("bike_routes", false)
+        Attribute("class", Find("network"))
+        Attribute("ref", Find("ref"))
       end
     end
+```
 
 
 ### Not supported

include/osm_lua_processing.h

@@ -16,6 +16,8 @@
 
 #include <boost/container/flat_map.hpp>
 
+class TagMap;
+
 // Lua
 extern "C" {
 	#include "lua.h"
@@ -31,6 +33,20 @@ extern bool verbose;
 class AttributeStore;
 class AttributeSet;
 
+// A string, which might be in `currentTags` as a value. If Lua
+// code refers to an absent value, it'll fallback to passing
+// it as a std::string.
+//
+// The intent is that Attribute("name", Find("name")) is a common
+// pattern, and we ought to avoid marshalling a string back and
+// forth from C++ to Lua when possible.
+struct PossiblyKnownTagValue {
+	bool found;
+	uint32_t index;
+	std::string fallback;
+};
+
+
 /**
 	\brief OsmLuaProcessing - converts OSM objects into OutputObjects.
 
@@ -71,31 +87,25 @@ class OsmLuaProcessing {
 	using tag_map_t = boost::container::flat_map<std::string, std::string>;
 
 	// Scan non-MP relation
-	bool scanRelation(WayID id, const tag_map_t &tags);
+	bool scanRelation(WayID id, const TagMap& tags);
 
 	/// \brief We are now processing a significant node
-	void setNode(NodeID id, LatpLon node, const tag_map_t &tags);
+	void setNode(NodeID id, LatpLon node, const TagMap& tags);
 
 	/// \brief We are now processing a way
-	void setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags);
+	void setWay(WayID wayId, LatpLonVec const &llVec, const TagMap& tags);
 
 	/** \brief We are now processing a relation
 	 * (note that we store relations as ways with artificial IDs, and that
 	 *  we use decrementing positive IDs to give a bit more space for way IDs)
 	 */
-	void setRelation(int64_t relationId, WayVec const &outerWayVec, WayVec const &innerWayVec, const tag_map_t &tags, bool isNativeMP, bool isInnerOuter);
+	void setRelation(int64_t relationId, WayVec const &outerWayVec, WayVec const &innerWayVec, const TagMap& tags, bool isNativeMP, bool isInnerOuter);
 
 	// ----	Metadata queries called from Lua
 
 	// Get the ID of the current object
 	std::string Id() const;
 
-	// Check if there's a value for a given key
-	bool Holds(const std::string& key) const;
-
-	// Get an OSM tag for a given key (or return empty string if none)
-	const std::string& Find(const std::string& key) const;
-
 	// ----	Spatial queries called from Lua
 
 	// Find intersecting shapefile layer
@@ -155,11 +165,8 @@ class OsmLuaProcessing {
 	void LayerAsCentroid(const std::string &layerName);
 
 	// Set attributes in a vector tile's Attributes table
-	void Attribute(const std::string &key, const std::string &val);
-	void AttributeWithMinZoom(const std::string &key, const std::string &val, const char minzoom);
-	void AttributeNumeric(const std::string &key, const float val);
+	void AttributeWithMinZoom(const std::string &key, const PossiblyKnownTagValue& val, const char minzoom);
 	void AttributeNumericWithMinZoom(const std::string &key, const float val, const char minzoom);
-	void AttributeBoolean(const std::string &key, const bool val);
 	void AttributeBooleanWithMinZoom(const std::string &key, const bool val, const char minzoom);
 	void MinZoom(const double z);
 	void ZOrder(const double z);
@@ -194,6 +201,7 @@ class OsmLuaProcessing {
 	inline AttributeStore &getAttributeStore() { return attributeStore; }
 
 	struct luaProcessingException :std::exception {};
+	const TagMap* currentTags;
 
 private:
 	/// Internal: clear current cached state
@@ -253,7 +261,6 @@ class OsmLuaProcessing {
 	class LayerDefinition &layers;
 
 	std::vector<std::pair<OutputObject, AttributeSet>> outputs;		// All output objects that have been created
-	const boost::container::flat_map<std::string, std::string>* currentTags;
 
 	std::vector<OutputObject> finalizeOutputs();
 };

include/read_pbf.h

@@ -11,6 +11,7 @@
 // Protobuf
 #include "osmformat.pb.h"
 #include "vector_tile.pb.h"
+#include "tag_map.h"
 
 class OsmLuaProcessing;
 
@@ -34,6 +35,7 @@ class PbfReader
 			pbfreader_generate_output const &generate_output);
 
 	// Read tags into a map from a way/node/relation
+	/*
 	using tag_map_t = boost::container::flat_map<std::string, std::string>;
 	template<typename T>
 	void readTags(T &pbfObject, PrimitiveBlock const &pb, tag_map_t &tags) {
@@ -44,6 +46,15 @@ class PbfReader
 			tags[pb.stringtable().s(keysPtr->Get(n))] = pb.stringtable().s(valsPtr->Get(n));
 		}
 	}
+	*/
+	template<typename T>
+	void readTags(T &pbfObject, PrimitiveBlock const &pb, TagMap& tags) {
+		auto keysPtr = pbfObject.mutable_keys();
+		auto valsPtr = pbfObject.mutable_vals();
+		for (uint n=0; n < pbfObject.keys_size(); n++) {
+			tags.addTag(pb.stringtable().s(keysPtr->Get(n)), pb.stringtable().s(valsPtr->Get(n)));
+		}
+	}
 
 private:
 	bool ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::pair<std::size_t, std::size_t> progress, std::size_t datasize, 

include/tag_map.h

@@ -0,0 +1,56 @@
+#ifndef _TAG_MAP_H
+#define _TAG_MAP_H
+
+#include <vector>
+#include <string>
+#include <boost/container/flat_map.hpp>
+
+// We track tags in a special structure, which enables some tricks when
+// doing Lua interop.
+//
+// The alternative is a std::map - but often, our map is quite small.
+// It's preferable to have a small set of vectors and do linear search.
+//
+// Further, we can avoid passing std::string from Lua -> C++ in some cases
+// by first checking to see if the string we would have passed is already
+// stored in our tag map, and passing a reference to its location.
+
+// Assumptions:
+// 1. Not thread-safe
+//      This is OK because we have 1 instance of OsmLuaProcessing per thread.
+// 2. Lifetime of map is less than lifetime of keys/values that are passed
+//      This is true since the strings are owned by the protobuf block reader
+// 3. Max number of tag values will fit in a short
+//      OSM limit is 5,000 tags per object
+class TagMap {
+public:
+	TagMap();
+	void reset();
+
+	void addTag(const std::string& key, const std::string& value);
+	const std::string* getTag(const std::string& key) const;
+
+	// Return -1 if key not found, else return its keyLoc.
+	int64_t getKey(const char* key, size_t size) const;
+
+	// Return -1 if value not found, else return its keyLoc.
+	int64_t getValue(const char* key, size_t size) const;
+
+	const std::string* getValueFromKey(uint32_t keyLoc) const;
+	const std::string* getValue(uint32_t valueLoc) const;
+
+	boost::container::flat_map<std::string, std::string> exportToBoostMap() const;
+
+private:
+	uint32_t ensureString(
+		std::vector<std::vector<const std::string*>>& vector,
+		const std::string& value
+	);
+
+
+	std::vector<std::vector<const std::string*>> keys;
+	std::vector<std::vector<uint32_t>> key2value;
+	std::vector<std::vector<const std::string*>> values;
+};
+
+#endif _TAG_MAP_H