Simple key/value store (#817)

Author: systemed

docs/CONFIGURATION.md

@@ -124,7 +124,7 @@ Your Lua file can supply these functions for tilemaker to call:
 2. (optional) `way_keys`, a list of those OSM tags which indicate that a way should be processed
 3. `node_function()`, a function to process an OSM node and add it to layers
 4. `way_function()`, a function to process an OSM way and add it to layers
-5. (optional) `init_function(name)`, a function to initialize Lua logic
+5. (optional) `init_function(name, is_first)`, a function to initialize Lua logic
 6. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics)
 7. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation 
 8. (optional) `relation_function`, a function to process an OSM relation and add it to layers
@@ -184,7 +184,7 @@ If your Lua file causes an error due to mistaken syntax, you can test it at the
 
 `way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}`
 
-`init_function(name)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data.
+`init_function(name, is_first)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. `is_first` will be true only the first time `init_function` is called.
 
 Other functions are described below and in RELATIONS.md.
 
@@ -243,3 +243,11 @@ To enable these functions, set `index` to true in your shapefile layer definitio
 `CoveredBy` and `FindCovering` work similarly but check if the object is covered by a shapefile layer object.
 
 `AreaIntersecting` returns the area of the current way's intersection with the shapefile layer. You can use this to find whether a water body is already represented in a shapefile ocean layer.
+
+### Lua key/value store
+
+tilemaker has a simple key/value store accessible from Lua which you can use to bring in external data. The same store is used across all processing threads.
+
+Read your data from file, using [Lua's I/O functions](https://www.lua.org/pil/21.1.html), in `init_function` (checking that `is_first` is set for the first run only). Set a key/value pair with `SetData(key,value)` - for example `SetData("name","Bill")`. Both key and value should be strings.
+
+You can then retrieve the value within `way_function` or similar with `GetData(key)`. If no value was found, the empty string is returned.

include/osm_lua_processing.h

@@ -6,6 +6,7 @@
 #include <string>
 #include <sstream>
 #include <map>
+#include <mutex>
 #include "geom.h"
 #include "osm_store.h"
 #include "shared_data.h"
@@ -56,7 +57,8 @@ class OsmLuaProcessing {
 		const class ShpMemTiles &shpMemTiles, 
 		class OsmMemTiles &osmMemTiles,
 		AttributeStore &attributeStore,
-		bool materializeGeometries
+		bool materializeGeometries,
+		bool isFirst
 	);
 	~OsmLuaProcessing();
 
@@ -239,6 +241,9 @@ class OsmLuaProcessing {
 	const TagMap* currentTags;
 	bool isPostScanRelation;				// processing a relation in postScanRelation
 
+	static std::unordered_map<std::string, std::string> dataStore;
+	static std::mutex dataStoreMutex;
+
 private:
 	/// Internal: clear current cached state
 	inline void reset() {

resources/process-openmaptiles.lua

@@ -17,7 +17,7 @@ additional_languages = { }
 --------
 
 -- Enter/exit Tilemaker
-function init_function()
+function init_function(name,is_first)
 end
 function exit_function()
 end

src/osm_lua_processing.cpp

@@ -18,6 +18,8 @@ thread_local kaguya::State *g_luaState = nullptr;
 thread_local OsmLuaProcessing* osmLuaProcessing = nullptr;
 
 std::mutex vectorLayerMetadataMutex;
+std::unordered_map<std::string, std::string> OsmLuaProcessing::dataStore;
+std::mutex OsmLuaProcessing::dataStoreMutex;
 
 void handleOsmLuaProcessingUserSignal(int signum) {
 	osmLuaProcessing->handleUserSignal(signum);
@@ -195,6 +197,14 @@ std::string rawFindInRelation(const std::string& key) { return osmLuaProcessing-
 void rawAccept() { return osmLuaProcessing->Accept(); }
 double rawAreaIntersecting(const std::string& layerName) { return osmLuaProcessing->AreaIntersecting(layerName); }
 
+void rawSetData(const std::string &key, const std::string &value) { 
+	std::lock_guard<std::mutex> lock(osmLuaProcessing->dataStoreMutex);
+	osmLuaProcessing->dataStore[key] = value;
+}
+std::string rawGetData(const std::string &key) {
+	auto r = osmLuaProcessing->dataStore.find(key);
+	return r==osmLuaProcessing->dataStore.end() ? "" : r->second;
+}
 
 bool supportsRemappingShapefiles = false;
 
@@ -217,7 +227,8 @@ OsmLuaProcessing::OsmLuaProcessing(
 	const class ShpMemTiles &shpMemTiles, 
 	class OsmMemTiles &osmMemTiles,
 	AttributeStore &attributeStore,
-	bool materializeGeometries):
+	bool materializeGeometries,
+	bool isFirst) :
 	osmStore(osmStore),
 	shpMemTiles(shpMemTiles),
 	osmMemTiles(osmMemTiles),
@@ -273,6 +284,8 @@ OsmLuaProcessing::OsmLuaProcessing(
 	luaState["NextRelation"] = &rawNextRelation;
 	luaState["RestartRelations"] = &rawRestartRelations;
 	luaState["FindInRelation"] = &rawFindInRelation;
+	luaState["SetData"] = &rawSetData;
+	luaState["GetData"] = &rawGetData;
 	supportsRemappingShapefiles = !!luaState["attribute_function"];
 	supportsReadingRelations    = !!luaState["relation_scan_function"];
 	supportsPostScanRelations   = !!luaState["relation_postscan_function"];
@@ -283,7 +296,7 @@ OsmLuaProcessing::OsmLuaProcessing(
 	// ---- Call init_function of Lua logic
 
 	if (!!luaState["init_function"]) {
-		luaState["init_function"](this->config.projectName);
+		luaState["init_function"](this->config.projectName, isFirst);
 	}
 }
 
@@ -1182,4 +1195,3 @@ std::vector<OutputObject> OsmLuaProcessing::finalizeOutputs() {
 	}
 	return list;
 }
-

src/tilemaker.cpp

@@ -253,7 +253,7 @@ int main(const int argc, const char* argv[]) {
 	shpMemTiles.open();
 
 	OsmLuaProcessing osmLuaProcessing(osmStore, config, layers, options.luaFile, 
-		shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries);
+		shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries, true);
 
 	// ---- Load external sources (shp/geojson)
 
@@ -311,7 +311,7 @@ int main(const int argc, const char* argv[]) {
 			[&]() {
 				thread_local std::pair<std::string, std::shared_ptr<OsmLuaProcessing>> osmLuaProcessing;
 				if (osmLuaProcessing.first != inputFile) {
-					osmLuaProcessing = std::make_pair(inputFile, std::make_shared<OsmLuaProcessing>(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries));
+					osmLuaProcessing = std::make_pair(inputFile, std::make_shared<OsmLuaProcessing>(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries, false));
 				}
 				return osmLuaProcessing.second;
 			},
@@ -323,6 +323,7 @@ int main(const int argc, const char* argv[]) {
 	attributeStore.finalize();
 	osmMemTiles.reportSize();
 	attributeStore.reportSize();
+	osmLuaProcessing.dataStore.clear(); // no longer needed
 
 	// ----	Initialise SharedData