be less chatty when run non-interactively (#767)

Author: cldellow

include/helpers.h

@@ -6,6 +6,12 @@
 #include <sstream>
 #include <vector>
 
+#ifdef _MSVC_LANG
+#define ISATTY true
+#else
+#define ISATTY isatty(1)
+#endif
+
 // General helper routines
 
 inline void endian_swap(unsigned int& x) {

include/tile_data.h

@@ -69,7 +69,7 @@ template<typename OO> void finalizeObjects(
 	int i = -1;
 	for (auto it = begin; it != end; it++) {
 		i++;
-		if (it->size() > 0 || i % 10 == 0 || i == 4095) {
+		if (it->size() > 0 || i % 50 == 0 || i == 4095) {
 			std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA;
 
 #ifdef CLOCK_MONOTONIC

src/pbf_processor.cpp

@@ -17,6 +17,11 @@ const std::string OptionSortTypeThenID = "Sort.Type_then_ID";
 const std::string OptionLocationsOnWays = "LocationsOnWays";
 std::atomic<uint64_t> blocksProcessed(0), blocksToProcess(0);
 
+// Access is guarded by ioMutex.
+// This counter decreases the chattiness of tilemaker's progress updates,
+// especially when run in a non-interactive context.
+uint64_t phaseProgress = 0;
+
 // Thread-local so that we can re-use buffers during parsing.
 thread_local PbfReader::PbfReader reader;
 
@@ -370,16 +375,27 @@ bool PbfProcessor::ReadBlock(
 		auto output_progress = [&]()
 		{
 			if (ioMutex.try_lock()) {
-				std::ostringstream str;
-				str << "\r";
-				void_mmap_allocator::reportStoreSize(str);
-				if (effectiveShards > 1)
-					str << std::to_string(shard + 1) << "/" << std::to_string(effectiveShards) << " ";
-
-				// TODO: revive showing the # of ways/relations?
-				str << "Block " << blocksProcessed.load() << "/" << blocksToProcess.load() << " ";
-				std::cout << str.str();
-				std::cout.flush();
+				// If we're interactive, show an update for each block.
+				// If we're not interactive, show an update for each 1% of blocks.
+				uint64_t blockProgress = blocksProcessed.load();
+				uint64_t minimumIncrement = blocksToProcess.load() / 100;
+				if (minimumIncrement < 1 || ISATTY)
+					minimumIncrement = 1;
+
+				if (phaseProgress == 0 || phaseProgress + minimumIncrement <= blockProgress) {
+					phaseProgress = blockProgress;
+
+					std::ostringstream str;
+					str << "\r";
+					void_mmap_allocator::reportStoreSize(str);
+					if (effectiveShards > 1)
+						str << std::to_string(shard + 1) << "/" << std::to_string(effectiveShards) << " ";
+
+					// TODO: revive showing the # of ways/relations?
+					str << "Block " << blocksProcessed.load() << "/" << blocksToProcess.load() << " ";
+					std::cout << str.str();
+					std::cout.flush();
+				}
 				ioMutex.unlock();
 			}
 		};
@@ -397,8 +413,13 @@ bool PbfProcessor::ReadBlock(
 			bool done = ScanWays(output, pg, pb, wayKeys);
 			if(done) { 
 				if (ioMutex.try_lock()) {
-					std::cout << "\r(Scanning for nodes used in ways: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
-					std::cout.flush();
+					size_t scanProgress = 100*blocksProcessed.load()/blocksToProcess.load();
+
+					if (scanProgress != phaseProgress) {
+						phaseProgress = scanProgress;
+						std::cout << "\r(Scanning for nodes used in ways: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
+						std::cout.flush();
+					}
 					ioMutex.unlock();
 				}
 				continue;
@@ -410,8 +431,13 @@ bool PbfProcessor::ReadBlock(
 			bool done = ScanRelations(output, pg, pb, wayKeys);
 			if(done) { 
 				if (ioMutex.try_lock()) {
-					std::cout << "\r(Scanning for ways used in relations: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
-					std::cout.flush();
+					size_t scanProgress = 100*blocksProcessed.load()/blocksToProcess.load();
+					
+					if (scanProgress != phaseProgress) {
+						phaseProgress = scanProgress;
+						std::cout << "\r(Scanning for ways used in relations: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
+						std::cout.flush();
+					}
 					ioMutex.unlock();
 				}
 				continue;
@@ -591,6 +617,7 @@ int PbfProcessor::ReadPbfFile(
 	all_phases.push_back(ReadPhase::Relations);
 
 	for(auto phase: all_phases) {
+		phaseProgress = 0;
 		uint effectiveShards = 1;
 
 		// On memory-constrained machines, we might read ways/relations

src/tilemaker.cpp

@@ -345,7 +345,7 @@ int main(const int argc, const char* argv[]) {
 	std::mutex io_mutex;
 
 	// Loop through tiles
-	std::atomic<uint64_t> tilesWritten(0);
+	std::atomic<uint64_t> tilesWritten(0), lastTilesWritten(0);
 
 	for (auto source : sources) {
 		source->finalize(options.threadNum);
@@ -518,7 +518,7 @@ int main(const int argc, const char* argv[]) {
 			batchSize++;
 		}
 
-		boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten]() {
+		boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten]() {
 			std::vector<std::string> tileTimings;
 			std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize);
 			for(std::size_t i = startIndex; i < endIndex; ++i) {
@@ -557,16 +557,21 @@ int main(const int argc, const char* argv[]) {
 			tilesWritten += (endIndex - startIndex); 
 
 			if (io_mutex.try_lock()) {
-				// Show progress grouped by z6 (or lower)
-				size_t z = tileCoordinates[startIndex].first;
-				size_t x = tileCoordinates[startIndex].second.x;
-				size_t y = tileCoordinates[startIndex].second.y;
-				if (z > CLUSTER_ZOOM) {
-					x = x / (1 << (z - CLUSTER_ZOOM));
-					y = y / (1 << (z - CLUSTER_ZOOM));
-					z = CLUSTER_ZOOM;
+				uint64_t written = tilesWritten.load();
+
+				if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) {
+					lastTilesWritten = written;
+					// Show progress grouped by z6 (or lower)
+					size_t z = tileCoordinates[startIndex].first;
+					size_t x = tileCoordinates[startIndex].second.x;
+					size_t y = tileCoordinates[startIndex].second.y;
+					if (z > CLUSTER_ZOOM) {
+						x = x / (1 << (z - CLUSTER_ZOOM));
+						y = y / (1 << (z - CLUSTER_ZOOM));
+						z = CLUSTER_ZOOM;
+					}
+					cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << "               \r" << std::flush;
 				}
-				cout << "z" << z << "/" << x << "/" << y << ", writing tile " << tilesWritten.load() << " of " << tileCoordinates.size() << "               \r" << std::flush;
 				io_mutex.unlock();
 			}
 		});