Merge pull request #601 from cldellow/warn-about-osmconvert-pbfs

warn the user when using sub-optimal PBFs

src/read_pbf.cpp

@@ -311,8 +311,12 @@ int PbfReader::ReadPbfFile(unordered_set<string> const &nodeKeys, unsigned int t
 
 	std::map<std::size_t, std::pair< std::size_t, std::size_t> > blocks;
 
+	// Track the filesize - note that we can't rely on tellg(), as
+	// its meant to be an opaque token useful only for seeking.
+	size_t filesize = 0;
 	while (true) {
 		BlobHeader bh = readHeader(*infile);
+		filesize += bh.datasize();
 		if (infile->eof()) {
 			break;
 		}
@@ -327,6 +331,21 @@ int PbfReader::ReadPbfFile(unordered_set<string> const &nodeKeys, unsigned int t
 
 	std::size_t total_blocks = blocks.size();
 
+	// PBFs generated by Osmium have 8,000 entities per block,
+	// and each block is about 64KB.
+	//
+	// PBFs generated by osmconvert (e.g., BBBike PBFs) have as
+	// many entities as fit in 31MB. Each block is about 16MB.
+	//
+	// Osmium PBFs seem to be processed about 3x faster than osmconvert
+	// PBFs, so try to hint to the user when they could speed up their
+	// pipeline.
+	if (filesize / total_blocks > 1000000) {
+		std::cout << "warning: PBF has very large blocks, which may slow processing" << std::endl;
+		std::cout << "         to fix: osmium cat -f pbf your-file.osm.pbf -o optimized.osm.pbf" << std::endl;
+	}
+
+
 	std::vector<ReadPhase> all_phases = { ReadPhase::Nodes, ReadPhase::RelationScan, ReadPhase::Ways, ReadPhase::Relations };
 	for(auto phase: all_phases) {
 		// Launch the pool with threadNum threads