From 815a69d8ade5f2b9e58002ed4ff282e5e0f4023a Mon Sep 17 00:00:00 2001
From: Rafael Troilo <rafael.troilo@heigit.org>
Date: Fri, 11 Nov 2022 20:42:33 +0100
Subject: [PATCH 1/2] initial setup

---
 oshdb-osm-source/pom.xml                      |  73 +++++
 .../heigit/ohsome/oshdb/TagTranslator.java    |  18 ++
 .../heigit/ohsome/oshdb/osm/OSMSource.java    |   9 +
 .../org/heigit/ohsome/oshdb/osm/pbf/Blob.java | 125 ++++++++
 .../heigit/ohsome/oshdb/osm/pbf/Block.java    | 295 ++++++++++++++++++
 .../ohsome/oshdb/osm/pbf/OSMPbfSource.java    | 172 ++++++++++
 .../ohsome/oshdb/mock/MockTranslator.java     |  77 +++++
 .../oshdb/osm/pbf/OSMPbfSourceTest.java       | 120 +++++++
 .../src/test/resources/sample.pbf             | Bin 0 -> 9653 bytes
 pom.xml                                       |   1 +
 10 files changed, 890 insertions(+)
 create mode 100644 oshdb-osm-source/pom.xml
 create mode 100644 oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/TagTranslator.java
 create mode 100644 oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/OSMSource.java
 create mode 100644 oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Blob.java
 create mode 100644 oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Block.java
 create mode 100644 oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSource.java
 create mode 100644 oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/mock/MockTranslator.java
 create mode 100644 oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSourceTest.java
 create mode 100644 oshdb-osm-source/src/test/resources/sample.pbf

diff --git a/oshdb-osm-source/pom.xml b/oshdb-osm-source/pom.xml
new file mode 100644
index 000000000..dd0128eef
--- /dev/null
+++ b/oshdb-osm-source/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>oshdb-parent</artifactId>
+        <groupId>org.heigit.ohsome</groupId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>oshdb-osm-source</artifactId>
+
+    <properties>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.projectreactor</groupId>
+                <artifactId>reactor-bom</artifactId>
+                <version>2020.0.24</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.heigit.ohsome</groupId>
+            <artifactId>oshdb-util</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.21.9</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openstreetmap.pbf</groupId>
+            <artifactId>osmpbf</artifactId>
+            <version>1.5.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>3.23.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/TagTranslator.java b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/TagTranslator.java
new file mode 100644
index 000000000..4716896d1
--- /dev/null
+++ b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/TagTranslator.java
@@ -0,0 +1,18 @@
+package org.heigit.ohsome.oshdb;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag;
+
+public interface TagTranslator {
+
+  Map<OSMTag, OSHDBTag> getOSHDBTagOf(Collection<? extends OSMTag> tags);
+
+  Map<OSHDBTag, OSMTag> lookupTag(Set<? extends OSHDBTag> tags);
+
+  Map<OSMRole, OSHDBRole> getOSHDBRoleOf(Collection<? extends OSMRole> values);
+
+  Map<OSHDBRole, OSMRole> lookupRole(Set<? extends OSHDBRole> roles);
+}
diff --git a/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/OSMSource.java b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/OSMSource.java
new file mode 100644
index 000000000..09b112dfc
--- /dev/null
+++ b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/OSMSource.java
@@ -0,0 +1,9 @@
+package org.heigit.ohsome.oshdb.osm;
+
+import org.heigit.ohsome.oshdb.TagTranslator;
+import reactor.core.publisher.Flux;
+
+public interface OSMSource {
+
+  Flux<OSMEntity> entities(TagTranslator translator);
+}
diff --git a/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Blob.java b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Blob.java
new file mode 100644
index 000000000..9f7991545
--- /dev/null
+++ b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Blob.java
@@ -0,0 +1,125 @@
+package org.heigit.ohsome.oshdb.osm.pbf;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import crosby.binary.Fileformat;
+import crosby.binary.Osmformat;
+import crosby.binary.file.FileFormatException;
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.NoSuchElementException;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+import reactor.core.publisher.Mono;
+
+public class Blob {
+
+  private static final int MAX_HEADER_SIZE = 64 * 1024;
+
+  public static Blob read(InputStream input) throws IOException {
+    DataInputStream dataInput = new DataInputStream(input);
+    var headerSize = dataInput.readInt();
+    if (headerSize > MAX_HEADER_SIZE) {
+      throw new FileFormatException(
+          "Unexpectedly long header " + MAX_HEADER_SIZE + " bytes. Possibly corrupt file.");
+    }
+
+    var buf = new byte[headerSize];
+    dataInput.readFully(buf);
+    var header = Fileformat.BlobHeader.parseFrom(buf);
+
+    var offset = position(input);
+
+    var data = new byte[header.getDatasize()];
+    dataInput.readFully(data);
+
+    return new Blob(header.getType(), offset, data);
+  }
+
+  private static long position(InputStream input) throws IOException {
+    if (input instanceof FileInputStream) {
+      return ((FileInputStream) input).getChannel().position();
+    }
+    return -1;
+  }
+
+  private final String type;
+  private final long offset;
+  private final byte[] data;
+
+  private Blob(String type, long offset, byte[] data) {
+    this.type = type;
+    this.offset = offset;
+    this.data = data;
+  }
+
+  public long offset() {
+    return offset;
+  }
+
+  public byte[] data() {
+    return data;
+  }
+
+  public boolean isHeader() {
+    return "OSMHeader".equals(type);
+  }
+
+  public Osmformat.HeaderBlock header() throws FileFormatException {
+    if (!isHeader()) {
+      throw new NoSuchElementException();
+    }
+
+    try {
+      return Osmformat.HeaderBlock.parseFrom(decompress());
+    } catch (InvalidProtocolBufferException e) {
+      throw new FileFormatException(e);
+    }
+  }
+
+  public boolean isData() {
+    return "OSMData".equals(type);
+  }
+
+  public Mono<Block> block() {
+    if (!isData()) {
+      return Mono.error(new NoSuchElementException());
+    }
+    return Mono.fromCallable(() -> Block.parse(this, decompress()));
+  }
+
+  private byte[] decompress() throws FileFormatException {
+    var blob = parseBlob();
+    if (blob.hasRaw()) {
+      return blob.getRaw().toByteArray();
+    }
+    if (blob.hasZlibData()) {
+      return decompress(blob);
+    }
+    throw new UnsupportedOperationException();
+  }
+
+  private static byte[] decompress(Fileformat.Blob blob) throws FileFormatException {
+    var buffer = new byte[blob.getRawSize()];
+    Inflater inflater = new Inflater();
+    try {
+      inflater.setInput(blob.getZlibData().toByteArray());
+      inflater.inflate(buffer);
+      assert (inflater.finished());
+    } catch (DataFormatException e) {
+      throw new FileFormatException(e);
+    } finally {
+      inflater.end();
+    }
+    return buffer;
+  }
+
+  private Fileformat.Blob parseBlob() throws FileFormatException {
+    try {
+      return Fileformat.Blob.parseFrom(data);
+    } catch (InvalidProtocolBufferException e) {
+      throw new FileFormatException(e);
+    }
+  }
+}
diff --git a/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Block.java b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Block.java
new file mode 100644
index 000000000..9c16b1199
--- /dev/null
+++ b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/Block.java
@@ -0,0 +1,295 @@
+package org.heigit.ohsome.oshdb.osm.pbf;
+
+import static java.lang.Boolean.TRUE;
+import static java.lang.Math.toIntExact;
+import static java.util.Spliterators.spliterator;
+import static java.util.function.Predicate.not;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.StreamSupport.stream;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import crosby.binary.Osmformat;
+import crosby.binary.Osmformat.DenseNodes;
+import crosby.binary.Osmformat.PrimitiveGroup;
+import crosby.binary.file.FileFormatException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.function.IntFunction;
+import java.util.stream.Stream;
+import org.heigit.ohsome.oshdb.OSHDBRole;
+import org.heigit.ohsome.oshdb.OSHDBTag;
+import org.heigit.ohsome.oshdb.osm.OSM;
+import org.heigit.ohsome.oshdb.osm.OSMEntity;
+import org.heigit.ohsome.oshdb.osm.OSMMember;
+import org.heigit.ohsome.oshdb.osm.OSMNode;
+import org.heigit.ohsome.oshdb.osm.OSMRelation;
+import org.heigit.ohsome.oshdb.osm.OSMType;
+import org.heigit.ohsome.oshdb.osm.OSMWay;
+import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag;
+
+public class Block {
+
+  public static Block parse(Blob blob, byte[] data) throws FileFormatException {
+    try {
+      var block = Osmformat.PrimitiveBlock.parseFrom(data);
+
+      var granularity = block.getGranularity();
+      var latOffset = block.getLatOffset();
+      var lonOffset = block.getLonOffset();
+      var dateGranularity = block.getDateGranularity();
+
+      if (granularity != 100) {
+        throw new OSHDBException("expected granularity must be 100! But got " + granularity);
+      }
+      if (dateGranularity != 1000) {
+        throw new OSHDBException(
+            "expected date granularity must be 1000! But got " + dateGranularity);
+      }
+      if (lonOffset != 0 || latOffset != 0) {
+        throw new OSHDBException(
+            "expected lon/lat offset must be 0! But got " + lonOffset + "/" + latOffset);
+      }
+
+      var stringTable = block.getStringtable();
+      var strings = new String[stringTable.getSCount()];
+      for (int i = 0; i < strings.length; i++) {
+        strings[i] = stringTable.getS(i).toStringUtf8();
+      }
+      return new Block(blob, block, strings);
+
+    } catch (InvalidProtocolBufferException e) {
+      throw new FileFormatException(e);
+    }
+  }
+
+  private final Blob blob;
+  private final Osmformat.PrimitiveBlock primitiveBlock;
+  private final String[] strings;
+  private final Map<OSHDBTag, OSMTag> blockTags = new HashMap<>();
+  private final Map<OSHDBRole, OSMRole> blockRoles = new HashMap<>();
+
+  private Block(Blob blob, Osmformat.PrimitiveBlock block, String[] strings) {
+    this.blob = blob;
+    this.primitiveBlock = block;
+    this.strings = strings;
+  }
+
+  public Blob getBlob() {
+    return blob;
+  }
+
+  public List<List<OSMEntity>> entities() {
+    return primitiveBlock.getPrimitivegroupList().stream()
+        .flatMap(this::groupToEntities)
+        .filter(not(List::isEmpty))
+        .collect(toList());
+  }
+
+  private Stream<List<OSMEntity>> groupToEntities(Osmformat.PrimitiveGroup group) {
+    return Stream.of(
+            denseToEntities(group),
+            group.getNodesList().stream().map(this::parse),
+            group.getWaysList().stream().map(this::parse),
+            group.getRelationsList().stream().map(this::parse))
+        .map(stream -> stream.collect(toList()));
+  }
+
+  private Stream<OSMEntity> denseToEntities(PrimitiveGroup group) {
+    if (!group.hasDense()) {
+      return Stream.empty();
+    }
+    var dense = group.getDense();
+    var itr = new DenseIterator(dense);
+    return stream(spliterator(itr, dense.getIdCount(), Spliterator.ORDERED), false);
+  }
+
+  public Map<OSHDBTag, OSMTag> getBlockTags() {
+    return blockTags;
+  }
+
+  public Map<OSHDBRole, OSMRole> getBlockRoles() {
+    return blockRoles;
+  }
+
+  private class DenseIterator implements Iterator<OSMEntity> {
+
+    private final Osmformat.DenseNodes dense;
+
+    private final List<Integer> versions;
+    private final List<Long> timestamps;
+    private final List<Long> changesets;
+    private final List<Integer> users;
+    private final IntFunction<Boolean> visibilities;
+    private final IntFunction<List<OSHDBTag>> keysVals;
+
+    private long id;
+    private long timestamp;
+    private long changeset;
+    private int user;
+    private long lon;
+    private long lat;
+
+    private int next = 0;
+
+    public DenseIterator(Osmformat.DenseNodes dense) {
+      this.dense = dense;
+      if (!dense.hasDenseinfo()) {
+        throw new OSHDBException("entity info is required for oshdb");
+      }
+
+      var info = dense.getDenseinfo();
+      versions = info.getVersionList();
+      timestamps = info.getTimestampList();
+      changesets = info.getChangesetList();
+      users = info.getUidList();
+      if (!info.getVisibleList().isEmpty()) {
+        visibilities = info.getVisibleList()::get;
+      } else {
+        visibilities = x -> true;
+      }
+
+      if (dense.getKeysValsList().isEmpty()) {
+        keysVals = x -> Collections.emptyList();
+      } else {
+        this.keysVals = buildKeyVals(dense)::get;
+      }
+    }
+
+    private List<List<OSHDBTag>> buildKeyVals(DenseNodes dense) {
+      var list = new ArrayList<List<OSHDBTag>>(dense.getIdCount());
+      var tags = new ArrayList<OSHDBTag>();
+      for (var i = 0; i < dense.getKeysValsCount(); i++) {
+        var key = dense.getKeysVals(i);
+        if (key == 0) {
+          addToBlockTags(tags);
+          list.add(List.copyOf(tags));
+          tags.clear();
+        } else {
+          var val = dense.getKeysVals(++i);
+          tags.add(new OSHDBTag(key, val));
+        }
+      }
+      return list;
+    }
+
+    @Override
+    public boolean hasNext() {
+      return next < dense.getIdCount();
+    }
+
+    @Override
+    public OSMEntity next() {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      return getNext(next++);
+    }
+
+    private OSMEntity getNext(int index) {
+      id += dense.getId(index);
+      timestamp += timestamps.get(index);
+      changeset += changesets.get(index);
+      user += users.get(index);
+
+      var visible = TRUE.equals(visibilities.apply(index)) ? 1 : -1;
+      var version = versions.get(index) * visible;
+
+      var tags = keysVals.apply(index);
+      lon += dense.getLon(index);
+      lat += dense.getLat(index);
+      return OSM.node(id, version, timestamp, changeset, user, tags, toIntExact(lon),
+          toIntExact(lat));
+    }
+  }
+
+  private OSMNode parse(Osmformat.Node entity) {
+    var id = entity.getId();
+    var lon = entity.getLon();
+    var lat = entity.getLat();
+
+    return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(),
+        (timestamp, changeset, user, version, tags) ->
+            OSM.node(id, version, timestamp, changeset, user, tags, toIntExact(lon),
+                toIntExact(lat)));
+  }
+
+  private OSMWay parse(Osmformat.Way entity) {
+    var id = entity.getId();
+    var members = new OSMMember[entity.getRefsCount()];
+    var memId = 0L;
+    for (var i = 0; i < members.length; i++) {
+      memId += entity.getRefs(i);
+      members[i] = new OSMMember(memId, OSMType.NODE, -1);
+    }
+    return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(),
+        (timestamp, changeset, user, version, tags) ->
+            OSM.way(id, version, timestamp, changeset, user, tags, members));
+  }
+
+  private OSMRelation parse(Osmformat.Relation entity) {
+    var id = entity.getId();
+    var members = new OSMMember[entity.getMemidsCount()];
+    var memId = 0L;
+    var roles = new HashSet<OSHDBRole>();
+    for (var i = 0; i < members.length; i++) {
+      memId += entity.getMemids(i);
+      var type = entity.getTypes(i);
+      var role = entity.getRolesSid(i);
+      var member =  new OSMMember(memId, OSMType.fromInt(type.getNumber()), role);
+      roles.add(member.getRole());
+      members[i] = member;
+    }
+    addToBlockRoles(roles);
+    return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(),
+        (timestamp, changeset, user, version, tags) ->
+            OSM.relation(id, version, timestamp, changeset, user, tags, members));
+  }
+
+
+  private <T> T withInfo(List<Integer> keys, List<Integer> values, Osmformat.Info info,
+      EntityInfo<T> metadata) {
+    var timestamp = info.getTimestamp();
+    var changeset = info.getChangeset();
+    var user = info.getUid();
+
+    var visible = info.hasVisible() && !info.getVisible() ? -1 : 1;
+    var version = info.getVersion() * visible;
+
+    var tags = new ArrayList<OSHDBTag>(keys.size());
+    for (var i = 0; i < keys.size(); i++) {
+      tags.add(new OSHDBTag(keys.get(i), values.get(i)));
+    }
+    addToBlockTags(tags);
+    return metadata.apply(timestamp, changeset, user, version, tags);
+  }
+
+  private interface EntityInfo<T> {
+    T apply(long timestamp, long changeset, int user, int version, List<OSHDBTag> tags);
+  }
+
+  private void addToBlockTags(List<OSHDBTag> tags) {
+    tags.forEach(tag -> blockTags.computeIfAbsent(tag,this::toOSMTag));
+  }
+
+  private OSMTag toOSMTag(OSHDBTag tag) {
+    return new OSMTag(strings[tag.getKey()], strings[tag.getValue()]);
+  }
+
+  private void addToBlockRoles(Set<OSHDBRole> roles) {
+    roles.forEach(role -> blockRoles.computeIfAbsent(role, this::toOSMRole));
+  }
+
+  private OSMRole toOSMRole(OSHDBRole role) {
+    return new OSMRole(strings[role.getId()]);
+  }
+}
diff --git a/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSource.java b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSource.java
new file mode 100644
index 000000000..4feb8741e
--- /dev/null
+++ b/oshdb-osm-source/src/main/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSource.java
@@ -0,0 +1,172 @@
+package org.heigit.ohsome.oshdb.osm.pbf;
+
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.collect.Maps;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+import java.util.stream.Stream;
+import org.heigit.ohsome.oshdb.OSHDBRole;
+import org.heigit.ohsome.oshdb.OSHDBTag;
+import org.heigit.ohsome.oshdb.TagTranslator;
+import org.heigit.ohsome.oshdb.osm.OSM;
+import org.heigit.ohsome.oshdb.osm.OSMEntity;
+import org.heigit.ohsome.oshdb.osm.OSMMember;
+import org.heigit.ohsome.oshdb.osm.OSMNode;
+import org.heigit.ohsome.oshdb.osm.OSMRelation;
+import org.heigit.ohsome.oshdb.osm.OSMSource;
+import org.heigit.ohsome.oshdb.osm.OSMWay;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.SynchronousSink;
+import reactor.core.scheduler.Scheduler;
+import reactor.core.scheduler.Schedulers;
+
+public class OSMPbfSource implements OSMSource {
+
+  private final Path path;
+
+  public OSMPbfSource(Path path) {
+    this.path = path;
+  }
+
+  @Override
+  public Flux<OSMEntity> entities(TagTranslator translator) {
+    return Flux.using(this::openSource,
+        source -> entities(source, translator),
+        this::closeQuietly);
+  }
+
+  private InputStream openSource() throws IOException {
+    return Files.newInputStream(path);
+  }
+
+  private void closeQuietly(InputStream input) {
+    try {
+      input.close();
+    } catch (IOException e) {
+      // ignite ioexception
+    }
+  }
+
+  private Flux<OSMEntity> entities(InputStream source, TagTranslator tagTranslator) {
+    return Flux.using(() -> Schedulers.newParallel("io"),
+        scheduler -> blobs(source)
+            .filter(Blob::isData)
+            .flatMapSequential(blob -> entities(blob, tagTranslator).subscribeOn(scheduler)),
+        Scheduler::dispose);
+  }
+
+  private Flux<Blob> blobs(InputStream source) {
+    return Flux.generate(sink -> readBlob(source, sink));
+  }
+
+  private static void readBlob(InputStream source, SynchronousSink<Blob> sink) {
+    try {
+      sink.next(Blob.read(source));
+    } catch (EOFException e) {
+      sink.complete();
+    } catch (IOException e) {
+      sink.error(e);
+    }
+  }
+
+  private Flux<OSMEntity> entities(Blob blob, TagTranslator translator) {
+    return blob.block().flatMapMany(block -> entities(block, translator));
+  }
+
+  private Flux<OSMEntity> entities(Block block, TagTranslator translator) {
+    var entities = block.entities();
+    var tags = mapTags(block, translator);
+    var roles = mapRoles(block, translator);
+    return Flux.fromIterable(entities)
+        .map(list -> rebuild(list, tags, roles))
+        .concatMap(Flux::fromStream);
+  }
+
+  private Map<OSHDBTag, OSHDBTag> mapTags(Block block, TagTranslator translator) {
+    var tags = block.getBlockTags();
+    var translated = translator.getOSHDBTagOf(tags.values());
+    var mapping = Maps.<OSHDBTag, OSHDBTag>newHashMapWithExpectedSize(tags.size());
+    tags.forEach((oshdb, osm) -> mapping.put(oshdb, translated.get(osm)));
+    return mapping;
+  }
+
+  private Map<OSHDBRole, OSHDBRole> mapRoles(Block block, TagTranslator translator) {
+    var roles = block.getBlockRoles();
+    var translated = translator.getOSHDBRoleOf(roles.values());
+    var mapping = Maps.<OSHDBRole, OSHDBRole>newHashMapWithExpectedSize(roles.size());
+    roles.forEach((oshdb, osm) -> mapping.put(oshdb, translated.get(osm)));
+    return mapping;
+  }
+
+  private Stream<OSMEntity> rebuild(List<OSMEntity> list, Map<OSHDBTag, OSHDBTag> mappingTags,
+      Map<OSHDBRole, OSHDBRole> mappingRoles) {
+    var type = list.get(0).getType();
+    switch (type) {
+      case NODE:
+        return rebuild(list, OSMNode.class, osm -> rebuild(osm, mappingTags));
+      case WAY:
+        return rebuild(list, OSMWay.class, osm -> rebuild(osm, mappingTags));
+      case RELATION:
+        return rebuild(list, OSMRelation.class, osm -> rebuild(osm, mappingTags, mappingRoles));
+      default:
+        throw new IllegalStateException();
+    }
+  }
+
+  private <T extends OSMEntity> Stream<OSMEntity> rebuild(List<OSMEntity> list, Class<T> type,
+      UnaryOperator<T> rebuild) {
+    return list.stream().map(type::cast).map(rebuild);
+  }
+
+  private OSMNode rebuild(OSMNode osm, Map<OSHDBTag, OSHDBTag> mappingTags) {
+    return rebuild(osm, mappingTags, (id, version, timestamp, changeset, user, tags) ->
+        OSM.node(id, version, timestamp, changeset, user, tags, osm.getLon(), osm.getLat()));
+  }
+
+  private OSMWay rebuild(OSMWay osm, Map<OSHDBTag, OSHDBTag> mappingTags) {
+    return rebuild(osm, mappingTags, (id, version, timestamp, changeset, user, tags) ->
+        OSM.way(id, version, timestamp, changeset, user, tags, osm.getMembers()));
+  }
+
+  private OSMRelation rebuild(OSMRelation osm, Map<OSHDBTag, OSHDBTag> mappingTags,
+      Map<OSHDBRole, OSHDBRole> mappingRoles) {
+    return rebuild(osm, mappingTags, (id, version, timestamp, changeset, user, tags) ->
+        OSM.relation(id, version, timestamp, changeset, user, tags,
+            rebuildMembers(osm, mappingRoles)));
+  }
+
+  private OSMMember[] rebuildMembers(OSMRelation osm, Map<OSHDBRole, OSHDBRole> mappingRoles) {
+    var members = osm.getMembers();
+    for (var i = 0; i < members.length; i++) {
+      var member = members[i];
+      var id = member.getId();
+      var type = member.getType();
+      var role = mappingRoles.get(member.getRole());
+      members[i] = new OSMMember(id, type, role.getId());
+    }
+    return members;
+  }
+
+  private <T extends OSMEntity> T rebuild(T entity, Map<OSHDBTag, OSHDBTag> mappingTags,
+      EntityCommon<T> entityCommon) {
+    var id = entity.getId();
+    var visible = entity.isVisible() ? 1 : -1;
+    var version = entity.getVersion() * visible;
+    var timestamp = entity.getEpochSecond();
+    var changeset = entity.getChangesetId();
+    var user = entity.getUserId();
+    var tags = entity.getTags().stream().map(mappingTags::get).sorted().collect(toList());
+    return entityCommon.apply(id, version, timestamp, changeset, user, tags);
+  }
+
+  private interface EntityCommon<T> {
+    T apply(long id, int version, long timestamp, long changeset, int user, List<OSHDBTag> tags);
+  }
+}
diff --git a/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/mock/MockTranslator.java b/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/mock/MockTranslator.java
new file mode 100644
index 000000000..4a302c762
--- /dev/null
+++ b/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/mock/MockTranslator.java
@@ -0,0 +1,77 @@
+package org.heigit.ohsome.oshdb.mock;
+
+import com.google.common.collect.Maps;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.heigit.ohsome.oshdb.OSHDBRole;
+import org.heigit.ohsome.oshdb.OSHDBTag;
+import org.heigit.ohsome.oshdb.TagTranslator;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag;
+
+public class MockTranslator implements TagTranslator {
+  private final Map<OSMTag, OSHDBTag> translate = new ConcurrentHashMap<>();
+  private final Map<OSHDBTag, OSMTag> lookup = new ConcurrentHashMap<>();
+  private final Map<OSMRole, OSHDBRole> translateRole = new ConcurrentHashMap<>();
+  private final Map<OSHDBRole, OSMRole> lookupRole = new ConcurrentHashMap<>();
+  private final Map<String, Integer> stringIds = new ConcurrentHashMap<>();
+  private final AtomicInteger nextStringId = new AtomicInteger();
+
+  @Override
+  public Map<OSMTag, OSHDBTag> getOSHDBTagOf(Collection<? extends OSMTag> tags) {
+    var map = Maps.<OSMTag, OSHDBTag>newHashMapWithExpectedSize(tags.size());
+    tags.forEach(tag -> map.put(tag, translate(tag)));
+    return map;
+  }
+
+  @Override
+  public Map<OSMRole, OSHDBRole> getOSHDBRoleOf(Collection<? extends OSMRole> roles) {
+    var map = Maps.<OSMRole, OSHDBRole>newHashMapWithExpectedSize(roles.size());
+    roles.forEach(role -> map.put(role, translate(role)));
+    return map;
+  }
+
+  @Override
+  public Map<OSHDBTag, OSMTag> lookupTag(Set<? extends OSHDBTag> tags) {
+    var map = Maps.<OSHDBTag, OSMTag>newHashMapWithExpectedSize(tags.size());
+    tags.forEach(tag -> map.put(tag, lookup.get(tag)));
+    return map;
+  }
+
+  @Override
+  public Map<OSHDBRole, OSMRole> lookupRole(Set<? extends OSHDBRole> roles) {
+    var map = Maps.<OSHDBRole, OSMRole>newHashMapWithExpectedSize(roles.size());
+    roles.forEach(role -> map.put(role, lookupRole.get(role)));
+    return map;
+  }
+
+  private OSHDBTag translate(OSMTag tag){
+    return translate.computeIfAbsent(tag, this::tag);
+  }
+
+  private OSHDBTag tag(OSMTag tag) {
+    var key = string(tag.getKey());
+    var val = string(tag.getValue());
+    var oshdb = new OSHDBTag(key,val);
+    lookup.put(oshdb, tag);
+    return oshdb;
+  }
+
+  private OSHDBRole translate(OSMRole role) {
+    return translateRole.computeIfAbsent(role, this::role);
+  }
+
+  private OSHDBRole role(OSMRole role) {
+    var oshdb = OSHDBRole.of(string(role.toString()));
+    lookupRole.put(oshdb, role);
+    return oshdb;
+  }
+
+  private int string(String string) {
+     return stringIds.computeIfAbsent(string, x -> nextStringId.getAndIncrement());
+  }
+
+}
diff --git a/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSourceTest.java b/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSourceTest.java
new file mode 100644
index 000000000..f815fad31
--- /dev/null
+++ b/oshdb-osm-source/src/test/java/org/heigit/ohsome/oshdb/osm/pbf/OSMPbfSourceTest.java
@@ -0,0 +1,120 @@
+package org.heigit.ohsome.oshdb.osm.pbf;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.heigit.ohsome.oshdb.TagTranslator;
+import org.heigit.ohsome.oshdb.mock.MockTranslator;
+import org.heigit.ohsome.oshdb.osm.OSMEntity;
+import org.heigit.ohsome.oshdb.osm.OSMMember;
+import org.heigit.ohsome.oshdb.osm.OSMNode;
+import org.heigit.ohsome.oshdb.osm.OSMRelation;
+import org.heigit.ohsome.oshdb.osm.OSMType;
+import org.heigit.ohsome.oshdb.osm.OSMWay;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole;
+import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag;
+import org.junit.jupiter.api.Test;
+
+class OSMPbfSourceTest {
+
+  private static final Path testResources = Paths.get("src", "test", "resources");
+  private static final Path SAMPLE_PBF = testResources.resolve("sample.pbf");
+
+  @Test
+  void openSource() {
+    var pbf = testResources.resolve("sample.pbf");
+    assertTrue(Files.exists(pbf));
+  }
+
+  @Test
+  void entities() {
+    var source = new OSMPbfSource(SAMPLE_PBF);
+    var tagTranslator = new MockTranslator();
+    var count = source.entities(tagTranslator).limitRate(10).count().block();
+    assertEquals(339, count);
+
+    var entities = new EnumMap<OSMType, Map<Long, OSMEntity>>(OSMType.class);
+
+    source.entities(tagTranslator)
+        .bufferUntilChanged(OSMEntity::getType)
+        .doOnNext(list -> {
+          var type = list.get(0).getType();
+          var map = entities.computeIfAbsent(type, x -> new HashMap<>());
+          list.forEach(entity -> map.put(entity.getId(), entity));
+        })
+        .then().block();
+    assertEquals(3, entities.size());
+
+    OSMEntity sample;
+
+    sample = entities.get(OSMType.NODE).get(647105170L);
+    assertNotNull(sample);
+    assertEquals(2, sample.getVersion());
+    assertEquals(4210769, sample.getChangesetId());
+    assertEquals(1269340001, sample.getEpochSecond());
+    assertEquals(234999, sample.getUserId());
+    assertEquals(-2344645, ((OSMNode) sample).getLon());
+    assertEquals(517635905, ((OSMNode) sample).getLat());
+    assertEquals(0, sample.getTags().size());
+
+    sample = entities.get(OSMType.WAY).get(49161822L);
+    assertNotNull(sample);
+    assertEquals(1, sample.getVersion());
+    assertEquals(3754726, sample.getChangesetId());
+    assertEquals(1264885069, sample.getEpochSecond());
+    assertEquals(508, sample.getUserId());
+    assertThat(Set.copyOf(tagTranslator.lookupTag(sample.getTags()).values()))
+        .hasSize(2)
+        .containsAll(List.of(
+            new OSMTag("highway", "residential"),
+            new OSMTag("name", "Worcester Road")
+        ));
+    assertThat(((OSMWay) sample).getMembers())
+        .hasSize(5)
+        .containsSequence(
+            new OSMMember(30983851, OSMType.NODE, -1),
+            new OSMMember(623624257, OSMType.NODE, -1),
+            new OSMMember(623624154, OSMType.NODE, -1),
+            new OSMMember(623624259, OSMType.NODE, -1),
+            new OSMMember(623624261, OSMType.NODE, -1));
+
+    sample = entities.get(OSMType.RELATION).get(31640L);
+
+    assertNotNull(sample);
+    assertEquals(81, sample.getVersion());
+    assertEquals(11640673, sample.getChangesetId());
+    assertEquals(1337419064, sample.getEpochSecond());
+    assertEquals(24119, sample.getUserId());
+    assertThat(Set.copyOf(tagTranslator.lookupTag(sample.getTags()).values()))
+        .hasSize(5)
+        .containsAll(List.of(
+            new OSMTag("type", "route"),
+            new OSMTag("ref", "61"),
+            new OSMTag("route", "bicycle"),
+            new OSMTag("network", "ncn"),
+            new OSMTag("name", "NCN National Route 61")
+        ));
+
+    assertThat(((OSMRelation) sample).getMembers())
+        .hasSize(234)
+        .contains(
+            new OSMMember(25896435, OSMType.WAY, roleIdOf("forward", tagTranslator)),
+            new OSMMember(121267847, OSMType.WAY, roleIdOf("", tagTranslator)));
+  }
+
+  private int roleIdOf(String role, TagTranslator translator) {
+    var osm = new OSMRole(role);
+    return translator.getOSHDBRoleOf(Set.of(osm)).get(osm).getId();
+  }
+
+}
\ No newline at end of file
diff --git a/oshdb-osm-source/src/test/resources/sample.pbf b/oshdb-osm-source/src/test/resources/sample.pbf
new file mode 100644
index 0000000000000000000000000000000000000000..8a22edfee9bf1514af8cecf183e0e994bc40041d
GIT binary patch
literal 9653
zcmV;mB}&==000gO2~Sf^NM&JUWpWsj0T6@%8jJyWoa2(>IKF+)14e=OH^1Fvlz4IP
z&W%Q9g&&KbU29}k;`1-g4NlHT%}vw|Gte_r;&MsND^B&xPf0D-5)95SD$xt6EJ%$n
z$w<wM_jGAu<j*h8P0r6NOD!r<Fwir1?_yNRC@Cqh($_C9FW1X20O~F&N=+@vO)SvM
zFG|->EXdRcnhOB(vN9?F000aM2TxN?L}7Gc7_=@B^g0@@E_j@sSb0=bXO^$;eeb=h
zdTcIej1WVjo#~`AiDFDlVvKt_b~HUnFEibLI7G`xM9Kihq?0*avIqsmz9Ue8?EAhW
z`@R+XTE)IE0tFO1b6<hP^qey@r=Q%f-n;AX{_cH+%FNIVhtaueeEpWt#K_#>%UedK
zx6PSdpO^{07MlHF@MmKa6Jw)0<_!MbH`f{Np9Lo7w~Pgh1_azTID60B!q}AI{a#=p
zF#FU($k11fe*cHFR~hat<F9YsH~N9$eRa?Lnz@DWE<>6c-4QTapBvq_FcSz2&YK9$
z1&ofV(OnB8(+|#CTo;}f-ViXFhR2Q`H9Yq2QNt6*jxyTY=5qogGh@@Q8FZD|y{$Sg
zymLordX>$&Yg_fH=?&xSMixRd1`F@r-O2r#k?D;)MrOAe?jKEWn+orn?&P?t{!rKD
z-7>Q<x_jsDeP-Xb@%JY8%mMsggr<To1UDJ(Z_obW(x;zYVD@j@zq}<hvv>{#arG>t
z^~}b$Xu)WGCb(mMQ)qU>;0vMAjU7PW3CxUd8VhbPgefp)F1%-UUGOaQn(>{xkeA__
z2#n3|nF$!?u8GkPUz-W<nL+{+;dLXEYituHf*W587|Q%UOAEeo`8>lr3xl~MxMRfN
z>xO2G_PKjzP{icMHq$%Ao;ZH=D1iO*cFRT<cNlG$s>znK8mOlEU7_&y4|fGdruQru
z{#I2pAc4601*3D>$o!76sleb9BeNR<Q}bVBKEa?bp)IqW<aZf-=f-VD>xz+u#Vx^o
z2(l2EGaC2I1!mU_PaJ>$#7Ty_Y5YC#XY_|##^we_U%|A1@*4(bM#d(vK?a~Szjya8
z^l-z#LTK=nz~HWtnT3Jyroq+oR{`%m3xUDA$KEvf>YjyxsnEi}$i##-8e6C%g$72Z
zpoOY8-eQ>Zw~XP*?40lih6^(@HMW2ah13?;z7l>9yDI$tngz7Y@Jx*E+*Ru^+cxj)
zXUC2j95*y%nCoT&HrKVUeqhMUpMQCU;axWqnwzszppDH<06fFHVtiY0?5N>8&ocwd
z{EU+Odp3^mSv}6UF)Qh<PstWZ8C^1;a3Ftr&JR~L1jgg&5dTW9cX&~{MqNx*AwJy*
zRuxEqQY9l(bLZnVXRB=rxw)fK88^Z@NY0Hb8T3UfNkwTWc{L)LJCs%`)eyDkD{ysS
zp#qO4L6RO@XKUQ<<Q0z>ix*P3^?jmBUU%OUTQr|jqm}p2_Ki}z&PvjcJ;Wv8@3r9=
zgZ+c5_s67<R+1rOQW@z}FuzYZQLHKPc`(FXUQx|+gU2cg@$fhtZ&pOP<lsv75Y;#T
z#(P?>w=OAcz-9C8g*d}+O^iw!ro_mR?I1iZC!6fAC_dBkxM2cs$SZ58h(uKv9bS+;
zMM`V4^|c>3S5=cw*kgf>ifx`xL7u~sH<CfREN$Od^09T}No#8sxGm$wn!$aGI>Vs#
z8*nb>R%f>_^F;wN2_ef~dR<ySDZ<m9{i)11dzTVd_uH%j>Q~&mJ1R*@gvT&BQkAVg
zQyp_4IREh&IXv~WR5OC@D~2s?=hm=59&3357;9|Gr5Z`ILCM^>>ZUc)jRopB2;_ON
zG&-_`rexP>g>MuVave6|$N*SManx%b%J_M1<vM8Enk$2_j6K{{lZKhm$@`srp^1ry
zvL=n(%(g;oTdkc5%*p+wQ&>@&^HHxO%*wyCHlI7u*WS#V_OibHYMYPs?LObKRw6wa
zM${>ZA+sD~$j<de{nfQa{p^}-{fe^wbUG|1_zAby+xm78TcUEUzDXlA-b#Us>VUC?
z8aQ%bjO&5xwPa>;dMjBvHYK7eR!ZEc#nk9A+BPT2oogkwzFqEZH-m?xT{ftPK@uOr
zO4PANJMXd{tDnFLqJdw<S`GZlA#dsRnWV9KZl6lj&P}jX=@0k=v?68L6t-C$)LZRc
z)F}g85g_etUB7X8469x24XYg}1!5%;Vpkj$4O`>3kZ{1V)t(G=4Av<v3Z25`<tTi$
zOsWw)2ejKEu|u$qqu54){oTW-uozNJ+t|LL%B<0P&~8sYg}0*?$QcA_hEj!vOV-p<
zOX?EY8SHY<%BgL)96Tl!nJ=n9)*w0!@|`Y)Wh?B1_05ijH%2@I<+2x(M###BkubEb
z%o`=`T^GVpE^AHj8z13WM&y`_lOmglHt&EDWcdX4Ko+0Z&MjVst+9t$#OFI7$n8{B
zaGUA}SGk2nUNJbfcc}y0HP@7K3%$Y{`JsM;F2n+0baD%5qCK6xqG(x^XA*9T_3y$P
z(38j!R6}z^9*moxBnC^0Q;z4%Oiz(6?7L&bZe}QZ%tj{fp8@fM>8Fcbv>wAd?8Gi_
zeJrpW?hY@X#9yS<1?HRLsO0^U$I%YF(sZ!7b)yy!XiLVDRLdTnHNGCyRDjR9hwduF
z+HhEvzIGJ7X|2Ff?3UYfD~IsI(TFjU{T4bH#lAj`VXYCG8PdrRGV3wSuy7cbVl5JQ
za%MYSYqKYVVb2p`&%IgP>Q|-`wr+l5P_ISSjO3Xxe%Hb{XH>)5*)PCF$F|*Xl)T7<
z5QrcMgxHh7h`liUL97mhK$rqi-xBI)_QYyOHb5PUh%KSi56=wSL2SM){kAKCXi$^S
zN~&|JeYbAa_XLHI8V$9boY+pW4H1FE7NV`B+rS7!Zg)WdM9kXM2f0<9HIeEn+gP@n
zRQn?}Xo9~DRh?2DPoO>Fyps0JO#X(oglMc|YLqp{!@8UdC0>rF(o_QvQ22`D1zu|%
z(^bbL4&)+9tL%e>&6*Ea7Uie99zQ-U_wv+^o$23I)>!gE4%mESe@aQJyHEpBzU9i7
zI-?U`-e{41ke20mDnxO`!$z0|NrGpTp9fmc-SY05eb>jkPOnS(T1J*X5!BvIxD9j=
zj;p%!z5yRut+#ws0N!~ZyhZno11gaAj0!Xck`Y-$JONSK`ROeA`}{g$$@*Zq!WYkk
z$tihv)kR4|>6g`kG5?wyR(MFNxMJzCao`bD3js{)Y{*3GuAz8)avWs6KNmT<HGLeB
z;P-KPOCD0pS)&cmJ*?~duzb<=uSK49de-j!cp+o-SiDn~3BhSEu0YO!j$&I(g$kaE
zD`g%{A4?loH3qXc_+qe0!4wGHNA~y`ZZxcFOmvpwEFdWo$j#~rJU-HYG8IT!1#%ih
z)=1Xc{9zM_vj3##sFYL$$DOH#$Ka*$vo*@iv-xFHWF+X_0B2jg*XdOUG0C<+X;Z+L
zhuHG-z^`*@`ken@)Q36c!>2GwNEBC`6<2H$!-%znFv2bSjBWNAFU1wnp!m3ZQQL98
z2c(*SR5muk`4pRvle*kK&WI{SlQlhhSq`#;nIIF30WbQ&P>TU5qg9a{$A8i%(^*d}
zeBsDp&&+<D4HA;%bvna)9n}|jew31XreWFfLP>Ay#lDVOVibt<QlnTJu0S5JQC*9$
zQB^RWHOTP*W}>KL3Y^am+x?i<(t6Qb(=PRq2;~7w<bd6e!7#Rw3bYEb;jjXwu|L_2
zp6IA!czC@3WOZ-leM<+)cZ!xmBElmXwbU^Rmt8$ByEeeR$L>cA(a{j)1Mmw0{!*#)
zkr5CbR_O)Gl%j~=H3qFPN=4I)1u!SEjc_#3?jx(Qwj))dme;zH{60=kaynJKHhiYH
zx|<q+8olF|r{a{dKlS^s9dt=5J>V7JanKXS+r1L-qFmm2kVUzu@H13`_`-b=ti(dn
zTp)@}3i!SlI?e#)W0+znh<)~%<fxz0K=Tl`rHg&lCT+#%r(;WzYxWs!_FZfC^*!aj
z1crD;)x}c4=oB4$s1_RU9qjmiHfoHL1W%JeHbPr4kGFSSfGrG!uxfw`G?B}8ZA5a>
z;OU$&=fkB?eN0w$aWWhT?t?zOJl?8;wPWALhydxe3cm$^q<Y$Ux)tVDY^7-fDeXX5
ze9d)CD5N~t#%;_U!m<+mbXOEFtER>p(kZ9r5?UFS!b}#nV-D!kTv2k9uC-GF-7%9(
zhgVnA394Rtwl)v>`SzpShB(>^Y%Tp-<;gvm4VsGd#X)Ovm<>tmE2g6}cy|7pNlm(A
z)05aEpp^D*o2BfRmkIeJvff*-DDc&l)gxVxiELRjZ9CXWLp5;bM8~%iQ6NRO`wQWx
z`fyS8VX3$VRb_;dEjA;Bv;FJSsh<USPT{q_P-?wmi1g_V#!(qOHjphb4LI0<#_nEJ
zo_(?+`y{0GSzV$lN9O3&I^FmjZ73xh&7+?VPViYkURLj@B&&GMFPh<S8ZBR7(Qw#y
zwllNADZa5N)Y1vDwr8CtJ7rR-pzrfNkpp~;1B;$@WmK&@oeur8^bF){kJjeVBOst{
zX?ahQv8sMr7)r;D;w-!=In&zm9$R22Vvtq_o`i0bAN|*W?S0mxGprOx2WHb<vvo9w
z!vRJC%|OD-g9$Hp&(K{nEXUc=CDcG)4aIqu#MWPPx_E}pv7e*GvHR-gM0hN0mKX<_
z8jusuro-X17XR2h-i=h9#RP}rY4r}UFbv_&-iKn8O>FWB80v7tG>)h~0-S9XUvb|5
zY*7MX1WORi4h!~WK8!1C*BSzUPRL__h9!hk->$V4v+TSdIEQ5k>!Dz=IGj3nTG?w$
zF1adkuhZex!o8pk976$P2)rST1998y$D&<~TEYt;oNZmpxHGRR#RGXa^%3kOpl3yF
zBP~y}B&TbFNOsb5MJ2RntR>ej>N7|$k`>W55rzD~4lCU4*GOci%o9m4zG<LDnlVQ}
zUUr(SYI|FP>-w402Sqf?52#>vFeqTXwO<P!?O+X95}rJ)!uGNIfGrkUt4rAqHBx|;
zn=8d^n?Z7KGHv$Cu+JHIBiPmTTJFVL9d5PGd7OTv(|eS5NSVNFLDq+5akIXN_OYY}
z^R=O-xM>W#Wt1>!(lI)+`bc>$V%^wUV1x;*74Cg?j+}1O=Wvu`dh=s?b=~^KvF|U|
zgMz~uT01zrb}-*wzrbF<O{LSW(%I5*7V7oE#UWZx60Ntb;&7&-^-4;0`$}{{$q8Q7
z3t82Jp>^xmcC*Q%hUn-ab|X2EDc+3_4_jH_l(gstCmdwMz-uePRypwQP|=+#HV$B<
z{wM2`pwg@Q8;6tH@gAFb3k2)tJEvPzb`o@3D(J^=PS?Rk7tn3>w%rfs=O$d5(2jKr
z&9icWkwicwvBSw$58Kyvwr$t#kL156{hgI(bU=b~NB(xp;1k(=m`$=(MV)4c>K$sp
z$k~F#dG2~Hg!A)M;$8LQT=msYPBt0=?V96Zh$RBDstOM$>;X(a&k~$$-<porgG!aw
z`v5$P^tnDBwePDL-^YU3DfWv+*m>BZc4u&``FVIyC)ZndYqUF!z<o*V`)USveBTHq
z;2VVfOD)C5@wg0MQ-jCl(A1@i@WJ8ymcSP|pZ$)*Au&8%{*?97JpCz6(O>Mo$hnL!
zYHEPy*4J<_r!I27G2(DA=3KhSxg{X*rHjPPYg__BUiOD3;GU*fXOr7d6aUO5_<$OI
z%)T}ImMv`Zin`RT>lm=8T>zrGJk24v&yQu>%-O<khiu_1ibqmsE^^3AhrVVT<DL2B
z@+ZjbTXhwq3!iR<Y_|*_mo7p()J`Jaxld_z2^!(;^ygyf#f_#ZtVH1rM7$D96Y%hB
z<)x@wOeH0uSt*tz@Ea0&iGNL~B%i2ogewxiSCWIQ9&znzsX{yC_TBhHs%@GIok2aN
zlp-9v6(H%&RzG_4i#1)(M6R_v9%{s47P0*(yo0FS6)r(V;ry(J8uL+H)e=85he?|t
z(<+eb=UoTW#i-0s(MpJWiKq2wcm(-Iq4GyKdX^j#6RE>w-y!)TRU{$@gk5-6fl2}i
zpG{OfPCoEM1v4l+ggewq*PZNGBMMvz%Q!S}dDaSLtYN23?s^#xg#XFuCL11Vjyn+r
zUUaY{x2lRvXd$di$aN{@@PeBgo=BmxY*F(9&P>C3ZlvoVaw(+ZR76-QYHP;X_Qc?0
z?EMHy6H)6j8t_Jbf3$zg>nSsosFC<m4jr^X<|klf8@3!JBvq(<0=t|Vj^!={;NW#K
z(GLX`kn2I@<RVd@L(0~v)mCiTLk<klc8*;4I?6?aRW4ZecJ@1id+kd|l@ia1Nk<>t
zVaE#((CC^b<Nw|3mFy`T7fH$-iSkfF?m^7Uh|TXOuXeQHeruv25jA}$i$Qf=WR{HT
zNh3BA$lz0b|6-qn3Kog=ca;%j+KH7c6yQgQ-c*XIA~6+MfpWfsUsFFVy7Mw>XyrSN
z+~F1~Y?e3N&g~f|8<oi2g%EiYPpeUZ6ImI_Yxd<Qxsl#-l>Kj_U@Xr@V^XxFlO9NP
zjFFSQ#Ds2IH@5b|BfBPgP;L^=*+kZ5sG^Kt=%d;FCl?!yglNY5wEbT?`iSOevZDzl
zS~0dubhS5;T8cgB2uJMWNOTM!hbELUPWTVtcoF3|Pdi1>0ll<hicW~&d2AAK3Y45k
z#0{a$M3iSmh*oe$7v)ikBuj+5j0o>1=BtR+8nV9~Z#oewDUy6xb*N<ySr(xN5z%Nx
zxE)bQ7;kGTE|?!`MFmCRmH{d{j_|8P0o8=fYjtOurTmUWUY0wu;w7An`MZOLoex=Z
zX}|`DHlWKD^kb<vF|<lgR?+TB<U>!~@VCSf)E=d^?zSuSAuh1tIV5Uz<)g@Q%Eg5U
zFdIv!%5tcjI_)ParmT+~0mS7xQz?6^igC4%rd<vh8_ey>#=Y%m>RZdwy_=nT3UaxX
zn{?4v!|}xQHx6$HJS7rsm;@_L*#Z$4z?GG~knwQea2{Roc76u(ibB5js5eU|O{&{G
z%8#hyraF)TMPyb7>U5?pdl*Xxa-?22V0ZpYK{LoJnf6Vlmp|_w;6^18Lj_3woiqps
zJtb2_lx%}r{eYJ}OXe5xrp9n|6g@ac#!L{yzFNywTHPxsFB^q#a8ok*H6l{3By1~C
zNFB;bLkfQ)EQ>c6!ej*E{uMkh%S#Qz$#dAFn(!>r^3Tv5y&sdot?wtint4y<gk3N(
z)k-Qx`GJNl;e1gZWm(I2ALZFbAk}}2U3F_B#@v~bE-E7r$$iM~6xwzcMHex})wq6{
zXz9gO%S64BC|@D%<@>y&c~dcTv%XU_cO;X_HcU#UtkbFL7Bbue2M=hCZ@y?F+B+P^
zJWb=q59&G$?0#~{`%t$#4oJhjGgMwWIkk)y`jEW~?d-=bOQz!j$aEi~+KPVY%8fDc
zn?b4&$}&>7IG^&a=G%l4jkYMJ7pDf|%?(YL1@34yRU${zgJ?j+m-tZ*Zzf)tK0hnQ
zeR;IyJk{!gHxDGcBAZUqT0tf_kQ<@sK{Tp)Ogc}Zl>lO~m6A{KDy39aFn=tP9BDy*
zwbVir&Pm{fBvFq7k;^)o$RvgnsPWU?gWA5WJozL!Dk80H7`J?)q7ql8F_KDt%f7i9
z?b+r%{=s;;kI2m3BQMkS*g#eAPj^?Yx16@^AVh<E9cOf{YmsLuSyoFfigA((=N!+S
z#a7=&4iLk8%H%}fn8tJ{QDjLDD$tZO>gXUsRk&@0P>6}LT-wT)3Rt2P6DU1|tcCH2
zkpB#~OvIE%Yb8b^WhYscyni8nU*`jA%AJ~XqbFWzlG2v;I?mRbG8eoOgJzrPo`Z7*
zgtQTDl%w7mJmO7$5rR4YaPjzzhNhOb&MuAJd-m$?+yBCW7Z2*`zjWwj1I{b2zI^x}
z-T%i0@xRu5UHgq)zgEZpMjijAI{ue#Z@tYN)jXzcxa;Wd<KXztbA|q?>*V|Yr^|<@
zK05vJng4&vv**+w=P&Ti|I(mFuc3Jf&}*pCfAUYG->&h$T>lMeIyJoj000aM2TxN?
zL}7Gc81oVkha4L15_p`AR(VhqXBOZ7x_hSi2AE+2I<~rwM_i0YB(55dbt9hfUNy05
ztG29WriEUbZmVa|rE05Ij3A;M8dOv?5*3g`jZx!)MnxeY#+y|j;HuyerMys3)O}wM
zA{olsoj-naz4yM~`@Q39Txr0+!}Qd@JFqcA3XfPU#9-qBRdtgO<{6SBLX`!VeNKd+
zM7pq178GZ+OT@--K@-A77dE)mXw4zQd$hJljKL;wiL{T6cDW@5n;n9tNni)T=4;{t
zOpV1{l<HPB5}k!$5P=#(KE;R{Mgv1IElX}p$H4Dgl<*JNLQ!;LP7&Oast7V>6d_8)
z=E;J4fh5XK``l<n5oK&p6><co$r<d}kl^Y4u-QXwWLp%otU3gFex#x<mPKbcP<DH~
zO`7PEoTB2E-~{s&!4)MbqJ5H}!EY|yY37{yGZ%`=c)=~qpX1g<(fy?)F2?>dg-H8c
z_-Y>~t3(l(s4bE}38O4ZAO^&Ds8La>g4q}$LRGQfWI=<kF>meXiLxs~5;1mYqTm*t
z^TT4W-z?QF3vNe*eej^6P8=}DtvVvZ;D7kV<A&!(i1z7{B1O5diF}?VIGtdG%@9xa
zsN%7tqnweL6T+h1Zc%e#AD8G*6{n!ZVE=cfvsJ-~nb43C!?9(aAiE?bJVJ<SjbNHC
zIi0eoTVigO2ys4ltJn~xNkl)+5#x|`-C4IL00k-p`;3*t1jRm&bdlmv^rE9-A=s!1
zl1xl}1o+<x^>P9Y6$<2ns2&<V6XZ5=!X!O7MhXZwU62=vn$r3o7ZxqaP9Tac6E#Ux
zoS@=D`(%~0rDc*PhO1hPeV!!Ck^mZU5FQE(S^G^Bqg)GAtu<-WTy?QxpGb0Zfp~&U
zc+7D`$VCUjWo(2#QzUl`wrHYT(nR>?w$G1Hq5CkLAu0k?4Jy#uWCE0^?Tg-{44ENo
zb`bv&SAMN(s+B@KMfduEqbxZENiB@+%HCb9eM@DxU@DYL+>ub@`-P8_AH)^Z7k9BD
z*2p6R2iC}P`BHohW81rI6|z6i^ABc1amBN;IF_RkHbd;P{}nrH8^r!6>*BxU(h^Th
zv6?XQGY5PS)Y-;&<4q=Ce~UG+leJ4%YcDJK7p~m-lCkAK`j)kyiVF4#?fLEf75$$s
z|0=_GRHU_UDi!_L9h7-Y0u^ZZo=T<8rkt6=59Eq&rSn!A1sQp|`|o?`JTk5?0oJ*>
zE8ns<mgleknnetlH|HjDR)m6h)ZGrow58D7*eAKLB%6AT{$k+!aod~k+4Ky1>XGvr
zYpWUCiyO_<+sH2xmp3y868Z6?Ox}umfq1?9=*F9F5b=IaXcNz;Vafe8*4Nk2ou9_#
z<N=XZ-XHeRsqFFAuWw5UP<co-)Sdg4Mh^%cymtJ6>3ag{RRDR756g^Di8mxI`VWv7
z9Sn&D!0{)4*mQ)kRn-RCTj=1vp@vmILae_0c<#WCHVOLa=RVk6`U_*r%7}YQ)L!x6
zbT+jVGt9k~Wi@mifg7&@st7QvF+Z3)-A?xp-=jNzTncB_-vHkH7%shC#vc(6renso
ziX!khk1D8qlHQwYfL(Wq{{Ze$?rZw3fBznOf9~@>(NKOM-G&f9l#9=5%W{Bjd)~Wt
z0jH9Rv#ASA`qC2@=%QE0FVOtQTox(#yDB2fCGQb1rLKgMlNSo$|HZxBUPxJ)K`6+^
zGu=NP90E=OmV&00uNd3rV@Fwg;zJxfFmztp{@H1nl|@`b{gNhhWz+Kv)1Bn?B~;4G
z7j+aWKfNQ_kn+Qx+svWl#w==GcG^?4aoxeERKd~mJ52JqCs|a%%M%-@>$d<Zubw(V
zugMQAJm0jMgG8USljReSEW5pBF_(cX3C*uxFt3^b5Y(@@UHyi5>bJQ={pv44HJ)B2
z`j6rY+tqJ~=b#S56Ljo6c&xe(9yhB2>Iooz2)E<(wy9R*&|%0NG;%a=v<@Hk>4;H(
z(#Jlt<PnVh#Vxt4{oj3q{}^g0OUbVA?ZXFiJK9elvLu(Cgg$+9a9LZmmyl9mek7Mc
zHrLxt#8|F+7tL1R2z%@D*3s3LS9?aI8Jm~pcZ<7y^#sj_a5d!zJGXYq7@g4B!)vM-
zTm9NP$}>9t6&s7Y=hv^_&zz+$QVjnISNqQ8&pNe3-a`1t$JShAcI`g&8TtkQ+IJVB
zNPmVvzOnRs)~lh0tXBrC%R8>MZ#p#lJ+jui!uVf%$0rn0K$eD<re7J`lB~0|Ck5la
zH@{qDU@$_se4WQQ64^SQ(-*+S4n4l=IpNV8bwUOxUea&x1=@1O+reD(6ddkR=n2(C
zJpql<DemId&B#M);v^syXnBy5{G72BeRrF&zZw=iAv8LD|Lok@;?Ak@Yqy({2;{~A
zoJd)>(@@t)`V^tZ8+NBqxA(^;QU@~u_7k|j_Ch0bmq5nyBPG;<{C)hh_zizMw><qQ
zdc8g+f!Z2dR?a@S@h}lp<lO1@RqgqDX8VyfH<?TVd<x={@03Ejusqcr(qS!^y~p)F
zm6*Leo@Eg8pYKMm-4ItpS68O~M5C6Y$v2H>e%e!KEG%7T)7$CHzB}FYQ8f9iI9t_h
zY(BOmkjYq8oy5h}7Ugh8KcB+nGo9|{1zsa_TT3Q*7n$WGGS$^ZYtq<)_~QO+OZ^$o
zP9d*#xv59!uL-0QNFtC%U^RgR0&xU%<0Di~&fYrX-7Em~p4XKX%Oa2X1$7SkVqn*%
z^ktFyk1MJAO8<myce+7Culhfl)AurIP6OljhI&TvpK^JP8*QykH(B?1e3s1TjYDno
z8QAa~?vZ+OX`E{UD1JsJZUA@nx6ci^-nRgbe+_>7vQ8c_!NpqEaTK<6xZ!V+QNRp8
zf;+k<zl)WfGA?NP44$28oz{obYuvJa4{Vu@SGTeD<MmAN=b`3`EyrV7A_SC~q%njm
zA#H&`1k}@`L5DqJUPhSJkYuU|@DsQ!vf0>Si0n0X_%O27^d0P-RM!Tw);8^JDZ*u&
z4lku2=WQ!B&iQfa@@7oI#dE4%mi^yzpZfd^YdroLOa?6Ql6DUE_bu?l$6y_AZy*bN
z5ciZkp-{bes<+45`}xH8VJtkKcsj-eTjlgl@<cNe@yO~a)P*wdeU;ICD~;_5m-{xN
z0z*{b86twN!7gtNIyGg+!hLs~Bt7>L5ulVv@n3|-^_Bnt01OHTPg6}qVRT^_x(E=3
z4H~ovc%0>weN0<b6u`N+x9`1PO27Cj9c~RKin>H*;^v6S9H=l7aWeiV&+>?c@=9J?
zm$@Zle3Y+&mDO#K;j}n6uuKD{G+P}_SIU==Zh&D!2C_m}oeLv`@gd@?V3x)Gv%mbq
zlbn-#?>WDmdveabxCi1e9G<Dw6sY%`3`Q->ZxKruGFX~U8}qb`E?1pxVp&Fqk?}|o
zg9+p2PWT>e(it$Br{@mSTrQ5zOwCkh(nhVGrFH5oy~)U^ckjX^%NP&qxdJR;H7piz
zOdf{(v5*S|VDdw)=7>heV9`M>TcCd1q$$7=@6!cFj$zcPI=z9xkt|(gq}k*{Mt#wK
zj-gZaxjgsUuB5a#_U`_dH#>Aily)%X5SDJaK4v&AUq>#6tR5qSmQO1(@ZmW$ACC!x
z-o$Ab9I+Xlqb)4bb4DylF>#y4Z{A`n;T|l_pbdpumQkniMKi47AJ6LwCr(JBCR>35
z3WO?%hy(>i%G#|hYe?yPXcMZvg^3^qLYP5ZvX*=fKo$fbp^u`@J^~7%0E;C;l~5EO
zA(bRVQXj~i{S)g*`Rg^GP~GVQiDXI;Kx$LqGs-gq|G8vzBiCd{?AOi<fbh4jXm5SH
zqP2h_e$_BzFFeU>6e@T29bfG1I}REyZAi$L6L$Sl%~gT5q%s_g-u6f=!M2ZVC)cCT
zxE4NH^W7!t=Jc?!maS{ddCPva_7ou39G-fRR1apip)p%W2^j6TTmr5HfLPU9`c;1E
z+?V;Nx58Tix<dw8*L!jX+JD}+;h5hs`;DNzj-0zMU0zngKg&;n`L>N?!);a&JFz@<
zbnde%fmu$83F{ylnF*g-bpzNa?e_}Hp1+emT_$phY;)edO}s+1h;IW(SRhgD4+`MB
z1eG2?UvX{E&Sv*~#KrExmts(cgc!1%R`>84=(E;=@{qBu-cZ3uD>qc<N<9d=yKh#3
zCcnE1{L$0^p$D<llmNbfFeM9+j!y}R7~0}IgD>~j?(_pn+1ny=jz;?*#>Lb%1LACp
zr5MaO%*6mz!XMUUFaQGMc4O5*zu^HwzRDUv+~FIt#82yLHGr_4a6Gs!gWrV=)WHS6
zSqvw;&0^bfS9<)t>8|wq*`Wp^6dJmv@^>yfvn}WCp!%F$2&Wwb`4_ys`4eXXBJZ$I
zW-h-67pFvUplm=+HCA-Ve;JOdES12ehJCKZ<l>Q6ZjJu<;uq^DUcF-O0PgR-q^Eh1
zbPZYKgQvS7INuyNF!q~|8=W0O$>U?IM}ubXu|UT~>4J5;JSGBQxhf(n2>w5k@FrE$
zB2$n8MO2JZB~+2}$mrO(ZA$(-fwGa#;5bs=z5Y8;SN$YRl%<4T8M)}QO$yxdEEpEX
r(<h&x<DYv>-+U@vAEHYXTXf|U^!6Q(>EWl+vmx5`ckaIcnH$TySRc&M

literal 0
HcmV?d00001

diff --git a/pom.xml b/pom.xml
index 6856123b1..34f034859 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,6 +28,7 @@
     <module>oshdb-api-ignite</module>
     <module>oshdb-tool</module>
     <module>oshdb-helpers</module>
+    <module>oshdb-osm-source</module>
   </modules>
 
   <properties>

From de4dc2662b39d22db40321db18f341cd03ff79ea Mon Sep 17 00:00:00 2001
From: Rafael Troilo <rafael.troilo@heigit.org>
Date: Fri, 11 Nov 2022 20:43:45 +0100
Subject: [PATCH 2/2] add sample.pbf to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 055cec09d..99d8ba8cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -298,3 +298,4 @@ temp_*
 *.trace.db
 !oshdb-api/src/test/resources/test-data.mv.db
 !oshdb-util/src/test/resources/test-data.mv.db
+!oshdb-osm-source/src/test/resources/sample.pbf