Initial landing for lobid-organisations

See hbz/lobid-organisations#419.
pelias · May 23, 2019 · 63ef8ae · 63ef8ae
1 parent 10ecbb7
commit 63ef8ae
Show file tree

Hide file tree

Showing 5 changed files with 335 additions and 0 deletions.
diff --git a/projects/lobid_DACH/.env b/projects/lobid_DACH/.env
@@ -0,0 +1,3 @@
+COMPOSE_PROJECT_NAME=pelias
+DOCKER_USER=1000
+DATA_DIR=/data/
diff --git a/projects/lobid_DACH/README.md b/projects/lobid_DACH/README.md
@@ -0,0 +1,49 @@
+
+# lobid organisations
+
+This project is configured to download(well, partly)/prepare/build a complete Pelias installation for https://lobid.org/organisations.
+
+See https://github.com/hbz/lobid-organisations/issues/419.
+
+Atm not all queries result accurate data, so there is room for proper configuration and data input.
+
+# Setup
+
+Please refer to the instructions at https://github.com/pelias/docker in order to install and configure your docker environment.
+
+The minimum configuration required in order to run this project are [installing prerequisites](https://github.com/pelias/docker#prerequisites), [install the pelias command](https://github.com/pelias/docker#installing-the-pelias-command) and [configure the environment](https://github.com/pelias/docker#configure-environment).
+
+Please ensure that's all working fine before continuing.
+
+# Run a Build
+
+To run a complete build, execute the following commands:
+
+```bash
+pelias compose down # if it was up before
+pelias compose pull
+pelias elastic start
+pelias elastic wait
+pelias elastic create
+pelias download all
+pelias prepare all
+pelias import all
+pelias compose up
+pelias compose ps # check all instances
+pelias test run
+```
+
+# Make an Example Query
+
+Residing in the hbz subnet, you can now make queries against this Pelias build.
+The following query is an example resulting not very accurate data:
+
+* @curl "http://gaia.hbz-nrw.de:4000/v1/search?layers=address&text=Juelicherstrasse%206,%2050674%20Koeln" |jq .@
+
+Inaccurate data can be identified when `confidence` is equal or less than `0.6`.
+
+In contrast, the following query results accurate geo data:
+
+* @curl "http://gaia.hbz-nrw.de:4000/v1/search?layers=address&text=Gleueler%20Str.%2050%2050931%20Koeln" |jq .@
+
+See that `confidence` has a value of `0.8`.
diff --git a/projects/lobid_DACH/docker-compose.yml b/projects/lobid_DACH/docker-compose.yml
@@ -0,0 +1,136 @@
+version: '3'
+networks:
+  default:
+    driver: bridge
+services:
+  libpostal:
+    image: pelias/libpostal-service
+    container_name: pelias_libpostal
+    user: "${DOCKER_USER}"
+    restart: always
+    ports: [ "4400:4400" ]
+  schema:
+    image: pelias/schema:master
+    container_name: pelias_schema
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+  api:
+    image: pelias/api:master
+    container_name: pelias_api
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4000" ]
+    ports: [ "4000:4000" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+  placeholder:
+    image: pelias/placeholder:master
+    container_name: pelias_placeholder
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4100" ]
+    ports: [ "4100:4100" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  whosonfirst:
+    image: pelias/whosonfirst:master
+    container_name: pelias_whosonfirst
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  openstreetmap:
+    image: pelias/openstreetmap:master
+    container_name: pelias_openstreetmap
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  openaddresses:
+    image: pelias/openaddresses:master
+    container_name: pelias_openaddresses
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  transit:
+    image: pelias/transit:master
+    container_name: pelias_transit
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  geonames:
+    image: pelias/geonames:master
+    container_name: pelias_geonames
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  csv-importer:
+    image: pelias/csv-importer:master
+    container_name: pelias_csv_importer
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  polylines:
+    image: pelias/polylines:master
+    container_name: pelias_polylines
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  interpolation:
+    image: pelias/interpolation:master
+    container_name: pelias_interpolation
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4300" ]
+    ports: [ "4300:4300" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  pip:
+    image: pelias/pip-service:master
+    container_name: pelias_pip-service
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4200" ]
+    ports: [ "4200:4200" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  elasticsearch:
+    image: pelias/elasticsearch:5.6.12
+    container_name: pelias_elasticsearch
+    user: "${DOCKER_USER}"
+    restart: always
+    ports: [ "9200:9200", "9300:9300" ]
+    volumes:
+      - "${DATA_DIR}/elasticsearch:/usr/share/elasticsearch/data"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add: [ "IPC_LOCK" ]
+  fuzzy-tester:
+    image: pelias/fuzzy-tester:master
+    container_name: pelias_fuzzy_tester
+    user: "${DOCKER_USER}"
+    restart: "no"
+    command: "--help"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "./test_cases:/code/pelias/fuzzy-tester/test_cases"
diff --git a/projects/lobid_DACH/pelias.json b/projects/lobid_DACH/pelias.json
@@ -0,0 +1,90 @@
+{
+  "logger": {
+    "level": "info",
+    "timestamp": false
+  },
+  "esclient": {
+    "apiVersion": "5.6",
+    "hosts": [
+      { "host": "elasticsearch" }
+    ]
+  },
+  "elasticsearch": {
+    "settings": {
+      "index": {
+        "refresh_interval": "100s",
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  },
+  "acceptance-tests": {
+    "endpoints": {
+      "docker": "http://api:4000/v1/"
+    }
+  },
+  "api": {
+    "textAnalyzer": "libpostal",
+    "services": {
+      "placeholder": { "url": "http://placeholder:4100" },
+      "pip": { "url": "http://pip:4200" },
+      "interpolation": { "url": "http://interpolation:4300" },
+      "libpostal": { "url": "http://libpostal:4400" }
+    },
+    "defaultParameters": {
+      "focus.point.lat": 50.938356,
+      "focus.point.lon": 6.95428
+    }
+  },
+  "imports": {
+    "adminLookup": {
+      "enabled": true
+    },
+    "geonames": {
+      "datapath": "/data/geonames",
+      "countryCode": "de" 
+    },
+    "openstreetmap": {
+      "leveldbpath": "/tmp",
+      "datapath": "/data/openstreetmap",
+      "import": [{
+        "filename": "switzerland-latest.osm.pbf",
+        "filename": "austria-latest.osm.pbf",
+        "filename": "germany-latest.osm.pbf"
+      }]
+     },
+    "openaddresses": {
+      "datapath": "/data/openaddresses",
+      "files": [
+        "/de/berlin.csv",
+        "/de/he/city_of_frankfurtammain.csv",
+        "/de/hh/statewide.csv",
+        "/de/ni/bentheim.csv",
+        "/de/nw/city_of_cologne.csv",
+        "/de/nw/city_of_wesel.csv",
+        "/de/nw/city_of_wuppertal.csv",
+        "/de/nw/kreis_viersen.csv",
+        "/de/nw/statewide.csv",
+        "/de/sn/statewide.csv",
+        "/de/th/statewide.csv",
+        "/at/31254.csv",
+        "/at/31255.csv",
+        "/at/31256.csv",
+        "/at/city_of_vienna.csv",
+        "/at/tirol.csv"
+      ]
+    },
+    "polyline": {
+      "datapath": "/data/polylines",
+      "files": ["extract.0sv"]
+    },
+    "whosonfirst": {
+	  "datapath": "/data/whosonfirst/",
+      "importVenues": false,
+      "importPostalcodes": true
+    },
+	 "interpolation": {
+        "datapath": "/data/interpolation/"
+  	}
+   }
+}
diff --git a/projects/lobid_DACH/test_cases/search_address.json b/projects/lobid_DACH/test_cases/search_address.json
@@ -0,0 +1,57 @@
+{
+   "name" : "/v1/search",
+   "tests" : [
+      {
+         "status" : "fail",
+         "in" : {
+            "layers" : "address",
+            "text" : "Juelicher Str. 6 50674 Koeln"
+         },
+         "id" : 1,
+         "expected" : {
+            "geometry" : {
+               "coordinates" : [
+                  6.93549,
+                  50.93410
+               ]
+            },
+            "properties" : {
+               "name" : "cologne",
+               "region" : "Nordrhein-Westfalen"
+            }
+         }
+      },
+      {
+         "status" : "pass",
+         "in" : {
+            "layers" : "address",
+            "text" : "Gleueler Str. 50 50931 Koeln"
+         },
+         "id" : 1,
+         "expected" : {
+            "geometry" : {
+               "coordinates" : [
+                  6.9165482,
+                  50.9259726
+               ]
+            },
+            "properties" : {
+               "name" : "GLEUELER STRAE 50",
+               "region" : "Nordrhein-Westfalen"
+            }
+         }
+      }
+
+   ],
+   "description" : "addresses in Germany",
+   "normalizers" : {
+      "name" : [
+         "toUpperCase",
+         "removeOrdinals",
+         "stripPunctuation",
+         "abbreviateDirectionals",
+         "abbreviateStreetSuffixes"
+      ]
+   },
+   "priorityThresh" : 1
+}