Serve robots.txt based on doNotCrawl in Strapi export (RPB-43)

app/controllers/HomeController.java

@@ -77,6 +77,7 @@
 import modules.IndexComponent;
 import play.Environment;
 import play.Logger;
+import play.cache.Cached;
 import play.libs.Json;
 import play.libs.ws.WSBodyReadables;
 import play.libs.ws.WSBodyWritables;
@@ -735,4 +736,14 @@ private static String findText(JsonNode info, String field) {
 		return node != null ? node.get("value").asText().replace("\n", " ").trim() : "";
 	}
 
+	@Cached(key = "robots", duration = 24 * 60 * 60) // One day
+	public Result robots() throws IOException {
+		return ok("User-agent: *\nDisallow: /"
+				+ Files.readAllLines(Paths.get("conf/rppd-export.jsonl")).stream()
+						.filter(line -> line.contains("doNotCrawl\":true"))
+						.map(line -> line.replaceAll(".*gndIdentifier\":\"(.+?)\".*", "$1")
+								.replaceAll("Keine GND-Ansetzung für ", ""))
+						.collect(Collectors.joining("\nDisallow: /")));
+	}
+
 }

app/modules/IndexComponent.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
 
 import controllers.HomeController;
+import net.sf.ehcache.CacheManager;
 import play.Logger;
 import play.inject.ApplicationLifecycle;
 
@@ -74,6 +75,7 @@ public ElasticsearchServer(ApplicationLifecycle lifecycle) {
 		});
 		lifecycle.addStopHook(() -> {
 			client.close();
+			CacheManager.getInstance().shutdown();
 			return null;
 		});
 	}

build.sbt

@@ -14,6 +14,8 @@ libraryDependencies += guice
 
 libraryDependencies += ws
 
+libraryDependencies += ehcache
+
 libraryDependencies += "com.github.jsonld-java" % "jsonld-java" % "0.12.0"
 
 libraryDependencies += "javax.mail" % "mail" % "1.4.1"

conf/routes

@@ -28,6 +28,8 @@ GET     /advanced       controllers.HomeController.advanced()
 
 GET     /cgi-bin/wwwalleg/:name.pl   controllers.HomeController.authorityPl(name, db ?= "rnam", index: Int ?= 1, zeilen: Int ?= 1, s1)
 
+GET     /robots.txt     controllers.HomeController.robots()
+
 GET     /:id.:format    controllers.HomeController.authorityDotFormat(id, format)
 
 GET     /:id            controllers.HomeController.authority(id, format ?= null)