diff --git a/.gitignore b/.gitignore index a4498b0..943bdbd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ target/ logback/ .DS_Store +jetty-base/ +jetty-home-11.0.15/ application.properties +jetty-home-11.0.15.tar.gz diff --git a/README.md b/README.md index 228c938..e0b5d0f 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,6 @@ Aristotle Aristotle is a [JSR 370] [JAX-RS] webservice of CRUD operations against a graph database. It supports Neo4J now. -Configuration -------------- - -- `NEO4J_URI` -- `NEO4J_USERNAME` -- `NEO4J_PASSWORD` -- `NEO4J_DATABASE` - Test ---- @@ -23,18 +15,59 @@ Test mvn clean verify ``` +Start Locally in Jetty +---------------------- + +Make sure port 8080 is not occupied and the following environment variables are set: + +```console +export NEO4J_URI= +export NEO4J_USERNAME= +export NEO4J_PASSWORD= +export NEO4J_DATABASE= +``` + +Then start webservice with: + +```bash +./jetty-start.sh +``` + + + Deployment ---------- -```bash +This is a one-person project. Agility outplays team scaling, so deployment is manual: + + +```console mvn clean package ``` +```console +export JETTY_HOME= +``` + +### Sending Logs to ELK Cloud + +Simply add Logstash integration and install agent on the production server. The logs will be available on integration +dashboard. + ### Gateway Registration ```bash export GATEWAY_PUBLIC_IP=52.53.186.26 +# healthcheck +curl -v -i -s -k -X POST https://api.paion-data.dev:8444/services \ + --data name=wilhelm-ws-healthcheck \ + --data url="http://${GATEWAY_PUBLIC_IP}:8080/v1/data/healthcheck" +curl -i -k -X POST https://api.paion-data.dev:8444/services/wilhelm-ws-healthcheck/routes \ + --data "paths[]=/wilhelm/healthcheck" \ + --data name=wilhelm-ws-healthcheck + +# language paged curl -v -i -s -k -X POST https://api.paion-data.dev:8444/services \ --data name=wilhelm-ws-languages \ --data url="http://${GATEWAY_PUBLIC_IP}:8080/v1/data/languages" @@ -42,6 +75,7 @@ curl -i -k -X POST https://api.paion-data.dev:8444/services/wilhelm-ws-languages --data "paths[]=/wilhelm/languages" \ --data name=wilhelm-ws-languages +# expand curl -v -i -s -k -X POST https://api.paion-data.dev:8444/services \ --data name=wilhelm-ws-expand \ --data url="http://${GATEWAY_PUBLIC_IP}:8080/v1/data/expand" @@ -54,8 +88,10 @@ We should see `HTTP/1.1 201 Created` as signs of success. #### Example requests: -- https://api.paion-data.dev/wilhelm/languages/german?perPage=100&page=1 -- https://api.paion-data.dev/wilhelm/expand/nämlich +- healthcheck: https://api.paion-data.dev/wilhelm/healthcheck +- vocabulary count: https://api.paion-data.dev/wilhelm/languages/german?perPage=100&page=1 +- query vocabulary paged: https://api.paion-data.dev/wilhelm/languages/german/count +- expand: https://api.paion-data.dev/wilhelm/expand/nämlich License ------- diff --git a/jetty-start.sh b/jetty-start.sh new file mode 100755 index 0000000..3aca33f --- /dev/null +++ b/jetty-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +set -e + +# Copyright Jiaqi Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mvn clean package -Dcheckstyle.skip -DskipTests + +wget -O jetty-home-11.0.15.tar.gz https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/11.0.15/jetty-home-11.0.15.tar.gz +tar -xzvf jetty-home-11.0.15.tar.gz +export JETTY_HOME=$(pwd)/jetty-home-11.0.15 + +mkdir -p jetty-base +cd jetty-base +java -jar $JETTY_HOME/start.jar --add-module=annotations,server,http,deploy,servlet,webapp,resources,jsp + +mv ../target/wilhelm-ws-1.0-SNAPSHOT.war webapps/ROOT.war +java -jar $JETTY_HOME/start.jar diff --git a/src/main/java/org/qubitpi/wilhelm/Graph.java b/src/main/java/org/qubitpi/wilhelm/Graph.java new file mode 100644 index 0000000..5fdd7f7 --- /dev/null +++ b/src/main/java/org/qubitpi/wilhelm/Graph.java @@ -0,0 +1,93 @@ +/* + * Copyright Jiaqi Liu + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.qubitpi.wilhelm; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A JSON-serializable object representation of a knowledge graph in wilhelm-ws. + */ +public class Graph { + + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + private final Set nodes; + private final Set links; + + /** + * All-args constructor. + * + * @param nodes + * @param links + */ + public Graph(final Set nodes, final Set links) { + this.nodes = nodes; + this.links = links; + } + + public static Graph emptyGraph() { + return new Graph(Collections.emptySet(), Collections.emptySet()); + } + + @JsonIgnore + public boolean isEmpty() { + return getNodes().isEmpty() && getLinks().isEmpty(); + } + + public List getUndirectedNeighborsOf(Node node) { + final Set neighborIds = getLinks().stream() + .filter(link -> node.getId().equals(link.getSourceNodeId()) || node.getId().equals(link.getTargetNodeId())) + .flatMap(link -> Stream.of(link.getSourceNodeId(), link.getTargetNodeId())) + .filter(id -> !node.getId().equals(id)) + .collect(Collectors.toUnmodifiableSet()); + + return getNodes().stream() + .filter(it -> neighborIds.contains(it.getId())) + .collect(Collectors.toUnmodifiableList()); + } + + public Graph merge(final Graph that) { + return new Graph( + Stream.of(this.getNodes(), that.getNodes()).flatMap(Set::stream).collect(Collectors.toSet()), + Stream.of(this.getLinks(), that.getLinks()).flatMap(Set::stream).collect(Collectors.toSet()) + ); + } + + public Set getNodes() { + return nodes; + } + + public Set getLinks() { + return links; + } + + @Override + public String toString() { + try { + return JSON_MAPPER.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/org/qubitpi/wilhelm/Language.java b/src/main/java/org/qubitpi/wilhelm/Language.java index ce5ebee..a1c874c 100644 --- a/src/main/java/org/qubitpi/wilhelm/Language.java +++ b/src/main/java/org/qubitpi/wilhelm/Language.java @@ -20,6 +20,7 @@ import net.jcip.annotations.ThreadSafe; import java.util.Arrays; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -58,6 +59,25 @@ public enum Language { this.databaseName = databaseName; } + private static Language valueOf(@NotNull final String language, Function nameExtractor) { + return Arrays.stream(values()) + .filter(value -> nameExtractor.apply(value).equals(language)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + String.format( + "'%s' is not a recognized language. Acceptable ones are %s", + language, + Arrays.stream(values()) + .map(nameExtractor) + .collect(Collectors.joining(", ")) + ) + )); + } + + public static Language ofDatabaseName(@NotNull final String language) { + return valueOf(language, Language::getDatabaseName); + } + /** * Constructs a {@link Language} from its client-side name. * @@ -68,16 +88,7 @@ public enum Language { * @throws IllegalArgumentException if the language name is not a valid one */ public static Language ofClientValue(@NotNull final String language) throws IllegalArgumentException { - return Arrays.stream(values()) - .filter(value -> value.pathName.equals(language)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - String.format( - "'%s' is not a recognized language. Acceptable ones are %s", - language, - Arrays.stream(values()).map(Language::getPathName).collect(Collectors.joining(", ") - ) - ))); + return valueOf(language, Language::getPathName); } @NotNull @@ -89,4 +100,9 @@ public String getPathName() { public String getDatabaseName() { return databaseName; } + + @Override + public String toString() { + return getDatabaseName(); + } } diff --git a/src/main/java/org/qubitpi/wilhelm/Link.java b/src/main/java/org/qubitpi/wilhelm/Link.java new file mode 100644 index 0000000..03579e1 --- /dev/null +++ b/src/main/java/org/qubitpi/wilhelm/Link.java @@ -0,0 +1,97 @@ +/* + * Copyright Jiaqi Liu + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.qubitpi.wilhelm; + +import org.neo4j.driver.types.Relationship; + +import jakarta.validation.constraints.NotNull; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class Link { + + private final String label; + private final String sourceNodeId; + private final String targetNodeId; + private final Map attributes; + + public Link( + @NotNull final String label, + @NotNull final String sourceNodeId, + @NotNull final String targetNodeId, + @NotNull final Map attributes + ) { + this.label = label; + this.sourceNodeId = sourceNodeId; + this.targetNodeId = targetNodeId; + this.attributes = attributes; + } + + public static Link valueOf(Relationship relationship) { + final String label = relationship.asMap().get("name").toString(); + final Map attributes = relationship.asMap().entrySet().stream() + .filter(entry -> !"name".equals(entry.getKey())) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + + return new Link(label, relationship.startNodeElementId(), relationship.endNodeElementId(), attributes); + } + + public String getLabel() { + return label; + } + + public String getSourceNodeId() { + return sourceNodeId; + } + + public String getTargetNodeId() { + return targetNodeId; + } + + public Map getAttributes() { + return attributes; + } + + @Override + public boolean equals(final Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + final Link that = (Link) other; + return Objects.equals(getLabel(), that.getLabel()) && Objects.equals( + getSourceNodeId(), + that.getSourceNodeId() + ) && Objects.equals( + getTargetNodeId(), + that.getTargetNodeId() + ) && Objects.equals(getAttributes(), that.getAttributes()); + } + + @Override + public int hashCode() { + return Objects.hash(getLabel(), getSourceNodeId(), getTargetNodeId(), getAttributes()); + } + + @Override + public String toString() { + return String.format("(%s)-%s-(%s)", getSourceNodeId(), getLabel(), getTargetNodeId()); + } +} diff --git a/src/main/java/org/qubitpi/wilhelm/Node.java b/src/main/java/org/qubitpi/wilhelm/Node.java new file mode 100644 index 0000000..09617a8 --- /dev/null +++ b/src/main/java/org/qubitpi/wilhelm/Node.java @@ -0,0 +1,85 @@ +/* + * Copyright Jiaqi Liu + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.qubitpi.wilhelm; + +import net.jcip.annotations.Immutable; +import net.jcip.annotations.ThreadSafe; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Immutable +@ThreadSafe +public class Node { + + private final String id; + private final String label; + private final Map attributes; + + public Node( + final String id, + final String label, + final Map attributes + ) { + this.id = id; + this.label = label; + this.attributes = attributes; + } + + public static Node valueOf(org.neo4j.driver.types.Node node) { + final String label = node.asMap().get("name").toString(); + final Map attributes = node.asMap().entrySet().stream() + .filter(entry -> !"name".equals(entry.getKey()) && !"language".equals(entry.getKey())) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + + return new Node(node.elementId(), label, attributes); + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public Map getAttributes() { + return attributes; + } + + @Override + public boolean equals(final Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + final Node that = (Node) other; + return Objects.equals(getId(), that.getId()); + } + + @Override + public int hashCode() { + return Objects.hash(getId()); + } + + @Override + public String toString() { + return getLabel(); + } +} diff --git a/src/main/java/org/qubitpi/wilhelm/web/endpoints/DataServlet.java b/src/main/java/org/qubitpi/wilhelm/web/endpoints/DataServlet.java index 53b687a..9ca27dd 100644 --- a/src/main/java/org/qubitpi/wilhelm/web/endpoints/DataServlet.java +++ b/src/main/java/org/qubitpi/wilhelm/web/endpoints/DataServlet.java @@ -23,13 +23,19 @@ import org.neo4j.driver.QueryConfig; import org.neo4j.driver.Value; import org.neo4j.driver.internal.types.InternalTypeSystem; +import org.qubitpi.wilhelm.Graph; import org.qubitpi.wilhelm.Language; import org.qubitpi.wilhelm.LanguageCheck; +import org.qubitpi.wilhelm.Link; +import org.qubitpi.wilhelm.Node; import org.qubitpi.wilhelm.config.ApplicationConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import jakarta.inject.Inject; import jakarta.inject.Singleton; import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; @@ -41,12 +47,10 @@ import net.jcip.annotations.ThreadSafe; import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import java.util.stream.StreamSupport; /** @@ -59,13 +63,13 @@ @Produces(MediaType.APPLICATION_JSON) public class DataServlet { + private static final Logger LOG = LoggerFactory.getLogger(DataServlet.class); private static final ApplicationConfig APPLICATION_CONFIG = ConfigFactory.create(ApplicationConfig.class); private static final String NEO4J_URL = APPLICATION_CONFIG.neo4jUrl(); private static final String NEO4J_USERNAME = APPLICATION_CONFIG.neo4jUsername(); private static final String NEO4J_PASSWORD = APPLICATION_CONFIG.neo4jPassword(); private static final String NEO4J_DATABASE = APPLICATION_CONFIG.neo4jDatabase(); - /** * Constructor for dependency injection. */ @@ -158,56 +162,98 @@ public Response getVocabularyByLanguagePaged( @Produces(MediaType.APPLICATION_JSON) @SuppressWarnings("MultipleStringLiterals") public Response expand(@NotNull @PathParam("word") final String word) { + return expandApoc(word, "3"); + } + + /** + * Recursively find all related terms and definitions of a word using multiple Cypher queries with a plain BFS + * algorithm. + *

+ * This is good for large sub-graph expand because it breaks huge memory consumption into sub-expand queries. But + * this endpoint sends multiple queries to database which incurs roundtrips and large Network I/O + * + * @param word The word to expand + * + * @return a JSON representation of the expanded sub-graph + */ + @GET + @Path("/expandBfs/{word}") + @Produces(MediaType.APPLICATION_JSON) + @SuppressWarnings("MultipleStringLiterals") + public Response expandBfs(@NotNull @PathParam("word") final String word) { + return Response + .status(Response.Status.OK) + .entity(expandBfs(word, new HashSet<>())) + .build(); + } + + private Graph expandBfs(final String label, final Set visited) { + if (visited.contains(label)) { + return Graph.emptyGraph(); + } + + visited.add(label); + final Graph oneHopExpand = (Graph) expandApoc(label, "1").getEntity(); + final Node wordNode = oneHopExpand.getNodes().stream() + .filter(node -> label.equals(node.getLabel())) + .findFirst() + .orElseThrow(() -> { + final String message = String.format("'%s' was not found in graph %s", label, oneHopExpand); + LOG.error(message); + return new IllegalArgumentException(message); + }); + return oneHopExpand.getUndirectedNeighborsOf(wordNode).stream() + .map(neighbor -> expandBfs(neighbor.getLabel(), visited)) + .reduce(oneHopExpand, Graph::merge); + } + + /** + * Recursively find all related terms and definitions of a word using a single Cypher query with apoc extension. + *

+ * This is bad for large sub-graph expand because it will exhaust memories allocated for the query in database. This + * is good for small-subgraph expand when WS and database are far away from each other. + * + * @param word The word to expand + * @param maxHops The max length of expanded path. Use "-1" for unlimitedly long path. + * + * @return a JSON representation of the expanded sub-graph. The format of the JSON would be + */ + @GET + @Path("/expandApoc/{word}") + @Produces(MediaType.APPLICATION_JSON) + @SuppressWarnings("MultipleStringLiterals") + public Response expandApoc( + @NotNull @PathParam("word") final String word, + @NotNull @QueryParam("maxHops") @DefaultValue("-1") final String maxHops + ) { + LOG.info("apoc expanding '{}' with max hops of {}", word, maxHops); + final String query = String.format( """ - MATCH (term:Term{name:'%s'}) - CALL apoc.path.expand(term, "RELATED|DEFINITION", null, 1, -1) + MATCH (node{name:'%s'}) + CALL apoc.path.expand(node, "LINK", null, 1, %s) YIELD path RETURN path, length(path) AS hops ORDER BY hops; """, - word + word.replace("'", "\\'"), maxHops ); final EagerResult result = executeNativeQuery(query); - final Map>> responseBody = Map.of( - "nodes", new ArrayList<>(), - "links", new ArrayList<>() - ); + final Set nodes = new HashSet<>(); + final Set links = new HashSet<>(); result.records().stream() .map(record -> record.get("path").asPath()) .forEach(path -> { - path.nodes().forEach(node -> responseBody.get("nodes").add( - Stream.of( - node.asMap(), - Collections.singletonMap("id", node.elementId()) - ) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)) - )); - path.relationships().forEach(relationship -> responseBody.get("links").add( - Stream.of( - relationship.asMap(), - Collections.singletonMap( - "sourceNodeId", - relationship.startNodeElementId() - ), - Collections.singletonMap( - "targetNodeId", - relationship.endNodeElementId() - ) - ) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)) - )); + path.nodes().forEach(node -> nodes.add(Node.valueOf(node))); + path.relationships().forEach(relationship -> links.add(Link.valueOf(relationship))); }); - return Response .status(Response.Status.OK) - .entity(responseBody) + .entity(new Graph(nodes, links)) .build(); } @@ -239,6 +285,8 @@ record -> record.keys() * @param query A standard cypher query string * * @return query's native result + * + * @throws IllegalStateException if a query execution error occurs */ @NotNull private EagerResult executeNativeQuery(@NotNull final String query) { diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 696f341..b9979b2 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -23,11 +23,14 @@ limitations under the License. - + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + - +