diff --git a/.gitmodules b/.gitmodules index ea53a5a847..978cea6d1a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "apoc-core"] path = apoc-core url = https://github.com/neo4j/apoc - branch = dev + branch = dev_remove_common diff --git a/extended-it/build.gradle b/extended-it/build.gradle index 9f8a74e91c..acc9bd8354 100644 --- a/extended-it/build.gradle +++ b/extended-it/build.gradle @@ -35,7 +35,6 @@ dependencies { testImplementation project(':core') testImplementation project(':test-utils') - testImplementation project(':common') testImplementation project(':extended') testImplementation project(':core').sourceSets.test.output compileOnly project(':extended').sourceSets.main.allJava diff --git a/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java b/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java index 71ee4ff0f7..2c2c72322d 100644 --- a/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java +++ b/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java @@ -1,6 +1,6 @@ package apoc; -import apoc.util.MissingDependencyException; +import apoc.util.MissingDependencyExceptionExtended; import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil; import org.junit.AfterClass; @@ -15,7 +15,6 @@ import static apoc.couchbase.Couchbase.COUCHBASE_MISSING_DEPS_ERROR; import static apoc.data.email.ExtractEmail.EMAIL_MISSING_DEPS_ERROR; -import static apoc.export.parquet.ParquetConfig.PARQUET_MISSING_DEPS_ERROR; import static apoc.export.xls.ExportXlsHandler.XLS_MISSING_DEPS_ERROR; import static apoc.load.LoadHtml.SELENIUM_MISSING_DEPS_ERROR; import static apoc.mongodb.MongoDBUtils.MONGO_MISSING_DEPS_ERROR; @@ -27,7 +26,7 @@ /** * This test verifies that, if the `extra-dependencies` jars are not present, - * the procedures that require them fail with {@link apoc.util.MissingDependencyException} + * the procedures that require them fail with {@link MissingDependencyExceptionExtended} */ public class MissingExtraDependenciesTest { private static Neo4jContainerExtension neo4jContainer; @@ -155,7 +154,7 @@ public static void assertFails(String query, Map params, String } catch (RuntimeException e) { String message = e.getMessage(); // String of type `apoc.util.MissingDependencyException: ` - String expected = "%s: %s".formatted(MissingDependencyException.class.getName(), errorMessage); + String expected = "%s: %s".formatted(MissingDependencyExceptionExtended.class.getName(), errorMessage); assertTrue("Actual error message is: " + message, message.contains(expected)); } } diff --git a/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java b/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java index 65502848e6..f5400915a9 100644 --- a/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java +++ b/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java @@ -42,7 +42,7 @@ import static apoc.couchbase.CouchbaseTestUtils.getNumConnections; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; diff --git a/extended-it/src/test/java/apoc/es/ElasticSearchTest.java b/extended-it/src/test/java/apoc/es/ElasticSearchTest.java index dbe8748a89..5be71cee39 100644 --- a/extended-it/src/test/java/apoc/es/ElasticSearchTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticSearchTest.java @@ -1,9 +1,8 @@ package apoc.es; -import apoc.es.ElasticSearch; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import com.fasterxml.jackson.core.JsonProcessingException; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; @@ -21,7 +20,7 @@ import java.util.function.Consumer; import static apoc.ApocConfig.apocConfig; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -96,7 +95,7 @@ public static void tearDown() { */ static Map createDefaultProcedureParametersWithPayloadAndId(String payload, String id) { try { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload, Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload, Map.class); return addPayloadAndIdToParams(paramsWithBasicAuth, mapPayload, id); } catch (IOException e) { throw new RuntimeException(e); @@ -104,7 +103,7 @@ static Map createDefaultProcedureParametersWithPayloadAndId(Stri } static Map addPayloadAndIdToParams(Map params, Object payload, String id) { - return Util.merge(params, Util.map("payload", payload, "id", id)); + return UtilExtended.merge(params, UtilExtended.map("payload", payload, "id", id)); } private static void insertDocuments() throws JsonProcessingException { @@ -289,9 +288,9 @@ public void testFullSearchWithOtherParametersAsAString() throws Exception { */ @Test public void testPutUpdateDocument() throws IOException{ - Map doc = JsonUtil.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); + Map doc = JsonUtilExtended.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); doc.put("tags", Arrays.asList("awesome")); - Map params = createDefaultProcedureParametersWithPayloadAndId(JsonUtil.OBJECT_MAPPER.writeValueAsString(doc), ES_ID); + Map params = createDefaultProcedureParametersWithPayloadAndId(JsonUtilExtended.OBJECT_MAPPER.writeValueAsString(doc), ES_ID); TestUtil.testCall(db, "CALL apoc.es.put($host,$index,$type,$id,'refresh=true',$payload, $config) yield value", params, r -> { Object updated = extractValueFromResponse(r, "$.result"); assertEquals("updated", updated); @@ -307,7 +306,7 @@ public void testPutUpdateDocument() throws IOException{ public void testPutUpdateDocumentWithAuthHeader() throws IOException{ String tags = UUID.randomUUID().toString(); - Map doc = JsonUtil.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); + Map doc = JsonUtilExtended.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); doc.put("tags", Arrays.asList(tags)); Map params = addPayloadAndIdToParams(paramsWithBasicAuth, doc, ES_ID); TestUtil.testCall(db, "CALL apoc.es.put($host,$index,$type,$id,'refresh=true',$payload, $config) yield value", @@ -330,8 +329,8 @@ public void testPostRawCreateDocument() throws IOException { String index = UUID.randomUUID().toString(); String type = getEsType(); String id = UUID.randomUUID().toString(); - Map payload = JsonUtil.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); - Map params = Util.map("host", HTTP_HOST_ADDRESS, + Map payload = JsonUtilExtended.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); + Map params = UtilExtended.map("host", HTTP_HOST_ADDRESS, "index", index, "suffix", index, "type", type, @@ -362,8 +361,8 @@ public void testPostRawCreateDocument() throws IOException { public void testPostCreateDocumentWithAuthHeader() throws IOException { String index = UUID.randomUUID().toString(); String type = getEsType(); - Map payload = JsonUtil.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); - Map params = Util.map("host", elastic.getHttpHostAddress(), + Map payload = JsonUtilExtended.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); + Map params = UtilExtended.map("host", elastic.getHttpHostAddress(), "index", index, "type", type, "payload", payload, diff --git a/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java b/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java index 46a60fc181..458f1333d4 100644 --- a/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java @@ -1,8 +1,8 @@ package apoc.es; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import com.fasterxml.jackson.core.JsonProcessingException; import com.nimbusds.jose.util.Pair; import org.junit.BeforeClass; @@ -22,7 +22,7 @@ public class ElasticVersionEightTest extends ElasticSearchTest { @BeforeClass public static void setUp() throws Exception { Map config = Map.of("headers", basicAuthHeader, VERSION_KEY, ElasticSearchHandler.Version.EIGHT.name()); - Map params = Util.map("index", ES_INDEX, + Map params = UtilExtended.map("index", ES_INDEX, "id", ES_ID, "type", ES_TYPE, "config", config); String tag = "8.14.3"; @@ -180,7 +180,7 @@ private void assertPutForRRF() { payloads.forEach(payload -> { try { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload.getRight(), Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload.getRight(), Map.class); paramsWithBasicAuth.put("payload", mapPayload); paramsWithBasicAuth.put("index", payload.getLeft()); TestUtil.testCall(db, "CALL apoc.es.put($host, $index, null, null, null, $payload, $config)", @@ -197,7 +197,7 @@ private void assertPutForRRF() { } private void setPayload(String payload, Map params) throws JsonProcessingException { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload, Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload, Map.class); params.put("payload", mapPayload); } diff --git a/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java b/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java index 61700b5f26..f7316edf7f 100644 --- a/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java @@ -1,7 +1,7 @@ package apoc.es; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.BeforeClass; import org.junit.Test; @@ -21,7 +21,7 @@ public class ElasticVersionSevenTest extends ElasticSearchTest { private final static String HOST = "localhost"; public static final ElasticSearchHandler DEFAULT_HANDLER = ElasticSearchHandler.Version.DEFAULT.get(); - private static final Map defaultParams = Util.map("index", ES_INDEX, "type", ES_TYPE, "id", ES_ID); + private static final Map defaultParams = UtilExtended.map("index", ES_INDEX, "type", ES_TYPE, "id", ES_ID); @BeforeClass public static void setUp() throws Exception { diff --git a/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java b/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java index 8ac4dffa2e..ff3d1203d4 100644 --- a/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java +++ b/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java @@ -2,7 +2,7 @@ import apoc.util.s3.MySQLContainerExtension; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -91,11 +91,11 @@ private static void testLoadJdbc(DbmsRule db, MySQLContainerExtension mysql) { // with the config {timezone: 'UTC'} and `preserveInstants=true&connectionTimeZone=SERVER` to make the result deterministic, // since `TIMESTAMP` values are automatically converted from the session time zone to UTC for storage, and vice versa. testCall(db, "CALL apoc.load.jdbc($url, $table, [], {timezone: 'UTC'})", - Util.map( + UtilExtended.map( "url", mysql.getJdbcUrl() + "&preserveInstants=true&connectionTimeZone=SERVER", "table", "country"), row -> { - Map expected = Util.map( + Map expected = UtilExtended.map( "Code", "NLD", "Name", "Netherlands", "Continent", "Europe", @@ -125,7 +125,7 @@ private static void testLoadJdbc(DbmsRule db, MySQLContainerExtension mysql) { private static void testIssue3496(DbmsRule db, MySQLContainerExtension mysql) { testCall(db, "CALL apoc.load.jdbc($url,'SELECT DATE(NOW()), NOW(), CURDATE(), CURTIME(), UTC_DATE(), UTC_TIME(), UTC_TIMESTAMP(), DATE(UTC_TIMESTAMP());')", - Util.map("url", mysql.getJdbcUrl()), + UtilExtended.map("url", mysql.getJdbcUrl()), r -> { Map row = (Map) r.get("row"); assertEquals(8, row.size()); diff --git a/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java b/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java index 5b3410c29e..f3cf69583e 100644 --- a/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java +++ b/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java @@ -3,7 +3,7 @@ import apoc.periodic.Periodic; import apoc.text.Strings; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -53,25 +53,25 @@ public static void tearDown() throws SQLException { @Test public void testLoadJdbc() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'PERSON',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'PERSON',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test public void testLoadJdbSelect() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test public void testLoadJdbSelectWithArrays() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM ARRAY_TABLE',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM ARRAY_TABLE',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (result) -> { Map row = (Map)result.get("row"); assertEquals("John", row.get("NAME")); @@ -85,27 +85,27 @@ public void testLoadJdbSelectWithArrays() throws Exception { @Test public void testLoadJdbcUpdate() throws Exception { testCall(db, "CALL apoc.load.jdbcUpdate($url,'UPDATE PERSON SET \"SURNAME\" = ? WHERE \"NAME\" = ?', ['DOE', 'John'], $config)", - Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), - (row) -> assertEquals( Util.map("count", 1 ), row.get("row"))); + UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + (row) -> assertEquals( UtilExtended.map("count", 1 ), row.get("row"))); } @Test public void testLoadJdbcParams() throws Exception { testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON WHERE \"NAME\" = ?',['John'], $config)", // YIELD row RETURN row - Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test @Ignore("flaky") public void testIssue4141PeriodicIterateWithJdbc() throws Exception { - var config = Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))); + var config = UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))); String query = "WITH range(0, 100) as list UNWIND list as l CREATE (n:MyNode{id: l})"; diff --git a/extended-it/src/test/java/apoc/model/ModelTest.java b/extended-it/src/test/java/apoc/model/ModelTest.java index 09a0697a35..a5447d01b9 100644 --- a/extended-it/src/test/java/apoc/model/ModelTest.java +++ b/extended-it/src/test/java/apoc/model/ModelTest.java @@ -1,8 +1,8 @@ package apoc.model; import apoc.util.TestUtil; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.junit.*; import org.junit.rules.TestName; import org.neo4j.graphdb.*; @@ -54,12 +54,12 @@ public void initDb() { @Test public void testLoadJdbcSchema() { testCall(db, "CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()))), + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()))), (row) -> { Long count = db.executeTransactionally("MATCH (n) RETURN count(n) AS count", Collections.emptyMap(), - result -> Iterators.single(result.columnAs("count"))); + result -> IteratorsExtended.single(result.columnAs("count"))); assertEquals(0L, count.longValue()); List nodes = (List) row.get("nodes"); List rels = (List) row.get("relationships"); @@ -107,16 +107,16 @@ public void testLoadJdbcSchema() { @Test public void testLoadJdbcSchemaWithWriteOperation() { db.executeTransactionally("CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", "write", true, - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()))), - innerResult -> Iterators.single(innerResult) + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()))), + innerResult -> IteratorsExtended.single(innerResult) ); try (Transaction tx = db.beginTx()) { - List nodes = Iterators.single(tx.execute("MATCH (n) RETURN collect(distinct n) AS nodes").columnAs("nodes")); - List rels = Iterators.single(tx.execute("MATCH ()-[r]-() RETURN collect(distinct r) AS rels").columnAs("rels")); + List nodes = IteratorsExtended.single(tx.execute("MATCH (n) RETURN collect(distinct n) AS nodes").columnAs("nodes")); + List rels = IteratorsExtended.single(tx.execute("MATCH ()-[r]-() RETURN collect(distinct r) AS rels").columnAs("rels")); assertEquals( 33, nodes.size()); assertEquals( 32, rels.size()); @@ -162,10 +162,10 @@ public void testLoadJdbcSchemaWithWriteOperation() { @Test public void testLoadJdbcSchemaWithFiltering() { testCall(db, "CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()), - "filters", Util.map("tables", Arrays.asList("country\\w*"), "columns", Arrays.asList("(?i)code", "(?i)name", "(?i)Language")))), + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()), + "filters", UtilExtended.map("tables", Arrays.asList("country\\w*"), "columns", Arrays.asList("(?i)code", "(?i)name", "(?i)Language")))), (row) -> { List nodes = (List) row.get("nodes"); List rels = (List) row.get("relationships"); diff --git a/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java b/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java index cb09af597d..f0c4a22f0d 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java @@ -1,7 +1,7 @@ package apoc.mongodb; import apoc.graph.Graphs; -import apoc.util.MapUtil; +import apoc.util.MapUtilExtended; import apoc.util.TestUtil; import apoc.util.UrlResolver; import com.mongodb.client.MongoClient; @@ -23,7 +23,7 @@ import java.util.Set; import java.util.stream.Collectors; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -91,7 +91,7 @@ public void testObjectIdToStringMapping() { String url = new UrlResolver("mongodb", mongo.getContainerIpAddress(), mongo.getMappedPort(MONGO_DEFAULT_PORT)) .getUrl("mongodb", mongo.getContainerIpAddress()); try (MongoDbCollInterface coll = MongoDbCollInterface.Factory.create(url, "test", "person", false, false, false)) { - Map document = coll.first(MapUtil.map("name", "Andrea Santurbano")); + Map document = coll.first(MapUtilExtended.map("name", "Andrea Santurbano")); assertTrue(document.get("_id") instanceof String); Collection bought = (Collection) document.get("bought"); assertEquals(2, bought.size()); @@ -107,7 +107,7 @@ public void testCompatibleValues() { String url = new UrlResolver("mongodb", mongo.getContainerIpAddress(), mongo.getMappedPort(MONGO_DEFAULT_PORT)) .getUrl("mongodb", mongo.getContainerIpAddress()); try (MongoDbCollInterface coll = MongoDbCollInterface.Factory.create(url, "test", "test", true, false, true)) { - Map document = coll.first(MapUtil.map("name", "testDocument")); + Map document = coll.first(MapUtilExtended.map("name", "testDocument")); assertNotNull(((Map) document.get("_id")).get("timestamp")); assertEquals(LocalDateTime.from(currentTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime()), document.get("date")); assertEquals(longValue, document.get("longValue")); diff --git a/extended-it/src/test/java/apoc/mongodb/MongoTest.java b/extended-it/src/test/java/apoc/mongodb/MongoTest.java index c5c6271dfd..9571a37560 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoTest.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoTest.java @@ -36,7 +36,7 @@ import java.util.function.Consumer; import static apoc.mongodb.MongoDBColl.ERROR_MESSAGE; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static org.junit.Assert.assertEquals; diff --git a/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java b/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java index 1c1a84ed7b..8d29d94d98 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java @@ -1,6 +1,6 @@ package apoc.mongodb; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -41,7 +41,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static org.junit.Assert.assertArrayEquals; @@ -185,7 +185,7 @@ public static Map getNumConnections(GenericContainer mongo, Stri .collect(Collectors.toList()); lists = lists.subList(lists.indexOf("{"), lists.size()); String jsonStr = String.join("", lists); - return JsonUtil.OBJECT_MAPPER.readValue(jsonStr, Map.class); + return JsonUtilExtended.OBJECT_MAPPER.readValue(jsonStr, Map.class); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java b/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java index 508f405d5b..d38fd3f704 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java @@ -9,7 +9,7 @@ import apoc.util.TestContainerUtil; import apoc.util.TestContainerUtil.ApocPackage; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -532,9 +532,9 @@ public void testExecuteCreateNodeStatistic() throws Exception { assertNotNull(r); Map row = r.next(); Map result = (Map) row.get("row"); - assertEquals(1L, (long) Util.toLong(result.get("nodesCreated"))); - assertEquals(1L, (long) Util.toLong(result.get("labelsAdded"))); - assertEquals(1L, (long) Util.toLong(result.get("propertiesSet"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("nodesCreated"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("labelsAdded"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("propertiesSet"))); assertEquals(false, r.hasNext()); }); } @@ -640,11 +640,11 @@ public void testLoadBigPathVirtual() throws Exception { public void testLoadFromLocal() { String localStatement = "RETURN 'foobar' AS foobar"; String remoteStatement = "CREATE (n: TestLoadFromLocalNode { m: foobar })"; - final Map map = Util.map( + final Map map = UtilExtended.map( "url", BOLT_URL, "localStatement", localStatement, "remoteStatement", remoteStatement, - "config", Util.map("readOnly", false)); + "config", UtilExtended.map("readOnly", false)); db.executeTransactionally("call apoc.bolt.load.fromLocal($url, $localStatement, $remoteStatement, $config) YIELD row return row", map); final long remoteCount = neo4jContainer.getSession().executeRead(tx -> (long) tx.run("MATCH (n: TestLoadFromLocalNode { m: 'foobar' }) RETURN count(n) AS count").single().asMap().get("count")); @@ -654,11 +654,11 @@ public void testLoadFromLocal() { @Test public void testLoadFromLocalStream() { String localStatement = "RETURN \"CREATE (n: TestLoadFromLocalStream)\" AS statement"; - final Map map = Util.map( + final Map map = UtilExtended.map( "url", BOLT_URL, "localStatement", localStatement, "remoteStatement", null, - "config", Util.map("readOnly", false, "streamStatements", true)); + "config", UtilExtended.map("readOnly", false, "streamStatements", true)); db.executeTransactionally("call apoc.bolt.load.fromLocal($url, $localStatement, $remoteStatement, $config)", map); final long remoteCount = neo4jContainer.getSession().executeRead(tx -> (long) tx.run("MATCH (n: TestLoadFromLocalStream) RETURN count(n) AS count").single().asMap().get("count")); diff --git a/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java b/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java index 864eca7c6a..804e302208 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java @@ -2,8 +2,8 @@ import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil.ApocPackage; -import apoc.util.collection.Iterables; -import apoc.util.collection.Iterators; +import apoc.util.collection.IterablesExtended; +import apoc.util.collection.IteratorsExtended; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.AfterClass; @@ -25,7 +25,7 @@ import static apoc.util.TestContainerUtil.testCall; import static apoc.util.TestContainerUtil.testCallEmpty; import static apoc.util.TestContainerUtil.testResult; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -407,7 +407,7 @@ public void testRunProcedureWithSetAndReturnResults(String query, Map(:Other {updated: 'true'}) RETURN *", - r -> assertEquals(4L, Iterators.count(r)) + r -> assertEquals(4L, IteratorsExtended.count(r)) ); } @@ -438,7 +438,7 @@ private void testCypherMapParallelCommon(String query, Map param public void assertOtherNodeAndRel(long id, Map result) { Node n = (Node) result.get("o"); - assertEquals(List.of("Other"), Iterables.asList(n.labels())); + assertEquals(List.of("Other"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("idOther", id), n.asMap()); Relationship rel = (Relationship) result.get("r"); @@ -471,7 +471,7 @@ private void assertReturnQueryNode(Map row, long id) { public void assertReturnQueryNode(long id, Map result) { Node n = result.get("n"); - assertEquals(List.of("ReturnQuery"), Iterables.asList(n.labels())); + assertEquals(List.of("ReturnQuery"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("id", id), n.asMap()); } @@ -482,7 +482,7 @@ private void assertRunProcNode(Map row, long id) { assertEquals(1, result.size()); Node n = result.get("n"); - assertEquals(List.of("Result"), Iterables.asList(n.labels())); + assertEquals(List.of("Result"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("id", id, "updated", true), n.asMap()); assertEquals(SET_RETURN_FILE, row.get("fileName")); diff --git a/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java b/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java index 90f48ae0e7..6b47658db9 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java @@ -3,7 +3,7 @@ import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil; import apoc.util.TestcontainersCausalCluster; -import apoc.util.collection.Iterators; +import apoc.util.collection.IteratorsExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Ignore; @@ -162,7 +162,7 @@ public void testCustomInstallAndDropAllowedOnlyInSysLeaderMember() { @Test public void testCustomShowAllowedInAllSysLeaderMembers() { final String query = "CALL apoc.custom.show"; - final BiConsumer testUuidShow = (session, name) -> testResult(session, query, Iterators::count); + final BiConsumer testUuidShow = (session, name) -> testResult(session, query, IteratorsExtended::count); customInSysLeaderMemberCommon(testUuidShow, true); } diff --git a/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java b/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java index 64ae293365..f63f1b8818 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java @@ -20,7 +20,7 @@ import static apoc.metrics.Metrics.OUTSIDE_DIR_ERR_MSG; import static apoc.util.ExtendedFileUtils.NEO4J_DIRECTORY_CONFIGURATION_SETTING_NAMES; import static apoc.util.TestContainerUtil.*; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.neo4j.test.assertion.Assert.assertEventually; diff --git a/extended-it/src/test/java/apoc/redis/RedisTest.java b/extended-it/src/test/java/apoc/redis/RedisTest.java index 0d316565a8..ccb44917c4 100644 --- a/extended-it/src/test/java/apoc/redis/RedisTest.java +++ b/extended-it/src/test/java/apoc/redis/RedisTest.java @@ -1,7 +1,5 @@ package apoc.redis; -import apoc.redis.Redis; -import apoc.redis.RedisConfig; import apoc.util.TestUtil; import org.apache.commons.lang3.StringUtils; import org.junit.After; @@ -24,7 +22,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; diff --git a/extended-it/src/test/java/apoc/s3/LoadS3Test.java b/extended-it/src/test/java/apoc/s3/LoadS3Test.java index b017bae225..a309ca8ab1 100644 --- a/extended-it/src/test/java/apoc/s3/LoadS3Test.java +++ b/extended-it/src/test/java/apoc/s3/LoadS3Test.java @@ -4,7 +4,7 @@ import apoc.load.LoadJson; import apoc.load.Xml; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import apoc.util.s3.S3BaseTest; import apoc.xml.XmlTestUtils; import org.junit.Assert; @@ -22,7 +22,7 @@ import static apoc.ApocConfig.apocConfig; import static apoc.load.LoadCsvTest.assertRow; import static apoc.util.ExtendedITUtil.EXTENDED_PATH; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static java.util.Arrays.asList; @@ -73,7 +73,7 @@ public void testLoadCsvS3() { String url = s3Container.putFile(EXTENDED_PATH + "src/test/resources/xml/books.xml"); url = removeRegionFromUrl(url); - testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", UtilExtended.map("url", url), (r) -> { Object value = Iterables.single(r.values()); Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); }); diff --git a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java index 2ceede5c3f..94de7b6d93 100644 --- a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java +++ b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java @@ -20,7 +20,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; import static apoc.util.ExtendedTestUtil.assertFails; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.CHROMA; diff --git a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java index d120da66ef..2137764306 100644 --- a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java +++ b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java @@ -2,7 +2,7 @@ import apoc.ml.Prompt; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -19,7 +19,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.MILVUS; @@ -173,7 +173,7 @@ public void deleteVector() { assertEquals(200L, value.get("code")); }); - Util.sleep(2000); + UtilExtended.sleep(2000); } @Test diff --git a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java index 3bce11f137..82fc0d5a7b 100644 --- a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java +++ b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java @@ -2,7 +2,7 @@ import apoc.ml.Prompt; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; @@ -22,7 +22,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; import static apoc.util.ExtendedTestUtil.assertFails; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.QDRANT; @@ -161,8 +161,8 @@ public void getVectorsWithReadOnlyApiKey() { public void writeOperationWithReadOnlyUser() { try { testCall(db, "CALL apoc.vectordb.qdrant.deleteCollection($host, 'test_collection', $conf)", - Util.map("host", HOST, - "conf", Util.map(HEADERS_KEY, READONLY_AUTHORIZATION) + UtilExtended.map("host", HOST, + "conf", UtilExtended.map(HEADERS_KEY, READONLY_AUTHORIZATION) ), r -> fail() ); diff --git a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java index 5b3aed4d27..6278c4f047 100644 --- a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java +++ b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java @@ -1,10 +1,9 @@ package apoc.vectordb; import apoc.ml.Prompt; -import apoc.util.MapUtil; +import apoc.util.MapUtilExtended; import apoc.util.TestUtil; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -26,7 +25,7 @@ import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; import static apoc.util.TestUtil.testResult; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static apoc.vectordb.VectorDbHandler.Type.WEAVIATE; import static apoc.vectordb.VectorDbTestUtil.EntityType.FALSE; import static apoc.vectordb.VectorDbTestUtil.EntityType.NODE; @@ -97,7 +96,7 @@ public static void setUp() throws Exception { TestUtil.registerProcedure(db, Weaviate.class, VectorDb.class, Prompt.class); testCall(db, "CALL apoc.vectordb.weaviate.createCollection($host, 'TestCollection', 'cosine', 4, $conf)", - MapUtil.map("host", HOST, "conf", ADMIN_HEADER_CONF), + MapUtilExtended.map("host", HOST, "conf", ADMIN_HEADER_CONF), r -> { Map value = (Map) r.get("value"); assertEquals("TestCollection", value.get("class")); @@ -113,7 +112,7 @@ public static void setUp() throws Exception { ], $conf) """, - MapUtil.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", ADMIN_HEADER_CONF), + MapUtilExtended.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", ADMIN_HEADER_CONF), r -> { ResourceIterator values = r.columnAs("value"); assertEquals(COLLECTION_NAME, values.next().get("class")); @@ -137,7 +136,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { testCallEmpty(db, "CALL apoc.vectordb.weaviate.deleteCollection($host, $collectionName, $conf)", - MapUtil.map("host", HOST, "collectionName", COLLECTION_NAME, "conf", ADMIN_HEADER_CONF) + MapUtilExtended.map("host", HOST, "collectionName", COLLECTION_NAME, "conf", ADMIN_HEADER_CONF) ); WEAVIATE_CONTAINER.stop(); @@ -364,9 +363,9 @@ public void getVectorsWithCreateNodeUsingExistingNode() { db.executeTransactionally("CREATE (:Test {myId: 'one'}), (:Test {myId: 'two'})"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, HEADERS_KEY, ADMIN_AUTHORIZATION, - MAPPING_KEY, MapUtil.map(EMBEDDING_KEY, "vect", + MAPPING_KEY, MapUtilExtended.map(EMBEDDING_KEY, "vect", NODE_LABEL, "Test", ENTITY_KEY, "myId", METADATA_KEY, "foo")); @@ -391,9 +390,9 @@ public void getVectorsWithCreateNodeUsingExistingNode() { public void getReadOnlyVectorsWithMapping() { db.executeTransactionally("CREATE (:Test {readID: 'one'}), (:Test {readID: 'two'})"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map( + MAPPING_KEY, MapUtilExtended.map( NODE_LABEL, "Test", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -401,7 +400,7 @@ public void getReadOnlyVectorsWithMapping() { testResult(db, "CALL apoc.vectordb.weaviate.get($host, 'TestCollection', [$id1, $id2], $conf) " + "YIELD vector, id, metadata, node RETURN * ORDER BY id", - MapUtil.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", conf), + MapUtilExtended.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", conf), r -> assertReadOnlyProcWithMappingResults(r, "node") ); } @@ -439,10 +438,10 @@ MAPPING_KEY, map(EMBEDDING_KEY, "vect", public void queryReadOnlyVectorsWithMapping() { db.executeTransactionally("CREATE (:Start)-[:TEST {readID: 'one'}]->(:End), (:Start)-[:TEST {readID: 'two'}]->(:End)"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, FIELDS_KEY, FIELDS, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map( + MAPPING_KEY, MapUtilExtended.map( REL_TYPE, "TEST", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -450,7 +449,7 @@ public void queryReadOnlyVectorsWithMapping() { testResult(db, "CALL apoc.vectordb.weaviate.query($host, 'TestCollection', [0.2, 0.1, 0.9, 0.7], null, 5, $conf) " + " YIELD score, vector, id, metadata, rel RETURN * ORDER BY id", - MapUtil.map("host", HOST, "conf", conf), + MapUtilExtended.map("host", HOST, "conf", conf), r -> assertReadOnlyProcWithMappingResults(r, "rel") ); } @@ -548,11 +547,11 @@ public void queryVectorsWithSystemDbStorage() { public void queryVectorsWithRag() { String openAIKey = ragSetup(db); - Map conf = MapUtil.map( + Map conf = MapUtilExtended.map( FIELDS_KEY, FIELDS, ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map(EMBEDDING_KEY, "vect", + MAPPING_KEY, MapUtilExtended.map(EMBEDDING_KEY, "vect", NODE_LABEL, "Rag", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -566,11 +565,11 @@ WITH collect(node) as paths RETURN value """ , - MapUtil.map( + MapUtilExtended.map( "host", HOST, "id1", ID_1, "conf", conf, - "confPrompt", MapUtil.map(API_KEY_CONF, openAIKey), + "confPrompt", MapUtilExtended.map(API_KEY_CONF, openAIKey), "attributes", List.of("city", "foo") ), VectorDbTestUtil::assertRagWithVectors); diff --git a/extended-it/src/test/java/apoc/xml/XmlTestUtils.java b/extended-it/src/test/java/apoc/xml/XmlTestUtils.java new file mode 100644 index 0000000000..408b086dd4 --- /dev/null +++ b/extended-it/src/test/java/apoc/xml/XmlTestUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.xml; + +import java.util.Arrays; +import java.util.Map; + +import static apoc.util.MapUtil.map; +import static java.util.Arrays.asList; + +public class XmlTestUtils { + + public static final Map XML_XPATH_AS_NESTED_MAP = map( + "_type", + "book", + "id", + "bk103", + "_children", + Arrays.asList( + map("_type", "author", "_text", "Corets, Eva"), + map("_type", "title", "_text", "Maeve Ascendant"), + map("_type", "genre", "_text", "Fantasy"), + map("_type", "price", "_text", "5.95"), + map("_type", "publish_date", "_text", "2000-11-17"), + map( + "_type", + "description", + "_text", + "After the collapse of a nanotechnology society in England, the young survivors lay the foundation for a new society."))); +} diff --git a/extended/build.gradle b/extended/build.gradle index ab2d688052..4c8e761fde 100644 --- a/extended/build.gradle +++ b/extended/build.gradle @@ -85,8 +85,8 @@ dependencies { } // These will be dependencies packaged with the .jar - implementation project(':common') implementation group: 'com.unboundid', name: 'unboundid-ldapsdk', version: '6.0.11' + api group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0' implementation group: 'org.jsoup', name: 'jsoup', version: '1.15.3' implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.10.0', { exclude group: 'org.apache.commons', module: 'commons-io' @@ -100,6 +100,8 @@ dependencies { compileOnly group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective // same version as the one included in neo4j `lib` compileOnly group: 'org.neo4j.driver', name: 'neo4j-java-driver', version: '5.20.0' + compileOnly group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1' + compileOnly group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' compileOnly group: 'org.apache.poi', name: 'poi', version: '5.1.0', { exclude group: 'org.apache.commons', module: 'commons-collections4' diff --git a/extended/src/main/java/apoc/ExtendedApocConfig.java b/extended/src/main/java/apoc/ExtendedApocConfig.java index f5249cda37..89da96d58f 100644 --- a/extended/src/main/java/apoc/ExtendedApocConfig.java +++ b/extended/src/main/java/apoc/ExtendedApocConfig.java @@ -1,10 +1,12 @@ package apoc; -import static apoc.ApocConfig.SUN_JAVA_COMMAND; - +import apoc.export.util.ExportConfigExtended; +import apoc.util.FileUtilsExtended; import apoc.util.SimpleRateLimiter; import java.io.File; +import java.io.IOException; +import java.net.URL; import java.time.Duration; import java.util.Iterator; import java.util.Map; @@ -19,13 +21,19 @@ import org.apache.commons.configuration2.ex.ConversionException; import org.apache.commons.configuration2.io.FileHandler; import org.apache.commons.configuration2.tree.OverrideCombiner; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.security.URLAccessChecker; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.Log; import org.neo4j.logging.internal.LogService; +import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; + public class ExtendedApocConfig extends LifecycleAdapter { + public static final String SUN_JAVA_COMMAND = "sun.java.command"; public static final String APOC_TTL_SCHEDULE = "apoc.ttl.schedule"; public static final String APOC_TTL_ENABLED = "apoc.ttl.enabled"; public static final String APOC_TTL_LIMIT = "apoc.ttl.limit"; @@ -44,6 +52,23 @@ public class ExtendedApocConfig extends LifecycleAdapter public static final String APOC_ML_WATSON_URL = "apoc.ml.watson.url"; public static final String APOC_AWS_KEY_ID = "apoc.aws.key.id"; public static final String APOC_AWS_SECRET_KEY = "apoc.aws.secret.key"; + + // From core + public static final String APOC_IMPORT_FILE_ENABLED = "apoc.import.file.enabled"; + public static final String APOC_EXPORT_FILE_ENABLED = "apoc.export.file.enabled"; + public static final String APOC_IMPORT_FILE_USE_NEO4J_CONFIG = "apoc.import.file.use_neo4j_config"; + public static final String APOC_TRIGGER_ENABLED = "apoc.trigger.enabled"; + public static final String APOC_IMPORT_FILE_ALLOW__READ__FROM__FILESYSTEM = + "apoc.import.file.allow_read_from_filesystem"; + public static final String APOC_CONFIG_JOBS_SCHEDULED_NUM_THREADS = "apoc.jobs.scheduled.num_threads"; + public static final String APOC_CONFIG_JOBS_POOL_NUM_THREADS = "apoc.jobs.pool.num_threads"; + public static final String APOC_CONFIG_JOBS_QUEUE_SIZE = "apoc.jobs.queue.size"; + public static final String APOC_CONFIG_INITIALIZER = "apoc.initializer"; + public static final String LOAD_FROM_FILE_ERROR = + "Import from files not enabled, please set apoc.import.file.enabled=true in your apoc.conf"; + public static final String APOC_MAX_DECOMPRESSION_RATIO = "apoc.max.decompression.ratio"; + public static final Integer DEFAULT_MAX_DECOMPRESSION_RATIO = 200; + public enum UuidFormatType { hex, base64 } // These were earlier added via the Neo4j config using the ApocSettings.java class @@ -56,6 +81,7 @@ public enum UuidFormatType { hex, base64 } ); private final Log log; + private final DatabaseManagementService databaseManagementService; private final String defaultConfigPath; @@ -66,6 +92,8 @@ public enum UuidFormatType { hex, base64 } private ExtendedApocConfig.LoggingType loggingType; private SimpleRateLimiter rateLimiter; + private GraphDatabaseService systemDb; + /** * keep track if this instance is already initialized so dependent class can wait if needed */ @@ -73,10 +101,14 @@ public enum UuidFormatType { hex, base64 } public static final String CONFIG_DIR = "config-dir="; - public ExtendedApocConfig(LogService log, GlobalProcedures globalProceduresRegistry, String defaultConfigPath) { - this.log = log.getInternalLog(ApocConfig.class); + public ExtendedApocConfig(LogService log, + GlobalProcedures globalProceduresRegistry, + String defaultConfigPath, + DatabaseManagementService databaseManagementService) { + this.log = log.getInternalLog(ExtendedApocConfig.class); this.defaultConfigPath = defaultConfigPath; theInstance = this; + this.databaseManagementService = databaseManagementService; // expose this config instance via `@Context ApocConfig config` globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); @@ -164,7 +196,7 @@ private static Configuration setupConfigurations(File propertyFile) throws Confi return combined; } - protected Configuration getConfig() { + public Configuration getConfig() { return config; } @@ -207,6 +239,49 @@ public boolean containsKey(String key) { return config.containsKey(key); } + public boolean getBoolean(String key) { + return getConfig().getBoolean(key); + } + + public void checkReadAllowed(String url, URLAccessChecker urlAccessChecker) throws IOException { + if (FileUtilsExtended.isFile(url)) { + isImportFileEnabled(); + } else { + checkAllowedUrlAndPinToIP(url, urlAccessChecker); + } + } + + // added because with binary file there isn't an url + public void isImportFileEnabled() { + if (!config.getBoolean(APOC_IMPORT_FILE_ENABLED)) { + throw new RuntimeException(LOAD_FROM_FILE_ERROR); + } + } + + public URL checkAllowedUrlAndPinToIP(String url, URLAccessChecker urlAccessChecker) throws IOException { + try { + URL parsedUrl = new URL(url); + return urlAccessChecker.checkURL(parsedUrl); + } catch (Exception e) { + throw new IOException(e); + } + } + + public boolean isImportFolderConfigured() { + // in case we're test database import path is TestDatabaseManagementServiceBuilder.EPHEMERAL_PATH + + String importFolder = getImportDir(); + if (importFolder == null) { + return false; + } else { + return !"/target/test data/neo4j".equals(importFolder); + } + } + + public String getImportDir() { + return extendedApocConfig().getString("server.directories.import"); + } + public boolean getBoolean(String key, boolean defaultValue) { return getConfig().getBoolean(key, defaultValue); } @@ -224,7 +299,7 @@ public > T getEnumProperty(String key, Class cls, T default } } - private int getInt(String key, int defaultValue) { + public int getInt(String key, int defaultValue) { try { return config.getInt(key, defaultValue); } catch ( ConversionException e) { @@ -236,4 +311,35 @@ private int getInt(String key, int defaultValue) { } } } + + + // Methods brought over from Core Config + + public static final String EXPORT_NOT_ENABLED_ERROR = + "Export to files not enabled, please set apoc.export.file.enabled=true in your apoc.conf."; + public static final String EXPORT_TO_FILE_ERROR = EXPORT_NOT_ENABLED_ERROR + + "\nOtherwise, if you are running in a cloud environment without filesystem access, use the `{stream:true}` config and null as a 'file' parameter to stream the export back to your client."; + + public String getString(String key) { + return getConfig().getString(key); + } + + public void checkWriteAllowed(ExportConfigExtended exportConfig, String fileName) { + if (!config.getBoolean(APOC_EXPORT_FILE_ENABLED)) { + if (exportConfig == null || (fileName != null && !fileName.isEmpty()) || !exportConfig.streamStatements()) { + throw new RuntimeException(EXPORT_TO_FILE_ERROR); + } + } + } + + public GraphDatabaseService getSystemDb() { + if (systemDb == null) { + try { + systemDb = databaseManagementService.database(SYSTEM_DATABASE_NAME); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return systemDb; + } } diff --git a/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java b/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java index e1dcec241d..8ec8ee6eaa 100644 --- a/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java +++ b/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java @@ -2,6 +2,7 @@ import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.configuration.Config; +import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.ExtensionType; @@ -18,6 +19,7 @@ public interface Dependencies { LogService log(); GlobalProcedures globalProceduresRegistry(); + DatabaseManagementService databaseManagementService(); Config config(); } @@ -32,7 +34,11 @@ public Lifecycle newInstance( ExtensionContext context, Dependencies dependencie .get(neo4j_home) .resolve(Config.DEFAULT_CONFIG_DIR_NAME) .toString(); - return new ExtendedApocConfig(dependencies.log(), dependencies.globalProceduresRegistry(), defaultConfigPath); + return new ExtendedApocConfig( + dependencies.log(), + dependencies.globalProceduresRegistry(), + defaultConfigPath, + dependencies.databaseManagementService()); } } \ No newline at end of file diff --git a/extended/src/main/java/apoc/ExtendedApocExtensionFactory.java b/extended/src/main/java/apoc/ExtendedApocExtensionFactory.java new file mode 100644 index 0000000000..ebfd0ac119 --- /dev/null +++ b/extended/src/main/java/apoc/ExtendedApocExtensionFactory.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.availability.AvailabilityGuard; +import org.neo4j.kernel.availability.AvailabilityListener; +import org.neo4j.kernel.extension.ExtensionFactory; +import org.neo4j.kernel.extension.ExtensionType; +import org.neo4j.kernel.extension.context.ExtensionContext; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.kernel.monitoring.DatabaseEventListeners; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.service.Services; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; + +/** + * @author mh + * @since 14.05.16 + */ +@ServiceProvider +public class ExtendedApocExtensionFactory extends ExtensionFactory { + + public ExtendedApocExtensionFactory() { + super(ExtensionType.DATABASE, "APOC"); + } + + public interface Dependencies { + GraphDatabaseAPI graphdatabaseAPI(); + + JobScheduler scheduler(); + + LogService log(); + + AvailabilityGuard availabilityGuard(); + + DatabaseManagementService databaseManagementService(); + + ExtendedApocConfig apocConfig(); + + DatabaseEventListeners databaseEventListeners(); + + @SuppressWarnings("unused") // used from extended + GlobalProcedures globalProceduresRegistry(); + + ExtendedRegisterComponentFactory.RegisterComponentLifecycle registerComponentLifecycle(); + + PoolsExtended pools(); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + GraphDatabaseAPI db = dependencies.graphdatabaseAPI(); + LogService log = dependencies.log(); + return new ApocLifecycle(log, db, dependencies); + } + + public static class ApocLifecycle extends LifecycleAdapter { + private final Log userLog; + private final GraphDatabaseAPI db; + private final Dependencies dependencies; + private final Map services = new HashMap<>(); + private final Collection apocGlobalComponents; + private final Collection registeredListeners = new ArrayList<>(); + + public ApocLifecycle(LogService log, GraphDatabaseAPI db, Dependencies dependencies) { + this.db = db; + this.dependencies = dependencies; + this.userLog = log.getUserLog(ExtendedApocExtensionFactory.class); + this.apocGlobalComponents = Services.loadAll(ExtendedApocGlobalComponents.class); + } + + public static void withNonSystemDatabase(GraphDatabaseService db, Consumer consumer) { + if (!SYSTEM_DATABASE_NAME.equals(db.databaseName())) { + consumer.accept(null); + } + } + + @Override + public void init() { + withNonSystemDatabase(db, aVoid -> { + for (ExtendedApocGlobalComponents c : apocGlobalComponents) { + services.putAll(c.getServices(db, dependencies)); + } + + String databaseName = db.databaseName(); + services.values().forEach(lifecycle -> dependencies + .registerComponentLifecycle() + .addResolver(databaseName, lifecycle.getClass(), lifecycle)); + }); + } + + @Override + public void start() { + withNonSystemDatabase(db, aVoid -> { + services.forEach((key, value) -> { + try { + value.start(); + } catch (Exception e) { + userLog.error("failed to start service " + key, e); + } + }); + }); + + AvailabilityGuard availabilityGuard = dependencies.availabilityGuard(); + for (ExtendedApocGlobalComponents c : apocGlobalComponents) { + for (AvailabilityListener listener : c.getListeners(db, dependencies)) { + registeredListeners.add(listener); + availabilityGuard.addListener(listener); + } + } + } + + @Override + public void stop() { + withNonSystemDatabase(db, aVoid -> { + services.forEach((key, value) -> { + try { + value.stop(); + } catch (Exception e) { + userLog.error("failed to stop service " + key, e); + } + }); + }); + + AvailabilityGuard availabilityGuard = dependencies.availabilityGuard(); + registeredListeners.forEach(availabilityGuard::removeListener); + registeredListeners.clear(); + } + + public Collection getRegisteredListeners() { + return registeredListeners; + } + } +} diff --git a/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java b/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java index 3ef25bb5b3..90c5567305 100644 --- a/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java +++ b/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java @@ -7,6 +7,7 @@ import apoc.ttl.TTLLifeCycle; import apoc.uuid.Uuid; import apoc.uuid.UuidHandler; +import org.neo4j.annotations.service.Service; import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.kernel.availability.AvailabilityListener; @@ -20,12 +21,12 @@ import java.util.concurrent.ConcurrentHashMap; @ServiceProvider -public class ExtendedApocGlobalComponents implements ApocGlobalComponents { +@Service +public class ExtendedApocGlobalComponents { private final Map cypherProcedureHandlers = new ConcurrentHashMap<>(); - @Override - public Map getServices(GraphDatabaseAPI db, ApocExtensionFactory.Dependencies dependencies) { + public Map getServices(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies) { CypherProceduresHandler cypherProcedureHandler = new CypherProceduresHandler( @@ -59,13 +60,11 @@ public Map getServices(GraphDatabaseAPI db, ApocExtensionFact ); } - @Override public Collection getContextClasses() { return List.of(CypherProceduresHandler.class, UuidHandler.class, LoadDirectoryHandler.class); } - @Override - public Iterable getListeners(GraphDatabaseAPI db, ApocExtensionFactory.Dependencies dependencies) { + public Iterable getListeners(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies) { CypherProceduresHandler cypherProceduresHandler = cypherProcedureHandlers.get(db); return cypherProceduresHandler==null ? Collections.emptyList() : Collections.singleton(cypherProceduresHandler); } diff --git a/extended/src/main/java/apoc/ExtendedSystemLabels.java b/extended/src/main/java/apoc/ExtendedSystemLabels.java index a50c89d442..66fcdb7fef 100644 --- a/extended/src/main/java/apoc/ExtendedSystemLabels.java +++ b/extended/src/main/java/apoc/ExtendedSystemLabels.java @@ -2,13 +2,14 @@ import org.neo4j.graphdb.Label; -public enum ExtendedSystemLabels implements Label +public enum ExtendedSystemLabels implements Label { ApocCypherProcedures, ApocCypherProceduresMeta, Procedure, Function, ApocUuid, + ApocTrigger, ApocUuidMeta, DataVirtualizationCatalog, VectorDb diff --git a/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java b/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java index 6d134e2c5c..b3d8445e18 100644 --- a/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java +++ b/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java @@ -2,6 +2,12 @@ public enum ExtendedSystemPropertyKeys { + database, + + // cypher stored procedures/functions + lastUpdated, + statement, + // cypher stored procedures/functions inputs, description, diff --git a/extended/src/main/java/apoc/JdbcRegistererInitFactory.java b/extended/src/main/java/apoc/JdbcRegistererInitFactory.java index e797c954b6..3e87077be9 100644 --- a/extended/src/main/java/apoc/JdbcRegistererInitFactory.java +++ b/extended/src/main/java/apoc/JdbcRegistererInitFactory.java @@ -1,8 +1,8 @@ package apoc; import apoc.load.Jdbc; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.ExtensionType; @@ -27,12 +27,12 @@ public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies @Override public void init() throws Exception { // we need to await initialization of ExtendedApocConfig. Unfortunately Neo4j's internal service loading tooling does *not* honor the order of service loader META-INF/services files. - Util.newDaemonThread(() -> { + UtilExtended.newDaemonThread(() -> { ExtendedApocConfig extendedApocConfig = dependencies.extendedApocConfig(); while (!extendedApocConfig.isInitialized()) { - Util.sleep(10); + UtilExtended.sleep(10); } - Iterators.stream(extendedApocConfig.getKeys("apoc.jdbc")) + IteratorsExtended.stream(extendedApocConfig.getKeys("apoc.jdbc")) .filter(k -> k.endsWith("driver")) .forEach( Jdbc::loadDriver ); }).start(); diff --git a/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java b/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java new file mode 100644 index 0000000000..910dce2508 --- /dev/null +++ b/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.extension.ExtensionFactory; +import org.neo4j.kernel.extension.ExtensionType; +import org.neo4j.kernel.extension.context.ExtensionContext; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.logging.internal.LogService; + +@ServiceProvider +@SuppressWarnings("unused") +public class PoolExtensionFactoryExtended extends ExtensionFactory { + + public PoolExtensionFactoryExtended() { + super(ExtensionType.GLOBAL, "APOC_POOLS"); + } + + public interface Dependencies { + GlobalProcedures globalProceduresRegistry(); + + LogService log(); + + ExtendedApocConfig apocConfig(); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + return new PoolsExtended(dependencies.log(), dependencies.globalProceduresRegistry(), dependencies.apocConfig()); + } +} diff --git a/extended/src/main/java/apoc/PoolsExtended.java b/extended/src/main/java/apoc/PoolsExtended.java new file mode 100644 index 0000000000..3f3a55cbcd --- /dev/null +++ b/extended/src/main/java/apoc/PoolsExtended.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import apoc.periodic.PeriodicUtilsExtended; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Transaction; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +public class PoolsExtended extends LifecycleAdapter { + + public static final int DEFAULT_SCHEDULED_THREADS = Runtime.getRuntime().availableProcessors() / 4; + public static final int DEFAULT_POOL_THREADS = Runtime.getRuntime().availableProcessors() * 2; + private final Log log; + private final ExtendedApocConfig apocConfig; + + private ExecutorService singleExecutorService; + private ScheduledExecutorService scheduledExecutorService; + private ExecutorService defaultExecutorService; + + private final Map jobList = new ConcurrentHashMap<>(); + + public PoolsExtended(LogService log, GlobalProcedures globalProceduresRegistry, ExtendedApocConfig apocConfig) { + + this.log = log.getInternalLog(PoolsExtended.class); + this.apocConfig = apocConfig; + + // expose this config instance via `@Context ApocConfig config` + globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); + this.log.info("successfully registered Pools for @Context"); + } + + @Override + public void init() { + + int threads = + Math.max(1, apocConfig.getInt(ExtendedApocConfig.APOC_CONFIG_JOBS_POOL_NUM_THREADS, DEFAULT_POOL_THREADS)); + + int queueSize = Math.max(1, apocConfig.getInt(ExtendedApocConfig.APOC_CONFIG_JOBS_QUEUE_SIZE, threads * 5)); + + // ensure we use daemon threads everywhere + ThreadFactory threadFactory = r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; + }; + this.singleExecutorService = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(queueSize), + threadFactory, + new CallerBlocksPolicy()); + + this.defaultExecutorService = new ThreadPoolExecutor( + threads / 2, + threads, + 30L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(queueSize), + threadFactory, + new CallerBlocksPolicy()); + + this.scheduledExecutorService = Executors.newScheduledThreadPool( + Math.max( + 1, + apocConfig.getInt( + ExtendedApocConfig.APOC_CONFIG_JOBS_SCHEDULED_NUM_THREADS, DEFAULT_SCHEDULED_THREADS)), + threadFactory); + + scheduledExecutorService.scheduleAtFixedRate( + () -> { + for (Iterator> it = + jobList.entrySet().iterator(); + it.hasNext(); ) { + Map.Entry entry = it.next(); + if (entry.getValue().isDone() || entry.getValue().isCancelled()) it.remove(); + } + }, + 10, + 10, + TimeUnit.SECONDS); + } + + @Override + public void shutdown() { + Stream.of(singleExecutorService, defaultExecutorService, scheduledExecutorService) + .forEach(service -> { + try { + service.shutdown(); + service.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Shutdown failed to complete with error: " + e.getMessage()); + } + }); + } + + public ExecutorService getSingleExecutorService() { + return singleExecutorService; + } + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + public ExecutorService getDefaultExecutorService() { + return defaultExecutorService; + } + + public Map getJobList() { + return jobList; + } + + static class CallerBlocksPolicy implements RejectedExecutionHandler { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + // Submit again by directly injecting the task into the work queue, waiting if necessary, but also + // periodically checking if the pool has been + // shut down. + FutureTask task = new FutureTask<>(r, null); + BlockingQueue queue = executor.getQueue(); + while (!executor.isShutdown()) { + try { + if (queue.offer(task, 250, TimeUnit.MILLISECONDS)) { + while (!executor.isShutdown()) { + try { + task.get(250, TimeUnit.MILLISECONDS); + return; // Success! + } catch (TimeoutException ignore) { + // This is fine an expected. We just want to check that the executor hasn't been shut + // down. + } + } + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } + } + + public Future processBatch(List batch, GraphDatabaseService db, BiConsumer action) { + return defaultExecutorService.submit(() -> { + try (Transaction tx = db.beginTx()) { + batch.forEach(t -> action.accept(tx, t)); + tx.commit(); + } + return null; + }); + } + + public static T force(Future future) throws ExecutionException { + while (true) { + try { + return future.get(); + } catch (InterruptedException e) { + Thread.interrupted(); + } + } + } +} diff --git a/extended/src/main/java/apoc/TTLConfig.java b/extended/src/main/java/apoc/TTLConfig.java index 1c737e7b6d..19c91e7715 100644 --- a/extended/src/main/java/apoc/TTLConfig.java +++ b/extended/src/main/java/apoc/TTLConfig.java @@ -5,11 +5,11 @@ import org.neo4j.kernel.lifecycle.LifecycleAdapter; public class TTLConfig extends LifecycleAdapter { - private final ApocConfig apocConfig; + private final ExtendedApocConfig apocConfig; public static final int DEFAULT_SCHEDULE = 60; private static TTLConfig theInstance; - public TTLConfig(ApocConfig apocConfig, GlobalProcedures globalProceduresRegistry) { + public TTLConfig(ExtendedApocConfig apocConfig, GlobalProcedures globalProceduresRegistry) { this.apocConfig = apocConfig; theInstance = this; globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); diff --git a/extended/src/main/java/apoc/TTLConfigExtensionFactory.java b/extended/src/main/java/apoc/TTLConfigExtensionFactory.java index 8a6e9f54a9..21a8f12352 100644 --- a/extended/src/main/java/apoc/TTLConfigExtensionFactory.java +++ b/extended/src/main/java/apoc/TTLConfigExtensionFactory.java @@ -14,7 +14,7 @@ public class TTLConfigExtensionFactory extends ExtensionFactory { public interface Dependencies { - ApocConfig config(); + ExtendedApocConfig config(); GlobalProcedures globalProceduresRegistry(); } diff --git a/extended/src/main/java/apoc/agg/AggregationExtended.java b/extended/src/main/java/apoc/agg/AggregationExtended.java index 128131c5fe..94e057ece7 100644 --- a/extended/src/main/java/apoc/agg/AggregationExtended.java +++ b/extended/src/main/java/apoc/agg/AggregationExtended.java @@ -1,8 +1,8 @@ package apoc.agg; import apoc.Extended; -import apoc.util.collection.Iterables; -import apoc.util.collection.Iterators; +import apoc.util.collection.IterablesExtended; +import apoc.util.collection.IteratorsExtended; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.procedure.Context; @@ -29,7 +29,7 @@ public class AggregationExtended { public RowFunction row() { BiPredicate curr = (current, value) -> db.executeTransactionally("RETURN " + value, Map.of("curr", current), - result -> Iterators.singleOrNull(result.columnAs(Iterables.single(result.columns())))); + result -> IteratorsExtended.singleOrNull(result.columnAs(IterablesExtended.single(result.columns())))); return new RowFunction(curr); } diff --git a/extended/src/main/java/apoc/agg/Rollup.java b/extended/src/main/java/apoc/agg/Rollup.java index 4a869861fe..9522611cfb 100644 --- a/extended/src/main/java/apoc/agg/Rollup.java +++ b/extended/src/main/java/apoc/agg/Rollup.java @@ -2,7 +2,7 @@ import apoc.Extended; import apoc.util.ExtendedListUtils; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.neo4j.graphdb.Entity; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -67,7 +67,7 @@ public void aggregate( @Name(value = "aggKeys") List aggKeys, @Name(value = "config", defaultValue = "{}") Map config) { - boolean cube = Util.toBoolean(config.get("cube")); + boolean cube = UtilExtended.toBoolean(config.get("cube")); Entity entity = (Entity) value; diff --git a/extended/src/main/java/apoc/algo/PathFindingExtended.java b/extended/src/main/java/apoc/algo/PathFindingExtended.java index f9eb496461..a0774eb0da 100644 --- a/extended/src/main/java/apoc/algo/PathFindingExtended.java +++ b/extended/src/main/java/apoc/algo/PathFindingExtended.java @@ -1,7 +1,7 @@ package apoc.algo; import apoc.Extended; -import apoc.result.WeightedPathResult; +import apoc.result.WeightedPathResultExtended; import org.neo4j.graphalgo.BasicEvaluationContext; import org.neo4j.graphalgo.CommonEvaluators; import org.neo4j.graphalgo.GraphAlgoFactory; @@ -16,7 +16,8 @@ import org.neo4j.procedure.Procedure; import java.util.stream.Stream; -import static apoc.algo.PathFindingUtils.buildPathExpander; + +import static apoc.algo.PathFindingExtendedUtils.buildPathExpander; @Extended public class PathFindingExtended { @@ -30,7 +31,7 @@ public class PathFindingExtended { @Procedure @Description("apoc.algo.aStarWithPoint(startNode, endNode, 'relTypesAndDirs', 'distance','pointProp') - " + "equivalent to apoc.algo.aStar but accept a Point type as a pointProperty instead of Number types as latitude and longitude properties") - public Stream aStarWithPoint( + public Stream aStarWithPoint( @Name("startNode") Node startNode, @Name("endNode") Node endNode, @Name("relationshipTypesAndDirections") String relTypesAndDirs, @@ -41,8 +42,8 @@ public Stream aStarWithPoint( new BasicEvaluationContext(tx, db), buildPathExpander(relTypesAndDirs), CommonEvaluators.doubleCostEvaluator(weightPropertyName), - new PathFindingUtils.GeoEstimateEvaluatorPointCustom(pointPropertyName)); - return WeightedPathResult.streamWeightedPathResult(startNode, endNode, algo); + new PathFindingExtendedUtils.GeoEstimateEvaluatorPointCustomExtended(pointPropertyName)); + return WeightedPathResultExtended.streamWeightedPathResult(startNode, endNode, algo); } } diff --git a/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java b/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java new file mode 100644 index 0000000000..deb7959f29 --- /dev/null +++ b/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.algo; + +import apoc.path.RelationshipTypeAndDirectionsExtended; +import org.apache.commons.lang3.tuple.Pair; +import org.neo4j.graphalgo.EstimateEvaluator; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.PathExpander; +import org.neo4j.graphdb.PathExpanderBuilder; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.values.storable.PointValue; + + +public class PathFindingExtendedUtils { + public static class GeoEstimateEvaluatorPointCustomExtended implements EstimateEvaluator { + // -- from org.neo4j.graphalgo.impl.util.GeoEstimateEvaluator + private static final double EARTH_RADIUS = 6371 * 1000; // Meters + private Node cachedGoal; + private final String pointPropertyKey; + private double[] cachedGoalCoordinates; + + public GeoEstimateEvaluatorPointCustomExtended(String pointPropertyKey) { + this.pointPropertyKey = pointPropertyKey; + } + + @Override + public Double getCost(Node node, Node goal) { + double[] nodeCoordinates = getCoordinates(node); + if (cachedGoal == null || !cachedGoal.equals(goal)) { + cachedGoalCoordinates = getCoordinates(goal); + cachedGoal = goal; + } + return distance(nodeCoordinates[0], nodeCoordinates[1], cachedGoalCoordinates[0], cachedGoalCoordinates[1]); + } + + private static double distance(double latitude1, double longitude1, double latitude2, double longitude2) { + latitude1 = Math.toRadians(latitude1); + longitude1 = Math.toRadians(longitude1); + latitude2 = Math.toRadians(latitude2); + longitude2 = Math.toRadians(longitude2); + double cLa1 = Math.cos(latitude1); + double xA = EARTH_RADIUS * cLa1 * Math.cos(longitude1); + double yA = EARTH_RADIUS * cLa1 * Math.sin(longitude1); + double zA = EARTH_RADIUS * Math.sin(latitude1); + double cLa2 = Math.cos(latitude2); + double xB = EARTH_RADIUS * cLa2 * Math.cos(longitude2); + double yB = EARTH_RADIUS * cLa2 * Math.sin(longitude2); + double zB = EARTH_RADIUS * Math.sin(latitude2); + return Math.sqrt((xA - xB) * (xA - xB) + (yA - yB) * (yA - yB) + (zA - zB) * (zA - zB)); + } + // -- end from org.neo4j.graphalgo.impl.util.GeoEstimateEvaluator + + private double[] getCoordinates(Node node) { + return ((PointValue) node.getProperty(pointPropertyKey)).coordinate(); + } + } + + public static PathExpander buildPathExpander(String relationshipsAndDirections) { + PathExpanderBuilder builder = PathExpanderBuilder.empty(); + for (Pair pair : RelationshipTypeAndDirectionsExtended.parse(relationshipsAndDirections)) { + if (pair.getLeft() == null) { + if (pair.getRight() == null) { + builder = PathExpanderBuilder.allTypesAndDirections(); + } else { + builder = PathExpanderBuilder.allTypes(pair.getRight()); + } + } else { + if (pair.getRight() == null) { + builder = builder.add(pair.getLeft()); + } else { + builder = builder.add(pair.getLeft(), pair.getRight()); + } + } + } + return builder.build(); + } +} diff --git a/extended/src/main/java/apoc/bolt/Bolt.java b/extended/src/main/java/apoc/bolt/Bolt.java index c630e72c61..5d26472d12 100644 --- a/extended/src/main/java/apoc/bolt/Bolt.java +++ b/extended/src/main/java/apoc/bolt/Bolt.java @@ -1,7 +1,7 @@ package apoc.bolt; import apoc.Extended; -import apoc.result.RowResult; +import apoc.result.RowResultExtended; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -32,23 +32,23 @@ private T withConnection(String url, Map config, Function load(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream load(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { return withConnection(url, config, conn -> conn.loadFromSession(statement, params)); } @Procedure(value = "apoc.bolt.load.fromLocal", mode = Mode.WRITE) - public Stream fromLocal(@Name("url") String url, - @Name("localStatement") String localStatement, - @Name("remoteStatement") String remoteStatement, - @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream fromLocal(@Name("url") String url, + @Name("localStatement") String localStatement, + @Name("remoteStatement") String remoteStatement, + @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { return withConnection(url, config, conn -> conn.loadFromLocal(localStatement, remoteStatement, db)); } @Procedure() @Description("apoc.bolt.execute(url-or-key, kernelTransaction, params, config) - access to other databases via bolt for reads and writes") - public Stream execute(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream execute(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { Map configuration = new HashMap<>(config); configuration.put("readOnly", false); return load(url, statement, params, configuration); diff --git a/extended/src/main/java/apoc/bolt/BoltConnection.java b/extended/src/main/java/apoc/bolt/BoltConnection.java index 0397c87335..1141977d31 100644 --- a/extended/src/main/java/apoc/bolt/BoltConnection.java +++ b/extended/src/main/java/apoc/bolt/BoltConnection.java @@ -1,11 +1,11 @@ package apoc.bolt; -import apoc.result.RowResult; -import apoc.result.VirtualNode; -import apoc.result.VirtualRelationship; +import apoc.result.RowResultExtended; +import apoc.result.VirtualNodeExtended; +import apoc.result.VirtualRelationshipExtended; import apoc.util.UriResolver; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.apache.commons.lang3.StringUtils; import org.neo4j.driver.Driver; import org.neo4j.driver.GraphDatabase; @@ -37,7 +37,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; public class BoltConnection { private final BoltConfig config; @@ -55,18 +55,18 @@ public static BoltConnection from(Map config, String url) throws } // methods from Bolt.java - public Stream loadFromSession(String statement, Map params) { + public Stream loadFromSession(String statement, Map params) { return withDriverAndSession(session -> { if (config.isAddStatistics()) { Result statementResult = session.run(statement, params); SummaryCounters counters = statementResult.consume().counters(); - return Stream.of(new RowResult(toMap(counters))); + return Stream.of(new RowResultExtended(toMap(counters))); } else return getRowResultStream(session, params, statement); }); } - public Stream loadFromLocal(String localStatement, String remoteStatement, GraphDatabaseService db) { + public Stream loadFromLocal(String localStatement, String remoteStatement, GraphDatabaseService db) { return withDriverAndSession(session -> { try (org.neo4j.graphdb.Transaction tx = db.beginTx()) { final org.neo4j.graphdb.Result localResult = tx.execute(localStatement, config.getLocalParams()); @@ -74,7 +74,7 @@ public Stream loadFromLocal(String localStatement, String remoteState .map(c -> "$" + c + " AS " + c) .collect(Collectors.joining(", ")) + "\n"; Map nodesCache = new HashMap<>(); - List response = new ArrayList<>(); + List response = new ArrayList<>(); while (localResult.hasNext()) { final Result statementResult; Map row = localResult.next(); @@ -87,7 +87,7 @@ public Stream loadFromLocal(String localStatement, String remoteState statementResult = session.run(withColumns + remoteStatement, row); } if (config.isStreamStatements()) { - response.add(new RowResult(toMap(statementResult.consume().counters()))); + response.add(new RowResultExtended(toMap(statementResult.consume().counters()))); } else { response.addAll( statementResult.stream() @@ -120,8 +120,8 @@ private Stream withTransaction(Session session, Function nodesCache) { - return new RowResult(record.asMap(value -> convertRecursive(value, nodesCache))); + private RowResultExtended buildRowResult(Record record, Map nodesCache) { + return new RowResultExtended(record.asMap(value -> convertRecursive(value, nodesCache))); } private Object convertRecursive(Object entity, Map nodesCache) { @@ -144,12 +144,12 @@ private Object toCollection(Collection entity, Map nodeCache) { return entity.stream().map(elem -> convertRecursive(elem, nodeCache)).collect(Collectors.toList()); } - private Stream getRowResultStream(Session session, Map params, String statement) { + private Stream getRowResultStream(Session session, Map params, String statement) { Map nodesCache = new HashMap<>(); return withTransaction(session, tx -> { ClosedAwareDelegatingIterator iterator = new ClosedAwareDelegatingIterator(tx.run(statement, params)); - return Iterators.stream(iterator).map(record -> buildRowResult(record, nodesCache)); + return IteratorsExtended.stream(iterator).map(record -> buildRowResult(record, nodesCache)); }); } @@ -159,23 +159,23 @@ private Object toNode(Object value, Map nodesCache) { if (config.isVirtual()) { List