From 5b0e9f291ccd865a83ecddd73e9666332bd0672f Mon Sep 17 00:00:00 2001 From: bale1017 Date: Tue, 31 Oct 2023 11:03:40 +0100 Subject: [PATCH] storage method name fix and indexing --- .../storage/StorageImplementation.java | 10 +- .../pathfinder/storage/StorageImpl.java | 4 +- .../storage/implementation/DebugStorage.java | 27 ++- .../storage/implementation/SqlStorage.java | 182 +++++++++++++----- .../storage/implementation/YmlStorage.java | 2 +- .../pathfinder/storage/StorageTest.java | 21 ++ 6 files changed, 181 insertions(+), 65 deletions(-) diff --git a/pathfinder-api/src/main/java/de/cubbossa/pathapi/storage/StorageImplementation.java b/pathfinder-api/src/main/java/de/cubbossa/pathapi/storage/StorageImplementation.java index 3abef3b6..f453d745 100644 --- a/pathfinder-api/src/main/java/de/cubbossa/pathapi/storage/StorageImplementation.java +++ b/pathfinder-api/src/main/java/de/cubbossa/pathapi/storage/StorageImplementation.java @@ -71,14 +71,16 @@ public interface StorageImplementation { Collection loadGroupsByMod(Collection key); default Optional loadGroup(NamespacedKey key) { - return loadGroupsByMod(Set.of(key)).stream().findAny(); + return loadGroups(Set.of(key)).stream().findAny(); } - Map> loadGroups(Collection ids); + Collection loadGroups(Collection keys); - List loadGroups(Range range); + Map> loadGroupsByNodes(Collection ids); + + Collection loadGroupsByNode(UUID node); - Collection loadGroups(UUID node); + List loadGroups(Range range); Collection loadGroups(NamespacedKey modifier); diff --git a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/StorageImpl.java b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/StorageImpl.java index 27158000..b6a4b32a 100644 --- a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/StorageImpl.java +++ b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/StorageImpl.java @@ -361,7 +361,7 @@ public CompletableFuture>> loadGroups(Collection }, () -> toLoad.add(uuid)); } if (toLoad.size() > 0) { - result.putAll(implementation.loadGroups(toLoad)); + result.putAll(implementation.loadGroupsByNodes(toLoad)); toLoad.forEach(uuid -> result.computeIfAbsent(uuid, u -> new HashSet<>())); } result.forEach((uuid, groups) -> { @@ -412,7 +412,7 @@ public CompletableFuture> loadGroups(UUID node) { return cached .map(CompletableFuture::completedFuture) .orElseGet(() -> asyncFuture(() -> { - Collection loaded = implementation.loadGroups(node); + Collection loaded = implementation.loadGroupsByNode(node); cache.getGroupCache().write(node, loaded); return loaded; })); diff --git a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/DebugStorage.java b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/DebugStorage.java index ab003547..b3181cd1 100644 --- a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/DebugStorage.java +++ b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/DebugStorage.java @@ -54,8 +54,17 @@ public Logger getLogger() { return logger; } + private Map threadStartMap = new HashMap<>(); + private void debug(String msg) { - logger.log(Level.INFO, msg + "\u001B[90m" + "(" + Thread.currentThread().getName() + ")" + "\u001B[0m"); + long id = Thread.currentThread().getId(); + Long dur = null; + if (!threadStartMap.containsKey(id)) { + threadStartMap.put(id, System.nanoTime()); + } else { + dur = System.nanoTime() - threadStartMap.remove(id); + } + logger.log(Level.INFO, msg + "\u001B[90m" + "(" + Thread.currentThread().getName() + (dur == null ? "" : ", " + (dur / 1_000_000.) + "ms") + ")" + "\u001B[0m"); } @Override @@ -134,9 +143,17 @@ public Collection loadGroupsByMod(Collection key) { } @Override - public Map> loadGroups(Collection ids) { + public Collection loadGroups(Collection keys) { + debug("> loadGroups(Collection keys)"); + var x = implementation.loadGroups(keys); + debug("< loadGroups(Collection keys)"); + return x; + } + + @Override + public Map> loadGroupsByNodes(Collection ids) { debug("> loadGroups(Collection ids " + ids.size() + ")"); - var x = implementation.loadGroups(ids); + var x = implementation.loadGroupsByNodes(ids); debug("< loadGroups(Collection ids)"); return x; } @@ -150,9 +167,9 @@ public List loadGroups(Range range) { } @Override - public Collection loadGroups(UUID node) { + public Collection loadGroupsByNode(UUID node) { debug("> loadGroups(UUID nodes)"); - var x = implementation.loadGroups(node); + var x = implementation.loadGroupsByNode(node); debug("< loadGroups(UUID nodes)"); return x; } diff --git a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/SqlStorage.java b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/SqlStorage.java index 9f3a3614..56d7b3bb 100644 --- a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/SqlStorage.java +++ b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/SqlStorage.java @@ -1,5 +1,15 @@ package de.cubbossa.pathfinder.storage.implementation; +import static de.cubbossa.pathfinder.jooq.Tables.PATHFINDER_VISUALIZER; +import static de.cubbossa.pathfinder.jooq.Tables.PATHFINDER_VISUALIZER_TYPE_RELATION; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderDiscoverings.PATHFINDER_DISCOVERINGS; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderEdges.PATHFINDER_EDGES; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderGroupModifierRelation.PATHFINDER_GROUP_MODIFIER_RELATION; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodeTypeRelation.PATHFINDER_NODE_TYPE_RELATION; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodegroupNodes.PATHFINDER_NODEGROUP_NODES; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodegroups.PATHFINDER_NODEGROUPS; +import static de.cubbossa.pathfinder.jooq.tables.PathfinderWaypoints.PATHFINDER_WAYPOINTS; + import de.cubbossa.pathapi.group.Modifier; import de.cubbossa.pathapi.group.ModifierRegistry; import de.cubbossa.pathapi.group.ModifierType; @@ -15,6 +25,7 @@ import de.cubbossa.pathapi.visualizer.VisualizerType; import de.cubbossa.pathapi.visualizer.VisualizerTypeRegistry; import de.cubbossa.pathfinder.jooq.tables.records.PathfinderEdgesRecord; +import de.cubbossa.pathfinder.jooq.tables.records.PathfinderGroupModifierRelationRecord; import de.cubbossa.pathfinder.jooq.tables.records.PathfinderNodegroupsRecord; import de.cubbossa.pathfinder.jooq.tables.records.PathfinderVisualizerRecord; import de.cubbossa.pathfinder.jooq.tables.records.PathfinderWaypointsRecord; @@ -22,6 +33,19 @@ import de.cubbossa.pathfinder.node.implementation.Waypoint; import de.cubbossa.pathfinder.nodegroup.SimpleNodeGroup; import de.cubbossa.pathfinder.util.HashedRegistry; +import java.io.StringReader; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.logging.Level; +import java.util.stream.Collectors; +import javax.sql.DataSource; import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; import org.jooq.DSLContext; @@ -32,23 +56,6 @@ import org.jooq.conf.Settings; import org.jooq.impl.DSL; -import javax.sql.DataSource; -import java.io.StringReader; -import java.time.LocalDateTime; -import java.util.*; -import java.util.logging.Level; -import java.util.stream.Collectors; - -import static de.cubbossa.pathfinder.jooq.Tables.PATHFINDER_VISUALIZER; -import static de.cubbossa.pathfinder.jooq.Tables.PATHFINDER_VISUALIZER_TYPE_RELATION; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderDiscoverings.PATHFINDER_DISCOVERINGS; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderEdges.PATHFINDER_EDGES; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderGroupModifierRelation.PATHFINDER_GROUP_MODIFIER_RELATION; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodeTypeRelation.PATHFINDER_NODE_TYPE_RELATION; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodegroupNodes.PATHFINDER_NODEGROUP_NODES; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderNodegroups.PATHFINDER_NODEGROUPS; -import static de.cubbossa.pathfinder.jooq.tables.PathfinderWaypoints.PATHFINDER_WAYPOINTS; - public abstract class SqlStorage extends CommonStorage { private final RecordMapper nodeMapper; @@ -172,27 +179,46 @@ private void createNodeTable() { } private void createEdgeTable() { - create - .createTableIfNotExists(PATHFINDER_EDGES) - .columns(PATHFINDER_EDGES.fields()) - .primaryKey(PATHFINDER_EDGES.START_ID, PATHFINDER_EDGES.END_ID) - .execute(); + create.transaction(c -> { + var ctx = DSL.using(c); + ctx.createTableIfNotExists(PATHFINDER_EDGES) + .columns(PATHFINDER_EDGES.fields()) + .primaryKey(PATHFINDER_EDGES.START_ID, PATHFINDER_EDGES.END_ID) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_EDGES) + .include(PATHFINDER_EDGES.START_ID) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_EDGES) + .include(PATHFINDER_EDGES.END_ID) + .execute(); + }); } private void createNodeGroupTable() { - create - .createTableIfNotExists(PATHFINDER_NODEGROUPS) + create.createTableIfNotExists(PATHFINDER_NODEGROUPS) .columns(PATHFINDER_NODEGROUPS.fields()) .primaryKey(PATHFINDER_NODEGROUPS.KEY) .execute(); } private void createNodeGroupNodesTable() { - create - .createTableIfNotExists(PATHFINDER_NODEGROUP_NODES) - .columns(PATHFINDER_NODEGROUP_NODES.fields()) - .primaryKey(PATHFINDER_NODEGROUP_NODES.fields()) - .execute(); + create.transaction(cfg -> { + var ctx = DSL.using(cfg); + ctx.createTableIfNotExists(PATHFINDER_NODEGROUP_NODES) + .columns(PATHFINDER_NODEGROUP_NODES.fields()) + .primaryKey(PATHFINDER_NODEGROUP_NODES.fields()) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_NODEGROUP_NODES) + .include(PATHFINDER_NODEGROUP_NODES.NODE_ID) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_NODEGROUP_NODES) + .include(PATHFINDER_NODEGROUP_NODES.GROUP_KEY) + .execute(); + }); } private void createPathVisualizerTable() { @@ -204,36 +230,74 @@ private void createPathVisualizerTable() { } private void createPathVisualizerTypeTable() { - create - .createTableIfNotExists(PATHFINDER_VISUALIZER_TYPE_RELATION) - .columns(PATHFINDER_VISUALIZER_TYPE_RELATION.fields()) - .primaryKey(PATHFINDER_VISUALIZER_TYPE_RELATION.VISUALIZER_KEY, PATHFINDER_VISUALIZER_TYPE_RELATION.TYPE_KEY) - .execute(); + create.transaction(cfg -> { + var ctx = DSL.using(cfg); + ctx.createTableIfNotExists(PATHFINDER_VISUALIZER_TYPE_RELATION) + .columns(PATHFINDER_VISUALIZER_TYPE_RELATION.fields()) + .primaryKey(PATHFINDER_VISUALIZER_TYPE_RELATION.VISUALIZER_KEY, PATHFINDER_VISUALIZER_TYPE_RELATION.TYPE_KEY) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_VISUALIZER_TYPE_RELATION) + .include(PATHFINDER_VISUALIZER_TYPE_RELATION.TYPE_KEY) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_VISUALIZER_TYPE_RELATION) + .include(PATHFINDER_VISUALIZER_TYPE_RELATION.VISUALIZER_KEY) + .execute(); + }); } private void createDiscoverInfoTable() { - create - .createTableIfNotExists(PATHFINDER_DISCOVERINGS) - .columns(PATHFINDER_DISCOVERINGS.fields()) - .primaryKey(PATHFINDER_DISCOVERINGS.PLAYER_ID, PATHFINDER_DISCOVERINGS.DISCOVER_KEY) - .execute(); + create.transaction(cfg -> { + var ctx = DSL.using(cfg); + ctx.createTableIfNotExists(PATHFINDER_DISCOVERINGS) + .columns(PATHFINDER_DISCOVERINGS.fields()) + .primaryKey(PATHFINDER_DISCOVERINGS.PLAYER_ID, PATHFINDER_DISCOVERINGS.DISCOVER_KEY) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_DISCOVERINGS) + .include(PATHFINDER_DISCOVERINGS.PLAYER_ID) + .execute(); + }); } private void createNodeTypeRelation() { - create - .createTableIfNotExists(PATHFINDER_NODE_TYPE_RELATION) - .columns(PATHFINDER_NODE_TYPE_RELATION.fields()) - .primaryKey(PATHFINDER_NODE_TYPE_RELATION.NODE_ID, PATHFINDER_NODE_TYPE_RELATION.NODE_TYPE) - .execute(); + + create.transaction(cfg -> { + var ctx = DSL.using(cfg); + ctx.createTableIfNotExists(PATHFINDER_NODE_TYPE_RELATION) + .columns(PATHFINDER_NODE_TYPE_RELATION.fields()) + .primaryKey(PATHFINDER_NODE_TYPE_RELATION.NODE_ID, PATHFINDER_NODE_TYPE_RELATION.NODE_TYPE) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_NODE_TYPE_RELATION) + .include(PATHFINDER_NODE_TYPE_RELATION.NODE_ID) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_NODE_TYPE_RELATION) + .include(PATHFINDER_NODE_TYPE_RELATION.NODE_TYPE) + .execute(); + }); + } private void createModifierGroupRelation() { - create - .createTableIfNotExists(PATHFINDER_GROUP_MODIFIER_RELATION) - .columns(PATHFINDER_GROUP_MODIFIER_RELATION.fields()) - .primaryKey(PATHFINDER_GROUP_MODIFIER_RELATION.GROUP_KEY, PATHFINDER_GROUP_MODIFIER_RELATION.MODIFIER_KEY) - .execute(); + create.transaction(cfg -> { + var ctx = DSL.using(cfg); + ctx.createTableIfNotExists(PATHFINDER_GROUP_MODIFIER_RELATION) + .columns(PATHFINDER_GROUP_MODIFIER_RELATION.fields()) + .primaryKey(PATHFINDER_GROUP_MODIFIER_RELATION.GROUP_KEY, PATHFINDER_GROUP_MODIFIER_RELATION.MODIFIER_KEY) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_GROUP_MODIFIER_RELATION) + .include(PATHFINDER_GROUP_MODIFIER_RELATION.GROUP_KEY) + .execute(); + ctx.createIndexIfNotExists() + .on(PATHFINDER_GROUP_MODIFIER_RELATION) + .include(PATHFINDER_GROUP_MODIFIER_RELATION.MODIFIER_KEY) + .execute(); + }); } @Override @@ -413,8 +477,7 @@ public Collection loadGroupNodes(NodeGroup group) { .fetch(Record1::value1); } - @Override - public Map> loadGroups(Collection ids) { + public Map> loadGroupsByNodes(Collection ids) { Map> result = new HashMap<>(); create.transaction(cfg -> { Map> mapping = new HashMap<>(); @@ -437,8 +500,21 @@ public Map> loadGroups(Collection ids) { @Override public Collection loadGroupsByMod(Collection key) { + return create.transactionResult(cfg -> { + var ctx = DSL.using(cfg); + Collection groups = ctx.selectFrom(PATHFINDER_GROUP_MODIFIER_RELATION) + .where(PATHFINDER_GROUP_MODIFIER_RELATION.MODIFIER_KEY.in(key)) + .fetch(PathfinderGroupModifierRelationRecord::getGroupKey); + return ctx.selectFrom(PATHFINDER_NODEGROUPS) + .where(PATHFINDER_NODEGROUPS.KEY.in(groups)) + .fetch(groupMapper); + }); + } + + @Override + public Collection loadGroups(Collection keys) { return create.selectFrom(PATHFINDER_NODEGROUPS) - .where(PATHFINDER_NODEGROUPS.KEY.in(key)) + .where(PATHFINDER_NODEGROUPS.KEY.in(keys)) .fetch(groupMapper); } @@ -451,7 +527,7 @@ public List loadGroups(Range range) { } @Override - public Collection loadGroups(UUID node) { + public Collection loadGroupsByNode(UUID node) { return create.select().from(PATHFINDER_NODEGROUPS) .join(PATHFINDER_NODEGROUP_NODES) .on(PATHFINDER_NODEGROUPS.KEY.eq(PATHFINDER_NODEGROUP_NODES.GROUP_KEY)) diff --git a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/YmlStorage.java b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/YmlStorage.java index 5179f1ab..8c29d8ff 100644 --- a/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/YmlStorage.java +++ b/pathfinder-core/src/main/java/de/cubbossa/pathfinder/storage/implementation/YmlStorage.java @@ -351,7 +351,7 @@ public List loadGroups(Range range) { } @Override - public Collection loadGroups(UUID node) { + public Collection loadGroupsByNode(UUID node) { return loadAllGroups().stream() .filter(g -> g.contains(node)) .collect(Collectors.toSet()); diff --git a/pathfinder-core/src/test/java/de/cubbossa/pathfinder/storage/StorageTest.java b/pathfinder-core/src/test/java/de/cubbossa/pathfinder/storage/StorageTest.java index 59d61716..db2ca28c 100644 --- a/pathfinder-core/src/test/java/de/cubbossa/pathfinder/storage/StorageTest.java +++ b/pathfinder-core/src/test/java/de/cubbossa/pathfinder/storage/StorageTest.java @@ -1,6 +1,7 @@ package de.cubbossa.pathfinder.storage; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import de.cubbossa.pathapi.group.Modifier; import de.cubbossa.pathapi.group.ModifierRegistry; import de.cubbossa.pathapi.group.NodeGroup; @@ -222,6 +223,26 @@ void loadGroupsByMod() { Assertions.assertTrue(Maps.difference(expected, result).areEqual()); } + @Test + @Timeout(value = 300, unit = TimeUnit.MILLISECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD) + void loadGroupsByMod2() { + NamespacedKey xKey = NamespacedKey.fromString("pathfinder:x"); + NamespacedKey yKey = NamespacedKey.fromString("pathfinder:y"); + + NodeGroup x = makeGroup(xKey); + NodeGroup y = makeGroup(yKey); + Set groups = Set.of(x, y); + + int i = 0; + for (NodeGroup g : groups) { + g.addModifier(new TestModifier(i++ + "")); + storage.saveGroup(g).join(); + } + + Set result = new HashSet<>(storage.loadGroupsByMod(Collections.singletonList(TestModifierType.KEY)).join()); + assertTrue(Sets.difference(groups, result).isEmpty()); + } + @Test @Timeout(value = 300, unit = TimeUnit.MILLISECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD) void getNodeGroupKeySet() {