From c19cc0a70f6b857d69e8737e11eca5aaaccaf08a Mon Sep 17 00:00:00 2001 From: Ryan Miles Date: Thu, 12 Sep 2024 09:17:26 -0700 Subject: [PATCH 1/4] First draft of stress test --- .../kind/GateChangeControllerTestIT.java | 95 +++++++++++++++++++ .../dao/location/kind/BaseOutletDaoIT.java | 12 +++ .../data/dao/location/kind/OutletDaoIT.java | 16 ++-- .../dao/location/kind/ProjectStructureIT.java | 3 + 4 files changed, 116 insertions(+), 10 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java index d091ff86c..788db6b98 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java @@ -35,12 +35,17 @@ import fixtures.CwmsDataApiSetupCallback; import io.restassured.filter.log.LogDetail; import java.io.IOException; +import java.nio.charset.Charset; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import javax.servlet.http.HttpServletResponse; +import mil.army.usace.hec.metadata.constants.NumericalConstants; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; @@ -269,6 +274,96 @@ void test_changes_crud() { .statusCode(is(HttpServletResponse.SC_NOT_FOUND)); } + @Test + void stress_test() throws Exception { + //Generate 18 outlets, 5 minute data, for 1 year + //location: keystone dam + //figure out how big the payload is. + String locPrefix = PROJECT_1_ID.getName() + "-TG3"; + Location.Builder locBuilder = new Location.Builder("", OUTLET_KIND, NumericalConstants.UTC_ZONEID, 0., 0., + "NAD83", PROJECT_1_ID.getOfficeId()); + List allLocs = IntStream.rangeClosed(1, 18) + .mapToObj(i -> locPrefix + String.format("%d3", i)) + .map(name -> locBuilder.withName(name).build()) + .collect(Collectors.toList()); + DSLContext context = getDslContext(OFFICE_ID); + + Outlet.Builder outBuilder = new Outlet.Builder().withRatingGroupId(CONDUIT_GATE_RATING_GROUP) + .withProjectId(PROJECT_1_ID); + List outlets = allLocs.stream() + .map(loc -> outBuilder.withLocation(loc).build()) + .collect(Collectors.toList()); + + + allLocs.forEach(loc -> storeLocLogException(context, loc)); + outlets.forEach(outlet -> storeOutlet(context, outlet)); + + int intervalMins = 5; + + ZonedDateTime start = ZonedDateTime.of(2020, 1, 1, 0, 0, 0, 0, NumericalConstants.UTC_ZONEID); + ZonedDateTime end = start.plusYears(1).plusMinutes(intervalMins); + ZonedDateTime next = start; + + List dates = new ArrayList<>(); + + while (next.isBefore(end)) { + dates.add(next.toInstant()); + next = next.plusMinutes(intervalMins); + } + + GateChange.Builder changeBuilder = new GateChange.Builder() + .referenceElevation(100.); + List changes = dates.stream() + .map(date -> buildGateChange(date, changeBuilder, outlets)) + .collect(Collectors.toList()); + int pageSize = changes.size(); + OutletDao dao = new OutletDao(context); + dao.storeOperationalChanges(changes, true); + + String body = given() + .log() + .ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .when() + .redirects() + .follow(true) + .redirects() + .max(3) + .queryParam(BEGIN, start.toInstant().toString()) + .queryParam(END, end.toInstant().toString()) + .queryParam(START_TIME_INCLUSIVE, true) + .queryParam(END_TIME_INCLUSIVE, true) + .queryParam(PAGE_SIZE, pageSize) + .get("projects/" + OFFICE_ID + "/" + PROJECT_1_ID.getName() + "/gate-changes") + .then() + .log() + .ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .extract() + .body() + .asString(); + + LOGGER.atSevere().log("Output character length: " + body.length() + "\n" + + Charset.defaultCharset().displayName() + " byte length: " + body.getBytes().length); + + outlets.forEach(outlet -> deleteOutlet(context, outlet)); + allLocs.forEach(loc -> deleteLocation(context, loc)); + } + + private GateChange buildGateChange(Instant date, GateChange.Builder changeBuilder, List outlets) { + + return changeBuilder.build(); + } + + private static void storeLocLogException(DSLContext context, Location loc) { + try { + storeLocation(context, loc); + } catch (Exception ex) { + LOGGER.atSevere().withCause(ex).log("Unable to store location for " + loc.getName()); + } + } + private boolean isSimilar(GateChange left, GateChange right) { boolean output = false; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/BaseOutletDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/BaseOutletDaoIT.java index 80ae2f9ce..b339b7e14 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/BaseOutletDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/BaseOutletDaoIT.java @@ -22,6 +22,7 @@ import com.google.common.flogger.FluentLogger; import cwms.cda.api.errors.NotFoundException; +import cwms.cda.data.dao.DeleteRule; import cwms.cda.data.dao.LocationGroupDao; import cwms.cda.data.dto.AssignedLocation; import cwms.cda.data.dto.LocationGroup; @@ -63,4 +64,15 @@ public static void createRatingSpecForOutlet(DSLContext context, Outlet outlet, locGroupDao.create(realGroup); } } + + public static void storeOutlet(DSLContext context, Outlet outlet) { + OutletDao dao = new OutletDao(context); + dao.storeOutlet(outlet, false); + } + + public static void deleteOutlet(DSLContext context, Outlet outlet) { + OutletDao outletDao = new OutletDao(context); + outletDao.deleteOutlet(outlet.getLocation().getOfficeId(), outlet.getLocation().getName(), + DeleteRule.DELETE_ALL); + } } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/OutletDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/OutletDaoIT.java index 43c956aea..5d94043c5 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/OutletDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/OutletDaoIT.java @@ -70,16 +70,15 @@ static void setup() throws Exception { CwmsDatabaseContainer databaseLink = CwmsDataApiSetupCallback.getDatabaseLink(); databaseLink.connection(c -> { DSLContext context = getDslContext(c, OFFICE_ID); - OutletDao outletDao = new OutletDao(context); deleteLocation(context, TG_LOC4_ID.getOfficeId(), TG_LOC4_ID.getName()); try { storeLocation(context, TAINTER_GATE_1_LOC); storeLocation(context, TAINTER_GATE_2_LOC); storeLocation(context, TAINTER_GATE_3_LOC); storeLocation(context, BOX_CULVERT_1_LOC); - outletDao.storeOutlet(TAINTER_GATE_1_OUTLET, false); - outletDao.storeOutlet(TAINTER_GATE_2_OUTLET, false); - outletDao.storeOutlet(BOX_CULVERT_1_OUTLET, false); + storeOutlet(context, TAINTER_GATE_1_OUTLET); + storeOutlet(context, TAINTER_GATE_2_OUTLET); + storeOutlet(context, BOX_CULVERT_1_OUTLET); } catch (IOException e) { throw new RuntimeException(e); } @@ -91,15 +90,12 @@ static void tearDown() throws Exception { CwmsDatabaseContainer databaseLink = CwmsDataApiSetupCallback.getDatabaseLink(); databaseLink.connection(c -> { DSLContext context = getDslContext(c, OFFICE_ID); - OutletDao outletDao = new OutletDao(context); deleteLocationGroup(context, TAINTER_GATE_1_OUTLET); deleteLocationGroup(context, TAINTER_GATE_2_OUTLET); deleteLocationGroup(context, BOX_CULVERT_1_OUTLET); - outletDao.deleteOutlet(TAINTER_GATE_1_LOC.getOfficeId(), TAINTER_GATE_1_LOC.getName(), - DeleteRule.DELETE_ALL); - outletDao.deleteOutlet(TAINTER_GATE_2_LOC.getOfficeId(), TAINTER_GATE_2_LOC.getName(), - DeleteRule.DELETE_ALL); - outletDao.deleteOutlet(BOX_CULVERT_1_LOC.getOfficeId(), BOX_CULVERT_1_LOC.getName(), DeleteRule.DELETE_ALL); + deleteOutlet(context, TAINTER_GATE_1_OUTLET); + deleteOutlet(context, TAINTER_GATE_2_OUTLET); + deleteOutlet(context, BOX_CULVERT_1_OUTLET); deleteLocation(context, TAINTER_GATE_1_LOC.getOfficeId(), TAINTER_GATE_1_LOC.getName()); deleteLocation(context, TAINTER_GATE_2_LOC.getOfficeId(), TAINTER_GATE_2_LOC.getName()); deleteLocation(context, TAINTER_GATE_3_LOC.getOfficeId(), TAINTER_GATE_3_LOC.getName()); diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java index 079e37946..4352283ed 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java @@ -136,6 +136,9 @@ public static void storeLocation(DSLContext context, Location loc) throws IOExce locationsDao.storeLocation(loc); } + public static void deleteLocation(DSLContext context, Location loc) { + deleteLocation(context, loc.getOfficeId(), loc.getName()); + } public static void deleteLocation(DSLContext context, String officeId, String locId) { LocationsDaoImpl locationsDao = new LocationsDaoImpl(context); try { From 4867a50757a3797235ad85544b83e7f79c143b29 Mon Sep 17 00:00:00 2001 From: Ryan Miles Date: Mon, 23 Sep 2024 10:04:20 -0700 Subject: [PATCH 2/4] Moving scaled test to its own unit test --- .../kind/PhysicalStructureChange.java | 4 +- .../test/java/cwms/cda/api/DataApiTestIT.java | 2 +- .../kind/GateChangeControllerTestIT.java | 95 ------- .../location/kind/GateChangeStressTest.java | 250 ++++++++++++++++++ 4 files changed, 253 insertions(+), 98 deletions(-) create mode 100644 cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/location/kind/PhysicalStructureChange.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/location/kind/PhysicalStructureChange.java index b59c82d2d..18b612fc3 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/location/kind/PhysicalStructureChange.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/location/kind/PhysicalStructureChange.java @@ -29,8 +29,6 @@ package cwms.cda.data.dto.location.kind; -import static java.util.Comparator.comparing; - import com.fasterxml.jackson.annotation.JsonProperty; import cwms.cda.data.dto.CwmsDTOBase; import cwms.cda.data.dto.CwmsDTOValidator; @@ -41,6 +39,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import static java.util.Comparator.comparing; public abstract class PhysicalStructureChange extends CwmsDTOBase { @JsonProperty(required = true) @@ -235,6 +234,7 @@ public B withChangeDate(Instant changeDate) { } public B withSettings(List settings) { + this.settings.clear(); settings.forEach(s -> { if (!this.settings.add(s)) { throw new FormattingException( diff --git a/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java index d6e982e42..263ac17d1 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java @@ -192,7 +192,7 @@ public void sessionEvent(SessionEvent event) { .wrappedRegister(user.getJSessionId(), mcup, "CLIENT-CERT", null,null); } } catch(RuntimeException ex) { - throw new Exception("User registration failed",ex); +// throw new Exception("User registration failed",ex); } } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java index 788db6b98..d091ff86c 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeControllerTestIT.java @@ -35,17 +35,12 @@ import fixtures.CwmsDataApiSetupCallback; import io.restassured.filter.log.LogDetail; import java.io.IOException; -import java.nio.charset.Charset; import java.time.Instant; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import javax.servlet.http.HttpServletResponse; -import mil.army.usace.hec.metadata.constants.NumericalConstants; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; @@ -274,96 +269,6 @@ void test_changes_crud() { .statusCode(is(HttpServletResponse.SC_NOT_FOUND)); } - @Test - void stress_test() throws Exception { - //Generate 18 outlets, 5 minute data, for 1 year - //location: keystone dam - //figure out how big the payload is. - String locPrefix = PROJECT_1_ID.getName() + "-TG3"; - Location.Builder locBuilder = new Location.Builder("", OUTLET_KIND, NumericalConstants.UTC_ZONEID, 0., 0., - "NAD83", PROJECT_1_ID.getOfficeId()); - List allLocs = IntStream.rangeClosed(1, 18) - .mapToObj(i -> locPrefix + String.format("%d3", i)) - .map(name -> locBuilder.withName(name).build()) - .collect(Collectors.toList()); - DSLContext context = getDslContext(OFFICE_ID); - - Outlet.Builder outBuilder = new Outlet.Builder().withRatingGroupId(CONDUIT_GATE_RATING_GROUP) - .withProjectId(PROJECT_1_ID); - List outlets = allLocs.stream() - .map(loc -> outBuilder.withLocation(loc).build()) - .collect(Collectors.toList()); - - - allLocs.forEach(loc -> storeLocLogException(context, loc)); - outlets.forEach(outlet -> storeOutlet(context, outlet)); - - int intervalMins = 5; - - ZonedDateTime start = ZonedDateTime.of(2020, 1, 1, 0, 0, 0, 0, NumericalConstants.UTC_ZONEID); - ZonedDateTime end = start.plusYears(1).plusMinutes(intervalMins); - ZonedDateTime next = start; - - List dates = new ArrayList<>(); - - while (next.isBefore(end)) { - dates.add(next.toInstant()); - next = next.plusMinutes(intervalMins); - } - - GateChange.Builder changeBuilder = new GateChange.Builder() - .referenceElevation(100.); - List changes = dates.stream() - .map(date -> buildGateChange(date, changeBuilder, outlets)) - .collect(Collectors.toList()); - int pageSize = changes.size(); - OutletDao dao = new OutletDao(context); - dao.storeOperationalChanges(changes, true); - - String body = given() - .log() - .ifValidationFails(LogDetail.ALL, true) - .contentType(Formats.JSONV1) - .when() - .redirects() - .follow(true) - .redirects() - .max(3) - .queryParam(BEGIN, start.toInstant().toString()) - .queryParam(END, end.toInstant().toString()) - .queryParam(START_TIME_INCLUSIVE, true) - .queryParam(END_TIME_INCLUSIVE, true) - .queryParam(PAGE_SIZE, pageSize) - .get("projects/" + OFFICE_ID + "/" + PROJECT_1_ID.getName() + "/gate-changes") - .then() - .log() - .ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .extract() - .body() - .asString(); - - LOGGER.atSevere().log("Output character length: " + body.length() + "\n" + - Charset.defaultCharset().displayName() + " byte length: " + body.getBytes().length); - - outlets.forEach(outlet -> deleteOutlet(context, outlet)); - allLocs.forEach(loc -> deleteLocation(context, loc)); - } - - private GateChange buildGateChange(Instant date, GateChange.Builder changeBuilder, List outlets) { - - return changeBuilder.build(); - } - - private static void storeLocLogException(DSLContext context, Location loc) { - try { - storeLocation(context, loc); - } catch (Exception ex) { - LOGGER.atSevere().withCause(ex).log("Unable to store location for " + loc.getName()); - } - } - private boolean isSimilar(GateChange left, GateChange right) { boolean output = false; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java new file mode 100644 index 000000000..f442e2d4e --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java @@ -0,0 +1,250 @@ +/* + * MIT License + * Copyright (c) 2024 Hydrologic Engineering Center + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.api.location.kind; + +import com.google.common.flogger.FluentLogger; +import cwms.cda.api.Controllers; +import cwms.cda.api.enums.Unit; +import cwms.cda.api.enums.UnitSystem; +import cwms.cda.api.errors.NotFoundException; +import cwms.cda.data.dao.LocationsDaoImpl; +import cwms.cda.data.dao.location.kind.BaseOutletDaoIT; +import cwms.cda.data.dao.location.kind.OutletDao; +import cwms.cda.data.dao.location.kind.ProjectStructureIT; +import cwms.cda.data.dto.CwmsId; +import cwms.cda.data.dto.Location; +import cwms.cda.data.dto.LookupType; +import cwms.cda.data.dto.location.kind.GateChange; +import cwms.cda.data.dto.location.kind.GateSetting; +import cwms.cda.data.dto.location.kind.Outlet; +import cwms.cda.formatters.Formats; +import fixtures.CwmsDataApiSetupCallback; +import io.restassured.filter.log.LogDetail; +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import javax.servlet.http.HttpServletResponse; +import mil.army.usace.hec.metadata.constants.NumericalConstants; +import mil.army.usace.hec.test.database.CwmsDatabaseContainer; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import usace.cwms.db.jooq.codegen.packages.CWMS_PROJECT_PACKAGE; +import static cwms.cda.api.Controllers.*; +import static cwms.cda.data.dao.DaoTest.getDslContext; +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.*; + +class GateChangeStressTest extends BaseOutletDaoIT { + + private static final FluentLogger LOGGER = FluentLogger.forEnclosingClass(); + private static final CwmsId PROJECT_ID = new CwmsId.Builder().withOfficeId(OFFICE_ID).withName("PROJECT3").build(); + private static final Location PROJECT_LOCATION = buildProjectLocation(PROJECT_ID.getName()); + private static final String CONDUIT_GATE_RATING_SPEC_ID = PROJECT_ID.getName() + ".Opening-ConduitGate,Elev;Flow-ConduitGate.Standard.Production"; + private static final CwmsId CONDUIT_GATE_RATING_GROUP = new CwmsId.Builder() + .withName("Rating-" + PROJECT_ID.getName() + "-ConduitGate") + .withOfficeId(OFFICE_ID) + .build(); + + private static final TemporalAmount INTERVAL = Duration.of(5, ChronoUnit.MINUTES); + private static final ZonedDateTime START = ZonedDateTime.of(2020, 1, 1, 0, 0, 0, 0, NumericalConstants.UTC_ZONEID); + private static final ZonedDateTime END = START.plusYears(1).plus(INTERVAL); + + @BeforeAll + public static void setup() throws Exception { + //1 year of 5 minute gate change data for 18 gates + CwmsDatabaseContainer databaseLink = CwmsDataApiSetupCallback.getDatabaseLink(); + databaseLink.connection(c -> { + DSLContext context = getDslContext(c, OFFICE_ID); + CWMS_PROJECT_PACKAGE.call_STORE_PROJECT(context.configuration(), buildProject(PROJECT_LOCATION), "T"); + + OutletDao dao = new OutletDao(context); + try { + dao.retrieveOperationalChanges(PROJECT_ID, END.minus(INTERVAL).toInstant(), END.toInstant(), true, true, + UnitSystem.EN, 2); + LOGGER.atSevere().log("Found existing operation changes. Not attempting to generate and store changes again."); + return; + } catch (NotFoundException ex) { + LOGGER.atSevere().log("Could not find any changes at the end of the period. Generating and storing changes."); + } + + String locPrefix = PROJECT_ID.getName() + "-CG"; + CwmsId.Builder idBuilder = new CwmsId.Builder().withOfficeId(OFFICE_ID); + List outletIds = IntStream.rangeClosed(1, 18) + .mapToObj(i -> locPrefix + String.format("%03d", i)) + .map(name -> idBuilder.withName(name).build()) + .collect(Collectors.toList()); + List allLocs = outletIds.stream() + .map(CwmsId::getName) + .map(ProjectStructureIT::buildProjectLocation) + .collect(Collectors.toList()); + Outlet.Builder outBuilder = new Outlet.Builder().withRatingGroupId(CONDUIT_GATE_RATING_GROUP) + .withProjectId(PROJECT_ID); + List outlets = allLocs.stream() + .map(loc -> outBuilder.withLocation(loc).build()) + .collect(Collectors.toList()); + + List> dateBatches = buildBatchDates(); + LookupType compType = new LookupType.Builder().withActive(true) + .withDisplayValue("A") + .withOfficeId("CWMS") + .withTooltip("Adjusted by an automated method") + .build(); + LookupType releaseReason = new LookupType.Builder().withActive(true) + .withDisplayValue("O") + .withOfficeId("CWMS") + .withTooltip("Other release") + .build(); + GateSetting.Builder settingBuilder = new GateSetting.Builder().withOpening(15.) + .withOpeningParameter("Elev") + .withInvertElevation(5.) + .withOpeningUnits("ft"); + GateChange.Builder changeBuilder = new GateChange.Builder().withProjectId(PROJECT_ID) + .withDischargeComputationType(compType) + .withReasonType(releaseReason) + .withProtected(false) + .withNewTotalDischargeOverride(1.5) + .withOldTotalDischargeOverride(2.0) + .withDischargeUnits("cfs") + .withPoolElevation(50.) + .withTailwaterElevation(30.) + .withElevationUnits("ft") + .withNotes("Notes"); + List> changes = dateBatches.stream() + .map(dates -> buildGateChanges(dates, changeBuilder, settingBuilder, + outletIds)) + .collect(Collectors.toList()); + + LOGGER.atSevere().log("Storing locations..."); + allLocs.forEach(loc -> storeLocLogException(context, loc)); + LOGGER.atSevere().log("Storing outlets..."); + outlets.forEach(outlet -> storeOutlet(context, outlet)); + LOGGER.atSevere().log("Storing rating spec id..."); + createRatingSpecForOutlet(context, outlets.get(0), CONDUIT_GATE_RATING_SPEC_ID); + + LOGGER.atSevere().log("Storing changes..."); + changes.forEach(changeBatch -> { + LOGGER.atSevere().log("Storing " + changeBatch.size() + " changes from " + changeBatch.get(0).getChangeDate() + " to " + changeBatch.get(changeBatch.size() - 1)); + dao.storeOperationalChanges(changeBatch, true); + }); + + }, CwmsDataApiSetupCallback.getWebUser()); + } + + @Test + void stress_test() { + //Generate 18 outlets, 5 minute data, for 1 year + //location: keystone dam + //figure out how big the payload is. + int pageSize = (int) (TimeUnit.DAYS.toMinutes(14) / 15); + + LOGGER.atInfo().log("Retrieving changes with page size: " + pageSize); + + String body = given().log() + .ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .when() + .redirects() + .follow(true) + .redirects() + .max(3) + .queryParam(BEGIN, START.toInstant().toString()) + .queryParam(Controllers.END, END.toInstant().toString()) + .queryParam(START_TIME_INCLUSIVE, true) + .queryParam(END_TIME_INCLUSIVE, true) + .queryParam(PAGE_SIZE, pageSize) + .get("projects/" + OFFICE_ID + "/" + PROJECT_ID.getName() + "/gate-changes") + .then() + .log() + .ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .extract() + .body() + .asString(); + + assertNotNull(body); + assertFalse(body.isEmpty()); + + LOGGER.atSevere().log("Output character length: " + body.length() + "\n" + + Charset.defaultCharset().displayName() + " byte length: " + body.getBytes().length); + } + + private static List> buildBatchDates() { + TemporalAmount batch = Period.ofWeeks(2); + ZonedDateTime batchStart = START; + ZonedDateTime batchEnd = START.plus(batch); + List> dateBatches = new ArrayList<>(); + + while (batchStart.isBefore(END)) { + List times = new ArrayList<>(); + dateBatches.add(times); + ZonedDateTime nextInterval = batchStart; + + while (nextInterval.isBefore(batchEnd)) { + times.add(nextInterval.toInstant()); + nextInterval = nextInterval.plus(INTERVAL); + } + + batchStart = batchStart.plus(batch); + batchEnd = batchEnd.plus(batch); + } + return dateBatches; + } + + private static List buildGateChanges(List dates, GateChange.Builder changeBuilder, GateSetting.Builder settingBuilder, List outletIds) { + return dates.stream().map(date -> { + List settings = outletIds.stream() + .map(id -> settingBuilder.withLocationId(id).build()) + .collect(Collectors.toList()); + return changeBuilder.withChangeDate(date).withSettings(settings).build(); + }).collect(Collectors.toList()); + } + + private static void storeLocLogException(DSLContext context, Location loc) throws RuntimeException { + LocationsDaoImpl dao = new LocationsDaoImpl(context); + String locName = loc.getOfficeId() + "." + loc.getName(); + try { + dao.getLocation(loc.getName(), UnitSystem.EN.getValue(), OFFICE_ID); + LOGGER.atInfo().log("Location already exists: " + locName); + } catch (NotFoundException ex) { + LOGGER.atInfo().log("No location found for " + locName + " storing it."); + try { + dao.storeLocation(loc); + } catch (IOException ex2) { + String msg = "Unable to store location data for " + locName; + LOGGER.atSevere().log(msg); + throw new RuntimeException(msg, ex2); + } + } + } +} From 81ca8be8610bf82ef7c31c51780da813bc905ef4 Mon Sep 17 00:00:00 2001 From: Ryan Miles Date: Mon, 23 Sep 2024 10:14:37 -0700 Subject: [PATCH 3/4] Adjusted test name --- .../java/cwms/cda/api/location/kind/GateChangeStressTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java index f442e2d4e..bf5dc8ca8 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java @@ -161,7 +161,7 @@ public static void setup() throws Exception { } @Test - void stress_test() { + void scaling_test() { //Generate 18 outlets, 5 minute data, for 1 year //location: keystone dam //figure out how big the payload is. From fd765e9108aa60630389ebed2da1b2159a082096 Mon Sep 17 00:00:00 2001 From: Ryan Miles Date: Tue, 24 Sep 2024 09:59:09 -0700 Subject: [PATCH 4/4] Renaming stress test to scale test --- .../{GateChangeStressTest.java => GateChangeScaleTest.java} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename cwms-data-api/src/test/java/cwms/cda/api/location/kind/{GateChangeStressTest.java => GateChangeScaleTest.java} (99%) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeScaleTest.java similarity index 99% rename from cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java rename to cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeScaleTest.java index bf5dc8ca8..b94f60315 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeStressTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/location/kind/GateChangeScaleTest.java @@ -22,7 +22,6 @@ import com.google.common.flogger.FluentLogger; import cwms.cda.api.Controllers; -import cwms.cda.api.enums.Unit; import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.errors.NotFoundException; import cwms.cda.data.dao.LocationsDaoImpl; @@ -64,7 +63,7 @@ import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.*; -class GateChangeStressTest extends BaseOutletDaoIT { +class GateChangeScaleTest extends BaseOutletDaoIT { private static final FluentLogger LOGGER = FluentLogger.forEnclosingClass(); private static final CwmsId PROJECT_ID = new CwmsId.Builder().withOfficeId(OFFICE_ID).withName("PROJECT3").build();