From acc4f377f7451cb47bc0413b5573081458e1fc54 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Tue, 22 Oct 2024 09:12:17 -0700 Subject: [PATCH 01/24] Implemented data entry data option for TS data retrieval, serialization bug in progress --- .../cwms/cda/api/TimeSeriesController.java | 16 ++- .../cda/data/dao/LocationLevelsDaoImpl.java | 2 +- .../java/cwms/cda/data/dao/TimeSeriesDao.java | 2 +- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 63 ++++++++-- .../java/cwms/cda/data/dto/TimeSeries.java | 39 +++++- .../cda/api/TimeSeriesControllerTest.java | 52 +++++++- .../cda/api/TimeseriesControllerTestIT.java | 112 ++++++++++++++++-- .../cwms/cda/data/dao/TimeSeriesDaoTest.java | 23 +--- .../cda/formatters/TimeSeriesTestBase.java | 2 +- 9 files changed, 251 insertions(+), 60 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index bf6b08246..ed6c8afd4 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -67,7 +67,6 @@ import io.javalin.plugin.openapi.annotations.OpenApiParam; import io.javalin.plugin.openapi.annotations.OpenApiRequestBody; import io.javalin.plugin.openapi.annotations.OpenApiResponse; -import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -84,7 +83,7 @@ public class TimeSeriesController implements CrudHandler { private static final Logger logger = Logger.getLogger(TimeSeriesController.class.getName()); - + private static final String INCLUDE_ENTRY_DATE = "include-entry-date"; public static final String TAG = "TimeSeries"; public static final String STORE_RULE_DESC = "The business rule to use " + "when merging the incoming with existing data\n" @@ -204,7 +203,7 @@ public void create(@NotNull Context ctx) { TimeSeries timeSeries = deserializeTimeSeries(ctx); dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (IOException | DataAccessException ex) { + } catch (DataAccessException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); @@ -382,6 +381,8 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "\n* `xml`" + "\n* `wml2` (only if name field is specified)" + "\n* `json` (default)"), + @OpenApiParam(name = INCLUDE_ENTRY_DATE, type = Boolean.class, description = "Specifies " + + "whether to include the data entry date in the response. Default is false."), @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " @@ -431,6 +432,9 @@ public void getAll(@NotNull Context ctx) { ZonedDateTime versionDate = queryParamAsZdt(ctx, VERSION_DATE); + boolean includeEntryDate = ctx.queryParamAsClass(INCLUDE_ENTRY_DATE, Boolean.class) + .getOrDefault(false); + // The following parameters are only used for jsonv2 and xmlv2 String cursor = queryParamAsClass(ctx, new String[]{PAGE, CURSOR}, String.class, "", metrics, name(TimeSeriesController.class.getName(), @@ -463,7 +467,7 @@ public void getAll(@NotNull Context ctx) { String office = requiredParam(ctx, OFFICE); TimeSeries ts = dao.getTimeseries(cursor, pageSize, names, office, unit, - beginZdt, endZdt, versionDate, trim.getOrDefault(true)); + beginZdt, endZdt, versionDate, trim.getOrDefault(true), includeEntryDate); results = Formats.format(contentType, ts); @@ -573,14 +577,14 @@ public void update(@NotNull Context ctx, @NotNull String id) { dao.store(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (IOException | DataAccessException ex) { + } catch (DataAccessException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); } } - private TimeSeries deserializeTimeSeries(Context ctx) throws IOException { + private TimeSeries deserializeTimeSeries(Context ctx) { String contentTypeHeader = ctx.req.getContentType(); ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeries.class); return Formats.parseContent(contentType, ctx.bodyAsInputStream(), TimeSeries.class); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java index 90fbcd55d..82b76ffc6 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java @@ -638,7 +638,7 @@ private static TimeSeries buildTimeSeries(ILocationLevelRef levelRef, Interval i if (qualityCode != null) { quality = qualityCode.intValue(); } - timeSeries.addValue(dateTime, value, quality); + timeSeries.addValue(dateTime, value, quality, null); } return timeSeries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 8750b24f3..29be65bb2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -27,7 +27,7 @@ void store(TimeSeries timeSeries, boolean createAsLrts, TimeSeries getTimeseries(String cursor, int pageSize, String names, String office, String unit, ZonedDateTime begin, ZonedDateTime end, - ZonedDateTime versionDate, boolean trim); + ZonedDateTime versionDate, boolean trim, boolean includeEntryDate); String getTimeseries(String format, String names, String office, String unit, String datum, ZonedDateTime begin, ZonedDateTime end, ZoneId timezone); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 30adf193c..1969a8dd7 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -64,6 +64,7 @@ import org.jooq.Record; import org.jooq.Record1; import org.jooq.Record3; +import org.jooq.Record4; import org.jooq.Record7; import org.jooq.Result; import org.jooq.SQL; @@ -166,7 +167,7 @@ public String getTimeseries(String format, String names, String office, String u public TimeSeries getTimeseries(String page, int pageSize, String names, String office, String units, ZonedDateTime beginTime, ZonedDateTime endTime, - ZonedDateTime versionDate, boolean shouldTrim) { + ZonedDateTime versionDate, boolean shouldTrim, boolean includeEntryDate) { TimeSeries retVal = null; String cursor = null; Timestamp tsCursor = null; @@ -238,7 +239,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // put all those columns together as "valid" CommonTableExpression<Record7<BigDecimal, String, String, String, String, BigDecimal, - String>> valid = + String>> valid = name("valid").fields("tscode", "tsid", "office_id", "loc_part", "units", "interval", "parm_part") .as( @@ -250,7 +251,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String unit.as("units"), ival.as("interval"), param.as("parm_part") - ).from(validTs) ); @@ -370,6 +370,8 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String ); }); + Field<Timestamp> dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + if (pageSize != 0) { SelectConditionStep<Record3<Timestamp, Double, BigDecimal>> query = dsl.select( @@ -392,14 +394,55 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String query.limit(DSL.val(pageSize + 1)); } - logger.fine(() -> query.getSQL(ParamType.INLINED)); + SelectConditionStep<Record3<Timestamp, Double, BigDecimal>> finalQuery = query; + logger.fine(() -> finalQuery.getSQL(ParamType.INLINED)); - query.forEach(tsRecord -> timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue() - ) - ); + if (includeEntryDate) { + SelectConditionStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( + dateTimeCol, + valueCol, + qualityNormCol, + dataEntryDate + ) + .from(AV_TSV_DQU.AV_TSV_DQU) + .where(dateTimeCol + .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.nvl(DSL.val(tsCursor == null ? null : + tsCursor.toInstant().toEpochMilli()), + DSL.val(beginTime.toInstant().toEpochMilli()))))) + .and(dateTimeCol + .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(endTime.toInstant().toEpochMilli()))) + .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) + .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) + .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) +// .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : +// Timestamp.from(versionDate.toInstant()))) + ); + + if (pageSize > 0) { + query2.limit(DSL.val(pageSize + 1)); + } + query2.forEach(tsRecord -> { + assert timeseries != null; + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate) + ); + }); + } else { + query.forEach(tsRecord -> { + assert timeseries != null; + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + null + ); + }); + } retVal = timeseries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index d14cc9108..eb092093a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat.Shape; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; @@ -23,9 +24,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Objects; @JsonRootName("timeseries") @JsonPropertyOrder(alphabetic = true) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) @FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) @FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) @@ -193,7 +196,7 @@ public List<Column> getValueColumnsJSON() { return getColumnDescriptor(); } - public boolean addValue(Timestamp dateTime, Double value, int qualityCode) { + public boolean addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set if ((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); @@ -202,7 +205,11 @@ public boolean addValue(Timestamp dateTime, Double value, int qualityCode) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); return false; } else { - return values.add(new Record(dateTime, value, qualityCode)); + if (dataEntryDate != null) { + return values.add(new Record(dateTime, value, qualityCode).withDataEntryDate(dataEntryDate)); + } else { + return values.add(new Record(dateTime, value, qualityCode)); + } } } @@ -239,6 +246,7 @@ private List<Column> getColumnDescriptor() { + "placeholder which can be important in irregular and psuedo regular timeseries." ) ) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @@ -252,6 +260,11 @@ public static class Record { @JsonProperty(value = "quality-code", index = 2) int qualityCode; +// @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate = null; + @SuppressWarnings("unused") // required so JAXB can initialize and marshal private Record() {} @@ -261,6 +274,12 @@ public Record(Timestamp dateTime, Double value, int qualityCode) { this.qualityCode = qualityCode; } + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + protected Record withDataEntryDate(Timestamp dataEntryDate) { + this.dataEntryDate = dataEntryDate; + return this; + } + // When serialized, the value is unix epoch at UTC. public Timestamp getDateTime() { return dateTime; @@ -274,6 +293,10 @@ public int getQualityCode() { return qualityCode; } + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + @Override public boolean equals(Object o) { @@ -286,17 +309,20 @@ public boolean equals(Object o) return false; } - final Record record = (Record) o; + final Record tsRecord = (Record) o; - if(getQualityCode() != record.getQualityCode()) + if(getQualityCode() != tsRecord.getQualityCode()) { return false; } - if(getDateTime() != null ? !getDateTime().equals(record.getDateTime()) : record.getDateTime() != null) + if(getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { return false; } - return getValue() != null ? getValue().equals(record.getValue()) : record.getValue() == null; + if (!Objects.equals(dataEntryDate, tsRecord.dataEntryDate)) { + return false; + } + return getValue() != null ? getValue().equals(tsRecord.getValue()) : tsRecord.getValue() == null; } @Override @@ -305,6 +331,7 @@ public int hashCode() int result = getDateTime() != null ? getDateTime().hashCode() : 0; result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); result = 31 * result + getQualityCode(); + result = 31 * result + (dataEntryDate != null ? dataEntryDate.hashCode() : 0); return result; } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index d528fb93e..5142a9530 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -22,11 +22,11 @@ import io.javalin.core.util.Header; import io.javalin.http.Context; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.Duration; +import java.time.Instant; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; @@ -67,7 +67,7 @@ void testDaoMock() throws JsonProcessingException { when( dao.getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true) )).thenReturn(expected); + isNotNull(), isNotNull(), isNull(), eq(true), eq(false))).thenReturn(expected); // build mock request and response @@ -113,7 +113,7 @@ protected TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { // Check that the controller accessed our mock dao in the expected way verify(dao, times(1)). getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true));// + isNotNull(), isNotNull(), isNull(), eq(true), eq(false));// // Make sure controller thought it was happy verify(response).setStatus(200); @@ -152,6 +152,19 @@ void testDeserializeTimeSeries(String format) { assertSimilar(fakeTs, ts2); } + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testDeserializeTimeSeriesWithEntryDate(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + @Test void testDeserializeTimeSeriesXmlUTC() { TimeZone aDefault = TimeZone.getDefault(); @@ -227,7 +240,38 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); + next = next.plus(minutes, ChronoUnit.MINUTES); + } + return ts; + } + + @NotNull + private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); + + long diff = end.toEpochSecond() - start.toEpochSecond(); + assertEquals(3600, diff); // just to make sure I've got the date parsing thing right. + + int minutes = 15; + int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? + // Also, should end be the last point or the next interval? + + TimeSeries ts = new TimeSeries(null, + -1, + 0, + tsId, + officeId, + start, + end, + "m", + Duration.ofMinutes(minutes)); + + ZonedDateTime next = start; + for(int i = 0; i < count; i++) { + Timestamp dateTime = Timestamp.from(next.toInstant()); + ts.addValue(dateTime, (double) i, 0, Timestamp.from(Instant.now())); next = next.plus(minutes, ChronoUnit.MINUTES); } return ts; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index c13a9ab1e..2135b45ed 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -21,6 +21,7 @@ import io.restassured.filter.log.LogDetail; import io.restassured.path.json.config.JsonPathConfig; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZonedDateTime; @@ -75,7 +76,6 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization",user.toHeaderValue()) .queryParam("office",officeId) .queryParam("units","cfs") @@ -173,7 +173,7 @@ void test_lrl_1day_bad_units() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset_bad_units.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -211,7 +211,7 @@ void test_lrl_1day_malicious_units() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset_malicious_units.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -242,6 +242,92 @@ void test_lrl_1day_malicious_units() throws Exception { } + @Test + void test_include_data_entry_date() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + final String includeDataEntryDate = "include-entry-date"; + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/num_ts_create2.json"); + assertNotNull(resource); + + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // 1675335600000 is Thursday, February 2, 2023 11:00:00 AM + // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM + + // get it back with the data entry date + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "CFS") + .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") + .queryParam(Controllers.END, "2010-02-03T11:00:00Z") + .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") + .queryParam(includeDataEntryDate, true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(4)) + .body("values[0][1]", equalTo(4.0F)) + .body("values[0].size()", equalTo(4)); + + // get it back without the data entry date + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "CFS") + .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") + .queryParam(Controllers.END, "2010-02-03T11:00:00Z") + .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(4)) + .body("values[0][1]", equalTo(4.0F)) + .body("values[0].size()", equalTo(3)); + } + @Test void test_delete_ts() throws Exception { @@ -251,7 +337,7 @@ void test_delete_ts() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -330,7 +416,7 @@ void test_no_office_permissions() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/timeseries/no_office_perms.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -385,7 +471,7 @@ void test_v1_cant_trim() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -422,7 +508,7 @@ void test_v1_cant_version() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -461,7 +547,7 @@ void test_v2_cant_datum() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -499,7 +585,7 @@ void test_lrl_trim() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -601,7 +687,7 @@ void test_big_create() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); String giantString = buildBigString(tsData, 200000); // 200k points looked like about 6MB. @@ -681,7 +767,7 @@ private String buildBigString(String tsData, int count) throws JsonProcessingExc StringBuilder sb = new StringBuilder(); for (int i = 0; i < count; i++) { long time = start2 + (diff * (i+1)); - sb.append(String.format(",\n [ %d, %d, %d]", time, count, 0)); + sb.append(String.format(",%n [ %d, %d, %d]", time, count, 0)); } return prefix + sb + "\n ]\n}"; @@ -694,7 +780,7 @@ void test_daylight_saving_retrieve()throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1hour.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); int count = 365 * 24 * 5; // 5 years of hourly data (43.8k points) @@ -814,7 +900,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java index 46e96baf5..5f9601ff3 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java @@ -1,6 +1,5 @@ package cwms.cda.data.dao; -import java.math.BigDecimal; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -24,7 +23,6 @@ import usace.cwms.db.dao.util.CwmsDatabaseVersionInfo; import usace.cwms.db.dao.util.TimeValueQuality; import usace.cwms.db.jooq.JooqCwmsDatabaseVersionInfoFactory; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2; import usace.cwms.db.jooq.codegen.tables.AV_LOC; import usace.cwms.db.jooq.dao.CwmsDbLocJooq; import usace.cwms.db.jooq.dao.CwmsDbTsJooq; @@ -114,7 +112,7 @@ public class TimeSeriesDaoTest @Test - public void testCreateEmpty() throws Exception + void testCreateEmpty() throws Exception { String officeId = "LRL"; @@ -144,7 +142,7 @@ public void testCreateEmpty() throws Exception } @Test - public void testCreateWithData() throws Exception + void testCreateWithData() throws Exception { String officeId = "LRL"; @@ -153,7 +151,6 @@ public void testCreateWithData() throws Exception DSLContext lrl = getDslContext(connection, officeId); TimeSeriesDao dao = new TimeSeriesDaoImpl(lrl); - Calendar instance = Calendar.getInstance(); String tsId = TIME_SERIES_ID; // Do I need to somehow check whether the location exists? Its not going to exist if I add the millis to it... if(!locationExists(connection, "RYAN3")) @@ -177,7 +174,7 @@ public void testCreateWithData() throws Exception for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.valueOf(next.toLocalDateTime()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); next = next.plus(minutes, ChronoUnit.MINUTES); } @@ -209,17 +206,8 @@ private void storeLocation(Connection connection, String officeId, String locati locationId, null, null, true, true); } - private BigDecimal retrieveTsCode(Connection connection, String tsId) throws Exception - { - BigDecimal bigD = DSL.using(connection).select(AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE).from( - AV_CWMS_TS_ID2.AV_CWMS_TS_ID2).where(AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.CWMS_TS_ID.eq(tsId)).fetchOptional( - AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE).orElse(null); - - return bigD; - } - @Test - public void testTimeSeriesStoreRetrieve() throws Exception + void testTimeSeriesStoreRetrieve() throws Exception { Connection connection = getConnection(); @@ -269,11 +257,10 @@ private void createTs(CwmsDbTsJooq cwmsTsJdbc, Connection connection) throws SQL } @Test - public void testVersion() throws SQLException + void testVersion() throws SQLException { JooqCwmsDatabaseVersionInfoFactory fac = new JooqCwmsDatabaseVersionInfoFactory(); - String officeId = "LRL"; try(Connection connection = getConnection()) { CwmsDatabaseVersionInfo info = fac.retrieveVersionInfo(connection); diff --git a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java index 3c50b8142..d195bde15 100644 --- a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java +++ b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java @@ -14,7 +14,7 @@ public abstract class TimeSeriesTestBase { protected TimeSeries getTimeSeries() { TimeSeries ts = new TimeSeries(null, -1, 0, "Test.Test.Elev.0.0.RAW", "SPK", ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"), ZonedDateTime.parse("2021-06-22T08:00:00-07:00[PST8PDT]"), null, Duration.ZERO); - ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0); + ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0, null); return ts; } From 23e38f6587649948db0abc3e7d5e9a381be2e901 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Fri, 25 Oct 2024 11:51:46 -0700 Subject: [PATCH 02/24] Created subclass of TimeSeries Record with custom deserializer to handle custom output, adds data entry date support --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 4 +- .../java/cwms/cda/data/dto/TimeSeries.java | 106 +++++++++--------- .../data/dto/TimeSeriesRecordWithDate.java | 81 +++++++++++++ .../TimeSeriesRecordDeserializer.java | 74 ++++++++++++ .../cda/api/TimeSeriesControllerTest.java | 67 +++++++++-- .../cda/api/TimeseriesControllerTestIT.java | 16 +-- 6 files changed, 271 insertions(+), 77 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 1969a8dd7..0049c4a8c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -416,8 +416,8 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) -// .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : -// Timestamp.from(versionDate.toInstant()))) + .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : + Timestamp.from(versionDate.toInstant()))) ); if (pageSize > 0) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index eb092093a..df3ae7461 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -4,16 +4,20 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat.Shape; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; +import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; @@ -24,7 +28,6 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; -import java.util.Objects; @JsonRootName("timeseries") @JsonPropertyOrder(alphabetic = true) @@ -44,10 +47,10 @@ public class TimeSeries extends CwmsDTOPaginated { @Schema(description = "The units of the time series data",required = true) String units; - @Schema(description = "The version type for the time series being queried. Can be in the form of MAX_AGGREGATE, SINGLE_VERSION, or UNVERSIONED. " + - "MAX_AGGREGATE will get the latest version date value for each value in the date range. SINGLE_VERSION must be called with a valid " + - "version date and will return the values for the version date provided. UNVERSIONED return values from an unversioned time series. " + - "Note that SINGLE_VERSION requires a valid version date while MAX_AGGREGATE and UNVERSIONED each require a null version date.") + @Schema(description = "The version type for the time series being queried. Can be in the form of MAX_AGGREGATE, SINGLE_VERSION, or UNVERSIONED. " + + "MAX_AGGREGATE will get the latest version date value for each value in the date range. SINGLE_VERSION must be called with a valid " + + "version date and will return the values for the version date provided. UNVERSIONED return values from an unversioned time series. " + + "Note that SINGLE_VERSION requires a valid version date while MAX_AGGREGATE and UNVERSIONED each require a null version date.") @JsonFormat(shape = Shape.STRING) VersionType dateVersionType; @@ -111,7 +114,7 @@ public TimeSeries(String page, int pageSize, Integer total, String name, String this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, null, null, null); } - public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, VersionType dateVersionType){ + public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, VersionType dateVersionType) { this(page, pageSize, total, name, officeId, begin, end, units, interval, info, null, null, versionDate, dateVersionType); } @@ -161,7 +164,8 @@ public ZonedDateTime getEnd() { } // Use the array shape to optimize data transfer to client - @JsonFormat(shape=JsonFormat.Shape.ARRAY) + @JsonFormat(shape = JsonFormat.Shape.ARRAY) + @JsonDeserialize(contentUsing = TimeSeriesRecordDeserializer.class) public List<Record> getValues() { return values; } @@ -171,8 +175,7 @@ public List<Record> getXmlValues() { return values; } - public VerticalDatumInfo getVerticalDatumInfo() - { + public VerticalDatumInfo getVerticalDatumInfo() { return verticalDatumInfo; } @@ -188,34 +191,35 @@ public ZonedDateTime getVersionDate() { return versionDate; } - public VersionType getDateVersionType() { return dateVersionType; } + public VersionType getDateVersionType() { + return dateVersionType; + } @JsonProperty(value = "value-columns") @Schema(name = "value-columns", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { - return getColumnDescriptor(); + return getColumnDescriptor((values != null && !values.isEmpty()) + && values.get(0) instanceof TimeSeriesRecordWithDate); } - public boolean addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set if ((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); } if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); - return false; } else { if (dataEntryDate != null) { - return values.add(new Record(dateTime, value, qualityCode).withDataEntryDate(dataEntryDate)); + values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); } else { - return values.add(new Record(dateTime, value, qualityCode)); + values.add(new Record(dateTime, value, qualityCode)); } } } - private List<Column> getColumnDescriptor() { + private List<Column> getColumnDescriptor(boolean includeDataEntryDate) { List<Column> columns = new ArrayList<>(); - for (Field f: Record.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); if(field != null) { @@ -224,7 +228,16 @@ private List<Column> getColumnDescriptor() { columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - + if (includeDataEntryDate) { + for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if(field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + } return columns; } @@ -234,10 +247,10 @@ private List<Column> getColumnDescriptor() { schema = @Schema( name = "TimeSeries.Record", description = "A representation of a time-series record in the form [dateTime, value, qualityCode]", - type="array" + type = "array" ), arraySchema = @Schema( - type="array", + type = "array", example = "[1509654000000, 54.3, 0]", description = "Time is Milliseconds since the UNIX Epoch. Value is Double (for missing data you " + "can use null, or -Float.MAX_VALUE (-340282346638528859811704183484516925440), " @@ -246,7 +259,16 @@ private List<Column> getColumnDescriptor() { + "placeholder which can be important in irregular and psuedo regular timeseries." ) ) - @JsonInclude(JsonInclude.Include.NON_DEFAULT) + + // This class is used to deserialize the time-series data JSON into an object + // Solves the issue of the deserializer getting stuck in a loop + // and throwing a StackOverflowError when trying to handle the Record class directly + @JsonDeserialize(using = JsonDeserializer.None.class) + public static final class RecordChild extends Record { + } + + @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) + @JsonIgnoreProperties(ignoreUnknown = true) public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @@ -260,11 +282,6 @@ public static class Record { @JsonProperty(value = "quality-code", index = 2) int qualityCode; -// @JsonProperty(value = "data-entry-date", index = 3) - @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - Timestamp dataEntryDate = null; - @SuppressWarnings("unused") // required so JAXB can initialize and marshal private Record() {} @@ -274,12 +291,6 @@ public Record(Timestamp dateTime, Double value, int qualityCode) { this.qualityCode = qualityCode; } - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - protected Record withDataEntryDate(Timestamp dataEntryDate) { - this.dataEntryDate = dataEntryDate; - return this; - } - // When serialized, the value is unix epoch at UTC. public Timestamp getDateTime() { return dateTime; @@ -293,51 +304,36 @@ public int getQualityCode() { return qualityCode; } - public Timestamp getDataEntryDate() { - return dataEntryDate; - } - @Override - public boolean equals(Object o) - { - if(this == o) - { + public boolean equals(Object o) { + if (this == o) { return true; } - if(o == null || getClass() != o.getClass()) - { + if (o == null || getClass() != o.getClass()) { return false; } final Record tsRecord = (Record) o; - if(getQualityCode() != tsRecord.getQualityCode()) - { - return false; - } - if(getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) - { + if (getQualityCode() != tsRecord.getQualityCode()) { return false; } - if (!Objects.equals(dataEntryDate, tsRecord.dataEntryDate)) { + if (getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { return false; } return getValue() != null ? getValue().equals(tsRecord.getValue()) : tsRecord.getValue() == null; } @Override - public int hashCode() - { + public int hashCode() { int result = getDateTime() != null ? getDateTime().hashCode() : 0; result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); result = 31 * result + getQualityCode(); - result = 31 * result + (dataEntryDate != null ? dataEntryDate.hashCode() : 0); return result; } @Override - public String toString() - { + public String toString() { return "Record{" + "dateTime=" + dateTime + ", value=" + value + ", qualityCode=" + qualityCode + '}'; } } @@ -349,7 +345,7 @@ private static class Column { public final Class<?> datatype; // JAXB seems to need a default ctor - private Column(){ + private Column() { this(null, 0,null); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java new file mode 100644 index 000000000..21066175d --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java @@ -0,0 +1,81 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.data.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.swagger.v3.oas.annotations.media.Schema; +import java.sql.Timestamp; +import java.util.Objects; + +/** + * TimeSeriesRecordWithDate is a subclass of TimeSeries.Record that includes a data entry date. + * The data entry date is the date that the data was entered into the database. + */ +@JsonDeserialize(using = JsonDeserializer.None.class) +public final class TimeSeriesRecordWithDate extends TimeSeries.Record { + @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate; + + // Default constructor for Jackson Deserialization + public TimeSeriesRecordWithDate() { + super(null, null, 0); + } + + public TimeSeriesRecordWithDate(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + super(dateTime, value, qualityCode); + this.dataEntryDate = dataEntryDate; + } + + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + TimeSeriesRecordWithDate that = (TimeSeriesRecordWithDate) o; + return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getDataEntryDate()); + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java new file mode 100644 index 000000000..e149ae441 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -0,0 +1,74 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.formatters.json.adapters; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import java.io.IOException; +import java.sql.Timestamp; +import java.time.Instant; + +/** + * A time-series record deserializer for properly deserializing JSON data. + * Requires {@link cwms.cda.data.dto.TimeSeries.RecordChild} class to avoid + * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer + * getting stuck in a loop if the Record class is used directly. + * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. + */ +public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { + @Override + public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.readValueAsTree(); + if (node.get("data-entry-date") != null) { + return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); + } + String nodeString = node.toString(); + if (nodeString.startsWith("[")) { + nodeString = nodeString.substring(1, nodeString.length() - 1); + String[] valList = nodeString.split(","); + if (valList.length == 4) { + Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + double value = Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + Timestamp entryDate = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[3]))); + return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); + } else if (valList.length == 3) { + Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + double value = Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + return new TimeSeries.Record(dateTime, value, quality); + } else { + throw new IOException("Invalid TimeSeries Record format"); + } + } + return jsonParser.getCodec().treeToValue(node, TimeSeries.RecordChild.class); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 5142a9530..cde473876 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -1,8 +1,6 @@ package cwms.cda.api; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNotNull; import static org.mockito.ArgumentMatchers.isNull; @@ -16,6 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -28,7 +27,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,9 +41,6 @@ import org.junit.jupiter.params.provider.ValueSource; class TimeSeriesControllerTest extends ControllerTest { - - - @Test void testDaoMock() throws JsonProcessingException { String officeId = "LRL"; @@ -134,11 +129,63 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { // Make sure ts we got back resembles the fakeTS our mock dao was supposed to return. assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); assertEquals(expected.getName(), actual.getName(), "names did not match"); - assertEquals(expected.getValues(), actual.getValues(), "values did not match"); + assertRecordsMatch(expected.getValues(), actual.getValues()); assertTrue(expected.getBegin().isEqual(actual.getBegin()), "begin dates not equal"); assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } + private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSeries.Record> actual) { + for (int i = 0; i < expected.size(); i++) { + if (expected.get(i) instanceof TimeSeriesRecordWithDate) { + if (!(actual.get(i) instanceof TimeSeriesRecordWithDate)) { + throw new AssertionError("Expected TimeSeriesRecordWithDate but got " + actual.get(i).getClass().getName()); + } + TimeSeriesRecordWithDate expectedRecord = new TimeSeriesRecordWithDate(expected.get(i).getDateTime(), + expected.get(i).getValue(), expected.get(i).getQualityCode(), + ((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate()); + TimeSeriesRecordWithDate actualRecord = new TimeSeriesRecordWithDate(actual.get(i).getDateTime(), + actual.get(i).getValue(), actual.get(i).getQualityCode(), + ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate()); + assertEquals(expectedRecord.getDataEntryDate(), + actualRecord.getDataEntryDate(), "Entry dates did not match"); + } + assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); + assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); + assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); + } + } + + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testSerializeTimeSeries(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeries(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + assertNotNull(formatted); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testSerializeTimeSeriesWithDataEntryDate(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + assertEquals(4, fakeTs.getValueColumnsJSON().size()); + assertInstanceOf(TimeSeriesRecordWithDate.class, fakeTs.getValues().get(0)); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + assertNotNull(formatted); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + + @ParameterizedTest @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) void testDeserializeTimeSeries(String format) { @@ -241,7 +288,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); ts.addValue(dateTime, (double) i, 0, null); - next = next.plus(minutes, ChronoUnit.MINUTES); + next = next.plusMinutes(minutes); } return ts; } @@ -272,7 +319,7 @@ private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); ts.addValue(dateTime, (double) i, 0, Timestamp.from(Instant.now())); - next = next.plus(minutes, ChronoUnit.MINUTES); + next = next.plusMinutes(minutes); } return ts; } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 2135b45ed..486b763c2 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -43,7 +43,7 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -109,7 +109,7 @@ void test_lrl_1day() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -143,7 +143,6 @@ void test_lrl_1day() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -328,7 +327,6 @@ void test_include_data_entry_date() throws Exception { .body("values[0].size()", equalTo(3)); } - @Test void test_delete_ts() throws Exception { ObjectMapper mapper = new ObjectMapper(); @@ -628,7 +626,6 @@ void test_lrl_trim() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -655,7 +652,6 @@ void test_lrl_trim() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -956,13 +952,13 @@ enum GetAllTest XMLV2(Formats.XMLV2, Formats.XMLV2), ; - final String _accept; - final String _expectedContentType; + final String accept; + final String expectedContentType; GetAllTest(String accept, String expectedContentType) { - _accept = accept; - _expectedContentType = expectedContentType; + this.accept = accept; + this.expectedContentType = expectedContentType; } } } From 61863629ca529368204a41da89bf9f0b1f438d43 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Mon, 4 Nov 2024 11:47:31 -0800 Subject: [PATCH 03/24] Fixes build error --- .../test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index e34953560..73e7ca651 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -247,7 +247,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = START; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); next = next.plusMinutes(minutes); } return ts; From b19d97236b0527102864c905b0f5ac7abe48745e Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Mon, 4 Nov 2024 14:09:16 -0800 Subject: [PATCH 04/24] Fixes test case failure --- .../formatters/json/adapters/TimeSeriesRecordDeserializer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java index e149ae441..a838018b1 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -62,6 +62,9 @@ public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationConte return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); } else if (valList.length == 3) { Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + if (valList[1].equalsIgnoreCase("null") || valList[1].isEmpty()) { + return new TimeSeries.Record(dateTime, null, Integer.parseInt(valList[2])); + } double value = Double.parseDouble(valList[1]); int quality = Integer.parseInt(valList[2]); return new TimeSeries.Record(dateTime, value, quality); From e460d5f4d6917853a15b64695f5e59bacb1548f5 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Tue, 5 Nov 2024 14:52:19 -0800 Subject: [PATCH 05/24] 634 TimeSeries Subclass update --- .../cda/data/dao/LocationLevelsDaoImpl.java | 4 +- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 231 ++++++++++-------- .../java/cwms/cda/data/dto/TimeSeries.java | 45 +--- .../cwms/cda/data/dto/TimeSeriesWithDate.java | 179 ++++++++++++++ .../TimeSeriesRecordDeserializer.java | 77 ------ .../cda/api/TimeSeriesControllerTest.java | 58 +++-- .../cda/api/TimeSeriesRecentControllerIT.java | 2 +- .../cwms/cda/data/dao/TimeSeriesDaoTest.java | 6 +- .../cda/formatters/TimeSeriesTestBase.java | 2 +- 9 files changed, 354 insertions(+), 250 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java delete mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java index 82b76ffc6..037fd3830 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java @@ -249,7 +249,7 @@ private static SEASONAL_VALUE_TAB_T getSeasonalValues(LocationLevel locationLeve SEASONAL_VALUE_TAB_T pSeasonalValues = null; if (seasonalValues != null && !seasonalValues.isEmpty()) { pSeasonalValues = new SEASONAL_VALUE_TAB_T(); - for(SeasonalValueBean seasonalValue : seasonalValues) { + for (SeasonalValueBean seasonalValue : seasonalValues) { SEASONAL_VALUE_T seasonalValueT = new SEASONAL_VALUE_T(); seasonalValueT.setOFFSET_MINUTES(toBigDecimal(seasonalValue.getOffsetMinutes())); if (seasonalValue.getOffsetMonths() != null) { @@ -638,7 +638,7 @@ private static TimeSeries buildTimeSeries(ILocationLevelRef levelRef, Interval i if (qualityCode != null) { quality = qualityCode.intValue(); } - timeSeries.addValue(dateTime, value, quality, null); + timeSeries.addValue(dateTime, value, quality); } return timeSeries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 0049c4a8c..45154ad83 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -24,6 +24,7 @@ import cwms.cda.data.dto.RecentValue; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesExtents; +import cwms.cda.data.dto.TimeSeriesWithDate; import cwms.cda.data.dto.Tsv; import cwms.cda.data.dto.TsvDqu; import cwms.cda.data.dto.TsvId; @@ -168,7 +169,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String String units, ZonedDateTime beginTime, ZonedDateTime endTime, ZonedDateTime versionDate, boolean shouldTrim, boolean includeEntryDate) { - TimeSeries retVal = null; String cursor = null; Timestamp tsCursor = null; Integer total = null; @@ -179,7 +179,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String logger.fine("Decoded cursor"); logger.finest(() -> { StringBuilder sb = new StringBuilder(); - for (String p: parts) { + for (String p : parts) { sb.append(p).append("\n"); } return sb.toString(); @@ -239,7 +239,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // put all those columns together as "valid" CommonTableExpression<Record7<BigDecimal, String, String, String, String, BigDecimal, - String>> valid = + String>> valid = name("valid").fields("tscode", "tsid", "office_id", "loc_part", "units", "interval", "parm_part") .as( @@ -259,9 +259,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field<Double> valueCol = field("VALUE", Double.class).as("VALUE"); Field<Integer> qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); - Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli(); String trim = formatBool(shouldTrim); @@ -278,18 +275,20 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String maxVersion = "T"; } + Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( + DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); // Now we're going to call the retrieve_ts_out_tab function to get the data and build an // internal table from it so we can manipulate it further // This code assumes the database timezone is in UTC (per Oracle recommendation) SQL retrieveSelectData = DSL.sql( - "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," - + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," - + "?,?,?,?,?," - + getVersionPart(versionDate) + ",?,?) ) retrieveTs", - tsId, unit, - beginTimeMilli, endTimeMilli, //tz hardcoded - trim, startInclusive, endInclusive, previous, next, - versionDateMilli, maxVersion, officeId); + "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," + + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," + + "?,?,?,?,?," + + getVersionPart(versionDate) + ",?,?) ) retrieveTs", + tsId, unit, + beginTimeMilli, endTimeMilli, //tz hardcoded + trim, startInclusive, endInclusive, previous, next, + versionDateMilli, maxVersion, officeId); Field<String> tzName = AV_CWMS_TS_ID2.TIME_ZONE_ID; @@ -352,27 +351,72 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String logger.fine(() -> metadataQuery.getSQL(ParamType.INLINED)); - VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); + TimeSeries timeseries = metadataQuery.fetchOne(tsMetadata -> { String vert = (String) tsMetadata.getValue("VERTICAL_DATUM"); VerticalDatumInfo verticalDatumInfo = parseVerticalDatumInfo(vert); - - return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", - Integer.class), tsMetadata.getValue("NAME", String.class), - tsMetadata.getValue("office_id", String.class), - beginTime, endTime, tsMetadata.getValue("units", String.class), - Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : - tsMetadata.getValue("interval", Long.class)), - verticalDatumInfo, - tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), - tsMetadata.getValue(tzName), - versionDate, finalDateVersionType - ); + VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); + if (!includeEntryDate) { + return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", + Integer.class), tsMetadata.getValue("NAME", String.class), + tsMetadata.getValue("office_id", String.class), + beginTime, endTime, tsMetadata.getValue("units", String.class), + Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : + tsMetadata.getValue("interval", Long.class)), + verticalDatumInfo, + tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), + tsMetadata.getValue(tzName), + versionDate, finalDateVersionType + ); + } else { + return new TimeSeriesWithDate(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", + Integer.class), tsMetadata.getValue("NAME", String.class), + tsMetadata.getValue("office_id", String.class), + beginTime, endTime, tsMetadata.getValue("units", String.class), + Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : + tsMetadata.getValue("interval", Long.class)), + verticalDatumInfo, + tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), + tsMetadata.getValue(tzName), + versionDate, finalDateVersionType + ); + } }); + if (includeEntryDate) { + timeseries = new TimeSeriesWithDate(timeseries); + } + + Field<Timestamp> dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + Condition whereCond = dateTimeCol + .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.nvl(DSL.val(tsCursor == null ? null : + tsCursor.toInstant().toEpochMilli()), + DSL.val(beginTime.toInstant().toEpochMilli())))) + .and(dateTimeCol + .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(endTime.toInstant().toEpochMilli()))) + .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) + .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) + .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit))); + TimeSeries retVal = null; if (pageSize != 0) { + if (versionDate != null) { + whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : + Timestamp.from(versionDate.toInstant()))); + } + + SelectConditionStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( + dateTimeCol, + valueCol, + qualityNormCol, + dataEntryDate + ) + .from(AV_TSV_DQU.AV_TSV_DQU) + .where(whereCond); + SelectConditionStep<Record3<Timestamp, Double, BigDecimal>> query = dsl.select( dateTimeCol, @@ -392,59 +436,29 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (pageSize > 0) { query.limit(DSL.val(pageSize + 1)); + query2.limit(DSL.val(pageSize + 1)); } - SelectConditionStep<Record3<Timestamp, Double, BigDecimal>> finalQuery = query; - logger.fine(() -> finalQuery.getSQL(ParamType.INLINED)); - if (includeEntryDate) { - SelectConditionStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( - dateTimeCol, - valueCol, - qualityNormCol, - dataEntryDate - ) - .from(AV_TSV_DQU.AV_TSV_DQU) - .where(dateTimeCol - .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.nvl(DSL.val(tsCursor == null ? null : - tsCursor.toInstant().toEpochMilli()), - DSL.val(beginTime.toInstant().toEpochMilli()))))) - .and(dateTimeCol - .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.val(endTime.toInstant().toEpochMilli()))) - .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) - .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) - .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) - .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : - Timestamp.from(versionDate.toInstant()))) - ); - - if (pageSize > 0) { - query2.limit(DSL.val(pageSize + 1)); - } - query2.forEach(tsRecord -> { - assert timeseries != null; - timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - tsRecord.getValue(dataEntryDate) - ); - }); + logger.fine(() -> query2.getSQL(ParamType.INLINED)); + final TimeSeriesWithDate timeSeries = new TimeSeriesWithDate(timeseries); + query2.forEach(tsRecord -> timeSeries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate) + )); + retVal = timeSeries; } else { - query.forEach(tsRecord -> { - assert timeseries != null; - timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - null - ); - }); + logger.fine(() -> query.getSQL(ParamType.INLINED)); + final TimeSeries finalTimeseries = timeseries; + query.forEach(tsRecord -> finalTimeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue() + )); + retVal = finalTimeseries; } - - retVal = timeseries; } return retVal; @@ -526,7 +540,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar String cursorOffice = null; Catalog.CatalogPage catPage = null; if (page == null || page.isEmpty()) { - CommonTableExpression<?> limiter = buildWithClause(inputParams, buildWhereConditions(inputParams), new ArrayList<>(), pageSize, true); + CommonTableExpression<?> limiter = buildWithClause(inputParams, buildWhereConditions(inputParams), + new ArrayList<>(), pageSize, true); SelectJoinStep<Record1<Integer>> totalQuery = dsl.with(limiter) .select(countDistinct(limiter.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.TS_CODE))) .from(limiter); @@ -575,7 +590,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .on(limiterCode .eq(AV_TS_EXTENTS_UTC.TS_CODE.coerce(limiterCode))); } - final SelectSeekStep2<?, String, String> overallQuery = tmpQuery.orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); + final SelectSeekStep2<?, String, String> overallQuery = tmpQuery.orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, + AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); logger.fine(() -> overallQuery.getSQL(ParamType.INLINED)); Result<?> result = overallQuery.fetch(); @@ -611,8 +627,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar } }); - List<? extends CatalogEntry> entries = tsIdExtentMap.entrySet().stream() - .map(e -> e.getValue().build()) + List<? extends CatalogEntry> entries = tsIdExtentMap.values().stream() + .map(TimeseriesCatalogEntry.Builder::build) .collect(Collectors.toList()); return new Catalog(catPage != null ? catPage.toString() : null, @@ -738,20 +754,23 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar TableLike<?> innerSelect = selectDistinct(selectFields) .from(fromTable) .where(whereConditions).and(DSL.and(pagingConditions)) - .orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID) + .orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, + AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID) .asTable("limiterInner"); if (forCount) { return name("limiter").as( select(asterisk()) .from(innerSelect) - .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) + .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), + innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) ); } else { return name("limiter").as( select(asterisk()) .from(innerSelect) .where(field("rownum").lessOrEqual(pageSize)) - .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) + .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), + innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) ); } } @@ -976,18 +995,18 @@ public List<RecentValue> findMostRecentsInRange(List<String> tsIds, Timestamp pa // Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp // and the generated sql refers to columns from the alias_??? table. Field[] queryFields = new Field[]{ - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), - unitField, - dateTimeField, - innerSelect.field(tsField) + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), + unitField, + dateTimeField, + innerSelect.field(tsField) }; SelectConditionStep<? extends Record> query = dsl.select(queryFields) @@ -1074,18 +1093,18 @@ public List<RecentValue> findRecentsInRange(String office, String categoryId, St Field<String> unit = innerSelect.field(tsvView.UNIT_ID); Field[] queryFields = new Field[]{ - innerSelect.field(tsvView.OFFICE_ID), - innerSelect.field(tsvView.TS_CODE), - innerSelect.field(tsvView.VERSION_DATE), - innerSelect.field(tsvView.DATA_ENTRY_DATE), - innerSelect.field(tsvView.VALUE), - innerSelect.field(tsvView.QUALITY_CODE), - innerSelect.field(tsvView.START_DATE), - innerSelect.field(tsvView.END_DATE), - dateTime, - unit, - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID), - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE)}; + innerSelect.field(tsvView.OFFICE_ID), + innerSelect.field(tsvView.TS_CODE), + innerSelect.field(tsvView.VERSION_DATE), + innerSelect.field(tsvView.DATA_ENTRY_DATE), + innerSelect.field(tsvView.VALUE), + innerSelect.field(tsvView.QUALITY_CODE), + innerSelect.field(tsvView.START_DATE), + innerSelect.field(tsvView.END_DATE), + dateTime, + unit, + innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID), + innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE)}; return dsl.select(queryFields) .from(innerSelect) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index df3ae7461..050682adf 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -9,15 +9,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; -import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; @@ -108,7 +105,7 @@ public class TimeSeries extends CwmsDTOPaginated { @SuppressWarnings("unused") // required so JAXB can initialize and marshal - private TimeSeries() {} + protected TimeSeries() {} public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval) { this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, null, null, null); @@ -165,7 +162,6 @@ public ZonedDateTime getEnd() { // Use the array shape to optimize data transfer to client @JsonFormat(shape = JsonFormat.Shape.ARRAY) - @JsonDeserialize(contentUsing = TimeSeriesRecordDeserializer.class) public List<Record> getValues() { return values; } @@ -198,11 +194,10 @@ public VersionType getDateVersionType() { @JsonProperty(value = "value-columns") @Schema(name = "value-columns", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { - return getColumnDescriptor((values != null && !values.isEmpty()) - && values.get(0) instanceof TimeSeriesRecordWithDate); + return getColumnDescriptor(); } - public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + public void addValue(Timestamp dateTime, Double value, int qualityCode) { // Set the current page, if not set if ((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); @@ -210,34 +205,20 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - if (dataEntryDate != null) { - values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); - } else { - values.add(new Record(dateTime, value, qualityCode)); - } + values.add(new Record(dateTime, value, qualityCode)); } } - private List<Column> getColumnDescriptor(boolean includeDataEntryDate) { + private List<Column> getColumnDescriptor() { List<Column> columns = new ArrayList<>(); for (Field f: Record.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); - if(field != null) { + if (field != null) { String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); int fieldIndex = field.index(); columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - if (includeDataEntryDate) { - for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { - JsonProperty field = f.getAnnotation(JsonProperty.class); - if(field != null) { - String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); - int fieldIndex = field.index(); - columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); - } - } - } return columns; } @@ -260,14 +241,6 @@ private List<Column> getColumnDescriptor(boolean includeDataEntryDate) { ) ) - // This class is used to deserialize the time-series data JSON into an object - // Solves the issue of the deserializer getting stuck in a loop - // and throwing a StackOverflowError when trying to handle the Record class directly - @JsonDeserialize(using = JsonDeserializer.None.class) - public static final class RecordChild extends Record { - } - - @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) @JsonIgnoreProperties(ignoreUnknown = true) public static class Record { // Explicitly set property order for array serialization @@ -283,9 +256,11 @@ public static class Record { int qualityCode; @SuppressWarnings("unused") // required so JAXB can initialize and marshal - private Record() {} + private Record() { + } public Record(Timestamp dateTime, Double value, int qualityCode) { + super(); this.dateTime = dateTime; this.value = value; this.qualityCode = qualityCode; @@ -339,7 +314,7 @@ public String toString() { } @Schema(hidden = true, name = "TimeSeries.Column", accessMode = Schema.AccessMode.READ_ONLY) - private static class Column { + protected static class Column { public final String name; public final int ordinal; public final Class<?> datatype; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java new file mode 100644 index 000000000..c35c3ab21 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java @@ -0,0 +1,179 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.data.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import cwms.cda.api.enums.VersionType; +import cwms.cda.formatters.Formats; +import cwms.cda.formatters.annotations.FormattableWith; +import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.xml.XMLv2; +import io.swagger.v3.oas.annotations.media.Schema; +import java.lang.reflect.Field; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) +@FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) +@FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) +public final class TimeSeriesWithDate extends TimeSeries { + + private List<TimeSeriesWithDate.Record> values; + + @Override + public List getValues() { + return values; + } + + TimeSeriesWithDate() { + super(); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(TimeSeries timeSeries) { + this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), timeSeries.getName(), + timeSeries.getOfficeId(), timeSeries.getBegin(), timeSeries.getEnd(), timeSeries.getUnits(), + timeSeries.getInterval(), timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), + timeSeries.getTimeZone(), timeSeries.getVersionDate(), timeSeries.getDateVersionType()); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, + ZonedDateTime begin, ZonedDateTime end, String units, Duration interval) { + this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, + null, null, null); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, + ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, + VersionType dateVersionType) { + this(page, pageSize, total, name, officeId, begin, end, units, interval, info, null, + null, versionDate, dateVersionType); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, + ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, Long intervalOffset, + String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { + super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, + timeZone, versionDate, dateVersionType); + values = new ArrayList<>(); + } + + public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + // Set the current page, if not set + if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { + page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); + } + if (pageSize > 0 && values.size() == pageSize) { + nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); + } else { + values.add(new Record(dateTime, value, qualityCode, dataEntryDate)); + } + } + + @Override + public List<Column> getValueColumnsJSON() { + return getColumnDescriptor(); + } + + private List<Column> getColumnDescriptor() { + List<Column> columns = new ArrayList<>(); + for (Field f: TimeSeries.Record.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + for (Field f: Record.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + + return columns; + } + + @JsonDeserialize(using = JsonDeserializer.None.class) + public static final class Record extends TimeSeries.Record { + @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate; + + // Default constructor for Jackson Deserialization + public Record() { + super(null, null, 0); + } + + public Record(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + super(dateTime, value, qualityCode); + this.dataEntryDate = dataEntryDate; + } + + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + Record that = (Record) o; + return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getDataEntryDate()); + } + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java deleted file mode 100644 index a838018b1..000000000 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * - * MIT License - * - * Copyright (c) 2024 Hydrologic Engineering Center - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE - * SOFTWARE. - */ - -package cwms.cda.formatters.json.adapters; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; -import java.io.IOException; -import java.sql.Timestamp; -import java.time.Instant; - -/** - * A time-series record deserializer for properly deserializing JSON data. - * Requires {@link cwms.cda.data.dto.TimeSeries.RecordChild} class to avoid - * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer - * getting stuck in a loop if the Record class is used directly. - * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. - */ -public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { - @Override - public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.readValueAsTree(); - if (node.get("data-entry-date") != null) { - return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); - } - String nodeString = node.toString(); - if (nodeString.startsWith("[")) { - nodeString = nodeString.substring(1, nodeString.length() - 1); - String[] valList = nodeString.split(","); - if (valList.length == 4) { - Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); - double value = Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - Timestamp entryDate = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[3]))); - return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); - } else if (valList.length == 3) { - Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); - if (valList[1].equalsIgnoreCase("null") || valList[1].isEmpty()) { - return new TimeSeries.Record(dateTime, null, Integer.parseInt(valList[2])); - } - double value = Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - return new TimeSeries.Record(dateTime, value, quality); - } else { - throw new IOException("Invalid TimeSeries Record format"); - } - } - return jsonParser.getCodec().treeToValue(node, TimeSeries.RecordChild.class); - } -} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index cde473876..839ea8a93 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,7 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import cwms.cda.data.dto.TimeSeriesWithDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -134,21 +134,27 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } + private void assertSimilarWithDate(TimeSeriesWithDate expected, TimeSeriesWithDate actual) + { + assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); + assertEquals(expected.getName(), actual.getName(), "names did not match"); + assertDateRecordsMatch(expected.getValues(), actual.getValues()); + assertTrue(expected.getBegin().isEqual(actual.getBegin()), "begin dates not equal"); + assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); + } + private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSeries.Record> actual) { for (int i = 0; i < expected.size(); i++) { - if (expected.get(i) instanceof TimeSeriesRecordWithDate) { - if (!(actual.get(i) instanceof TimeSeriesRecordWithDate)) { - throw new AssertionError("Expected TimeSeriesRecordWithDate but got " + actual.get(i).getClass().getName()); - } - TimeSeriesRecordWithDate expectedRecord = new TimeSeriesRecordWithDate(expected.get(i).getDateTime(), - expected.get(i).getValue(), expected.get(i).getQualityCode(), - ((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate()); - TimeSeriesRecordWithDate actualRecord = new TimeSeriesRecordWithDate(actual.get(i).getDateTime(), - actual.get(i).getValue(), actual.get(i).getQualityCode(), - ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate()); - assertEquals(expectedRecord.getDataEntryDate(), - actualRecord.getDataEntryDate(), "Entry dates did not match"); - } + assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); + assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); + assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); + } + } + + private void assertDateRecordsMatch(List<? extends TimeSeries.Record> expected, List<? extends TimeSeries.Record> actual) { + for (int i = 0; i < expected.size(); i++) { + assertEquals(((TimeSeriesWithDate.Record) expected.get(i)).getDataEntryDate(), + ((TimeSeriesWithDate.Record) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); @@ -174,15 +180,15 @@ void testSerializeTimeSeries(String format) { void testSerializeTimeSeriesWithDataEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); assertEquals(4, fakeTs.getValueColumnsJSON().size()); - assertInstanceOf(TimeSeriesRecordWithDate.class, fakeTs.getValues().get(0)); - ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + assertInstanceOf(TimeSeriesWithDate.Record.class, fakeTs.getValues().get(0)); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + assertSimilarWithDate(fakeTs, ts2); } @@ -204,12 +210,12 @@ void testDeserializeTimeSeries(String format) { void testDeserializeTimeSeriesWithEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + assertSimilarWithDate(fakeTs, ts2); } @Test @@ -287,14 +293,14 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plusMinutes(minutes); } return ts; } @NotNull - private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { + private TimeSeriesWithDate buildTimeSeriesWithEntryDate(String officeId, String tsId) { ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); @@ -305,7 +311,7 @@ private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeries ts = new TimeSeries(null, + TimeSeriesWithDate ts = new TimeSeriesWithDate(null, -1, 0, tsId, diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index 73e7ca651..e34953560 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -247,7 +247,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = START; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plusMinutes(minutes); } return ts; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java index 5f9601ff3..dc68ee4f8 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java @@ -13,6 +13,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import cwms.cda.data.dto.TimeSeriesWithDate; import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.impl.DSL; @@ -168,13 +169,13 @@ void testCreateWithData() throws Exception int count = 60 / 15; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeries ts = new TimeSeries(null, -1, 0, tsId, officeId, start, end, "m", Duration.ofMinutes(minutes)); + TimeSeriesWithDate ts = new TimeSeriesWithDate(null, -1, 0, tsId, officeId, start, end, "m", Duration.ofMinutes(minutes)); ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.valueOf(next.toLocalDateTime()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plus(minutes, ChronoUnit.MINUTES); } @@ -253,6 +254,7 @@ private void createTs(CwmsDbTsJooq cwmsTsJdbc, Connection connection) throws SQL } catch(Exception e) { + LOGGER.log(Level.CONFIG, "Unable to create TimeSeries: " + e.getMessage()); } } diff --git a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java index d195bde15..3c50b8142 100644 --- a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java +++ b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java @@ -14,7 +14,7 @@ public abstract class TimeSeriesTestBase { protected TimeSeries getTimeSeries() { TimeSeries ts = new TimeSeries(null, -1, 0, "Test.Test.Elev.0.0.RAW", "SPK", ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"), ZonedDateTime.parse("2021-06-22T08:00:00-07:00[PST8PDT]"), null, Duration.ZERO); - ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0, null); + ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0); return ts; } From e7f2d9ec4ad1e00e6d8d14957b06798f06e55eac Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Tue, 5 Nov 2024 15:31:42 -0800 Subject: [PATCH 06/24] 634 TimeSeries Subclass update - added Mixin test --- .../cwms/cda/data/dto/TimeSeriesWithDate.java | 4 +--- .../cwms/cda/api/TimeSeriesControllerTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java index c35c3ab21..78ce3a57b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java @@ -28,9 +28,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; @@ -55,6 +53,7 @@ public final class TimeSeriesWithDate extends TimeSeries { private List<TimeSeriesWithDate.Record> values; + // list of TimeSeriesWithDate.Record, uses raw to avoid typing errors @Override public List getValues() { return values; @@ -135,7 +134,6 @@ private List<Column> getColumnDescriptor() { return columns; } - @JsonDeserialize(using = JsonDeserializer.None.class) public static final class Record extends TimeSeries.Record { @JsonProperty(value = "data-entry-date", index = 3) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 839ea8a93..58de111bc 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -218,6 +218,22 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { assertSimilarWithDate(fakeTs, ts2); } + @Test + void testXMLSerializeDeserializeTimeSeries() + { + String format = Formats.XMLV2; + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); + String formatted = Formats.format(contentType, fakeTs); + assertTrue(formatted.contains("quality-code")); + assertTrue(formatted.contains("data-entry-date")); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); + assertNotNull(ts2); + assertSimilarWithDate(fakeTs, ts2); + } + @Test void testDeserializeTimeSeriesXmlUTC() { TimeZone aDefault = TimeZone.getDefault(); From 3844e6098e79c003ed459875093d39280f46815d Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Mon, 2 Dec 2024 13:36:00 -0800 Subject: [PATCH 07/24] Updated max version handling, added test case --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 111 +++++++++++------- .../cda/api/TimeseriesControllerTestIT.java | 110 ++++++++++++++++- .../cda/api/lrl/1day_offset_version_date.json | 18 +++ 3 files changed, 194 insertions(+), 45 deletions(-) create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date.json diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 45154ad83..a01b0e6f8 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -48,7 +48,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -72,6 +71,7 @@ import org.jooq.SelectConditionStep; import org.jooq.SelectHavingStep; import org.jooq.SelectJoinStep; +import org.jooq.SelectSeekStep1; import org.jooq.SelectSeekStep2; import org.jooq.Table; import org.jooq.TableField; @@ -275,21 +275,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String maxVersion = "T"; } - Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - // Now we're going to call the retrieve_ts_out_tab function to get the data and build an - // internal table from it so we can manipulate it further - // This code assumes the database timezone is in UTC (per Oracle recommendation) - SQL retrieveSelectData = DSL.sql( - "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," - + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," - + "?,?,?,?,?," - + getVersionPart(versionDate) + ",?,?) ) retrieveTs", - tsId, unit, - beginTimeMilli, endTimeMilli, //tz hardcoded - trim, startInclusive, endInclusive, previous, next, - versionDateMilli, maxVersion, officeId); - Field<String> tzName = AV_CWMS_TS_ID2.TIME_ZONE_ID; Field<Integer> totalField; @@ -387,7 +372,21 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String timeseries = new TimeSeriesWithDate(timeseries); } + // Now we're going to call the retrieve_ts_out_tab function to get the data and build an + // internal table from it so we can manipulate it further + // This code assumes the database timezone is in UTC (per Oracle recommendation) + SQL retrieveSelectData = DSL.sql( + "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," + + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," + + "?,?,?,?,?," + + getVersionPart(versionDate) + ",?,?) ) retrieveTs", + tsId, unit, + beginTimeMilli, endTimeMilli, //tz hardcoded + trim, startInclusive, endInclusive, previous, next, + versionDateMilli, maxVersion, officeId); + Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( + DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); Field<Timestamp> dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); Condition whereCond = dateTimeCol .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( @@ -401,11 +400,32 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit))); + Field<Timestamp> versionDateCol = field("VERSION_DATE", Timestamp.class).as("VERSION_DATE"); TimeSeries retVal = null; if (pageSize != 0) { if (versionDate != null) { - whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : - Timestamp.from(versionDate.toInstant()))); + whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE + .eq(Timestamp.from(versionDate.toInstant()))); + } else { + SelectSeekStep1<Record4<Timestamp, Double, BigDecimal, Timestamp>, Timestamp> verQuery = + dsl.select( + dateTimeCol, + valueCol, + qualityNormCol, + versionDateCol + ) + .from(AV_TSV_DQU.AV_TSV_DQU) + .where(whereCond) + .orderBy(versionDateCol.desc()); + Result<Record4<Timestamp, Double, BigDecimal, Timestamp>> result = verQuery.fetch(); + Timestamp lastVersionDate = null; + if (!result.isEmpty()) { + lastVersionDate = result.get(0).getValue(versionDateCol); + } + + if (lastVersionDate != null) { + whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(lastVersionDate)); + } } SelectConditionStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( @@ -443,19 +463,19 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String logger.fine(() -> query2.getSQL(ParamType.INLINED)); final TimeSeriesWithDate timeSeries = new TimeSeriesWithDate(timeseries); query2.forEach(tsRecord -> timeSeries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - tsRecord.getValue(dataEntryDate) + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate) )); retVal = timeSeries; } else { logger.fine(() -> query.getSQL(ParamType.INLINED)); final TimeSeries finalTimeseries = timeseries; query.forEach(tsRecord -> finalTimeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue() + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue() )); retVal = finalTimeseries; } @@ -582,7 +602,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar SelectJoinStep<?> tmpQuery = dsl.with(limiter) .select(pageEntryFields) .from(limiter) - .join(AV_CWMS_TS_ID.AV_CWMS_TS_ID).on(limiterCode.eq(AV_CWMS_TS_ID.AV_CWMS_TS_ID.TS_CODE)); + .join(AV_CWMS_TS_ID.AV_CWMS_TS_ID) + .on(limiterCode.eq(AV_CWMS_TS_ID.AV_CWMS_TS_ID.TS_CODE)); if (params.isIncludeExtents()) { @@ -590,8 +611,9 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .on(limiterCode .eq(AV_TS_EXTENTS_UTC.TS_CODE.coerce(limiterCode))); } - final SelectSeekStep2<?, String, String> overallQuery = tmpQuery.orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, - AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); + final SelectSeekStep2<?, String, String> overallQuery = tmpQuery + .orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, + AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); logger.fine(() -> overallQuery.getSQL(ParamType.INLINED)); Result<?> result = overallQuery.fetch(); @@ -684,15 +706,16 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar return conditions; } - private static @NotNull CommonTableExpression<?> buildWithClause(CatalogRequestParameters params, List<Condition> whereConditions, List<Condition> pagingConditions, int pageSize, boolean forCount) { + private static @NotNull CommonTableExpression<?> buildWithClause(CatalogRequestParameters params, + List<Condition> whereConditions, List<Condition> pagingConditions, int pageSize, boolean forCount) { TableLike<?> fromTable = AV_CWMS_TS_ID.AV_CWMS_TS_ID; - TableOnConditionStep<Record> on = null; List<Field<?>> selectFields = new ArrayList<>(); selectFields.add(fromTable.field(cwmsTsIdView.TS_CODE)); selectFields.add(fromTable.field(cwmsTsIdView.DB_OFFICE_ID)); selectFields.add(fromTable.field(cwmsTsIdView.CWMS_TS_ID)); + TableOnConditionStep<Record> on = null; if (params.needs(tsGroupView)) { on = AV_CWMS_TS_ID.AV_CWMS_TS_ID @@ -795,11 +818,11 @@ private Collection<? extends Condition> buildExtentsConditions(CatalogRequestPar if (params.isExcludeEmpty()) { retval.add(DSL.or( - AV_TS_EXTENTS_UTC.VERSION_TIME.isNotNull(), - AV_TS_EXTENTS_UTC.EARLIEST_TIME.isNotNull(), - AV_TS_EXTENTS_UTC.LATEST_TIME.isNotNull(), - AV_TS_EXTENTS_UTC.LAST_UPDATE.isNotNull()) - ); + AV_TS_EXTENTS_UTC.VERSION_TIME.isNotNull(), + AV_TS_EXTENTS_UTC.EARLIEST_TIME.isNotNull(), + AV_TS_EXTENTS_UTC.LATEST_TIME.isNotNull(), + AV_TS_EXTENTS_UTC.LAST_UPDATE.isNotNull()) + ); } return retval; @@ -1189,15 +1212,15 @@ private void store(Connection connection, String officeId, String tsId, String u final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); if (values != null && !values.isEmpty()) { - Iterator<TimeSeries.Record> iter = values.iterator(); - while (iter.hasNext()) { - TimeSeries.Record value = iter.next(); - Double dataValue = value.getValue(); - if (dataValue != null && dataValue == -Float.MAX_VALUE) { - dataValue = null; - } - tsvArray.add(new ZTSV_TYPE(value.getDateTime(), dataValue, BigDecimal.valueOf(value.getQualityCode()))); - } + for(TimeSeries.Record value : values) + { + Double dataValue = value.getValue(); + if(dataValue != null && dataValue == -Float.MAX_VALUE) + { + dataValue = null; + } + tsvArray.add(new ZTSV_TYPE(value.getDateTime(), dataValue, BigDecimal.valueOf(value.getQualityCode()))); + } } if (versionDate != null) { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 486b763c2..56d8535ef 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -9,7 +9,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.isNull; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -165,6 +164,115 @@ void test_lrl_1day() throws Exception { } + @Test + void test_lrl_1day_max_version() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/1day_offset_version_date.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + String secondVersionDate = "1604786000000"; + tsData = tsData.replace("1594786000000", secondVersionDate).replace("35,", "47.5,"); + // inserting the second time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back + String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or + String versionDate = "2020-07-15T04:06:40Z"; + // 1675335600000 + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .queryParam("version-date", versionDate) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) // one point + .body("values[0].size()", equalTo(3)) // time, value, quality + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(35, 0.0001)) + .body("version-date", equalTo(versionDate)) + ; + + // get again as max version + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) // one point + .body("values[0].size()", equalTo(3)) // time, value, quality + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(47.5, 0.0001)) + ; + } + @Test void test_lrl_1day_bad_units() throws Exception { ObjectMapper mapper = new ObjectMapper(); diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date.json new file mode 100644 index 000000000..ca4c45ee8 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date.json @@ -0,0 +1,18 @@ +{ + "name": "Buckhorn.Temp-Water.Inst.1Day.0.cda-test", + "office-id": "SPK", + "units": "F", + "version-date": 1594786000000, + "values": [ + [ + 1675335600000, + 35, + 0 + ], + [ + 1675422000000, + 36, + 0 + ] + ] +} \ No newline at end of file From c6a7d6158cdea46e40aab86d413c29b803591835 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 18 Dec 2024 16:48:08 -0800 Subject: [PATCH 08/24] Updated based on Adam's feedback. Added serializer test on file, cleaned up DTO --- .../main/java/cwms/cda/api/Controllers.java | 1 + .../cwms/cda/api/TimeSeriesController.java | 52 ++++------------- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 8 +-- .../java/cwms/cda/data/dto/TimeSeries.java | 12 +++- ....java => TimeSeriesWithDataEntryDate.java} | 58 +++++++------------ .../cda/api/TimeSeriesControllerTest.java | 52 +++++++++++------ .../cda/api/TimeseriesControllerTestIT.java | 32 +++++----- .../cwms/cda/data/dao/TimeSeriesDaoTest.java | 5 +- .../1day_offset_version_date_roundtrip.json | 22 +++++++ 9 files changed, 124 insertions(+), 118 deletions(-) rename cwms-data-api/src/main/java/cwms/cda/data/dto/{TimeSeriesWithDate.java => TimeSeriesWithDataEntryDate.java} (76%) create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json diff --git a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java index d9a4eeea1..cf099edaa 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java @@ -75,6 +75,7 @@ public final class Controllers { public static final String UNIT_SYSTEM = "unit-system"; public static final String TIMESERIES_CATEGORY_LIKE = "timeseries-category-like"; + public static final String INCLUDE_ENTRY_DATE = "include-entry-date"; public static final String LOCATION_CATEGORY_LIKE = "location-category-like"; public static final String LOCATION_GROUP_LIKE = "location-group-like"; diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index ed6c8afd4..c024be4bc 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -1,45 +1,7 @@ package cwms.cda.api; import static com.codahale.metrics.MetricRegistry.name; -import static cwms.cda.api.Controllers.BEGIN; -import static cwms.cda.api.Controllers.CREATE; -import static cwms.cda.api.Controllers.CREATE_AS_LRTS; -import static cwms.cda.api.Controllers.CURSOR; -import static cwms.cda.api.Controllers.DATE_FORMAT; -import static cwms.cda.api.Controllers.DATUM; -import static cwms.cda.api.Controllers.DELETE; -import static cwms.cda.api.Controllers.END; -import static cwms.cda.api.Controllers.END_TIME_INCLUSIVE; -import static cwms.cda.api.Controllers.EXAMPLE_DATE; -import static cwms.cda.api.Controllers.FORMAT; -import static cwms.cda.api.Controllers.GET_ALL; -import static cwms.cda.api.Controllers.GET_ONE; -import static cwms.cda.api.Controllers.MAX_VERSION; -import static cwms.cda.api.Controllers.NAME; -import static cwms.cda.api.Controllers.NOT_SUPPORTED_YET; -import static cwms.cda.api.Controllers.OFFICE; -import static cwms.cda.api.Controllers.OVERRIDE_PROTECTION; -import static cwms.cda.api.Controllers.PAGE; -import static cwms.cda.api.Controllers.PAGE_SIZE; -import static cwms.cda.api.Controllers.RESULTS; -import static cwms.cda.api.Controllers.SIZE; -import static cwms.cda.api.Controllers.START_TIME_INCLUSIVE; -import static cwms.cda.api.Controllers.STATUS_200; -import static cwms.cda.api.Controllers.STATUS_400; -import static cwms.cda.api.Controllers.STATUS_404; -import static cwms.cda.api.Controllers.STATUS_501; -import static cwms.cda.api.Controllers.STORE_RULE; -import static cwms.cda.api.Controllers.TIMESERIES; -import static cwms.cda.api.Controllers.TIMEZONE; -import static cwms.cda.api.Controllers.UNIT; -import static cwms.cda.api.Controllers.UPDATE; -import static cwms.cda.api.Controllers.VERSION; -import static cwms.cda.api.Controllers.VERSION_DATE; -import static cwms.cda.api.Controllers.addDeprecatedContentTypeWarning; -import static cwms.cda.api.Controllers.queryParamAsClass; -import static cwms.cda.api.Controllers.queryParamAsZdt; -import static cwms.cda.api.Controllers.requiredParam; -import static cwms.cda.api.Controllers.requiredZdt; +import static cwms.cda.api.Controllers.*; import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; @@ -53,6 +15,7 @@ import cwms.cda.data.dao.TimeSeriesDaoImpl; import cwms.cda.data.dao.TimeSeriesDeleteOptions; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DateUtils; @@ -83,7 +46,6 @@ public class TimeSeriesController implements CrudHandler { private static final Logger logger = Logger.getLogger(TimeSeriesController.class.getName()); - private static final String INCLUDE_ENTRY_DATE = "include-entry-date"; public static final String TAG = "TimeSeries"; public static final String STORE_RULE_DESC = "The business rule to use " + "when merging the incoming with existing data\n" @@ -201,6 +163,9 @@ public void create(@NotNull Context ctx) { TimeSeriesDao dao = getTimeSeriesDao(dsl); TimeSeries timeSeries = deserializeTimeSeries(ctx); + if (timeSeries instanceof TimeSeriesWithDataEntryDate) { + throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); + } dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException ex) { @@ -382,7 +347,9 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "\n* `wml2` (only if name field is specified)" + "\n* `json` (default)"), @OpenApiParam(name = INCLUDE_ENTRY_DATE, type = Boolean.class, description = "Specifies " - + "whether to include the data entry date in the response. Default is false."), + + "whether to include the data entry date of each value in the response. Including the data entry " + + "date will increase the size of the array containing each data value from three to four." + + " Default is false."), @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " @@ -566,6 +533,9 @@ public void update(@NotNull Context ctx, @NotNull String id) { TimeSeriesDao dao = getTimeSeriesDao(dsl); TimeSeries timeSeries = deserializeTimeSeries(ctx); + if (timeSeries instanceof TimeSeriesWithDataEntryDate) { + throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); + } boolean createAsLrts = ctx.queryParamAsClass(CREATE_AS_LRTS, Boolean.class) .getOrDefault(false); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index a01b0e6f8..e6ad25477 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -24,7 +24,7 @@ import cwms.cda.data.dto.RecentValue; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesExtents; -import cwms.cda.data.dto.TimeSeriesWithDate; +import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.data.dto.Tsv; import cwms.cda.data.dto.TsvDqu; import cwms.cda.data.dto.TsvId; @@ -354,7 +354,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String versionDate, finalDateVersionType ); } else { - return new TimeSeriesWithDate(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", + return new TimeSeriesWithDataEntryDate(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", Integer.class), tsMetadata.getValue("NAME", String.class), tsMetadata.getValue("office_id", String.class), beginTime, endTime, tsMetadata.getValue("units", String.class), @@ -369,7 +369,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String }); if (includeEntryDate) { - timeseries = new TimeSeriesWithDate(timeseries); + timeseries = new TimeSeriesWithDataEntryDate(timeseries); } // Now we're going to call the retrieve_ts_out_tab function to get the data and build an @@ -461,7 +461,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (includeEntryDate) { logger.fine(() -> query2.getSQL(ParamType.INLINED)); - final TimeSeriesWithDate timeSeries = new TimeSeriesWithDate(timeseries); + final TimeSeriesWithDataEntryDate timeSeries = new TimeSeriesWithDataEntryDate(timeseries); query2.forEach(tsRecord -> timeSeries.addValue( tsRecord.getValue(dateTimeCol), tsRecord.getValue(valueCol), diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 050682adf..42e28135a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -79,7 +79,8 @@ public class TimeSeries extends CwmsDTOPaginated { @ArraySchema( schema = @Schema( - description = "List of retrieved time-series values", + description = "List of retrieved time-series values. Contains [dateTime, value, qualityCode]. " + + "Refer to the value-columns property for more information.", implementation = Record.class ) ) @@ -192,7 +193,14 @@ public VersionType getDateVersionType() { } @JsonProperty(value = "value-columns") - @Schema(name = "value-columns", accessMode = AccessMode.READ_ONLY) + @Schema(name = "value-columns", + description = "The columns of the time-series data array returned, this property is used to describe " + + "the data structure of the records array. Contains [name, ordinal, datatype]. " + + "Name corresponds to the variable described by the data, " + + "ordinal is the order of the column in the list returned (starting at index 1), " + + "and datatype is the class name of the data type for the variable. Since the records array " + + "can be of variable length, the column index value is used to identify the data in the array.", + accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { return getColumnDescriptor(); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java similarity index 76% rename from cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java rename to cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java index 78ce3a57b..54cc05965 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java @@ -49,70 +49,56 @@ @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) @FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) @FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) -public final class TimeSeriesWithDate extends TimeSeries { +public final class TimeSeriesWithDataEntryDate extends TimeSeries { - private List<TimeSeriesWithDate.Record> values; - - // list of TimeSeriesWithDate.Record, uses raw to avoid typing errors - @Override - public List getValues() { - return values; - } - - TimeSeriesWithDate() { + // Default constructor for Jackson Deserialization + public TimeSeriesWithDataEntryDate() { super(); - values = new ArrayList<>(); + valuesWithEntryDate = new ArrayList<>(); } - public TimeSeriesWithDate(TimeSeries timeSeries) { + public TimeSeriesWithDataEntryDate(TimeSeries timeSeries) { this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), timeSeries.getName(), timeSeries.getOfficeId(), timeSeries.getBegin(), timeSeries.getEnd(), timeSeries.getUnits(), timeSeries.getInterval(), timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), timeSeries.getTimeZone(), timeSeries.getVersionDate(), timeSeries.getDateVersionType()); - values = new ArrayList<>(); - } - - public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, - ZonedDateTime begin, ZonedDateTime end, String units, Duration interval) { - this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, - null, null, null); - values = new ArrayList<>(); + valuesWithEntryDate = new ArrayList<>(); } - public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, - ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, - VersionType dateVersionType) { - this(page, pageSize, total, name, officeId, begin, end, units, interval, info, null, - null, versionDate, dateVersionType); - values = new ArrayList<>(); - } - - public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, + public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, Long intervalOffset, String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, timeZone, versionDate, dateVersionType); - values = new ArrayList<>(); + valuesWithEntryDate = new ArrayList<>(); + } + + @JsonProperty(value = "values") + private List<TimeSeriesWithDataEntryDate.Record> valuesWithEntryDate; + + @Override + public List<TimeSeries.Record> getValues() { + return new ArrayList<>(valuesWithEntryDate); } public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set - if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { + if ((page == null || page.isEmpty()) && (valuesWithEntryDate == null || valuesWithEntryDate.isEmpty())) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); } - if (pageSize > 0 && values.size() == pageSize) { + if (pageSize > 0 && valuesWithEntryDate.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - values.add(new Record(dateTime, value, qualityCode, dataEntryDate)); + valuesWithEntryDate.add(new Record(dateTime, value, qualityCode, dataEntryDate)); } } @Override public List<Column> getValueColumnsJSON() { - return getColumnDescriptor(); + return getColumnDescriptorWithEntryDate(); } - private List<Column> getColumnDescriptor() { + private List<Column> getColumnDescriptorWithEntryDate() { List<Column> columns = new ArrayList<>(); for (Field f: TimeSeries.Record.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); @@ -138,7 +124,7 @@ public static final class Record extends TimeSeries.Record { @JsonProperty(value = "data-entry-date", index = 3) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") @JsonInclude(JsonInclude.Include.NON_DEFAULT) - Timestamp dataEntryDate; + private Timestamp dataEntryDate; // Default constructor for Jackson Deserialization public Record() { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 58de111bc..79cf0a194 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,7 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesWithDate; +import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -134,7 +134,7 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } - private void assertSimilarWithDate(TimeSeriesWithDate expected, TimeSeriesWithDate actual) + private void assertSimilarWithDate(TimeSeriesWithDataEntryDate expected, TimeSeriesWithDataEntryDate actual) { assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); assertEquals(expected.getName(), actual.getName(), "names did not match"); @@ -153,8 +153,8 @@ private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSerie private void assertDateRecordsMatch(List<? extends TimeSeries.Record> expected, List<? extends TimeSeries.Record> actual) { for (int i = 0; i < expected.size(); i++) { - assertEquals(((TimeSeriesWithDate.Record) expected.get(i)).getDataEntryDate(), - ((TimeSeriesWithDate.Record) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); + assertEquals(((TimeSeriesWithDataEntryDate.Record) expected.get(i)).getDataEntryDate(), + ((TimeSeriesWithDataEntryDate.Record) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); @@ -180,13 +180,14 @@ void testSerializeTimeSeries(String format) { void testSerializeTimeSeriesWithDataEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); assertEquals(4, fakeTs.getValueColumnsJSON().size()); - assertInstanceOf(TimeSeriesWithDate.Record.class, fakeTs.getValues().get(0)); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); + assertInstanceOf(TimeSeriesWithDataEntryDate.Record.class, + fakeTs.getValues().get(0)); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); + TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -210,10 +211,22 @@ void testDeserializeTimeSeries(String format) { void testDeserializeTimeSeriesWithEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); + TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); + TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); + assertNotNull(ts2); + assertSimilarWithDate(fakeTs, ts2); + } + + @Test + void testDeserializeTimeSeriesWithEntryDateFromFile() { + InputStream inputStream = this.getClass() + .getResourceAsStream("/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json"); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeriesWithDataEntryDate.class); + TimeSeriesWithDataEntryDate fakeTs = Formats.parseContent(contentType, inputStream, TimeSeriesWithDataEntryDate.class); + String formatted = Formats.format(contentType, fakeTs); + TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -224,12 +237,12 @@ void testXMLSerializeDeserializeTimeSeries() String format = Formats.XMLV2; String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); + TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); String formatted = Formats.format(contentType, fakeTs); assertTrue(formatted.contains("quality-code")); assertTrue(formatted.contains("data-entry-date")); - TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); + TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -316,7 +329,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { } @NotNull - private TimeSeriesWithDate buildTimeSeriesWithEntryDate(String officeId, String tsId) { + private TimeSeriesWithDataEntryDate buildTimeSeriesWithEntryDate(String officeId, String tsId) { ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); @@ -327,7 +340,7 @@ private TimeSeriesWithDate buildTimeSeriesWithEntryDate(String officeId, String int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeriesWithDate ts = new TimeSeriesWithDate(null, + TimeSeriesWithDataEntryDate ts = new TimeSeriesWithDataEntryDate(null, -1, 0, tsId, @@ -335,7 +348,12 @@ private TimeSeriesWithDate buildTimeSeriesWithEntryDate(String officeId, String start, end, "m", - Duration.ofMinutes(minutes)); + Duration.ofMinutes(minutes), + null, + null, + null, + null, + null); ZonedDateTime next = start; for(int i = 0; i < count; i++) { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 56d8535ef..a5f5cc836 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -352,7 +352,6 @@ void test_lrl_1day_malicious_units() throws Exception { @Test void test_include_data_entry_date() throws Exception { ObjectMapper mapper = new ObjectMapper(); - final String includeDataEntryDate = "include-entry-date"; InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/spk/num_ts_create2.json"); @@ -369,20 +368,21 @@ void test_include_data_entry_date() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + ; // 1675335600000 is Thursday, February 2, 2023 11:00:00 AM // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM @@ -398,7 +398,7 @@ void test_include_data_entry_date() throws Exception { .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") .queryParam(Controllers.END, "2010-02-03T11:00:00Z") .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") - .queryParam(includeDataEntryDate, true) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, true) .when() .redirects().follow(true) .redirects().max(3) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java index dc68ee4f8..242237a50 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java @@ -13,7 +13,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import cwms.cda.data.dto.TimeSeriesWithDate; import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.impl.DSL; @@ -169,7 +168,9 @@ void testCreateWithData() throws Exception int count = 60 / 15; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeriesWithDate ts = new TimeSeriesWithDate(null, -1, 0, tsId, officeId, start, end, "m", Duration.ofMinutes(minutes)); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, + officeId, start, end, "m", Duration.ofMinutes(minutes), null, + null, null, null, null); ZonedDateTime next = start; for(int i = 0; i < count; i++) diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json new file mode 100644 index 000000000..e7eed0bfa --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json @@ -0,0 +1,22 @@ +{ + "name": "Buckhorn.Temp-Water.Inst.1Day.0.cda-test", + "office-id": "SPK", + "units": "F", + "begin": 1675335600000, + "end": 1675422000000, + "version-date": 1594786000000, + "values": [ + [ + 1675335600000, + 35, + 0, + 1734566146000 + ], + [ + 1675422000000, + 36, + 0, + 1734566147000 + ] + ] +} \ No newline at end of file From 24b18494603029b9e853bda48d39bbcd98f0c83e Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Thu, 19 Dec 2024 11:40:14 -0800 Subject: [PATCH 09/24] Updated based on Adam's feedback. Added error when attempting to store values with data entry date --- .../cwms/cda/api/TimeSeriesController.java | 34 +- .../java/cwms/cda/data/dto/TimeSeries.java | 5 +- .../data/dto/TimeSeriesWithDataEntryDate.java | 7 +- .../cda/api/TimeseriesControllerTestIT.java | 690 +++++++++--------- .../1day_offset_version_date_roundtrip.json | 19 + 5 files changed, 408 insertions(+), 347 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index c024be4bc..590358881 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -124,7 +124,8 @@ private Timer.Context markAndTime(String subject) { @OpenApi( description = "Used to create and save time-series data. Data to be stored must have " - + "time stamps in UTC represented as epoch milliseconds ", + + "time stamps in UTC represented as epoch milliseconds. If data entry date is included in the " + + "request, it will be dropped. ", requestBody = @OpenApiRequestBody( content = { @OpenApiContent(from = TimeSeries.class, type = Formats.JSONV2), @@ -162,10 +163,8 @@ public void create(@NotNull Context ctx) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); - TimeSeries timeSeries = deserializeTimeSeries(ctx); - if (timeSeries instanceof TimeSeriesWithDataEntryDate) { - throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); - } + TimeSeriesWithDataEntryDate timeSeries = deserializeTimeSeries(ctx); + checkForEntryDate(timeSeries); dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException ex) { @@ -348,8 +347,8 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "\n* `json` (default)"), @OpenApiParam(name = INCLUDE_ENTRY_DATE, type = Boolean.class, description = "Specifies " + "whether to include the data entry date of each value in the response. Including the data entry " - + "date will increase the size of the array containing each data value from three to four." - + " Default is false."), + + "date will increase the size of the array containing each data value from three to four, " + + "changing the format of the response. Default is false."), @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " @@ -532,11 +531,8 @@ public void update(@NotNull Context ctx, @NotNull String id) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); - TimeSeries timeSeries = deserializeTimeSeries(ctx); - if (timeSeries instanceof TimeSeriesWithDataEntryDate) { - throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); - } - + TimeSeriesWithDataEntryDate timeSeries = deserializeTimeSeries(ctx); + checkForEntryDate(timeSeries); boolean createAsLrts = ctx.queryParamAsClass(CREATE_AS_LRTS, Boolean.class) .getOrDefault(false); StoreRule storeRule = ctx.queryParamAsClass(STORE_RULE, StoreRule.class) @@ -554,10 +550,18 @@ public void update(@NotNull Context ctx, @NotNull String id) { } } - private TimeSeries deserializeTimeSeries(Context ctx) { + private TimeSeriesWithDataEntryDate deserializeTimeSeries(Context ctx) { String contentTypeHeader = ctx.req.getContentType(); - ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeries.class); - return Formats.parseContent(contentType, ctx.bodyAsInputStream(), TimeSeries.class); + ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeriesWithDataEntryDate.class); + return Formats.parseContent(contentType, ctx.bodyAsInputStream(), TimeSeriesWithDataEntryDate.class); + } + + private void checkForEntryDate(TimeSeriesWithDataEntryDate timeSeries) { + for (TimeSeries.Record rec : timeSeries.getValues()) { + if (((TimeSeriesWithDataEntryDate.Record) rec).getDataEntryDate() != null) { + throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); + } + } } /** diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 42e28135a..80f7aff05 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -197,9 +197,10 @@ public VersionType getDateVersionType() { description = "The columns of the time-series data array returned, this property is used to describe " + "the data structure of the records array. Contains [name, ordinal, datatype]. " + "Name corresponds to the variable described by the data, " - + "ordinal is the order of the column in the list returned (starting at index 1), " + + "ordinal is the order of the column in the data value array returned (starts at index 1), " + "and datatype is the class name of the data type for the variable. Since the records array " - + "can be of variable length, the column index value is used to identify the data in the array.", + + "can be of variable length, the column index value is used to identify the position of the " + + "data in the array.", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { return getColumnDescriptor(); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java index 54cc05965..a49b59577 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java @@ -65,9 +65,9 @@ public TimeSeriesWithDataEntryDate(TimeSeries timeSeries) { valuesWithEntryDate = new ArrayList<>(); } - public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, - ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, Long intervalOffset, - String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { + public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, String name, String officeId, + ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, + Long intervalOffset, String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, timeZone, versionDate, dateVersionType); valuesWithEntryDate = new ArrayList<>(); @@ -93,6 +93,7 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam } } + @JsonProperty(value = "value-columns") @Override public List<Column> getValueColumnsJSON() { return getColumnDescriptorWithEntryDate(); diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index a5f5cc836..ca43cae42 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -120,46 +120,46 @@ void test_lrl_1day() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); // get it back String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values.size()", equalTo(1)) // one point - .body("values[0].size()", equalTo(3)) // time, value, quality - .body("values[0][0]", equalTo(1675335600000L)) // time - .body("values[0][1]", closeTo(35, 0.0001)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) // one point + .body("values[0].size()", equalTo(3)) // time, value, quality + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(35, 0.0001)) ; } @@ -292,23 +292,22 @@ void test_lrl_1day_bad_units() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) - .body("details.message", containsString("The unit: m is not a recognized CWMS " - + "Database unit for the Temp Parameter")); - + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .body("details.message", containsString("The unit: m is not a recognized CWMS " + + "Database unit for the Temp Parameter")); } @Test @@ -331,22 +330,21 @@ void test_lrl_1day_malicious_units() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) - .body("details.message", equalTo("Invalid Units.")); - + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .body("details.message", equalTo("Invalid Units.")); } @Test @@ -435,6 +433,43 @@ void test_include_data_entry_date() throws Exception { .body("values[0].size()", equalTo(3)); } + @Test + void test_attempt_store_with_entry_date() throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json"); + assertNotNull(resource); + + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + ; + } + @Test void test_delete_ts() throws Exception { ObjectMapper mapper = new ObjectMapper(); @@ -454,65 +489,65 @@ void test_delete_ts() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("begin", "2023-02-02T11:00:00+00:00") - .queryParam("end", "2023-02-02T11:00:00+00:00") - .queryParam("start-time-inclusive", "true") - .queryParam("end-time-inclusive", "true") - .queryParam("override-protection", "true") - .when() - .redirects().follow(true) - .redirects().max(3) - .delete("/timeseries/" + ts.get("name").asText()) - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("begin", "2023-02-02T11:00:00+00:00") + .queryParam("end", "2023-02-02T11:00:00+00:00") + .queryParam("start-time-inclusive", "true") + .queryParam("end-time-inclusive", "true") + .queryParam("override-protection", "true") + .when() + .redirects().follow(true) + .redirects().max(3) + .delete("/timeseries/" + ts.get("name").asText()) + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); // 1675335600000 is Thursday, February 2, 2023 11:00:00 AM // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM // get it back given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) // .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", "2023-02-02T11:00:00+00:00") - .queryParam("end", "2023-02-03T11:00:00+00:00") - .queryParam(Controllers.TRIM, false) - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values.size()", equalTo(2)) - .body("values[0][1]", nullValue()); + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", "2023-02-02T11:00:00+00:00") + .queryParam("end", "2023-02-03T11:00:00+00:00") + .queryParam(Controllers.TRIM, false) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(2)) + .body("values[0][1]", nullValue()); } @Test @@ -533,26 +568,26 @@ void test_no_office_permissions() throws Exception { TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; // inserting the time series - given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_UNAUTHORIZED)) - .body("message", is("User not authorized for this office.")); - } - - @Test - void test_invalid_office() { + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_UNAUTHORIZED)) + .body("message", is("User not authorized for this office.")); + } + + @Test + void test_invalid_office() { given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -588,23 +623,23 @@ void test_v1_cant_trim() throws Exception { String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV1) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) - .queryParam("trim", "true") - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV1) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .queryParam("trim", "true") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) ; } @@ -664,23 +699,23 @@ void test_v2_cant_datum() throws Exception { String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) - .queryParam("datum", "NAVD88") - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .queryParam("datum", "NAVD88") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) ; } @@ -704,20 +739,20 @@ void test_lrl_trim() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); // The ts we created has two values 1675335600000, 1675422000000, @@ -731,54 +766,54 @@ void test_lrl_trim() throws Exception { // without trim we should get extra null point given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", false) - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values.size()", equalTo(2)) - .body("values[0].size()", equalTo(3)) // time, value, quality - .body("values[1][0]", equalTo(1675335600000L)) // time - .body("values[0][1]", nullValue()) - .body("values[1][1]", closeTo(35, 0.0001)); + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", dayBeforeFirst.toInstant().toString()) + .queryParam("end", firstPoint) + .queryParam("trim", false) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(2)) + .body("values[0].size()", equalTo(3)) // time, value, quality + .body("values[1][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", nullValue()) + .body("values[1][1]", closeTo(35, 0.0001)); // with trim the null should get trimmed. given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", true) - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values.size()", equalTo(1)) - .body("values[0].size()", equalTo(3)) // time, value, quality - .body("values[0][0]", equalTo(1675335600000L)) // time - .body("values[0][1]", closeTo(35, 0.0001)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", dayBeforeFirst.toInstant().toString()) + .queryParam("end", firstPoint) + .queryParam("trim", true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) + .body("values[0].size()", equalTo(3)) // time, value, quality + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(35, 0.0001)) ; } catch (SQLException ex) { throw new RuntimeException("Unable to create location for TS", ex); @@ -810,20 +845,20 @@ void test_big_create() throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(giantString) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(giantString) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); } /** @@ -910,66 +945,67 @@ void test_daylight_saving_retrieve()throws Exception { // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(giantString) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(giantString) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + ; // this doesn't cross Daylight savings - should work given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL,true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units","mm") - .queryParam("name", name) - .queryParam("begin","2021-02-08T08:00:00Z") - .queryParam("end","2021-03-08T08:00:00Z") - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values[1][1]",closeTo(1724.4,0.1)) - .body("values[0][1]",closeTo(1724.4,0.1)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units","mm") + .queryParam("name", name) + .queryParam("begin","2021-02-08T08:00:00Z") + .queryParam("end","2021-03-08T08:00:00Z") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values[1][1]",closeTo(1724.4,0.1)) + .body("values[0][1]",closeTo(1724.4,0.1)) ; // these dates do cross daylight savings - won't work if seessiontimezone isn't set in 24.04.05 given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL,true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units","mm") - .queryParam("name", name) - .queryParam("begin","2021-03-08T08:00:00Z") - .queryParam("end","2021-03-15T08:00:00Z") - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) - .body("values[1][1]",closeTo(1724.4,0.1)) - .body("values[0][1]",closeTo(1724.4,0.1)) - ; + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units","mm") + .queryParam("name", name) + .queryParam("begin","2021-03-08T08:00:00Z") + .queryParam("end","2021-03-15T08:00:00Z") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values[1][1]",closeTo(1724.4,0.1)) + .body("values[0][1]",closeTo(1724.4,0.1)) + ; } private static void deleteLocation(String location, String officeId) throws SQLException { @@ -1016,38 +1052,38 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); // get it back given() - .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("units", "F") - .queryParam("name", ts.get("name").asText()) - .when() - .redirects().follow(true) - .redirects().max(3) - .get("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)) + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("units", "F") + .queryParam("name", ts.get("name").asText()) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) ; } diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json index e7eed0bfa..3e0d8ce88 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json @@ -5,6 +5,25 @@ "begin": 1675335600000, "end": 1675422000000, "version-date": 1594786000000, + "value-columns" : [ + { + "name" : "date-time", + "ordinal" : 1, + "datatype" : "java.sql.Timestamp" + }, { + "name" : "value", + "ordinal" : 2, + "datatype" : "java.lang.Double" + }, { + "name" : "quality-code", + "ordinal" : 3, + "datatype" : "int" + }, { + "name" : "data-entry-date", + "ordinal" : 4, + "datatype" : "java.sql.Timestamp" + } + ], "values": [ [ 1675335600000, From 49f4e0b4360af99a711935ee1fc9dc5053bdd773 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Thu, 19 Dec 2024 16:09:01 -0800 Subject: [PATCH 10/24] Renamed json file, fixed documentation --- cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java | 2 +- .../src/test/java/cwms/cda/api/TimeSeriesControllerTest.java | 2 +- .../src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java | 2 +- ...ate_roundtrip.json => timeseries_with_data_entry_dates.json} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename cwms-data-api/src/test/resources/cwms/cda/api/lrl/{1day_offset_version_date_roundtrip.json => timeseries_with_data_entry_dates.json} (100%) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 80f7aff05..79b1230cd 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -79,7 +79,7 @@ public class TimeSeries extends CwmsDTOPaginated { @ArraySchema( schema = @Schema( - description = "List of retrieved time-series values. Contains [dateTime, value, qualityCode]. " + description = "List of retrieved time-series values. The values-columns property describes the structure of the data value array. " + "Refer to the value-columns property for more information.", implementation = Record.class ) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 79cf0a194..e66d872de 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -222,7 +222,7 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { @Test void testDeserializeTimeSeriesWithEntryDateFromFile() { InputStream inputStream = this.getClass() - .getResourceAsStream("/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json"); + .getResourceAsStream("/cwms/cda/api/lrl/timeseries_with_data_entry_dates.json"); ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeriesWithDataEntryDate.class); TimeSeriesWithDataEntryDate fakeTs = Formats.parseContent(contentType, inputStream, TimeSeriesWithDataEntryDate.class); String formatted = Formats.format(contentType, fakeTs); diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index ca43cae42..40c0b5253 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -439,7 +439,7 @@ void test_attempt_store_with_entry_date() throws Exception ObjectMapper mapper = new ObjectMapper(); InputStream resource = this.getClass().getResourceAsStream( - "/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json"); + "/cwms/cda/api/lrl/timeseries_with_data_entry_dates.json"); assertNotNull(resource); String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/timeseries_with_data_entry_dates.json similarity index 100% rename from cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_roundtrip.json rename to cwms-data-api/src/test/resources/cwms/cda/api/lrl/timeseries_with_data_entry_dates.json From 5b6f145c845f2691f9909313fdbfd3fdaec3dfb8 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Mon, 6 Jan 2025 10:01:02 -0800 Subject: [PATCH 11/24] Fixed data entry date request body checking --- .../cwms/cda/api/TimeSeriesController.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 590358881..9c38477c5 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -15,7 +15,6 @@ import cwms.cda.data.dao.TimeSeriesDaoImpl; import cwms.cda.data.dao.TimeSeriesDeleteOptions; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DateUtils; @@ -30,6 +29,8 @@ import io.javalin.plugin.openapi.annotations.OpenApiParam; import io.javalin.plugin.openapi.annotations.OpenApiRequestBody; import io.javalin.plugin.openapi.annotations.OpenApiResponse; +import java.io.IOException; +import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -40,6 +41,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; @@ -163,11 +165,10 @@ public void create(@NotNull Context ctx) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); - TimeSeriesWithDataEntryDate timeSeries = deserializeTimeSeries(ctx); - checkForEntryDate(timeSeries); + TimeSeries timeSeries = deserializeTimeSeries(ctx); dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (DataAccessException ex) { + } catch (DataAccessException | IOException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); @@ -531,8 +532,7 @@ public void update(@NotNull Context ctx, @NotNull String id) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); - TimeSeriesWithDataEntryDate timeSeries = deserializeTimeSeries(ctx); - checkForEntryDate(timeSeries); + TimeSeries timeSeries = deserializeTimeSeries(ctx); boolean createAsLrts = ctx.queryParamAsClass(CREATE_AS_LRTS, Boolean.class) .getOrDefault(false); StoreRule storeRule = ctx.queryParamAsClass(STORE_RULE, StoreRule.class) @@ -543,25 +543,24 @@ public void update(@NotNull Context ctx, @NotNull String id) { dao.store(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (DataAccessException ex) { + } catch (DataAccessException | IOException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); } } - private TimeSeriesWithDataEntryDate deserializeTimeSeries(Context ctx) { + private TimeSeries deserializeTimeSeries(Context ctx) throws IOException + { String contentTypeHeader = ctx.req.getContentType(); - ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeriesWithDataEntryDate.class); - return Formats.parseContent(contentType, ctx.bodyAsInputStream(), TimeSeriesWithDataEntryDate.class); - } - - private void checkForEntryDate(TimeSeriesWithDataEntryDate timeSeries) { - for (TimeSeries.Record rec : timeSeries.getValues()) { - if (((TimeSeriesWithDataEntryDate.Record) rec).getDataEntryDate() != null) { - throw new IllegalArgumentException("Data Entry Date is not allowed in the request when storing data"); - } + StringWriter writer = new StringWriter(); + IOUtils.copy(ctx.bodyAsInputStream(), writer, StandardCharsets.UTF_8); + if (writer.toString().contains("data-entry-date")) + { + throw new IllegalArgumentException("Data entry date is not allowed in the request"); } + ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeries.class); + return Formats.parseContent(contentType, writer.toString(), TimeSeries.class); } /** From eabc472ec5f5b01a1fb0c4f3b2a6cb3889ba24e8 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 12 Feb 2025 13:48:51 -0800 Subject: [PATCH 12/24] Cleaned up TS retrieval with data entry dates. Added integration tests with data. Fixed unit parameter usage in integration tests --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 47 +--- .../cda/api/TimeseriesControllerTestIT.java | 240 +++++++++++++++++- .../api/lrl/1day_offset_version_date_max.json | 18 ++ .../1day_offset_with_data_entry_dates.json | 17 ++ 4 files changed, 266 insertions(+), 56 deletions(-) create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_max.json create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_with_data_entry_dates.json diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index e6ad25477..47f13929d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -71,7 +71,6 @@ import org.jooq.SelectConditionStep; import org.jooq.SelectHavingStep; import org.jooq.SelectJoinStep; -import org.jooq.SelectSeekStep1; import org.jooq.SelectSeekStep2; import org.jooq.Table; import org.jooq.TableField; @@ -376,7 +375,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // internal table from it so we can manipulate it further // This code assumes the database timezone is in UTC (per Oracle recommendation) SQL retrieveSelectData = DSL.sql( - "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," + "table(cwms_20.cwms_ts.retrieve_ts_entry_out_tab(?,?," + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," + "?,?,?,?,?," + getVersionPart(versionDate) + ",?,?) ) retrieveTs", @@ -388,54 +387,16 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); Field<Timestamp> dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); - Condition whereCond = dateTimeCol - .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.nvl(DSL.val(tsCursor == null ? null : - tsCursor.toInstant().toEpochMilli()), - DSL.val(beginTime.toInstant().toEpochMilli())))) - .and(dateTimeCol - .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.val(endTime.toInstant().toEpochMilli()))) - .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) - .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) - .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit))); - - Field<Timestamp> versionDateCol = field("VERSION_DATE", Timestamp.class).as("VERSION_DATE"); + TimeSeries retVal = null; if (pageSize != 0) { - if (versionDate != null) { - whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE - .eq(Timestamp.from(versionDate.toInstant()))); - } else { - SelectSeekStep1<Record4<Timestamp, Double, BigDecimal, Timestamp>, Timestamp> verQuery = - dsl.select( - dateTimeCol, - valueCol, - qualityNormCol, - versionDateCol - ) - .from(AV_TSV_DQU.AV_TSV_DQU) - .where(whereCond) - .orderBy(versionDateCol.desc()); - Result<Record4<Timestamp, Double, BigDecimal, Timestamp>> result = verQuery.fetch(); - Timestamp lastVersionDate = null; - if (!result.isEmpty()) { - lastVersionDate = result.get(0).getValue(versionDateCol); - } - - if (lastVersionDate != null) { - whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(lastVersionDate)); - } - } - - SelectConditionStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( + SelectJoinStep<Record4<Timestamp, Double, BigDecimal, Timestamp>> query2 = dsl.select( dateTimeCol, valueCol, qualityNormCol, dataEntryDate ) - .from(AV_TSV_DQU.AV_TSV_DQU) - .where(whereCond); + .from(retrieveSelectData); SelectConditionStep<Record3<Timestamp, Double, BigDecimal>> query = dsl.select( diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 40c0b5253..f6caaaf16 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -77,7 +77,7 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .accept(Formats.JSONV2) .header("Authorization",user.toHeaderValue()) .queryParam("office",officeId) - .queryParam("units","cfs") + .queryParam("unit","cfs") .queryParam("name",ts.get("name").asText()) .queryParam("begin","2023-01-11T12:00:00-00:00") .queryParam("end","2023-01-11T13:00:00-00:00") @@ -144,7 +144,7 @@ void test_lrl_1day() throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -227,7 +227,7 @@ void test_lrl_1day_max_version() throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -254,7 +254,7 @@ void test_lrl_1day_max_version() throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -273,6 +273,117 @@ void test_lrl_1day_max_version() throws Exception { ; } + @Test + void test_lrl_1day_max_version_with_entry_date() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/1day_offset_version_date_max.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + String secondVersionDate = "1604786000000"; + tsData = tsData.replace("1594786000000", secondVersionDate).replace("35,", "47.5,"); + // inserting the second time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam("office",officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back + String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or + String versionDate = "2020-07-15T04:06:40Z"; + // 1675335600000 + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("unit", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .queryParam("version-date", versionDate) + .queryParam("include-entry-date", true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) // one point + .body("values[0].size()", equalTo(4)) // time, value, quality, entry date + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(35, 0.0001)) + .body("version-date", equalTo(versionDate)) + ; + + // get again as max version + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("unit", "F") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", firstPoint) + .queryParam("end", firstPoint) + .queryParam("include-entry-date", true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) // one point + .body("values[0].size()", equalTo(4)) // time, value, quality, entry date + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(47.5, 0.0001)) + ; + } + @Test void test_lrl_1day_bad_units() throws Exception { ObjectMapper mapper = new ObjectMapper(); @@ -533,7 +644,7 @@ void test_delete_ts() throws Exception { // .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", "2023-02-02T11:00:00+00:00") .queryParam("end", "2023-02-03T11:00:00+00:00") @@ -627,7 +738,7 @@ void test_v1_cant_trim() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV1) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -666,7 +777,7 @@ void test_v1_cant_version() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV1) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -703,7 +814,7 @@ void test_v2_cant_datum() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", firstPoint) .queryParam("end", firstPoint) @@ -771,7 +882,7 @@ void test_lrl_trim() throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", dayBeforeFirst.toInstant().toString()) .queryParam("end", firstPoint) @@ -797,7 +908,7 @@ void test_lrl_trim() throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .queryParam("begin", dayBeforeFirst.toInstant().toString()) .queryParam("end", firstPoint) @@ -820,6 +931,109 @@ void test_lrl_trim() throws Exception { } } + @Test + void test_lrl_trim_with_data_entry_date() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/1day_offset_with_data_entry_dates.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + + // The ts we created has two values 1675335600000, 1675422000000, + + // get it back + String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or + // 1675335600000 + + ZonedDateTime beginZdt = ZonedDateTime.parse(firstPoint); + ZonedDateTime dayBeforeFirst = beginZdt.minusDays(1); + + // without trim we should get extra null point + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("unit", "m2") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", dayBeforeFirst.toInstant().toString()) + .queryParam("end", firstPoint) + .queryParam("trim", false) + .queryParam("include-entry-date", true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(2)) + .body("values[0].size()", equalTo(4)) // time, value, quality, data entry date + .body("values[1][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", nullValue()) + .body("values[1][1]", closeTo(35, 0.0001)); + + // with trim the null should get trimmed. + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .queryParam("unit", "m2") + .queryParam("name", ts.get("name").asText()) + .queryParam("begin", dayBeforeFirst.toInstant().toString()) + .queryParam("end", firstPoint) + .queryParam("trim", true) + .queryParam("include-entry-date", true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(1)) + .body("values[0].size()", equalTo(4)) // time, value, quality, data entry date + .body("values[0][0]", equalTo(1675335600000L)) // time + .body("values[0][1]", closeTo(35, 0.0001)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + @Test void test_big_create() throws Exception { @@ -968,7 +1182,7 @@ void test_daylight_saving_retrieve()throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units","mm") + .queryParam("unit","mm") .queryParam("name", name) .queryParam("begin","2021-02-08T08:00:00Z") .queryParam("end","2021-03-08T08:00:00Z") @@ -991,7 +1205,7 @@ void test_daylight_saving_retrieve()throws Exception { .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units","mm") + .queryParam("unit","mm") .queryParam("name", name) .queryParam("begin","2021-03-08T08:00:00Z") .queryParam("end","2021-03-15T08:00:00Z") @@ -1074,7 +1288,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) - .queryParam("units", "F") + .queryParam("unit", "F") .queryParam("name", ts.get("name").asText()) .when() .redirects().follow(true) diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_max.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_max.json new file mode 100644 index 000000000..3044e0122 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_version_date_max.json @@ -0,0 +1,18 @@ +{ + "name": "Buckhorn.Temp-Water.Inst.1Day.0.cda-version-test", + "office-id": "SPK", + "units": "F", + "version-date": 1594786000000, + "values": [ + [ + 1675335600000, + 35, + 0 + ], + [ + 1675422000000, + 36, + 0 + ] + ] +} \ No newline at end of file diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_with_data_entry_dates.json b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_with_data_entry_dates.json new file mode 100644 index 000000000..ba8dedd41 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lrl/1day_offset_with_data_entry_dates.json @@ -0,0 +1,17 @@ +{ + "name": "Buckhorn.Area.Inst.1Day.0.cda-trim-test", + "office-id": "SPK", + "units": "m2", + "values": [ + [ + 1675335600000, + 35, + 0 + ], + [ + 1675422000000, + 36, + 0 + ] + ] +} \ No newline at end of file From 910b6afe6bca7e7616d0fe8ad30f0b2054a89727 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Fri, 14 Feb 2025 15:47:37 -0800 Subject: [PATCH 13/24] Updated comment --- .../src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 47f13929d..39e3af2ff 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -371,7 +371,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String timeseries = new TimeSeriesWithDataEntryDate(timeseries); } - // Now we're going to call the retrieve_ts_out_tab function to get the data and build an + // Now we're going to call the retrieve_ts_entry_out_tab function to get the data and build an // internal table from it so we can manipulate it further // This code assumes the database timezone is in UTC (per Oracle recommendation) SQL retrieveSelectData = DSL.sql( From 8b069f475197beb766a0977de3931d3aaf852970 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 23 Apr 2025 14:32:55 -0700 Subject: [PATCH 14/24] Added null check --- .../java/cwms/cda/api/TimeseriesControllerTestIT.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index f6caaaf16..f47f8e990 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -25,6 +25,9 @@ import java.sql.SQLException; import java.time.ZonedDateTime; import javax.servlet.http.HttpServletResponse; + +import io.restassured.response.ExtractableResponse; +import io.restassured.response.Response; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Disabled; @@ -497,7 +500,7 @@ void test_include_data_entry_date() throws Exception { // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM // get it back with the data entry date - given() + ExtractableResponse<Response> response = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) @@ -518,7 +521,10 @@ void test_include_data_entry_date() throws Exception { .statusCode(is(HttpServletResponse.SC_OK)) .body("values.size()", equalTo(4)) .body("values[0][1]", equalTo(4.0F)) - .body("values[0].size()", equalTo(4)); + .body("values[0].size()", equalTo(4)) + .extract(); + + assertNotNull(response.body().path("values[0][3]")); // get it back without the data entry date given() From c0b2a470929a827b133cbcb0221486149813a4e2 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Mon, 19 May 2025 15:18:05 -0700 Subject: [PATCH 15/24] Added database compatibility check --- .../java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 39e3af2ff..295b02752 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -9,6 +9,7 @@ import static org.jooq.impl.DSL.partitionBy; import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; +import static org.jooq.impl.DSL.table; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -172,6 +173,16 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Timestamp tsCursor = null; Integer total = null; + if (includeEntryDate) { + Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) + .where(field("TYPE_NAME").eq("ZTSV_ENTRY_TYPE")) + .and(field("OWNER").eq("CWMS_20")).fetchOne(); + + if (entryDateSupport == null) { + throw new DataAccessException("Data entry date retrieval is not supported by this database"); + } + } + if (page != null && !page.isEmpty()) { final String[] parts = CwmsDTOPaginated.decodeCursor(page); @@ -298,7 +309,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String valid.field("office_id", String.class) )); - totalField = DSL.selectCount().from(DSL.table(retrieveSelectCount)).asField("TOTAL"); + totalField = DSL.selectCount().from(table(retrieveSelectCount)).asField("TOTAL"); } SelectJoinStep<?> metadataQuery = From 49df2d4b82e73bc325ed10af25ad455bbe134a82 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 21 May 2025 13:08:33 -0700 Subject: [PATCH 16/24] Replaced strings with constants, cleaned up DAO --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 12 +- .../java/cwms/cda/data/dto/TimeSeries.java | 1 - .../cda/api/TimeseriesControllerTestIT.java | 274 +++++++++--------- 3 files changed, 140 insertions(+), 147 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 295b02752..673329b7f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -378,10 +378,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String } }); - if (includeEntryDate) { - timeseries = new TimeSeriesWithDataEntryDate(timeseries); - } - // Now we're going to call the retrieve_ts_entry_out_tab function to get the data and build an // internal table from it so we can manipulate it further // This code assumes the database timezone is in UTC (per Oracle recommendation) @@ -433,7 +429,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (includeEntryDate) { logger.fine(() -> query2.getSQL(ParamType.INLINED)); - final TimeSeriesWithDataEntryDate timeSeries = new TimeSeriesWithDataEntryDate(timeseries); + final TimeSeriesWithDataEntryDate timeSeries = (TimeSeriesWithDataEntryDate) timeseries; query2.forEach(tsRecord -> timeSeries.addValue( tsRecord.getValue(dateTimeCol), tsRecord.getValue(valueCol), @@ -1184,11 +1180,9 @@ private void store(Connection connection, String officeId, String tsId, String u final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); if (values != null && !values.isEmpty()) { - for(TimeSeries.Record value : values) - { + for (TimeSeries.Record value : values) { Double dataValue = value.getValue(); - if(dataValue != null && dataValue == -Float.MAX_VALUE) - { + if (dataValue != null && dataValue == -Float.MAX_VALUE) { dataValue = null; } tsvArray.add(new ZTSV_TYPE(value.getDateTime(), dataValue, BigDecimal.valueOf(value.getQualityCode()))); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 79b1230cd..18f86d323 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -269,7 +269,6 @@ private Record() { } public Record(Timestamp dateTime, Double value, int qualityCode) { - super(); this.dateTime = dateTime; this.value = value; this.qualityCode = qualityCode; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index f47f8e990..5fb8a52eb 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -48,7 +48,7 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); try { @@ -63,7 +63,7 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -79,11 +79,11 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) - .queryParam("unit","cfs") - .queryParam("name",ts.get("name").asText()) - .queryParam("begin","2023-01-11T12:00:00-00:00") - .queryParam("end","2023-01-11T13:00:00-00:00") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") .when() .redirects().follow(true) .redirects().max(3) @@ -114,7 +114,7 @@ void test_lrl_1day() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -128,7 +128,7 @@ void test_lrl_1day() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -146,11 +146,11 @@ void test_lrl_1day() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .when() .redirects().follow(true) .redirects().max(3) @@ -177,7 +177,7 @@ void test_lrl_1day_max_version() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -191,7 +191,7 @@ void test_lrl_1day_max_version() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -210,7 +210,7 @@ void test_lrl_1day_max_version() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -229,11 +229,11 @@ void test_lrl_1day_max_version() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .queryParam("version-date", versionDate) .when() .redirects().follow(true) @@ -256,11 +256,11 @@ void test_lrl_1day_max_version() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .when() .redirects().follow(true) .redirects().max(3) @@ -286,7 +286,7 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -300,7 +300,7 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -319,7 +319,7 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -338,13 +338,13 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .queryParam("version-date", versionDate) - .queryParam("include-entry-date", true) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, true) .when() .redirects().follow(true) .redirects().max(3) @@ -366,12 +366,12 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) - .queryParam("include-entry-date", true) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, true) .when() .redirects().follow(true) .redirects().max(3) @@ -397,7 +397,7 @@ void test_lrl_1day_bad_units() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -411,7 +411,7 @@ void test_lrl_1day_bad_units() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -435,7 +435,7 @@ void test_lrl_1day_malicious_units() throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -449,7 +449,7 @@ void test_lrl_1day_malicious_units() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -472,7 +472,7 @@ void test_include_data_entry_date() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -485,7 +485,7 @@ void test_include_data_entry_date() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -506,7 +506,7 @@ void test_include_data_entry_date() throws Exception { .header("Authorization", user.toHeaderValue()) .queryParam(Controllers.OFFICE, officeId) .queryParam(Controllers.UNIT, "CFS") - .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") .queryParam(Controllers.END, "2010-02-03T11:00:00Z") .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") @@ -533,7 +533,7 @@ void test_include_data_entry_date() throws Exception { .header("Authorization", user.toHeaderValue()) .queryParam(Controllers.OFFICE, officeId) .queryParam(Controllers.UNIT, "CFS") - .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") .queryParam(Controllers.END, "2010-02-03T11:00:00Z") .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") @@ -562,7 +562,7 @@ void test_attempt_store_with_entry_date() throws Exception String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -598,7 +598,7 @@ void test_delete_ts() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -611,7 +611,7 @@ void test_delete_ts() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -625,16 +625,16 @@ void test_delete_ts() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("begin", "2023-02-02T11:00:00+00:00") - .queryParam("end", "2023-02-02T11:00:00+00:00") - .queryParam("start-time-inclusive", "true") - .queryParam("end-time-inclusive", "true") - .queryParam("override-protection", "true") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.BEGIN, "2023-02-02T11:00:00+00:00") + .queryParam(Controllers.END, "2023-02-02T11:00:00+00:00") + .queryParam(Controllers.START_TIME_INCLUSIVE, "true") + .queryParam(Controllers.END_TIME_INCLUSIVE, "true") + .queryParam(Controllers.OVERRIDE_PROTECTION, "true") .when() .redirects().follow(true) .redirects().max(3) - .delete("/timeseries/" + ts.get("name").asText()) + .delete("/timeseries/" + ts.get(Controllers.NAME).asText()) .then() .log().ifValidationFails(LogDetail.ALL, true) .assertThat() @@ -649,11 +649,11 @@ void test_delete_ts() throws Exception { .accept(Formats.JSONV2) // .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", "2023-02-02T11:00:00+00:00") - .queryParam("end", "2023-02-03T11:00:00+00:00") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, "2023-02-02T11:00:00+00:00") + .queryParam(Controllers.END, "2023-02-03T11:00:00+00:00") .queryParam(Controllers.TRIM, false) .when() .redirects().follow(true) @@ -677,7 +677,7 @@ void test_no_office_permissions() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -710,8 +710,8 @@ void test_invalid_office() { .accept(Formats.JSONV2) .contentType(Formats.JSONV2) //Purposefully misspelled office id - .queryParam("office", "NWDW") - .queryParam("name", "Buckhorn.Temp-Water.Inst.1Day.0.cda-test") + .queryParam(Controllers.OFFICE, "NWDW") + .queryParam(Controllers.NAME, "Buckhorn.Temp-Water.Inst.1Day.0.cda-test") .when() .redirects().follow(true) .redirects().max(3) @@ -733,7 +733,7 @@ void test_v1_cant_trim() throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -743,12 +743,12 @@ void test_v1_cant_trim() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV1) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) - .queryParam("trim", "true") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.TRIM, "true") .when() .redirects().follow(true) .redirects().max(3) @@ -770,7 +770,7 @@ void test_v1_cant_version() throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -782,11 +782,11 @@ void test_v1_cant_version() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV1) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .queryParam("version-date", version) .when() .redirects().follow(true) @@ -809,7 +809,7 @@ void test_v2_cant_datum() throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -819,11 +819,11 @@ void test_v2_cant_datum() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", firstPoint) - .queryParam("end", firstPoint) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, firstPoint) + .queryParam(Controllers.END, firstPoint) .queryParam("datum", "NAVD88") .when() .redirects().follow(true) @@ -846,7 +846,7 @@ void test_lrl_trim() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); try { @@ -861,7 +861,7 @@ void test_lrl_trim() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -887,12 +887,12 @@ void test_lrl_trim() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", false) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, dayBeforeFirst.toInstant().toString()) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.TRIM, false) .when() .redirects().follow(true) .redirects().max(3) @@ -913,12 +913,12 @@ void test_lrl_trim() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", true) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, dayBeforeFirst.toInstant().toString()) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.TRIM, true) .when() .redirects().follow(true) .redirects().max(3) @@ -947,7 +947,7 @@ void test_lrl_trim_with_data_entry_date() throws Exception { String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); try { @@ -962,7 +962,7 @@ void test_lrl_trim_with_data_entry_date() throws Exception { .contentType(Formats.JSONV2) .body(tsData) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -988,13 +988,13 @@ void test_lrl_trim_with_data_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "m2") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", false) - .queryParam("include-entry-date", true) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "m2") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, dayBeforeFirst.toInstant().toString()) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.TRIM, false) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, true) .when() .redirects().follow(true) .redirects().max(3) @@ -1015,13 +1015,13 @@ void test_lrl_trim_with_data_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "m2") - .queryParam("name", ts.get("name").asText()) - .queryParam("begin", dayBeforeFirst.toInstant().toString()) - .queryParam("end", firstPoint) - .queryParam("trim", true) - .queryParam("include-entry-date", true) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "m2") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN, dayBeforeFirst.toInstant().toString()) + .queryParam(Controllers.END, firstPoint) + .queryParam(Controllers.TRIM, true) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, true) .when() .redirects().follow(true) .redirects().max(3) @@ -1056,7 +1056,7 @@ void test_big_create() throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -1070,7 +1070,7 @@ void test_big_create() throws Exception { .contentType(Formats.JSONV2) .body(giantString) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -1084,7 +1084,7 @@ void test_big_create() throws Exception { /** * Input looks like: * { - * "name": "Buckhorn.Temp-Water.Inst.1Day.0.cda-test", + * Controllers.NAME: "Buckhorn.Temp-Water.Inst.1Day.0.cda-test", * "office-id": "SPK", * "units": "F", * "values": [ @@ -1150,7 +1150,7 @@ void test_daylight_saving_retrieve()throws Exception { ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); - String name = ts.get("name").asText(); + String name = ts.get(Controllers.NAME).asText(); String location = name.split("\\.")[0]; String officeId = ts.get("office-id").asText(); @@ -1170,7 +1170,7 @@ void test_daylight_saving_retrieve()throws Exception { .contentType(Formats.JSONV2) .body(giantString) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) + .queryParam(Controllers.OFFICE, officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -1187,11 +1187,11 @@ void test_daylight_saving_retrieve()throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit","mm") - .queryParam("name", name) - .queryParam("begin","2021-02-08T08:00:00Z") - .queryParam("end","2021-03-08T08:00:00Z") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"mm") + .queryParam(Controllers.NAME, name) + .queryParam(Controllers.BEGIN,"2021-02-08T08:00:00Z") + .queryParam(Controllers.END,"2021-03-08T08:00:00Z") .when() .redirects().follow(true) .redirects().max(3) @@ -1210,11 +1210,11 @@ void test_daylight_saving_retrieve()throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit","mm") - .queryParam("name", name) - .queryParam("begin","2021-03-08T08:00:00Z") - .queryParam("end","2021-03-15T08:00:00Z") + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"mm") + .queryParam(Controllers.NAME, name) + .queryParam(Controllers.BEGIN,"2021-03-08T08:00:00Z") + .queryParam(Controllers.END,"2021-03-15T08:00:00Z") .when() .redirects().follow(true) .redirects().max(3) @@ -1263,7 +1263,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); - String location = ts.get("name").asText().split("\\.")[0]; + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); @@ -1277,7 +1277,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception .contentType(Formats.JSONV2) .body(tsData) .header("Authorization",user.toHeaderValue()) - .queryParam("office",officeId) + .queryParam(Controllers.OFFICE,officeId) .when() .redirects().follow(true) .redirects().max(3) @@ -1293,9 +1293,9 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) .header("Authorization", user.toHeaderValue()) - .queryParam("office", officeId) - .queryParam("unit", "F") - .queryParam("name", ts.get("name").asText()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "F") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) .when() .redirects().follow(true) .redirects().max(3) From ce9a30aa6ed940f2b8a7b373cb8091bb75695f12 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 21 May 2025 16:00:54 -0700 Subject: [PATCH 17/24] Fixed scoping issue --- cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index e767c7a3a..d90587efa 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -322,7 +322,7 @@ public String toString() { } @Schema(hidden = true, name = "TimeSeries.Column", accessMode = Schema.AccessMode.READ_ONLY) - protected static class Column { + public static class Column { public final String name; public final int ordinal; public final Class<?> datatype; From b4195a16d081d2072909928fbdbdad9d002f9339 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Thu, 5 Jun 2025 14:13:02 -0700 Subject: [PATCH 18/24] Removed duplicate value list, added custom deserializer to handle subtypes of Record class --- .../main/java/cwms/cda/data/dao/RateDao.java | 4 +- .../java/cwms/cda/data/dto/TimeSeries.java | 21 ++++- .../data/dto/TimeSeriesWithDataEntryDate.java | 64 ++-------------- .../TimeSeriesRecordDeserializer.java | 76 +++++++++++++++++++ .../cda/api/TimeSeriesControllerTest.java | 5 +- .../TimeSeriesProfileInstanceDaoIT.java | 2 +- .../dto/rating/RatedOutputTimeSeriesTest.java | 10 +-- 7 files changed, 110 insertions(+), 72 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java index 51da2c0a5..9c6ab4689 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java @@ -132,7 +132,7 @@ public RatedOutput rate(String officeId, String ratingId, RateInputTimeSeries in officeId); }); List<TimeSeries.Record> records = ztsvTypes.stream() - .map(z -> new TimeSeries.Record(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) + .map(z -> new TimeSeries.StandardRecord(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) .collect(toList()); return new RatedOutputTimeSeries(CwmsId.buildCwmsId(officeId, ratingId), records, input.getOutputUnit()); } @@ -153,7 +153,7 @@ public RatedOutput reverseRate(String officeId, String ratingId, RateInputTimeSe officeId); }); List<TimeSeries.Record> records = ztsvTypes.stream() - .map(z -> new TimeSeries.Record(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) + .map(z -> new TimeSeries.StandardRecord(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) .collect(toList()); return new RatedOutputTimeSeries(CwmsId.buildCwmsId(officeId, ratingId), records, input.getOutputUnit()); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index d90587efa..c015402eb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -9,12 +9,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; +import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; @@ -214,7 +217,7 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode) { if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - values.add(new Record(dateTime, value, qualityCode)); + values.add(new StandardRecord(dateTime, value, qualityCode)); } } @@ -232,7 +235,6 @@ public static List<Column> getColumnDescriptor() { } - @ArraySchema( schema = @Schema( name = "TimeSeries.Record", @@ -250,8 +252,9 @@ public static List<Column> getColumnDescriptor() { ) ) + @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) @JsonIgnoreProperties(ignoreUnknown = true) - public static class Record { + public abstract static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") @@ -321,6 +324,18 @@ public String toString() { } } + @JsonDeserialize(using = JsonDeserializer.None.class) + public static final class StandardRecord extends Record { + // unused - required for Jackson to deserialize + private StandardRecord() + { + } + + public StandardRecord(Timestamp dateTime, Double value, int qualityCode) { + super(dateTime, value, qualityCode); + } + } + @Schema(hidden = true, name = "TimeSeries.Column", accessMode = Schema.AccessMode.READ_ONLY) public static class Column { public final String name; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java index a49b59577..603aefe3f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java @@ -35,15 +35,12 @@ import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; import cwms.cda.formatters.xml.XMLv2; -import io.swagger.v3.oas.annotations.media.Schema; import java.lang.reflect.Field; import java.sql.Timestamp; import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; -import java.util.Objects; - @JsonInclude(JsonInclude.Include.NON_NULL) @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) @@ -54,15 +51,14 @@ public final class TimeSeriesWithDataEntryDate extends TimeSeries { // Default constructor for Jackson Deserialization public TimeSeriesWithDataEntryDate() { super(); - valuesWithEntryDate = new ArrayList<>(); } + // Unused constructor required for Jackson Deserialization public TimeSeriesWithDataEntryDate(TimeSeries timeSeries) { this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), timeSeries.getName(), timeSeries.getOfficeId(), timeSeries.getBegin(), timeSeries.getEnd(), timeSeries.getUnits(), timeSeries.getInterval(), timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), timeSeries.getTimeZone(), timeSeries.getVersionDate(), timeSeries.getDateVersionType()); - valuesWithEntryDate = new ArrayList<>(); } public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, String name, String officeId, @@ -70,26 +66,17 @@ public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, Str Long intervalOffset, String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, timeZone, versionDate, dateVersionType); - valuesWithEntryDate = new ArrayList<>(); - } - - @JsonProperty(value = "values") - private List<TimeSeriesWithDataEntryDate.Record> valuesWithEntryDate; - - @Override - public List<TimeSeries.Record> getValues() { - return new ArrayList<>(valuesWithEntryDate); } public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set - if ((page == null || page.isEmpty()) && (valuesWithEntryDate == null || valuesWithEntryDate.isEmpty())) { + if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); } - if (pageSize > 0 && valuesWithEntryDate.size() == pageSize) { + if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - valuesWithEntryDate.add(new Record(dateTime, value, qualityCode, dataEntryDate)); + values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); } } @@ -109,7 +96,7 @@ private List<Column> getColumnDescriptorWithEntryDate() { columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - for (Field f: Record.class.getDeclaredFields()) { + for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); if (field != null) { String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); @@ -120,45 +107,4 @@ private List<Column> getColumnDescriptorWithEntryDate() { return columns; } - - public static final class Record extends TimeSeries.Record { - @JsonProperty(value = "data-entry-date", index = 3) - @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - private Timestamp dataEntryDate; - - // Default constructor for Jackson Deserialization - public Record() { - super(null, null, 0); - } - - public Record(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { - super(dateTime, value, qualityCode); - this.dataEntryDate = dataEntryDate; - } - - public Timestamp getDataEntryDate() { - return dataEntryDate; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - Record that = (Record) o; - return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), getDataEntryDate()); - } - } } diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java new file mode 100644 index 000000000..bd3c29d02 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -0,0 +1,76 @@ +/* + * + * MIT License + * + * Copyright (c) 2025 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.formatters.json.adapters; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import java.io.IOException; +import java.sql.Timestamp; + +/** + * A time-series record deserializer for properly deserializing JSON data. + * Requires {@link cwms.cda.data.dto.TimeSeries.StandardRecord} class to avoid + * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer + * getting stuck in a loop if the Record class is used directly. + * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. + */ +public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { + private static final String DATA_ENTRY_DATE = "data-entry-date"; + @Override + public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.readValueAsTree(); + if (node.get(DATA_ENTRY_DATE) != null) { + return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); + } + String nodeString = node.toString(); + if (nodeString.startsWith("[")) { + nodeString = nodeString.substring(1, nodeString.length() - 1); + String[] valList = nodeString.split(","); + if (valList.length == 4) { + Timestamp dateTime = new Timestamp(Long.parseLong(valList[0])); + Double value = valList[1] == null || valList[1].equalsIgnoreCase("null") + ? null : Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + Timestamp entryDate = new Timestamp(Long.parseLong(valList[3])); + return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); + } else if (valList.length == 3) { + Timestamp dateTime = new Timestamp(Long.parseLong(valList[0])); + Double value = valList[1] == null || valList[1].equalsIgnoreCase("null") + ? null : Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + return new TimeSeries.StandardRecord(dateTime, value, quality); + } else { + throw new IOException("Invalid TimeSeries Record format"); + } + } + return jsonParser.getCodec().treeToValue(node, TimeSeries.StandardRecord.class); + } +} \ No newline at end of file diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index e66d872de..c6832830f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; @@ -153,8 +154,8 @@ private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSerie private void assertDateRecordsMatch(List<? extends TimeSeries.Record> expected, List<? extends TimeSeries.Record> actual) { for (int i = 0; i < expected.size(); i++) { - assertEquals(((TimeSeriesWithDataEntryDate.Record) expected.get(i)).getDataEntryDate(), - ((TimeSeriesWithDataEntryDate.Record) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); + assertEquals(((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate(), + ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java index 73a155393..df1ed8a72 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java @@ -934,7 +934,7 @@ private static TimeSeriesProfileInstance buildTestTimeSeriesProfileInstance(Stri List<TimeSeries.Record> timeValuePairList = new ArrayList<>(); for (int i = 0; i < dateTimeArray.length; i++) { - TimeSeries.Record timeValuePair = new TimeSeries.Record(Timestamp.from(dateTimeArray[i]), + TimeSeries.Record timeValuePair = new TimeSeries.StandardRecord(Timestamp.from(dateTimeArray[i]), valueArray[i], 0); timeValuePairList.add(timeValuePair); } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java index 83edfe2b1..596ee633a 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java @@ -52,11 +52,11 @@ void testSerializationRoundTrip() throws Exception { RatedOutputTimeSeries deserialized = Formats.parseContent(contentType, json, RatedOutputTimeSeries.class); CwmsId cwmsId = CwmsId.buildCwmsId("NWDP", "DOTW.Stage;Flow.Logarithmic.USGS-NWIS"); List<TimeSeries.Record> depValues = - Arrays.asList(new TimeSeries.Record(new Timestamp(1672531200000L), 137.90304290304002, 0), - new TimeSeries.Record(new Timestamp(1577836800000L), 167.0693948928, 0), - new TimeSeries.Record(new Timestamp(1546300800000L), null, 5), - new TimeSeries.Record(new Timestamp(1451606400000L), 0.0269010042624, 0), - new TimeSeries.Record(new Timestamp(1388534400000L), 1786.7930199552, 0)); + Arrays.asList(new TimeSeries.StandardRecord(new Timestamp(1672531200000L), 137.90304290304002, 0), + new TimeSeries.StandardRecord(new Timestamp(1577836800000L), 167.0693948928, 0), + new TimeSeries.StandardRecord(new Timestamp(1546300800000L), null, 5), + new TimeSeries.StandardRecord(new Timestamp(1451606400000L), 0.0269010042624, 0), + new TimeSeries.StandardRecord(new Timestamp(1388534400000L), 1786.7930199552, 0)); String outputUnit = "cfs"; assertEquals(depValues, deserialized.getValues()); assertMatch(cwmsId, deserialized.getRatingId()); From d14893375526df9d6e6d3cfa6691ed0f507897e2 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Fri, 6 Jun 2025 15:04:02 -0700 Subject: [PATCH 19/24] Renamed class --- ...ithDate.java => TimeSeriesRecordWithEntryDate.java} | 10 +++++----- .../cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java | 4 ++-- .../json/adapters/TimeSeriesRecordDeserializer.java | 9 +++++---- .../java/cwms/cda/api/TimeSeriesControllerTest.java | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) rename cwms-data-api/src/main/java/cwms/cda/data/dto/{TimeSeriesRecordWithDate.java => TimeSeriesRecordWithEntryDate.java} (86%) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java similarity index 86% rename from cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java rename to cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java index 21066175d..908b3999d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java @@ -35,22 +35,22 @@ import java.util.Objects; /** - * TimeSeriesRecordWithDate is a subclass of TimeSeries.Record that includes a data entry date. + * TimeSeriesRecordWithEntryDate is a subclass of TimeSeries.Record that includes a data entry date. * The data entry date is the date that the data was entered into the database. */ @JsonDeserialize(using = JsonDeserializer.None.class) -public final class TimeSeriesRecordWithDate extends TimeSeries.Record { +public final class TimeSeriesRecordWithEntryDate extends TimeSeries.Record { @JsonProperty(value = "data-entry-date", index = 3) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") @JsonInclude(JsonInclude.Include.NON_DEFAULT) Timestamp dataEntryDate; // Default constructor for Jackson Deserialization - public TimeSeriesRecordWithDate() { + public TimeSeriesRecordWithEntryDate() { super(null, null, 0); } - public TimeSeriesRecordWithDate(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + public TimeSeriesRecordWithEntryDate(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { super(dateTime, value, qualityCode); this.dataEntryDate = dataEntryDate; } @@ -70,7 +70,7 @@ public boolean equals(Object o) { if (!super.equals(o)) { return false; } - TimeSeriesRecordWithDate that = (TimeSeriesRecordWithDate) o; + TimeSeriesRecordWithEntryDate that = (TimeSeriesRecordWithEntryDate) o; return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java index 603aefe3f..ff701a391 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java @@ -76,7 +76,7 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); + values.add(new TimeSeriesRecordWithEntryDate(dateTime, value, qualityCode, dataEntryDate)); } } @@ -96,7 +96,7 @@ private List<Column> getColumnDescriptorWithEntryDate() { columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { + for (Field f: TimeSeriesRecordWithEntryDate.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); if (field != null) { String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java index bd3c29d02..8afcedb5e 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -31,7 +31,8 @@ import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; + import java.io.IOException; import java.sql.Timestamp; @@ -40,7 +41,7 @@ * Requires {@link cwms.cda.data.dto.TimeSeries.StandardRecord} class to avoid * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer * getting stuck in a loop if the Record class is used directly. - * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. + * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithEntryDate}. */ public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { private static final String DATA_ENTRY_DATE = "data-entry-date"; @@ -48,7 +49,7 @@ public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSer public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { JsonNode node = jsonParser.readValueAsTree(); if (node.get(DATA_ENTRY_DATE) != null) { - return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); + return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithEntryDate.class); } String nodeString = node.toString(); if (nodeString.startsWith("[")) { @@ -60,7 +61,7 @@ public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationConte ? null : Double.parseDouble(valList[1]); int quality = Integer.parseInt(valList[2]); Timestamp entryDate = new Timestamp(Long.parseLong(valList[3])); - return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); + return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); } else if (valList.length == 3) { Timestamp dateTime = new Timestamp(Long.parseLong(valList[0])); Double value = valList[1] == null || valList[1].equalsIgnoreCase("null") diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index c6832830f..2534d9b19 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,7 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; @@ -154,8 +154,8 @@ private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSerie private void assertDateRecordsMatch(List<? extends TimeSeries.Record> expected, List<? extends TimeSeries.Record> actual) { for (int i = 0; i < expected.size(); i++) { - assertEquals(((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate(), - ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); + assertEquals(((TimeSeriesRecordWithEntryDate) expected.get(i)).getDataEntryDate(), + ((TimeSeriesRecordWithEntryDate) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); From efffc133aa776acc8208af7ca03378f1ace58f93 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Fri, 6 Jun 2025 15:30:42 -0700 Subject: [PATCH 20/24] Fixed broken ratings test --- .../java/cwms/cda/api/rating/RatingsControllerTestIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingsControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingsControllerTestIT.java index c8404a8cf..4a456fbaa 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingsControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingsControllerTestIT.java @@ -267,7 +267,7 @@ void test_get_one_latest() { effectiveDate = response.path("simple-rating.effective-date"); } assertNotNull(effectiveDate); - assertEquals("2016-06-06T00:00:00Z", effectiveDate); + assertEquals("2025-06-06T00:00:00Z", effectiveDate); // get latest xml response = given() @@ -290,7 +290,7 @@ void test_get_one_latest() { effectiveDate = response.path("simple-rating.effective-date"); } assertNotNull(effectiveDate); - assertEquals("2016-06-06T00:00:00Z", effectiveDate); + assertEquals("2025-06-06T00:00:00Z", effectiveDate); } enum GetOneTest From c93ed4580f36fee1c101dacb204a241e9ae326dd Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Tue, 10 Jun 2025 11:03:08 -0700 Subject: [PATCH 21/24] Cleaned up subclasses, cleaned up deserializer. Added more unit test coverage of ts record classes --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 18 +-- .../java/cwms/cda/data/dto/TimeSeries.java | 43 ++++++- .../dto/TimeSeriesRecordWithEntryDate.java | 3 - .../data/dto/TimeSeriesWithDataEntryDate.java | 110 ----------------- .../TimeSeriesRecordDeserializer.java | 60 ++++++---- .../cda/api/TimeSeriesControllerTest.java | 33 +++--- .../cda/data/dto/TimeSeriesRecordTest.java | 111 ++++++++++++++++++ .../cwms/cda/data/dto/TimeSeriesTest.java | 7 +- 8 files changed, 210 insertions(+), 175 deletions(-) delete mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 673329b7f..dce0f3344 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -25,7 +25,6 @@ import cwms.cda.data.dto.RecentValue; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesExtents; -import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.data.dto.Tsv; import cwms.cda.data.dto.TsvDqu; import cwms.cda.data.dto.TsvId; @@ -351,7 +350,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String String vert = (String) tsMetadata.getValue("VERTICAL_DATUM"); VerticalDatumInfo verticalDatumInfo = parseVerticalDatumInfo(vert); VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); - if (!includeEntryDate) { return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", Integer.class), tsMetadata.getValue("NAME", String.class), tsMetadata.getValue("office_id", String.class), @@ -363,20 +361,8 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String tsMetadata.getValue(tzName), versionDate, finalDateVersionType ); - } else { - return new TimeSeriesWithDataEntryDate(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", - Integer.class), tsMetadata.getValue("NAME", String.class), - tsMetadata.getValue("office_id", String.class), - beginTime, endTime, tsMetadata.getValue("units", String.class), - Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : - tsMetadata.getValue("interval", Long.class)), - verticalDatumInfo, - tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), - tsMetadata.getValue(tzName), - versionDate, finalDateVersionType - ); } - }); + ); // Now we're going to call the retrieve_ts_entry_out_tab function to get the data and build an // internal table from it so we can manipulate it further @@ -429,7 +415,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (includeEntryDate) { logger.fine(() -> query2.getSQL(ParamType.INLINED)); - final TimeSeriesWithDataEntryDate timeSeries = (TimeSeriesWithDataEntryDate) timeseries; + final TimeSeries timeSeries = timeseries; query2.forEach(tsRecord -> timeSeries.addValue( tsRecord.getValue(dateTimeCol), tsRecord.getValue(valueCol), diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index c015402eb..0f58d92db 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; @@ -206,6 +205,9 @@ public VersionType getDateVersionType() { + "data in the array.", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { + if (values != null && !values.isEmpty() && values.get(0) instanceof TimeSeriesRecordWithEntryDate) { + return getColumnDescriptorWithEntryDate(); + } return getColumnDescriptor(); } @@ -221,6 +223,18 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode) { } } + public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + // Set the current page, if not set + if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { + page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); + } + if (pageSize > 0 && values.size() == pageSize) { + nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); + } else { + values.add(new TimeSeriesRecordWithEntryDate(dateTime, value, qualityCode, dataEntryDate)); + } + } + public static List<Column> getColumnDescriptor() { List<Column> columns = new ArrayList<>(); for (Field f: Record.class.getDeclaredFields()) { @@ -234,6 +248,28 @@ public static List<Column> getColumnDescriptor() { return columns; } + private List<Column> getColumnDescriptorWithEntryDate() { + List<Column> columns = new ArrayList<>(); + for (Field f: TimeSeries.Record.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + for (Field f: TimeSeriesRecordWithEntryDate.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + + return columns; + } + @ArraySchema( schema = @Schema( @@ -324,12 +360,9 @@ public String toString() { } } - @JsonDeserialize(using = JsonDeserializer.None.class) public static final class StandardRecord extends Record { // unused - required for Jackson to deserialize - private StandardRecord() - { - } + private StandardRecord() {} public StandardRecord(Timestamp dateTime, Double value, int qualityCode) { super(dateTime, value, qualityCode); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java index 908b3999d..c023f1fa5 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java @@ -28,8 +28,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.swagger.v3.oas.annotations.media.Schema; import java.sql.Timestamp; import java.util.Objects; @@ -38,7 +36,6 @@ * TimeSeriesRecordWithEntryDate is a subclass of TimeSeries.Record that includes a data entry date. * The data entry date is the date that the data was entered into the database. */ -@JsonDeserialize(using = JsonDeserializer.None.class) public final class TimeSeriesRecordWithEntryDate extends TimeSeries.Record { @JsonProperty(value = "data-entry-date", index = 3) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java deleted file mode 100644 index ff701a391..000000000 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDataEntryDate.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * - * MIT License - * - * Copyright (c) 2024 Hydrologic Engineering Center - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE - * SOFTWARE. - */ - -package cwms.cda.data.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import cwms.cda.api.enums.VersionType; -import cwms.cda.formatters.Formats; -import cwms.cda.formatters.annotations.FormattableWith; -import cwms.cda.formatters.json.JsonV2; -import cwms.cda.formatters.xml.XMLv2; -import java.lang.reflect.Field; -import java.sql.Timestamp; -import java.time.Duration; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.List; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) -@FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) -@FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) -public final class TimeSeriesWithDataEntryDate extends TimeSeries { - - // Default constructor for Jackson Deserialization - public TimeSeriesWithDataEntryDate() { - super(); - } - - // Unused constructor required for Jackson Deserialization - public TimeSeriesWithDataEntryDate(TimeSeries timeSeries) { - this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), timeSeries.getName(), - timeSeries.getOfficeId(), timeSeries.getBegin(), timeSeries.getEnd(), timeSeries.getUnits(), - timeSeries.getInterval(), timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), - timeSeries.getTimeZone(), timeSeries.getVersionDate(), timeSeries.getDateVersionType()); - } - - public TimeSeriesWithDataEntryDate(String page, int pageSize, Integer total, String name, String officeId, - ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, - Long intervalOffset, String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { - super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, - timeZone, versionDate, dateVersionType); - } - - public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { - // Set the current page, if not set - if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { - page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); - } - if (pageSize > 0 && values.size() == pageSize) { - nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); - } else { - values.add(new TimeSeriesRecordWithEntryDate(dateTime, value, qualityCode, dataEntryDate)); - } - } - - @JsonProperty(value = "value-columns") - @Override - public List<Column> getValueColumnsJSON() { - return getColumnDescriptorWithEntryDate(); - } - - private List<Column> getColumnDescriptorWithEntryDate() { - List<Column> columns = new ArrayList<>(); - for (Field f: TimeSeries.Record.class.getDeclaredFields()) { - JsonProperty field = f.getAnnotation(JsonProperty.class); - if (field != null) { - String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); - int fieldIndex = field.index(); - columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); - } - } - for (Field f: TimeSeriesRecordWithEntryDate.class.getDeclaredFields()) { - JsonProperty field = f.getAnnotation(JsonProperty.class); - if (field != null) { - String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); - int fieldIndex = field.index(); - columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); - } - } - - return columns; - } -} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java index 8afcedb5e..20fa9453d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -30,6 +30,8 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; @@ -45,33 +47,45 @@ */ public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { private static final String DATA_ENTRY_DATE = "data-entry-date"; + private static final String DATE_TIME = "date-time"; + private static final String VALUE = "value"; + private static final String QUALITY = "quality"; + @Override public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { JsonNode node = jsonParser.readValueAsTree(); - if (node.get(DATA_ENTRY_DATE) != null) { - return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithEntryDate.class); + if (node instanceof ObjectNode) { + return parseObjectNode((ObjectNode) node); + } else if (node instanceof ArrayNode) { + return parseArrayNode((ArrayNode) node); + } else { + throw new IOException("Unexpected JSON node type: " + node.getNodeType()); + } + } + + private TimeSeries.Record parseObjectNode(ObjectNode node) { + Timestamp dateTime = node.get(DATE_TIME) == null ? null : new Timestamp(node.get(DATE_TIME).asLong()); + Double value = node.get(VALUE) == null || node.get(VALUE).asText().equalsIgnoreCase("null") + ? null : node.get(VALUE).asDouble(); + int quality = node.get(QUALITY) == null ? 0 : node.get(QUALITY).asInt(); + if (node.size() == 4) { + Timestamp entryDate = new Timestamp(node.get(DATA_ENTRY_DATE).asLong()); + return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); + } else { + return new TimeSeries.StandardRecord(dateTime, value, quality); } - String nodeString = node.toString(); - if (nodeString.startsWith("[")) { - nodeString = nodeString.substring(1, nodeString.length() - 1); - String[] valList = nodeString.split(","); - if (valList.length == 4) { - Timestamp dateTime = new Timestamp(Long.parseLong(valList[0])); - Double value = valList[1] == null || valList[1].equalsIgnoreCase("null") - ? null : Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - Timestamp entryDate = new Timestamp(Long.parseLong(valList[3])); - return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); - } else if (valList.length == 3) { - Timestamp dateTime = new Timestamp(Long.parseLong(valList[0])); - Double value = valList[1] == null || valList[1].equalsIgnoreCase("null") - ? null : Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - return new TimeSeries.StandardRecord(dateTime, value, quality); - } else { - throw new IOException("Invalid TimeSeries Record format"); - } + } + + private TimeSeries.Record parseArrayNode(ArrayNode aNode) { + Timestamp dateTime = aNode.get(0) == null ? null : new Timestamp(aNode.get(0).asLong()); + Double value = aNode.get(1) == null || aNode.get(1).asText().equalsIgnoreCase("null") + ? null : aNode.get(1).asDouble(); + int quality = aNode.get(2) == null ? 0 : aNode.get(2).asInt(); + if (aNode.size() == 4) { + Timestamp entryDate = new Timestamp(aNode.get(3).asLong()); + return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); + } else { + return new TimeSeries.StandardRecord(dateTime, value, quality); } - return jsonParser.getCodec().treeToValue(node, TimeSeries.StandardRecord.class); } } \ No newline at end of file diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 2534d9b19..6fe772899 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -15,7 +15,6 @@ import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; -import cwms.cda.data.dto.TimeSeriesWithDataEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -135,7 +134,7 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } - private void assertSimilarWithDate(TimeSeriesWithDataEntryDate expected, TimeSeriesWithDataEntryDate actual) + private void assertSimilarWithDate(TimeSeries expected, TimeSeries actual) { assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); assertEquals(expected.getName(), actual.getName(), "names did not match"); @@ -181,14 +180,14 @@ void testSerializeTimeSeries(String format) { void testSerializeTimeSeriesWithDataEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); assertEquals(4, fakeTs.getValueColumnsJSON().size()); - assertInstanceOf(TimeSeriesWithDataEntryDate.Record.class, + assertInstanceOf(TimeSeries.Record.class, fakeTs.getValues().get(0)); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -212,10 +211,10 @@ void testDeserializeTimeSeries(String format) { void testDeserializeTimeSeriesWithEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -224,10 +223,10 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { void testDeserializeTimeSeriesWithEntryDateFromFile() { InputStream inputStream = this.getClass() .getResourceAsStream("/cwms/cda/api/lrl/timeseries_with_data_entry_dates.json"); - ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeriesWithDataEntryDate.class); - TimeSeriesWithDataEntryDate fakeTs = Formats.parseContent(contentType, inputStream, TimeSeriesWithDataEntryDate.class); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries fakeTs = Formats.parseContent(contentType, inputStream, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -238,12 +237,12 @@ void testXMLSerializeDeserializeTimeSeries() String format = Formats.XMLV2; String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeriesWithDataEntryDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDataEntryDate.class); + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertTrue(formatted.contains("quality-code")); assertTrue(formatted.contains("data-entry-date")); - TimeSeriesWithDataEntryDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDataEntryDate.class); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); assertSimilarWithDate(fakeTs, ts2); } @@ -330,7 +329,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { } @NotNull - private TimeSeriesWithDataEntryDate buildTimeSeriesWithEntryDate(String officeId, String tsId) { + private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); @@ -341,7 +340,7 @@ private TimeSeriesWithDataEntryDate buildTimeSeriesWithEntryDate(String officeId int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeriesWithDataEntryDate ts = new TimeSeriesWithDataEntryDate(null, + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java new file mode 100644 index 000000000..8940e79fa --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java @@ -0,0 +1,111 @@ +/* + * + * MIT License + * + * Copyright (c) 2025 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.data.dto; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.formatters.json.JsonV2; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +final class TimeSeriesRecordTest +{ + @Test + void testRecordRoundTrip() throws Exception { + TimeSeries ts = buildTimeSeries(); + TimeSeries.Record tsRecord = ts.values.get(0); + + ObjectMapper om = JsonV2.buildObjectMapper(); + + String tsBody = om.writeValueAsString(tsRecord); + assertNotNull(tsBody); + + TimeSeries.Record tsRecordReturned = om.readValue(tsBody, TimeSeries.Record.class); + assertNotNull(tsRecordReturned); + + assertAll(() -> { + assertEquals(tsRecord.getDateTime(), tsRecordReturned.getDateTime()); + assertEquals(tsRecord.getValue(), tsRecordReturned.getValue()); + assertEquals(tsRecord.getQualityCode(), tsRecordReturned.getQualityCode()); + }); + } + + @Test + void testRecordWithEntryDateRoundTrip() throws Exception { + TimeSeries ts = buildTimeSeriesWithEntryDate(); + TimeSeriesRecordWithEntryDate tsRecord = (TimeSeriesRecordWithEntryDate) ts.values.get(0); + + ObjectMapper om = JsonV2.buildObjectMapper(); + + String tsBody = om.writeValueAsString(tsRecord); + assertNotNull(tsBody); + + TimeSeriesRecordWithEntryDate tsRecordReturned = om.readValue(tsBody, TimeSeriesRecordWithEntryDate.class); + assertNotNull(tsRecordReturned); + + assertAll(() -> + { + assertEquals(tsRecord.getDateTime(), tsRecordReturned.getDateTime()); + assertEquals(tsRecord.getValue(), tsRecordReturned.getValue()); + assertEquals(tsRecord.getQualityCode(), tsRecordReturned.getQualityCode()); + assertEquals(tsRecord.getDataEntryDate(), tsRecordReturned.getDataEntryDate()); + }); + } + + + private TimeSeries buildTimeSeries() + { + String tsId = "TS-Record-Test.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); + ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 13.45678, 0); + return ts; + } + + private TimeSeries buildTimeSeriesWithEntryDate() + { + String tsId = "TS-Record-Test.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); + ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0, Timestamp.from(Instant.now().minusSeconds(60))); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 13.45678, 0, Timestamp.from(Instant.now())); + return ts; + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java index 227e3d60d..39a0adc52 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java @@ -1,6 +1,8 @@ package cwms.cda.data.dto; import cwms.cda.formatters.json.JsonV2; + +import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; @@ -90,7 +92,10 @@ private TimeSeries buildTimeSeries(VerticalDatumInfo vdi) ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); - return new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, vdi, versionDate, null); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, vdi, versionDate, null); + ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 12.34567, 0); + return ts; } VerticalDatumInfo buildVerticalDatumInfo() From 35cdefa4eb1c19c9a3f5cd3cea6340a05c986b21 Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 11 Jun 2025 10:02:41 -0700 Subject: [PATCH 22/24] Removed deserializer, added custom serializer for ts records, added and updated tests --- .../cwms/cda/api/TimeSeriesController.java | 6 +- .../main/java/cwms/cda/data/dao/RateDao.java | 14 +-- .../java/cwms/cda/data/dto/TimeSeries.java | 67 ++++++++------ .../dto/TimeSeriesRecordWithEntryDate.java | 78 ---------------- .../TimeSeriesRecordDeserializer.java | 91 ------------------- .../adapters/TimeSeriesRecordSerializer.java | 64 +++++++++++++ .../cda/api/TimeSeriesControllerTest.java | 61 +++++-------- .../cda/api/TimeseriesControllerTestIT.java | 6 +- .../TimeSeriesProfileInstanceDaoIT.java | 2 +- .../cda/data/dto/TimeSeriesRecordTest.java | 24 +---- .../cwms/cda/data/dto/TimeSeriesTest.java | 22 +++++ .../dto/rating/RatedOutputTimeSeriesTest.java | 10 +- 12 files changed, 171 insertions(+), 274 deletions(-) delete mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java delete mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 9c38477c5..998b4dfa2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -550,13 +550,11 @@ public void update(@NotNull Context ctx, @NotNull String id) { } } - private TimeSeries deserializeTimeSeries(Context ctx) throws IOException - { + private TimeSeries deserializeTimeSeries(Context ctx) throws IOException { String contentTypeHeader = ctx.req.getContentType(); StringWriter writer = new StringWriter(); IOUtils.copy(ctx.bodyAsInputStream(), writer, StandardCharsets.UTF_8); - if (writer.toString().contains("data-entry-date")) - { + if (writer.toString().contains("data-entry-date")) { throw new IllegalArgumentException("Data entry date is not allowed in the request"); } ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeries.class); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java index 9c6ab4689..ca22b869c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RateDao.java @@ -78,18 +78,18 @@ public RatedOutput rate(String officeId, String ratingId, RateInputValues input) } private void validateReverseRateInput(RateInput input) { - if(input instanceof RateInputTimeSeries && ((RateInputTimeSeries) input).getTimeSeriesIds().size() > 1) { + if (input instanceof RateInputTimeSeries && ((RateInputTimeSeries) input).getTimeSeriesIds().size() > 1) { throw new IllegalArgumentException("Reverse Rating only supports one dependent parameter"); } - if(input instanceof RateInputValues) { + if (input instanceof RateInputValues) { List<List<Double>> values = ((RateInputValues) input).getValues(); - if(values.size() > 1) { + if (values.size() > 1) { throw new IllegalArgumentException("Reverse Rating only supports one time series at a time"); } List<Double> inputValues = new ArrayList<>(values.get(0)); Collections.sort(inputValues); for (int i = 1; i < inputValues.size(); i++) { - if(inputValues.get(i) == null) { + if (inputValues.get(i) == null) { throw new IllegalArgumentException("Input values must be non-null"); } if (inputValues.get(i) < inputValues.get(i - 1)) { @@ -132,7 +132,7 @@ public RatedOutput rate(String officeId, String ratingId, RateInputTimeSeries in officeId); }); List<TimeSeries.Record> records = ztsvTypes.stream() - .map(z -> new TimeSeries.StandardRecord(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) + .map(z -> new TimeSeries.Record(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) .collect(toList()); return new RatedOutputTimeSeries(CwmsId.buildCwmsId(officeId, ratingId), records, input.getOutputUnit()); } @@ -153,7 +153,7 @@ public RatedOutput reverseRate(String officeId, String ratingId, RateInputTimeSe officeId); }); List<TimeSeries.Record> records = ztsvTypes.stream() - .map(z -> new TimeSeries.StandardRecord(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) + .map(z -> new TimeSeries.Record(z.getDATE_TIME(), z.getVALUE(), z.getQUALITY_CODE().intValue())) .collect(toList()); return new RatedOutputTimeSeries(CwmsId.buildCwmsId(officeId, ratingId), records, input.getOutputUnit()); } @@ -161,7 +161,7 @@ public RatedOutput reverseRate(String officeId, String ratingId, RateInputTimeSe private <R> R connectionResult(ConnectionCallable<R> callable) { try { return connectionResult(dsl, callable); - } catch(DataAccessException ex) { + } catch (DataAccessException ex) { throw handleRateDbError(ex); } } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 0f58d92db..e7997f8f1 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -10,13 +10,13 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; -import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; +import cwms.cda.formatters.json.adapters.TimeSeriesRecordSerializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; @@ -205,7 +205,7 @@ public VersionType getDateVersionType() { + "data in the array.", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { - if (values != null && !values.isEmpty() && values.get(0) instanceof TimeSeriesRecordWithEntryDate) { + if (values != null && !values.isEmpty() && values.get(0) != null) { return getColumnDescriptorWithEntryDate(); } return getColumnDescriptor(); @@ -219,7 +219,7 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode) { if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - values.add(new StandardRecord(dateTime, value, qualityCode)); + values.add(new Record(dateTime, value, qualityCode)); } } @@ -231,7 +231,7 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - values.add(new TimeSeriesRecordWithEntryDate(dateTime, value, qualityCode, dataEntryDate)); + values.add(new Record(dateTime, value, qualityCode, dataEntryDate)); } } @@ -241,6 +241,10 @@ public static List<Column> getColumnDescriptor() { JsonProperty field = f.getAnnotation(JsonProperty.class); if (field != null) { String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + if (fieldName.equals("data-entry-date")) { + // Skip data entry date + continue; + } int fieldIndex = field.index(); columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } @@ -258,15 +262,6 @@ private List<Column> getColumnDescriptorWithEntryDate() { columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - for (Field f: TimeSeriesRecordWithEntryDate.class.getDeclaredFields()) { - JsonProperty field = f.getAnnotation(JsonProperty.class); - if (field != null) { - String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); - int fieldIndex = field.index(); - columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); - } - } - return columns; } @@ -288,9 +283,10 @@ private List<Column> getColumnDescriptorWithEntryDate() { ) ) - @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) + @JsonSerialize(using = TimeSeriesRecordSerializer.class) @JsonIgnoreProperties(ignoreUnknown = true) - public abstract static class Record { + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") @@ -303,6 +299,9 @@ public abstract static class Record { @JsonProperty(value = "quality-code", index = 2) int qualityCode; + @JsonProperty(value = "data-entry-date", index = 3) + Timestamp dataEntryDate; + @SuppressWarnings("unused") // required so JAXB can initialize and marshal private Record() { } @@ -311,6 +310,14 @@ public Record(Timestamp dateTime, Double value, int qualityCode) { this.dateTime = dateTime; this.value = value; this.qualityCode = qualityCode; + this.dataEntryDate = null; + } + + public Record(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + this.dateTime = dateTime; + this.value = value; + this.qualityCode = qualityCode; + this.dataEntryDate = dataEntryDate; } // When serialized, the value is unix epoch at UTC. @@ -326,6 +333,10 @@ public int getQualityCode() { return qualityCode; } + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -340,7 +351,12 @@ public boolean equals(Object o) { if (getQualityCode() != tsRecord.getQualityCode()) { return false; } - if (getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { + if (getDateTime() != null + ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { + return false; + } + if (getDataEntryDate() != null + ? !getDataEntryDate().equals(tsRecord.getDataEntryDate()) : tsRecord.getDataEntryDate() != null) { return false; } return getValue() != null ? getValue().equals(tsRecord.getValue()) : tsRecord.getValue() == null; @@ -351,24 +367,20 @@ public int hashCode() { int result = getDateTime() != null ? getDateTime().hashCode() : 0; result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); result = 31 * result + getQualityCode(); + result = 31 * result + (getDataEntryDate() != null ? getDataEntryDate().hashCode() : 0); return result; } @Override public String toString() { + if (dataEntryDate != null) { + return "Record{" + "dateTime=" + dateTime + ", value=" + value + + ", qualityCode=" + qualityCode + ", dataEntryDate=" + dataEntryDate + '}'; + } return "Record{" + "dateTime=" + dateTime + ", value=" + value + ", qualityCode=" + qualityCode + '}'; } } - public static final class StandardRecord extends Record { - // unused - required for Jackson to deserialize - private StandardRecord() {} - - public StandardRecord(Timestamp dateTime, Double value, int qualityCode) { - super(dateTime, value, qualityCode); - } - } - @Schema(hidden = true, name = "TimeSeries.Column", accessMode = Schema.AccessMode.READ_ONLY) public static class Column { public final String name; @@ -381,7 +393,8 @@ private Column() { } @JsonCreator - protected Column(@JsonProperty("name") String name, @JsonProperty("ordinal") int number, @JsonProperty("datatype") Class<?> datatype) { + protected Column(@JsonProperty("name") String name, + @JsonProperty("ordinal") int number, @JsonProperty("datatype") Class<?> datatype) { this.name = name; this.ordinal = number; this.datatype = datatype; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java deleted file mode 100644 index c023f1fa5..000000000 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithEntryDate.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * MIT License - * - * Copyright (c) 2024 Hydrologic Engineering Center - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE - * SOFTWARE. - */ - -package cwms.cda.data.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.media.Schema; -import java.sql.Timestamp; -import java.util.Objects; - -/** - * TimeSeriesRecordWithEntryDate is a subclass of TimeSeries.Record that includes a data entry date. - * The data entry date is the date that the data was entered into the database. - */ -public final class TimeSeriesRecordWithEntryDate extends TimeSeries.Record { - @JsonProperty(value = "data-entry-date", index = 3) - @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - Timestamp dataEntryDate; - - // Default constructor for Jackson Deserialization - public TimeSeriesRecordWithEntryDate() { - super(null, null, 0); - } - - public TimeSeriesRecordWithEntryDate(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { - super(dateTime, value, qualityCode); - this.dataEntryDate = dataEntryDate; - } - - public Timestamp getDataEntryDate() { - return dataEntryDate; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - TimeSeriesRecordWithEntryDate that = (TimeSeriesRecordWithEntryDate) o; - return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), getDataEntryDate()); - } -} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java deleted file mode 100644 index 20fa9453d..000000000 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * - * MIT License - * - * Copyright (c) 2025 Hydrologic Engineering Center - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE - * SOFTWARE. - */ - -package cwms.cda.formatters.json.adapters; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; - -import java.io.IOException; -import java.sql.Timestamp; - -/** - * A time-series record deserializer for properly deserializing JSON data. - * Requires {@link cwms.cda.data.dto.TimeSeries.StandardRecord} class to avoid - * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer - * getting stuck in a loop if the Record class is used directly. - * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithEntryDate}. - */ -public final class TimeSeriesRecordDeserializer extends JsonDeserializer<TimeSeries.Record> { - private static final String DATA_ENTRY_DATE = "data-entry-date"; - private static final String DATE_TIME = "date-time"; - private static final String VALUE = "value"; - private static final String QUALITY = "quality"; - - @Override - public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.readValueAsTree(); - if (node instanceof ObjectNode) { - return parseObjectNode((ObjectNode) node); - } else if (node instanceof ArrayNode) { - return parseArrayNode((ArrayNode) node); - } else { - throw new IOException("Unexpected JSON node type: " + node.getNodeType()); - } - } - - private TimeSeries.Record parseObjectNode(ObjectNode node) { - Timestamp dateTime = node.get(DATE_TIME) == null ? null : new Timestamp(node.get(DATE_TIME).asLong()); - Double value = node.get(VALUE) == null || node.get(VALUE).asText().equalsIgnoreCase("null") - ? null : node.get(VALUE).asDouble(); - int quality = node.get(QUALITY) == null ? 0 : node.get(QUALITY).asInt(); - if (node.size() == 4) { - Timestamp entryDate = new Timestamp(node.get(DATA_ENTRY_DATE).asLong()); - return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); - } else { - return new TimeSeries.StandardRecord(dateTime, value, quality); - } - } - - private TimeSeries.Record parseArrayNode(ArrayNode aNode) { - Timestamp dateTime = aNode.get(0) == null ? null : new Timestamp(aNode.get(0).asLong()); - Double value = aNode.get(1) == null || aNode.get(1).asText().equalsIgnoreCase("null") - ? null : aNode.get(1).asDouble(); - int quality = aNode.get(2) == null ? 0 : aNode.get(2).asInt(); - if (aNode.size() == 4) { - Timestamp entryDate = new Timestamp(aNode.get(3).asLong()); - return new TimeSeriesRecordWithEntryDate(dateTime, value, quality, entryDate); - } else { - return new TimeSeries.StandardRecord(dateTime, value, quality); - } - } -} \ No newline at end of file diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java new file mode 100644 index 000000000..1a6d0b6e6 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java @@ -0,0 +1,64 @@ +/* + * + * MIT License + * + * Copyright (c) 2025 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.formatters.json.adapters; + + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import cwms.cda.data.dto.TimeSeries; +import java.io.IOException; + +public class TimeSeriesRecordSerializer extends StdSerializer<TimeSeries.Record> { + // Default constructor for Jackson + public TimeSeriesRecordSerializer() { + this(null); + } + + public TimeSeriesRecordSerializer(Class<TimeSeries.Record> t) { + super(t); + } + + @Override + public void serialize(TimeSeries.Record recordValue, JsonGenerator gen, SerializerProvider provider) throws IOException { + if (recordValue != null) { + gen.writeStartArray(); + gen.writeNumber(recordValue.getDateTime().getTime()); + // Handle null values for value and qualityCode + if (recordValue.getValue() == null) { + gen.writeNull(); + } else { + gen.writeNumber(recordValue.getValue()); + } + gen.writeNumber(recordValue.getQualityCode()); + if (recordValue.getDataEntryDate() != null) { + gen.writeNumber(recordValue.getDataEntryDate().getTime()); + } + gen.writeEndArray(); + } + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 6fe772899..be1d8ad00 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,7 +14,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithEntryDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -134,15 +133,6 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } - private void assertSimilarWithDate(TimeSeries expected, TimeSeries actual) - { - assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); - assertEquals(expected.getName(), actual.getName(), "names did not match"); - assertDateRecordsMatch(expected.getValues(), actual.getValues()); - assertTrue(expected.getBegin().isEqual(actual.getBegin()), "begin dates not equal"); - assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); - } - private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSeries.Record> actual) { for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); @@ -151,16 +141,6 @@ private void assertRecordsMatch(List<TimeSeries.Record> expected, List<TimeSerie } } - private void assertDateRecordsMatch(List<? extends TimeSeries.Record> expected, List<? extends TimeSeries.Record> actual) { - for (int i = 0; i < expected.size(); i++) { - assertEquals(((TimeSeriesRecordWithEntryDate) expected.get(i)).getDataEntryDate(), - ((TimeSeriesRecordWithEntryDate) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); - assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); - assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); - assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); - } - } - @ParameterizedTest @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) void testSerializeTimeSeries(String format) { @@ -170,9 +150,11 @@ void testSerializeTimeSeries(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + if (format.equalsIgnoreCase(Formats.JSONV2)) { + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } } @ParameterizedTest @@ -187,9 +169,12 @@ void testSerializeTimeSeriesWithDataEntryDate(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilarWithDate(fakeTs, ts2); + assertTrue(formatted.contains("data-entry-date")); + if (format.equalsIgnoreCase(Formats.JSONV2)) { + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } } @@ -201,9 +186,12 @@ void testDeserializeTimeSeries(String format) { TimeSeries fakeTs = buildTimeSeries(officeId, tsId); ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + assertNotNull(formatted); + if (format.equalsIgnoreCase(Formats.JSONV2)) { + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } } @ParameterizedTest @@ -214,9 +202,12 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilarWithDate(fakeTs, ts2); + assertNotNull(formatted); + if (format.equalsIgnoreCase(Formats.JSONV2)) { + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } } @Test @@ -228,7 +219,7 @@ void testDeserializeTimeSeriesWithEntryDateFromFile() { String formatted = Formats.format(contentType, fakeTs); TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); - assertSimilarWithDate(fakeTs, ts2); + assertSimilar(fakeTs, ts2); } @Test @@ -240,11 +231,9 @@ void testXMLSerializeDeserializeTimeSeries() TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); + assertNotNull(formatted); assertTrue(formatted.contains("quality-code")); assertTrue(formatted.contains("data-entry-date")); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilarWithDate(fakeTs, ts2); } @Test diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 5fb8a52eb..10f01d527 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -30,6 +30,7 @@ import io.restassured.response.Response; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.apache.commons.io.IOUtils; +import org.hamcrest.Matchers; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -1004,10 +1005,11 @@ void test_lrl_trim_with_data_entry_date() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) .body("values.size()", equalTo(2)) - .body("values[0].size()", equalTo(4)) // time, value, quality, data entry date + .body("values[1].size()", equalTo(4)) // time, value, quality, data entry date .body("values[1][0]", equalTo(1675335600000L)) // time .body("values[0][1]", nullValue()) - .body("values[1][1]", closeTo(35, 0.0001)); + .body("values[1][1]", closeTo(35, 0.0001)) + .body("values[1][3]", Matchers.notNullValue()); // data entry date // with trim the null should get trimmed. given() diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java index df1ed8a72..73a155393 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/timeseriesprofile/TimeSeriesProfileInstanceDaoIT.java @@ -934,7 +934,7 @@ private static TimeSeriesProfileInstance buildTestTimeSeriesProfileInstance(Stri List<TimeSeries.Record> timeValuePairList = new ArrayList<>(); for (int i = 0; i < dateTimeArray.length; i++) { - TimeSeries.Record timeValuePair = new TimeSeries.StandardRecord(Timestamp.from(dateTimeArray[i]), + TimeSeries.Record timeValuePair = new TimeSeries.Record(Timestamp.from(dateTimeArray[i]), valueArray[i], 0); timeValuePairList.add(timeValuePair); } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java index 8940e79fa..ba0c59033 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java @@ -36,8 +36,6 @@ import cwms.cda.formatters.json.JsonV2; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; final class TimeSeriesRecordTest @@ -51,37 +49,17 @@ void testRecordRoundTrip() throws Exception { String tsBody = om.writeValueAsString(tsRecord); assertNotNull(tsBody); - - TimeSeries.Record tsRecordReturned = om.readValue(tsBody, TimeSeries.Record.class); - assertNotNull(tsRecordReturned); - - assertAll(() -> { - assertEquals(tsRecord.getDateTime(), tsRecordReturned.getDateTime()); - assertEquals(tsRecord.getValue(), tsRecordReturned.getValue()); - assertEquals(tsRecord.getQualityCode(), tsRecordReturned.getQualityCode()); - }); } @Test void testRecordWithEntryDateRoundTrip() throws Exception { TimeSeries ts = buildTimeSeriesWithEntryDate(); - TimeSeriesRecordWithEntryDate tsRecord = (TimeSeriesRecordWithEntryDate) ts.values.get(0); + TimeSeries.Record tsRecord = ts.values.get(0); ObjectMapper om = JsonV2.buildObjectMapper(); String tsBody = om.writeValueAsString(tsRecord); assertNotNull(tsBody); - - TimeSeriesRecordWithEntryDate tsRecordReturned = om.readValue(tsBody, TimeSeriesRecordWithEntryDate.class); - assertNotNull(tsRecordReturned); - - assertAll(() -> - { - assertEquals(tsRecord.getDateTime(), tsRecordReturned.getDateTime()); - assertEquals(tsRecord.getValue(), tsRecordReturned.getValue()); - assertEquals(tsRecord.getQualityCode(), tsRecordReturned.getQualityCode()); - assertEquals(tsRecord.getDataEntryDate(), tsRecordReturned.getDataEntryDate()); - }); } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java index 39a0adc52..03df8c23f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java @@ -1,5 +1,7 @@ package cwms.cda.data.dto; +import cwms.cda.formatters.ContentType; +import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; import java.sql.Timestamp; @@ -77,6 +79,12 @@ void testRoundtripJsonVertical() throws JsonProcessingException assertEquals("NGVD-29", ts.getVerticalDatumInfo().getNativeDatum()); } + @Test + void testSerializerWithNulls() { + TimeSeries ts = buildTimeSeriesWithNulls(); + String tsBody = Formats.format(new ContentType(Formats.JSONV2), ts); + assertNotNull(tsBody); + } @NotNull private TimeSeries buildTimeSeries() @@ -98,6 +106,20 @@ private TimeSeries buildTimeSeries(VerticalDatumInfo vdi) return ts; } + private TimeSeries buildTimeSeriesWithNulls() + { + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); + ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 12.34567, 0); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(120)), null, 0); + return ts; + } + VerticalDatumInfo buildVerticalDatumInfo() { VerticalDatumInfo.Builder builder = new VerticalDatumInfo.Builder() diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java index 596ee633a..83edfe2b1 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/rating/RatedOutputTimeSeriesTest.java @@ -52,11 +52,11 @@ void testSerializationRoundTrip() throws Exception { RatedOutputTimeSeries deserialized = Formats.parseContent(contentType, json, RatedOutputTimeSeries.class); CwmsId cwmsId = CwmsId.buildCwmsId("NWDP", "DOTW.Stage;Flow.Logarithmic.USGS-NWIS"); List<TimeSeries.Record> depValues = - Arrays.asList(new TimeSeries.StandardRecord(new Timestamp(1672531200000L), 137.90304290304002, 0), - new TimeSeries.StandardRecord(new Timestamp(1577836800000L), 167.0693948928, 0), - new TimeSeries.StandardRecord(new Timestamp(1546300800000L), null, 5), - new TimeSeries.StandardRecord(new Timestamp(1451606400000L), 0.0269010042624, 0), - new TimeSeries.StandardRecord(new Timestamp(1388534400000L), 1786.7930199552, 0)); + Arrays.asList(new TimeSeries.Record(new Timestamp(1672531200000L), 137.90304290304002, 0), + new TimeSeries.Record(new Timestamp(1577836800000L), 167.0693948928, 0), + new TimeSeries.Record(new Timestamp(1546300800000L), null, 5), + new TimeSeries.Record(new Timestamp(1451606400000L), 0.0269010042624, 0), + new TimeSeries.Record(new Timestamp(1388534400000L), 1786.7930199552, 0)); String outputUnit = "cfs"; assertEquals(depValues, deserialized.getValues()); assertMatch(cwmsId, deserialized.getRatingId()); From d42b9729fc29828dcf1df29ed7ed87b871fc058b Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 11 Jun 2025 11:40:06 -0700 Subject: [PATCH 23/24] Updated serializer design, improved unit tests. XML deserialization still in progress --- .../java/cwms/cda/data/dto/TimeSeries.java | 10 ++- .../adapters/TimeSeriesRecordSerializer.java | 38 ++++++----- .../cda/api/TimeSeriesControllerTest.java | 42 ++++++------- .../cda/data/dto/TimeSeriesRecordTest.java | 21 +++++-- .../cwms/cda/data/dto/TimeSeriesTest.java | 63 ++++++++++++------- 5 files changed, 103 insertions(+), 71 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index e7997f8f1..7254a4928 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -12,6 +12,8 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; @@ -205,7 +207,7 @@ public VersionType getDateVersionType() { + "data in the array.", accessMode = AccessMode.READ_ONLY) public List<Column> getValueColumnsJSON() { - if (values != null && !values.isEmpty() && values.get(0) != null) { + if (values != null && !values.isEmpty() && values.get(0) != null && values.get(0).getDataEntryDate() != null) { return getColumnDescriptorWithEntryDate(); } return getColumnDescriptor(); @@ -285,20 +287,24 @@ private List<Column> getColumnDescriptorWithEntryDate() { @JsonSerialize(using = TimeSeriesRecordSerializer.class) @JsonIgnoreProperties(ignoreUnknown = true) - @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + @JacksonXmlRootElement public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) + @JacksonXmlProperty(localName = "date-time") @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") Timestamp dateTime; @JsonProperty(index = 1) + @JacksonXmlProperty(localName = "value") @Schema(description = "Requested time-series data value") Double value; + @JacksonXmlProperty(localName = "quality-code") @JsonProperty(value = "quality-code", index = 2) int qualityCode; + @JsonIgnore @JsonProperty(value = "data-entry-date", index = 3) Timestamp dataEntryDate; diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java index 1a6d0b6e6..92ef8e759 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java @@ -36,29 +36,27 @@ public class TimeSeriesRecordSerializer extends StdSerializer<TimeSeries.Record> { // Default constructor for Jackson public TimeSeriesRecordSerializer() { - this(null); - } - - public TimeSeriesRecordSerializer(Class<TimeSeries.Record> t) { - super(t); + super(TimeSeries.Record.class); } @Override - public void serialize(TimeSeries.Record recordValue, JsonGenerator gen, SerializerProvider provider) throws IOException { - if (recordValue != null) { - gen.writeStartArray(); - gen.writeNumber(recordValue.getDateTime().getTime()); - // Handle null values for value and qualityCode - if (recordValue.getValue() == null) { - gen.writeNull(); - } else { - gen.writeNumber(recordValue.getValue()); - } - gen.writeNumber(recordValue.getQualityCode()); - if (recordValue.getDataEntryDate() != null) { - gen.writeNumber(recordValue.getDataEntryDate().getTime()); - } - gen.writeEndArray(); + public void serialize(TimeSeries.Record recordValue, JsonGenerator gen, SerializerProvider provider) + throws IOException { + + gen.writeStartArray(); + gen.writeNumber(recordValue.getDateTime().getTime()); + if (recordValue.getValue() == null) { + gen.writeNull(); + } else { + gen.writeNumber(recordValue.getValue()); + } + gen.writeNumber(recordValue.getQualityCode()); + // Used to include the dataEntryDate in the serialized output if requested. Modifies length of the output array. + // If the dataEntryDate is requested, it will always be non-null + // Without the dataEntryDate, the array will have 3 elements: [dateTime, value, qualityCode] + if (recordValue.getDataEntryDate() != null) { + gen.writeNumber(recordValue.getDataEntryDate().getTime()); } + gen.writeEndArray(); } } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index be1d8ad00..17b8995bf 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -150,11 +150,9 @@ void testSerializeTimeSeries(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - if (format.equalsIgnoreCase(Formats.JSONV2)) { - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); - } + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); } @ParameterizedTest @@ -170,11 +168,9 @@ void testSerializeTimeSeriesWithDataEntryDate(String format) { String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); assertTrue(formatted.contains("data-entry-date")); - if (format.equalsIgnoreCase(Formats.JSONV2)) { - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); - } + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); } @@ -187,11 +183,9 @@ void testDeserializeTimeSeries(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - if (format.equalsIgnoreCase(Formats.JSONV2)) { - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); - } + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); } @ParameterizedTest @@ -203,11 +197,9 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - if (format.equalsIgnoreCase(Formats.JSONV2)) { - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); - assertNotNull(ts2); - assertSimilar(fakeTs, ts2); - } + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); } @Test @@ -234,6 +226,9 @@ void testXMLSerializeDeserializeTimeSeries() assertNotNull(formatted); assertTrue(formatted.contains("quality-code")); assertTrue(formatted.contains("data-entry-date")); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); } @Test @@ -265,11 +260,10 @@ void testDeserializeTimeSeriesXml() { InputStream inputStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); ContentType contentType = Formats.parseHeader(Formats.XMLV2, TimeSeries.class); TimeSeries ts = Formats.parseContent(contentType, inputStream, TimeSeries.class); + assertNotNull(ts); - assertNotNull(ts); - - TimeSeries fakeTs = buildTimeSeries("LRL", "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"); - assertSimilar(fakeTs, ts); + TimeSeries fakeTs = buildTimeSeries("LRL", "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"); + assertSimilar(fakeTs, ts); } @Test diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java index ba0c59033..df254dd65 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesRecordTest.java @@ -26,6 +26,7 @@ package cwms.cda.data.dto; +import com.fasterxml.jackson.databind.JsonNode; import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; @@ -36,6 +37,7 @@ import cwms.cda.formatters.json.JsonV2; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; final class TimeSeriesRecordTest @@ -49,6 +51,12 @@ void testRecordRoundTrip() throws Exception { String tsBody = om.writeValueAsString(tsRecord); assertNotNull(tsBody); + JsonNode tsNode = om.readTree(tsBody); + assertNotNull(tsNode); + assertNotNull(tsNode.get(0)); + assertEquals(1749211200000L, tsNode.get(0).asLong()); + assertEquals(12.34567, tsNode.get(1).asDouble(), 0.00001); + assertEquals(0, tsNode.get(2).asInt()); } @Test @@ -60,6 +68,11 @@ void testRecordWithEntryDateRoundTrip() throws Exception { String tsBody = om.writeValueAsString(tsRecord); assertNotNull(tsBody); + JsonNode tsNode = om.readTree(tsBody); + assertNotNull(tsNode); + assertEquals(1749211200000L, tsNode.get(0).asLong()); + assertEquals(12.34567, tsNode.get(1).asDouble(), 0.00001); + assertEquals(0, tsNode.get(2).asInt()); } @@ -70,8 +83,8 @@ private TimeSeries buildTimeSeries() ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); - ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0); - ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 13.45678, 0); + ts.addValue(Timestamp.from(Instant.parse("2025-06-06T12:00:00Z")), 12.34567, 0); + ts.addValue(Timestamp.from(Instant.parse("2025-06-06T12:00:00Z").plusSeconds(60)), 13.45678, 0); return ts; } @@ -82,8 +95,8 @@ private TimeSeries buildTimeSeriesWithEntryDate() ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); - ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0, Timestamp.from(Instant.now().minusSeconds(60))); - ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 13.45678, 0, Timestamp.from(Instant.now())); + ts.addValue(Timestamp.from(Instant.parse("2025-06-06T12:00:00Z")), 12.34567, 0, Timestamp.from(Instant.now().minusSeconds(60))); + ts.addValue(Timestamp.from(Instant.parse("2025-06-06T12:00:00Z").plusSeconds(60)), 13.45678, 0, Timestamp.from(Instant.now())); return ts; } } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java index 03df8c23f..dd3ef84bb 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java @@ -19,6 +19,7 @@ import cwms.cda.formatters.xml.XMLv2; +import java.util.List; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; @@ -28,12 +29,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -public class TimeSeriesTest -{ - +public class TimeSeriesTest { @Test - void testRoundtripJson() throws JsonProcessingException - { + void testRoundtripJson() throws JsonProcessingException { TimeSeries ts = buildTimeSeries(); ObjectMapper om = buildObjectMapper(); @@ -55,8 +53,7 @@ void testRoundtripJson() throws JsonProcessingException } @Test - void testRoundtripJsonVertical() throws JsonProcessingException - { + void testRoundtripJsonVertical() throws JsonProcessingException { TimeSeries ts = buildTimeSeries(buildVerticalDatumInfo()); ObjectMapper om = buildObjectMapper(); @@ -87,14 +84,12 @@ void testSerializerWithNulls() { } @NotNull - private TimeSeries buildTimeSeries() - { + private TimeSeries buildTimeSeries() { return buildTimeSeries(null); } @NotNull - private TimeSeries buildTimeSeries(VerticalDatumInfo vdi) - { + private TimeSeries buildTimeSeries(VerticalDatumInfo vdi) { String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); @@ -106,8 +101,7 @@ private TimeSeries buildTimeSeries(VerticalDatumInfo vdi) return ts; } - private TimeSeries buildTimeSeriesWithNulls() - { + private TimeSeries buildTimeSeriesWithNulls() { String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); @@ -120,8 +114,20 @@ private TimeSeries buildTimeSeriesWithNulls() return ts; } - VerticalDatumInfo buildVerticalDatumInfo() - { + private TimeSeries buildTimeSeriesWithEntryDates() { + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-22T14:00:00-07:00[PST8PDT]"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "LRL", start, end, null, Duration.ZERO, null, versionDate, null); + ts.addValue(Timestamp.from(Instant.now()), 12.34567, 0, Timestamp.from(Instant.now())); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(60)), 12.34567, 0, Timestamp.from(Instant.now())); + ts.addValue(Timestamp.from(Instant.now().plusSeconds(120)), null, 0, Timestamp.from(Instant.now())); + return ts; + } + + VerticalDatumInfo buildVerticalDatumInfo() { VerticalDatumInfo.Builder builder = new VerticalDatumInfo.Builder() .withOffice("LRL").withUnit("m").withLocation("Buckhorn") .withNativeDatum("NGVD-29").withElevation(230.7).withOffset( @@ -130,8 +136,25 @@ VerticalDatumInfo buildVerticalDatumInfo() } @Test - void testFormatter() - { + void testColumns() { + TimeSeries ts = buildTimeSeries(); + + List<TimeSeries.Column> columnList = ts.getValueColumnsJSON(); + assertNotNull(columnList); + assertEquals(3, columnList.size()); + } + + @Test + void testColumnsWithEntryDates() { + TimeSeries ts = buildTimeSeriesWithEntryDates(); + + List<TimeSeries.Column> columnList = ts.getValueColumnsJSON(); + assertNotNull(columnList); + assertEquals(4, columnList.size()); + } + + @Test + void testFormatter() { ZonedDateTime start = ZonedDateTime.parse("2021-06-21T14:00:00-07:00[PST8PDT]"); DateTimeFormatter formatter1 = DateTimeFormatter.ofPattern(TimeSeries.ZONED_DATE_TIME_FORMAT); @@ -147,8 +170,7 @@ public static ObjectMapper buildObjectMapper() } @NotNull - public static ObjectMapper buildObjectMapper(ObjectMapper om) - { + public static ObjectMapper buildObjectMapper(ObjectMapper om) { ObjectMapper retval = om.copy(); retval.setPropertyNamingStrategy(PropertyNamingStrategies.KEBAB_CASE); @@ -160,8 +182,7 @@ public static ObjectMapper buildObjectMapper(ObjectMapper om) @Test - void test_xml_value_columns() - { + void test_xml_value_columns() { TimeSeries ts = buildTimeSeries(); XMLv2 xmlV2 = new XMLv2(); From 3112831cde8b99b3dfff9958fa2cd2d925e27b1a Mon Sep 17 00:00:00 2001 From: zack-rma <zack@rmanet.com> Date: Wed, 11 Jun 2025 13:11:03 -0700 Subject: [PATCH 24/24] Fixed custom serializer to handle XML serialization --- .../java/cwms/cda/data/dto/TimeSeries.java | 6 --- .../adapters/TimeSeriesRecordSerializer.java | 43 +++++++++++++------ .../cda/api/TimeSeriesControllerTest.java | 1 + 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 7254a4928..64778d37e 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -12,8 +12,6 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; @@ -287,20 +285,16 @@ private List<Column> getColumnDescriptorWithEntryDate() { @JsonSerialize(using = TimeSeriesRecordSerializer.class) @JsonIgnoreProperties(ignoreUnknown = true) - @JacksonXmlRootElement public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) - @JacksonXmlProperty(localName = "date-time") @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") Timestamp dateTime; @JsonProperty(index = 1) - @JacksonXmlProperty(localName = "value") @Schema(description = "Requested time-series data value") Double value; - @JacksonXmlProperty(localName = "quality-code") @JsonProperty(value = "quality-code", index = 2) int qualityCode; diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java index 92ef8e759..2091c3cf7 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordSerializer.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.dataformat.xml.ser.XmlSerializerProvider; import cwms.cda.data.dto.TimeSeries; import java.io.IOException; @@ -43,20 +44,36 @@ public TimeSeriesRecordSerializer() { public void serialize(TimeSeries.Record recordValue, JsonGenerator gen, SerializerProvider provider) throws IOException { - gen.writeStartArray(); - gen.writeNumber(recordValue.getDateTime().getTime()); - if (recordValue.getValue() == null) { - gen.writeNull(); + if (provider instanceof XmlSerializerProvider) { + // Handle XML serialization + + gen.writeStartObject(); + gen.writeNumberField("date-time", recordValue.getDateTime().getTime()); + if (recordValue.getValue() == null) { + gen.writeNullField("value"); + } else { + gen.writeNumberField("value", recordValue.getValue()); + } + gen.writeNumberField("quality-code", recordValue.getQualityCode()); + gen.writeEndObject(); } else { - gen.writeNumber(recordValue.getValue()); - } - gen.writeNumber(recordValue.getQualityCode()); - // Used to include the dataEntryDate in the serialized output if requested. Modifies length of the output array. - // If the dataEntryDate is requested, it will always be non-null - // Without the dataEntryDate, the array will have 3 elements: [dateTime, value, qualityCode] - if (recordValue.getDataEntryDate() != null) { - gen.writeNumber(recordValue.getDataEntryDate().getTime()); + // Handle JSON serialization + + gen.writeStartArray(); + gen.writeNumber(recordValue.getDateTime().getTime()); + if (recordValue.getValue() == null) { + gen.writeNull(); + } else { + gen.writeNumber(recordValue.getValue()); + } + gen.writeNumber(recordValue.getQualityCode()); + // Used to include the dataEntryDate in the serialized output if requested. Modifies length of the output array. + // If the dataEntryDate is requested, it will always be non-null + // Without the dataEntryDate, the array will have 3 elements: [dateTime, value, qualityCode] + if (recordValue.getDataEntryDate() != null) { + gen.writeNumber(recordValue.getDataEntryDate().getTime()); + } + gen.writeEndArray(); } - gen.writeEndArray(); } } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 17b8995bf..8987a6fbb 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -150,6 +150,7 @@ void testSerializeTimeSeries(String format) { ContentType contentType = Formats.parseHeader(format, TimeSeries.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); + assertFalse(formatted.contains("null")); TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); assertNotNull(ts2); assertSimilar(fakeTs, ts2);