From 95361d7b25668594e25b2cf1d3ae7d906aeea10b Mon Sep 17 00:00:00 2001 From: zack-rma Date: Tue, 22 Oct 2024 09:12:17 -0700 Subject: [PATCH 1/6] Implemented data entry data option for TS data retrieval, serialization bug in progress --- .../cwms/cda/api/TimeSeriesController.java | 16 ++- .../cda/data/dao/LocationLevelsDaoImpl.java | 3 +- .../java/cwms/cda/data/dao/TimeSeriesDao.java | 2 +- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 63 ++++++++-- .../java/cwms/cda/data/dto/TimeSeries.java | 39 +++++- .../cda/api/TimeSeriesControllerTest.java | 52 +++++++- .../cda/api/TimeseriesControllerTestIT.java | 112 ++++++++++++++++-- .../cwms/cda/data/dao/TimeSeriesDaoTest.java | 23 +--- .../cda/formatters/TimeSeriesTestBase.java | 2 +- 9 files changed, 251 insertions(+), 61 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index bf6b08246..ed6c8afd4 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -67,7 +67,6 @@ import io.javalin.plugin.openapi.annotations.OpenApiParam; import io.javalin.plugin.openapi.annotations.OpenApiRequestBody; import io.javalin.plugin.openapi.annotations.OpenApiResponse; -import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -84,7 +83,7 @@ public class TimeSeriesController implements CrudHandler { private static final Logger logger = Logger.getLogger(TimeSeriesController.class.getName()); - + private static final String INCLUDE_ENTRY_DATE = "include-entry-date"; public static final String TAG = "TimeSeries"; public static final String STORE_RULE_DESC = "The business rule to use " + "when merging the incoming with existing data\n" @@ -204,7 +203,7 @@ public void create(@NotNull Context ctx) { TimeSeries timeSeries = deserializeTimeSeries(ctx); dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (IOException | DataAccessException ex) { + } catch (DataAccessException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); @@ -382,6 +381,8 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "\n* `xml`" + "\n* `wml2` (only if name field is specified)" + "\n* `json` (default)"), + @OpenApiParam(name = INCLUDE_ENTRY_DATE, type = Boolean.class, description = "Specifies " + + "whether to include the data entry date in the response. Default is false."), @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " @@ -431,6 +432,9 @@ public void getAll(@NotNull Context ctx) { ZonedDateTime versionDate = queryParamAsZdt(ctx, VERSION_DATE); + boolean includeEntryDate = ctx.queryParamAsClass(INCLUDE_ENTRY_DATE, Boolean.class) + .getOrDefault(false); + // The following parameters are only used for jsonv2 and xmlv2 String cursor = queryParamAsClass(ctx, new String[]{PAGE, CURSOR}, String.class, "", metrics, name(TimeSeriesController.class.getName(), @@ -463,7 +467,7 @@ public void getAll(@NotNull Context ctx) { String office = requiredParam(ctx, OFFICE); TimeSeries ts = dao.getTimeseries(cursor, pageSize, names, office, unit, - beginZdt, endZdt, versionDate, trim.getOrDefault(true)); + beginZdt, endZdt, versionDate, trim.getOrDefault(true), includeEntryDate); results = Formats.format(contentType, ts); @@ -573,14 +577,14 @@ public void update(@NotNull Context ctx, @NotNull String id) { dao.store(timeSeries, createAsLrts, storeRule, overrideProtection); ctx.status(HttpServletResponse.SC_OK); - } catch (IOException | DataAccessException ex) { + } catch (DataAccessException ex) { CdaError re = new CdaError("Internal Error"); logger.log(Level.SEVERE, re.toString(), ex); ctx.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR).json(re); } } - private TimeSeries deserializeTimeSeries(Context ctx) throws IOException { + private TimeSeries deserializeTimeSeries(Context ctx) { String contentTypeHeader = ctx.req.getContentType(); ContentType contentType = Formats.parseHeader(contentTypeHeader, TimeSeries.class); return Formats.parseContent(contentType, ctx.bodyAsInputStream(), TimeSeries.class); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java index 470e89000..e6f1ed7c2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java @@ -62,7 +62,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TimeZone; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -623,7 +622,7 @@ private static TimeSeries buildTimeSeries(ILocationLevelRef levelRef, Interval i if (qualityCode != null) { quality = qualityCode.intValue(); } - timeSeries.addValue(dateTime, value, quality); + timeSeries.addValue(dateTime, value, quality, null); } return timeSeries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 8750b24f3..29be65bb2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -27,7 +27,7 @@ void store(TimeSeries timeSeries, boolean createAsLrts, TimeSeries getTimeseries(String cursor, int pageSize, String names, String office, String unit, ZonedDateTime begin, ZonedDateTime end, - ZonedDateTime versionDate, boolean trim); + ZonedDateTime versionDate, boolean trim, boolean includeEntryDate); String getTimeseries(String format, String names, String office, String unit, String datum, ZonedDateTime begin, ZonedDateTime end, ZoneId timezone); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index d0262dc42..de8845369 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -64,6 +64,7 @@ import org.jooq.Record; import org.jooq.Record1; import org.jooq.Record3; +import org.jooq.Record4; import org.jooq.Record7; import org.jooq.Result; import org.jooq.SQL; @@ -164,7 +165,7 @@ public String getTimeseries(String format, String names, String office, String u public TimeSeries getTimeseries(String page, int pageSize, String names, String office, String units, ZonedDateTime beginTime, ZonedDateTime endTime, - ZonedDateTime versionDate, boolean shouldTrim) { + ZonedDateTime versionDate, boolean shouldTrim, boolean includeEntryDate) { TimeSeries retVal = null; String cursor = null; Timestamp tsCursor = null; @@ -236,7 +237,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // put all those columns together as "valid" CommonTableExpression> valid = + String>> valid = name("valid").fields("tscode", "tsid", "office_id", "loc_part", "units", "interval", "parm_part") .as( @@ -248,7 +249,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String unit.as("units"), ival.as("interval"), param.as("parm_part") - ).from(validTs) ); @@ -368,6 +368,8 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String ); }); + Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + if (pageSize != 0) { SelectConditionStep> query = dsl.select( @@ -390,14 +392,55 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String query.limit(DSL.val(pageSize + 1)); } - logger.fine(() -> query.getSQL(ParamType.INLINED)); + SelectConditionStep> finalQuery = query; + logger.fine(() -> finalQuery.getSQL(ParamType.INLINED)); - query.forEach(tsRecord -> timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue() - ) - ); + if (includeEntryDate) { + SelectConditionStep> query2 = dsl.select( + dateTimeCol, + valueCol, + qualityNormCol, + dataEntryDate + ) + .from(AV_TSV_DQU.AV_TSV_DQU) + .where(dateTimeCol + .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.nvl(DSL.val(tsCursor == null ? null : + tsCursor.toInstant().toEpochMilli()), + DSL.val(beginTime.toInstant().toEpochMilli()))))) + .and(dateTimeCol + .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(endTime.toInstant().toEpochMilli()))) + .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) + .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) + .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) +// .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : +// Timestamp.from(versionDate.toInstant()))) + ); + + if (pageSize > 0) { + query2.limit(DSL.val(pageSize + 1)); + } + query2.forEach(tsRecord -> { + assert timeseries != null; + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate) + ); + }); + } else { + query.forEach(tsRecord -> { + assert timeseries != null; + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + null + ); + }); + } retVal = timeseries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 2ba41ed1b..3c928946f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat.Shape; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; @@ -24,9 +25,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Objects; @JsonRootName("timeseries") @JsonPropertyOrder(alphabetic = true) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) @FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) @FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) @@ -194,7 +197,7 @@ public List getValueColumnsJSON() { return getColumnDescriptor(); } - public boolean addValue(Timestamp dateTime, Double value, int qualityCode) { + public boolean addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set if((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); @@ -203,7 +206,11 @@ public boolean addValue(Timestamp dateTime, Double value, int qualityCode) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); return false; } else { - return values.add(new Record(dateTime, value, qualityCode)); + if (dataEntryDate != null) { + return values.add(new Record(dateTime, value, qualityCode).withDataEntryDate(dataEntryDate)); + } else { + return values.add(new Record(dateTime, value, qualityCode)); + } } } @@ -235,6 +242,7 @@ private List getColumnDescriptor() { example = "[1509654000000, 54.3, 0]" ) ) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @@ -248,6 +256,11 @@ public static class Record { @JsonProperty(value = "quality-code", index = 2) int qualityCode; +// @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate = null; + @SuppressWarnings("unused") // required so JAXB can initialize and marshal private Record() {} @@ -257,6 +270,12 @@ protected Record(Timestamp dateTime, Double value, int qualityCode) { this.qualityCode = qualityCode; } + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + protected Record withDataEntryDate(Timestamp dataEntryDate) { + this.dataEntryDate = dataEntryDate; + return this; + } + // When serialized, the value is unix epoch at UTC. public Timestamp getDateTime() { return dateTime; @@ -270,6 +289,10 @@ public int getQualityCode() { return qualityCode; } + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + @Override public boolean equals(Object o) { @@ -282,17 +305,20 @@ public boolean equals(Object o) return false; } - final Record record = (Record) o; + final Record tsRecord = (Record) o; - if(getQualityCode() != record.getQualityCode()) + if(getQualityCode() != tsRecord.getQualityCode()) { return false; } - if(getDateTime() != null ? !getDateTime().equals(record.getDateTime()) : record.getDateTime() != null) + if(getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { return false; } - return getValue() != null ? getValue().equals(record.getValue()) : record.getValue() == null; + if (!Objects.equals(dataEntryDate, tsRecord.dataEntryDate)) { + return false; + } + return getValue() != null ? getValue().equals(tsRecord.getValue()) : tsRecord.getValue() == null; } @Override @@ -301,6 +327,7 @@ public int hashCode() int result = getDateTime() != null ? getDateTime().hashCode() : 0; result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); result = 31 * result + getQualityCode(); + result = 31 * result + (dataEntryDate != null ? dataEntryDate.hashCode() : 0); return result; } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index d528fb93e..5142a9530 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -22,11 +22,11 @@ import io.javalin.core.util.Header; import io.javalin.http.Context; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.Duration; +import java.time.Instant; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; @@ -67,7 +67,7 @@ void testDaoMock() throws JsonProcessingException { when( dao.getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true) )).thenReturn(expected); + isNotNull(), isNotNull(), isNull(), eq(true), eq(false))).thenReturn(expected); // build mock request and response @@ -113,7 +113,7 @@ protected TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { // Check that the controller accessed our mock dao in the expected way verify(dao, times(1)). getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true));// + isNotNull(), isNotNull(), isNull(), eq(true), eq(false));// // Make sure controller thought it was happy verify(response).setStatus(200); @@ -152,6 +152,19 @@ void testDeserializeTimeSeries(String format) { assertSimilar(fakeTs, ts2); } + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testDeserializeTimeSeriesWithEntryDate(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + @Test void testDeserializeTimeSeriesXmlUTC() { TimeZone aDefault = TimeZone.getDefault(); @@ -227,7 +240,38 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); + next = next.plus(minutes, ChronoUnit.MINUTES); + } + return ts; + } + + @NotNull + private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { + ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); + ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); + + long diff = end.toEpochSecond() - start.toEpochSecond(); + assertEquals(3600, diff); // just to make sure I've got the date parsing thing right. + + int minutes = 15; + int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? + // Also, should end be the last point or the next interval? + + TimeSeries ts = new TimeSeries(null, + -1, + 0, + tsId, + officeId, + start, + end, + "m", + Duration.ofMinutes(minutes)); + + ZonedDateTime next = start; + for(int i = 0; i < count; i++) { + Timestamp dateTime = Timestamp.from(next.toInstant()); + ts.addValue(dateTime, (double) i, 0, Timestamp.from(Instant.now())); next = next.plus(minutes, ChronoUnit.MINUTES); } return ts; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 06a9a0f98..58f9fdc38 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -20,6 +20,7 @@ import io.restassured.filter.log.LogDetail; import io.restassured.path.json.config.JsonPathConfig; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZonedDateTime; @@ -74,7 +75,6 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization",user.toHeaderValue()) .queryParam("office",officeId) .queryParam("units","cfs") @@ -169,7 +169,7 @@ void test_lrl_1day_bad_units() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset_bad_units.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -207,7 +207,7 @@ void test_lrl_1day_malicious_units() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset_malicious_units.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -238,6 +238,92 @@ void test_lrl_1day_malicious_units() throws Exception { } + @Test + void test_include_data_entry_date() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + final String includeDataEntryDate = "include-entry-date"; + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/num_ts_create2.json"); + assertNotNull(resource); + + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get("name").asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam("office", officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // 1675335600000 is Thursday, February 2, 2023 11:00:00 AM + // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM + + // get it back with the data entry date + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "CFS") + .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") + .queryParam(Controllers.END, "2010-02-03T11:00:00Z") + .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") + .queryParam(includeDataEntryDate, true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(4)) + .body("values[0][1]", equalTo(4.0F)) + .body("values[0].size()", equalTo(4)); + + // get it back without the data entry date + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT, "CFS") + .queryParam(Controllers.NAME, ts.get("name").asText()) + .queryParam(Controllers.BEGIN, "2007-02-02T11:00:00Z") + .queryParam(Controllers.END, "2010-02-03T11:00:00Z") + .queryParam(Controllers.VERSION_DATE, "2021-06-20T08:00:00-0000[UTC]") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("values.size()", equalTo(4)) + .body("values[0][1]", equalTo(4.0F)) + .body("values[0].size()", equalTo(3)); + } + @Test void test_delete_ts() throws Exception { @@ -247,7 +333,7 @@ void test_delete_ts() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -326,7 +412,7 @@ void test_no_office_permissions() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/timeseries/no_office_perms.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -361,7 +447,7 @@ void test_v1_cant_trim() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -398,7 +484,7 @@ void test_v1_cant_version() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -437,7 +523,7 @@ void test_v2_cant_datum() throws Exception { "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode ts = mapper.readTree(tsData); @@ -475,7 +561,7 @@ void test_lrl_trim() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -577,7 +663,7 @@ void test_big_create() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); String giantString = buildBigString(tsData, 200000); // 200k points looked like about 6MB. @@ -657,7 +743,7 @@ private String buildBigString(String tsData, int count) throws JsonProcessingExc StringBuilder sb = new StringBuilder(); for (int i = 0; i < count; i++) { long time = start2 + (diff * (i+1)); - sb.append(String.format(",\n [ %d, %d, %d]", time, count, 0)); + sb.append(String.format(",%n [ %d, %d, %d]", time, count, 0)); } return prefix + sb + "\n ]\n}"; @@ -670,7 +756,7 @@ void test_daylight_saving_retrieve()throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1hour.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); int count = 365 * 24 * 5; // 5 years of hourly data (43.8k points) @@ -790,7 +876,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java index 46e96baf5..5f9601ff3 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java @@ -1,6 +1,5 @@ package cwms.cda.data.dao; -import java.math.BigDecimal; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -24,7 +23,6 @@ import usace.cwms.db.dao.util.CwmsDatabaseVersionInfo; import usace.cwms.db.dao.util.TimeValueQuality; import usace.cwms.db.jooq.JooqCwmsDatabaseVersionInfoFactory; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2; import usace.cwms.db.jooq.codegen.tables.AV_LOC; import usace.cwms.db.jooq.dao.CwmsDbLocJooq; import usace.cwms.db.jooq.dao.CwmsDbTsJooq; @@ -114,7 +112,7 @@ public class TimeSeriesDaoTest @Test - public void testCreateEmpty() throws Exception + void testCreateEmpty() throws Exception { String officeId = "LRL"; @@ -144,7 +142,7 @@ public void testCreateEmpty() throws Exception } @Test - public void testCreateWithData() throws Exception + void testCreateWithData() throws Exception { String officeId = "LRL"; @@ -153,7 +151,6 @@ public void testCreateWithData() throws Exception DSLContext lrl = getDslContext(connection, officeId); TimeSeriesDao dao = new TimeSeriesDaoImpl(lrl); - Calendar instance = Calendar.getInstance(); String tsId = TIME_SERIES_ID; // Do I need to somehow check whether the location exists? Its not going to exist if I add the millis to it... if(!locationExists(connection, "RYAN3")) @@ -177,7 +174,7 @@ public void testCreateWithData() throws Exception for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.valueOf(next.toLocalDateTime()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); next = next.plus(minutes, ChronoUnit.MINUTES); } @@ -209,17 +206,8 @@ private void storeLocation(Connection connection, String officeId, String locati locationId, null, null, true, true); } - private BigDecimal retrieveTsCode(Connection connection, String tsId) throws Exception - { - BigDecimal bigD = DSL.using(connection).select(AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE).from( - AV_CWMS_TS_ID2.AV_CWMS_TS_ID2).where(AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.CWMS_TS_ID.eq(tsId)).fetchOptional( - AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE).orElse(null); - - return bigD; - } - @Test - public void testTimeSeriesStoreRetrieve() throws Exception + void testTimeSeriesStoreRetrieve() throws Exception { Connection connection = getConnection(); @@ -269,11 +257,10 @@ private void createTs(CwmsDbTsJooq cwmsTsJdbc, Connection connection) throws SQL } @Test - public void testVersion() throws SQLException + void testVersion() throws SQLException { JooqCwmsDatabaseVersionInfoFactory fac = new JooqCwmsDatabaseVersionInfoFactory(); - String officeId = "LRL"; try(Connection connection = getConnection()) { CwmsDatabaseVersionInfo info = fac.retrieveVersionInfo(connection); diff --git a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java index 3c50b8142..d195bde15 100644 --- a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java +++ b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java @@ -14,7 +14,7 @@ public abstract class TimeSeriesTestBase { protected TimeSeries getTimeSeries() { TimeSeries ts = new TimeSeries(null, -1, 0, "Test.Test.Elev.0.0.RAW", "SPK", ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"), ZonedDateTime.parse("2021-06-22T08:00:00-07:00[PST8PDT]"), null, Duration.ZERO); - ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0); + ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0, null); return ts; } From e04b7e42a299bb9acb51a1b3ed92886287d4e475 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Fri, 25 Oct 2024 11:51:46 -0700 Subject: [PATCH 2/6] Created subclass of TimeSeries Record with custom deserializer to handle custom output, adds data entry date support --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 4 +- .../java/cwms/cda/data/dto/TimeSeries.java | 115 +++++++++--------- .../data/dto/TimeSeriesRecordWithDate.java | 81 ++++++++++++ .../TimeSeriesRecordDeserializer.java | 74 +++++++++++ .../cda/api/TimeSeriesControllerTest.java | 67 ++++++++-- .../cda/api/TimeseriesControllerTestIT.java | 16 +-- 6 files changed, 275 insertions(+), 82 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index de8845369..5bb8e85d4 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -414,8 +414,8 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) -// .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : -// Timestamp.from(versionDate.toInstant()))) + .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : + Timestamp.from(versionDate.toInstant()))) ); if (pageSize > 0) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 3c928946f..3f6fb43b1 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -4,28 +4,30 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat.Shape; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; +import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema.AccessMode; - import java.lang.reflect.Field; import java.sql.Timestamp; import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; -import java.util.Objects; @JsonRootName("timeseries") @JsonPropertyOrder(alphabetic = true) @@ -45,10 +47,10 @@ public class TimeSeries extends CwmsDTOPaginated { @Schema(description = "The units of the time series data",required = true) String units; - @Schema(description = "The version type for the time series being queried. Can be in the form of MAX_AGGREGATE, SINGLE_VERSION, or UNVERSIONED. " + - "MAX_AGGREGATE will get the latest version date value for each value in the date range. SINGLE_VERSION must be called with a valid " + - "version date and will return the values for the version date provided. UNVERSIONED return values from an unversioned time series. " + - "Note that SINGLE_VERSION requires a valid version date while MAX_AGGREGATE and UNVERSIONED each require a null version date.") + @Schema(description = "The version type for the time series being queried. Can be in the form of MAX_AGGREGATE, SINGLE_VERSION, or UNVERSIONED. " + + "MAX_AGGREGATE will get the latest version date value for each value in the date range. SINGLE_VERSION must be called with a valid " + + "version date and will return the values for the version date provided. UNVERSIONED return values from an unversioned time series. " + + "Note that SINGLE_VERSION requires a valid version date while MAX_AGGREGATE and UNVERSIONED each require a null version date.") @JsonFormat(shape = Shape.STRING) VersionType dateVersionType; @@ -94,7 +96,7 @@ public class TimeSeries extends CwmsDTOPaginated { @Schema( accessMode = AccessMode.READ_ONLY, - description="Offset from top of interval" + description = "Offset from top of interval" ) private Long intervalOffset; @@ -112,7 +114,7 @@ public TimeSeries(String page, int pageSize, Integer total, String name, String this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, null, null, null); } - public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, VersionType dateVersionType){ + public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, VersionType dateVersionType) { this(page, pageSize, total, name, officeId, begin, end, units, interval, info, null, null, versionDate, dateVersionType); } @@ -162,7 +164,8 @@ public ZonedDateTime getEnd() { } // Use the array shape to optimize data transfer to client - @JsonFormat(shape=JsonFormat.Shape.ARRAY) + @JsonFormat(shape = JsonFormat.Shape.ARRAY) + @JsonDeserialize(contentUsing = TimeSeriesRecordDeserializer.class) public List getValues() { return values; } @@ -172,8 +175,7 @@ public List getXmlValues() { return values; } - public VerticalDatumInfo getVerticalDatumInfo() - { + public VerticalDatumInfo getVerticalDatumInfo() { return verticalDatumInfo; } @@ -189,34 +191,35 @@ public ZonedDateTime getVersionDate() { return versionDate; } - public VersionType getDateVersionType() { return dateVersionType; } + public VersionType getDateVersionType() { + return dateVersionType; + } @JsonProperty(value = "value-columns") @Schema(name = "value-columns", accessMode = AccessMode.READ_ONLY) public List getValueColumnsJSON() { - return getColumnDescriptor(); + return getColumnDescriptor((values != null && !values.isEmpty()) + && values.get(0) instanceof TimeSeriesRecordWithDate); } - public boolean addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { // Set the current page, if not set - if((page == null || page.isEmpty()) && values.isEmpty()) { + if ((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); } - if(pageSize > 0 && values.size() == pageSize) { + if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); - return false; } else { if (dataEntryDate != null) { - return values.add(new Record(dateTime, value, qualityCode).withDataEntryDate(dataEntryDate)); + values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); } else { - return values.add(new Record(dateTime, value, qualityCode)); + values.add(new Record(dateTime, value, qualityCode)); } } } - private List getColumnDescriptor() { + private List getColumnDescriptor(boolean includeDataEntryDate) { List columns = new ArrayList<>(); - for (Field f: Record.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); if(field != null) { @@ -225,7 +228,16 @@ private List getColumnDescriptor() { columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - + if (includeDataEntryDate) { + for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if(field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + } return columns; } @@ -235,14 +247,23 @@ private List getColumnDescriptor() { schema = @Schema( name = "TimeSeries.Record", description = "A representation of a time-series record in the form [dateTime, value, qualityCode]", - type="array" + type = "array" ), arraySchema = @Schema( - type="array", + type = "array", example = "[1509654000000, 54.3, 0]" ) ) - @JsonInclude(JsonInclude.Include.NON_DEFAULT) + + // This class is used to deserialize the time-series data JSON into an object + // Solves the issue of the deserializer getting stuck in a loop + // and throwing a StackOverflowError when trying to handle the Record class directly + @JsonDeserialize(using = JsonDeserializer.None.class) + public static final class RecordChild extends Record { + } + + @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) + @JsonIgnoreProperties(ignoreUnknown = true) public static class Record { // Explicitly set property order for array serialization @JsonProperty(value = "date-time", index = 0) @@ -256,26 +277,15 @@ public static class Record { @JsonProperty(value = "quality-code", index = 2) int qualityCode; -// @JsonProperty(value = "data-entry-date", index = 3) - @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - Timestamp dataEntryDate = null; - @SuppressWarnings("unused") // required so JAXB can initialize and marshal private Record() {} - protected Record(Timestamp dateTime, Double value, int qualityCode) { + public Record(Timestamp dateTime, Double value, int qualityCode) { this.dateTime = dateTime; this.value = value; this.qualityCode = qualityCode; } - @JsonInclude(JsonInclude.Include.NON_DEFAULT) - protected Record withDataEntryDate(Timestamp dataEntryDate) { - this.dataEntryDate = dataEntryDate; - return this; - } - // When serialized, the value is unix epoch at UTC. public Timestamp getDateTime() { return dateTime; @@ -289,51 +299,36 @@ public int getQualityCode() { return qualityCode; } - public Timestamp getDataEntryDate() { - return dataEntryDate; - } - @Override - public boolean equals(Object o) - { - if(this == o) - { + public boolean equals(Object o) { + if (this == o) { return true; } - if(o == null || getClass() != o.getClass()) - { + if (o == null || getClass() != o.getClass()) { return false; } final Record tsRecord = (Record) o; - if(getQualityCode() != tsRecord.getQualityCode()) - { - return false; - } - if(getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) - { + if (getQualityCode() != tsRecord.getQualityCode()) { return false; } - if (!Objects.equals(dataEntryDate, tsRecord.dataEntryDate)) { + if (getDateTime() != null ? !getDateTime().equals(tsRecord.getDateTime()) : tsRecord.getDateTime() != null) { return false; } return getValue() != null ? getValue().equals(tsRecord.getValue()) : tsRecord.getValue() == null; } @Override - public int hashCode() - { + public int hashCode() { int result = getDateTime() != null ? getDateTime().hashCode() : 0; result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); result = 31 * result + getQualityCode(); - result = 31 * result + (dataEntryDate != null ? dataEntryDate.hashCode() : 0); return result; } @Override - public String toString() - { + public String toString() { return "Record{" + "dateTime=" + dateTime + ", value=" + value + ", qualityCode=" + qualityCode + '}'; } } @@ -345,7 +340,7 @@ private static class Column { public final Class datatype; // JAXB seems to need a default ctor - private Column(){ + private Column() { this(null, 0,null); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java new file mode 100644 index 000000000..21066175d --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesRecordWithDate.java @@ -0,0 +1,81 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.data.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.swagger.v3.oas.annotations.media.Schema; +import java.sql.Timestamp; +import java.util.Objects; + +/** + * TimeSeriesRecordWithDate is a subclass of TimeSeries.Record that includes a data entry date. + * The data entry date is the date that the data was entered into the database. + */ +@JsonDeserialize(using = JsonDeserializer.None.class) +public final class TimeSeriesRecordWithDate extends TimeSeries.Record { + @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate; + + // Default constructor for Jackson Deserialization + public TimeSeriesRecordWithDate() { + super(null, null, 0); + } + + public TimeSeriesRecordWithDate(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + super(dateTime, value, qualityCode); + this.dataEntryDate = dataEntryDate; + } + + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + TimeSeriesRecordWithDate that = (TimeSeriesRecordWithDate) o; + return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getDataEntryDate()); + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java new file mode 100644 index 000000000..e149ae441 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -0,0 +1,74 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.formatters.json.adapters; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import java.io.IOException; +import java.sql.Timestamp; +import java.time.Instant; + +/** + * A time-series record deserializer for properly deserializing JSON data. + * Requires {@link cwms.cda.data.dto.TimeSeries.RecordChild} class to avoid + * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer + * getting stuck in a loop if the Record class is used directly. + * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. + */ +public final class TimeSeriesRecordDeserializer extends JsonDeserializer { + @Override + public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.readValueAsTree(); + if (node.get("data-entry-date") != null) { + return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); + } + String nodeString = node.toString(); + if (nodeString.startsWith("[")) { + nodeString = nodeString.substring(1, nodeString.length() - 1); + String[] valList = nodeString.split(","); + if (valList.length == 4) { + Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + double value = Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + Timestamp entryDate = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[3]))); + return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); + } else if (valList.length == 3) { + Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + double value = Double.parseDouble(valList[1]); + int quality = Integer.parseInt(valList[2]); + return new TimeSeries.Record(dateTime, value, quality); + } else { + throw new IOException("Invalid TimeSeries Record format"); + } + } + return jsonParser.getCodec().treeToValue(node, TimeSeries.RecordChild.class); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 5142a9530..cde473876 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -1,8 +1,6 @@ package cwms.cda.api; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNotNull; import static org.mockito.ArgumentMatchers.isNull; @@ -16,6 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.TimeSeriesRecordWithDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -28,7 +27,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,9 +41,6 @@ import org.junit.jupiter.params.provider.ValueSource; class TimeSeriesControllerTest extends ControllerTest { - - - @Test void testDaoMock() throws JsonProcessingException { String officeId = "LRL"; @@ -134,11 +129,63 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { // Make sure ts we got back resembles the fakeTS our mock dao was supposed to return. assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); assertEquals(expected.getName(), actual.getName(), "names did not match"); - assertEquals(expected.getValues(), actual.getValues(), "values did not match"); + assertRecordsMatch(expected.getValues(), actual.getValues()); assertTrue(expected.getBegin().isEqual(actual.getBegin()), "begin dates not equal"); assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } + private void assertRecordsMatch(List expected, List actual) { + for (int i = 0; i < expected.size(); i++) { + if (expected.get(i) instanceof TimeSeriesRecordWithDate) { + if (!(actual.get(i) instanceof TimeSeriesRecordWithDate)) { + throw new AssertionError("Expected TimeSeriesRecordWithDate but got " + actual.get(i).getClass().getName()); + } + TimeSeriesRecordWithDate expectedRecord = new TimeSeriesRecordWithDate(expected.get(i).getDateTime(), + expected.get(i).getValue(), expected.get(i).getQualityCode(), + ((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate()); + TimeSeriesRecordWithDate actualRecord = new TimeSeriesRecordWithDate(actual.get(i).getDateTime(), + actual.get(i).getValue(), actual.get(i).getQualityCode(), + ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate()); + assertEquals(expectedRecord.getDataEntryDate(), + actualRecord.getDataEntryDate(), "Entry dates did not match"); + } + assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); + assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); + assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); + } + } + + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testSerializeTimeSeries(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeries(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + assertNotNull(formatted); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + + @ParameterizedTest + @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) + void testSerializeTimeSeriesWithDataEntryDate(String format) { + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + assertEquals(4, fakeTs.getValueColumnsJSON().size()); + assertInstanceOf(TimeSeriesRecordWithDate.class, fakeTs.getValues().get(0)); + ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + String formatted = Formats.format(contentType, fakeTs); + assertNotNull(formatted); + TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + assertNotNull(ts2); + assertSimilar(fakeTs, ts2); + } + + @ParameterizedTest @ValueSource(strings = {Formats.XMLV2, Formats.JSONV2}) void testDeserializeTimeSeries(String format) { @@ -241,7 +288,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); ts.addValue(dateTime, (double) i, 0, null); - next = next.plus(minutes, ChronoUnit.MINUTES); + next = next.plusMinutes(minutes); } return ts; } @@ -272,7 +319,7 @@ private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); ts.addValue(dateTime, (double) i, 0, Timestamp.from(Instant.now())); - next = next.plus(minutes, ChronoUnit.MINUTES); + next = next.plusMinutes(minutes); } return ts; } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 58f9fdc38..6dc12b6ec 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -42,7 +42,7 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -105,7 +105,7 @@ void test_lrl_1day() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); - String tsData = IOUtils.toString(resource, "UTF-8"); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); JsonNode ts = mapper.readTree(tsData); String location = ts.get("name").asText().split("\\.")[0]; @@ -139,7 +139,6 @@ void test_lrl_1day() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -324,7 +323,6 @@ void test_include_data_entry_date() throws Exception { .body("values[0].size()", equalTo(3)); } - @Test void test_delete_ts() throws Exception { ObjectMapper mapper = new ObjectMapper(); @@ -604,7 +602,6 @@ void test_lrl_trim() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -631,7 +628,6 @@ void test_lrl_trim() throws Exception { .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) -// .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam("office", officeId) .queryParam("units", "F") @@ -932,13 +928,13 @@ enum GetAllTest XMLV2(Formats.XMLV2, Formats.XMLV2), ; - final String _accept; - final String _expectedContentType; + final String accept; + final String expectedContentType; GetAllTest(String accept, String expectedContentType) { - _accept = accept; - _expectedContentType = expectedContentType; + this.accept = accept; + this.expectedContentType = expectedContentType; } } } From b91145cef8d1484fc912fba083543e973d733f08 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Mon, 4 Nov 2024 11:47:31 -0800 Subject: [PATCH 3/6] Fixes build error --- .../test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index 4de1262df..fe5d7f765 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -247,7 +247,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = START; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0); + ts.addValue(dateTime, (double) i, 0, null); next = next.plusMinutes(minutes); } return ts; From 20615dacb074e3487a51149d6ad54e0928890bd5 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Mon, 4 Nov 2024 14:09:16 -0800 Subject: [PATCH 4/6] Fixes test case failure --- .../formatters/json/adapters/TimeSeriesRecordDeserializer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java index e149ae441..a838018b1 100644 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java @@ -62,6 +62,9 @@ public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationConte return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); } else if (valList.length == 3) { Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); + if (valList[1].equalsIgnoreCase("null") || valList[1].isEmpty()) { + return new TimeSeries.Record(dateTime, null, Integer.parseInt(valList[2])); + } double value = Double.parseDouble(valList[1]); int quality = Integer.parseInt(valList[2]); return new TimeSeries.Record(dateTime, value, quality); From 1d0d3892f5ad41104a4cdd6ab2fd4091060bb3e0 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Tue, 5 Nov 2024 14:52:19 -0800 Subject: [PATCH 5/6] 634 TimeSeries Subclass update --- .../cda/data/dao/LocationLevelsDaoImpl.java | 11 +- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 231 ++++++++++-------- .../java/cwms/cda/data/dto/TimeSeries.java | 47 +--- .../cwms/cda/data/dto/TimeSeriesWithDate.java | 179 ++++++++++++++ .../TimeSeriesRecordDeserializer.java | 77 ------ .../cda/api/TimeSeriesControllerTest.java | 58 +++-- .../cda/api/TimeSeriesRecentControllerIT.java | 2 +- .../cwms/cda/data/dao/TimeSeriesDaoTest.java | 6 +- .../cda/formatters/TimeSeriesTestBase.java | 2 +- 9 files changed, 358 insertions(+), 255 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java delete mode 100644 cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java index e6f1ed7c2..1406c5d1b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationLevelsDaoImpl.java @@ -241,13 +241,12 @@ private static SEASONAL_VALUE_TAB_T getSeasonalValues(LocationLevel locationLeve List seasonalValues = locationLevel.getSeasonalValues(); SEASONAL_VALUE_TAB_T pSeasonalValues = null; - if(seasonalValues != null && !seasonalValues.isEmpty()) { + if (seasonalValues != null && !seasonalValues.isEmpty()) { pSeasonalValues = new SEASONAL_VALUE_TAB_T(); - for(SeasonalValueBean seasonalValue : seasonalValues) - { + for (SeasonalValueBean seasonalValue : seasonalValues) { SEASONAL_VALUE_T seasonalValueT = new SEASONAL_VALUE_T(); seasonalValueT.setOFFSET_MINUTES(toBigDecimal(seasonalValue.getOffsetMinutes())); - if(seasonalValue.getOffsetMonths() != null) { + if (seasonalValue.getOffsetMonths() != null) { seasonalValueT.setOFFSET_MONTHS(seasonalValue.getOffsetMonths().byteValue()); } seasonalValueT.setVALUE(toBigDecimal(seasonalValue.getValue())); @@ -468,7 +467,7 @@ private void addSeasonalValue(Record r, String calOffset = r.get(view.CALENDAR_OFFSET); String timeOffset = r.get(view.TIME_OFFSET); JDomSeasonalIntervalImpl newSeasonalOffset = buildSeasonalOffset(calOffset, timeOffset); - SeasonalValueBean seasonalValue = buildSeasonalValueBean(seasonalLevel, newSeasonalOffset) ; + SeasonalValueBean seasonalValue = buildSeasonalValueBean(seasonalLevel, newSeasonalOffset); builder.withSeasonalValue(seasonalValue); } } @@ -622,7 +621,7 @@ private static TimeSeries buildTimeSeries(ILocationLevelRef levelRef, Interval i if (qualityCode != null) { quality = qualityCode.intValue(); } - timeSeries.addValue(dateTime, value, quality, null); + timeSeries.addValue(dateTime, value, quality); } return timeSeries; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index a407e52da..cdf9c22e0 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -23,6 +23,7 @@ import cwms.cda.data.dto.RecentValue; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.TimeSeriesExtents; +import cwms.cda.data.dto.TimeSeriesWithDate; import cwms.cda.data.dto.Tsv; import cwms.cda.data.dto.TsvDqu; import cwms.cda.data.dto.TsvId; @@ -167,7 +168,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String String units, ZonedDateTime beginTime, ZonedDateTime endTime, ZonedDateTime versionDate, boolean shouldTrim, boolean includeEntryDate) { - TimeSeries retVal = null; String cursor = null; Timestamp tsCursor = null; Integer total = null; @@ -178,7 +178,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String logger.fine("Decoded cursor"); logger.finest(() -> { StringBuilder sb = new StringBuilder(); - for (String p: parts) { + for (String p : parts) { sb.append(p).append("\n"); } return sb.toString(); @@ -238,7 +238,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // put all those columns together as "valid" CommonTableExpression> valid = + String>> valid = name("valid").fields("tscode", "tsid", "office_id", "loc_part", "units", "interval", "parm_part") .as( @@ -258,9 +258,6 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field valueCol = field("VALUE", Double.class).as("VALUE"); Field qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); - Field qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli(); String trim = formatBool(shouldTrim); @@ -277,18 +274,20 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String maxVersion = "T"; } + Field qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( + DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); // Now we're going to call the retrieve_ts_out_tab function to get the data and build an // internal table from it so we can manipulate it further // This code assumes the database timezone is in UTC (per Oracle recommendation) SQL retrieveSelectData = DSL.sql( - "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," - + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," - + "?,?,?,?,?," - + getVersionPart(versionDate) + ",?,?) ) retrieveTs", - tsId, unit, - beginTimeMilli, endTimeMilli, //tz hardcoded - trim, startInclusive, endInclusive, previous, next, - versionDateMilli, maxVersion, officeId); + "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," + + "cwms_20.cwms_util.to_timestamp(?), cwms_20.cwms_util.to_timestamp(?), 'UTC'," + + "?,?,?,?,?," + + getVersionPart(versionDate) + ",?,?) ) retrieveTs", + tsId, unit, + beginTimeMilli, endTimeMilli, //tz hardcoded + trim, startInclusive, endInclusive, previous, next, + versionDateMilli, maxVersion, officeId); Field tzName = AV_CWMS_TS_ID2.TIME_ZONE_ID; @@ -351,27 +350,72 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String logger.fine(() -> metadataQuery.getSQL(ParamType.INLINED)); - VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); + TimeSeries timeseries = metadataQuery.fetchOne(tsMetadata -> { String vert = (String) tsMetadata.getValue("VERTICAL_DATUM"); VerticalDatumInfo verticalDatumInfo = parseVerticalDatumInfo(vert); - - return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", - Integer.class), tsMetadata.getValue("NAME", String.class), - tsMetadata.getValue("office_id", String.class), - beginTime, endTime, tsMetadata.getValue("units", String.class), - Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : - tsMetadata.getValue("interval", Long.class)), - verticalDatumInfo, - tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), - tsMetadata.getValue(tzName), - versionDate, finalDateVersionType - ); + VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); + if (!includeEntryDate) { + return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", + Integer.class), tsMetadata.getValue("NAME", String.class), + tsMetadata.getValue("office_id", String.class), + beginTime, endTime, tsMetadata.getValue("units", String.class), + Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : + tsMetadata.getValue("interval", Long.class)), + verticalDatumInfo, + tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), + tsMetadata.getValue(tzName), + versionDate, finalDateVersionType + ); + } else { + return new TimeSeriesWithDate(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", + Integer.class), tsMetadata.getValue("NAME", String.class), + tsMetadata.getValue("office_id", String.class), + beginTime, endTime, tsMetadata.getValue("units", String.class), + Duration.ofMinutes(tsMetadata.get("interval") == null ? 0 : + tsMetadata.getValue("interval", Long.class)), + verticalDatumInfo, + tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET).longValue(), + tsMetadata.getValue(tzName), + versionDate, finalDateVersionType + ); + } }); + if (includeEntryDate) { + timeseries = new TimeSeriesWithDate(timeseries); + } + + Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + Condition whereCond = dateTimeCol + .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.nvl(DSL.val(tsCursor == null ? null : + tsCursor.toInstant().toEpochMilli()), + DSL.val(beginTime.toInstant().toEpochMilli())))) + .and(dateTimeCol + .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(endTime.toInstant().toEpochMilli()))) + .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) + .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) + .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit))); + TimeSeries retVal = null; if (pageSize != 0) { + if (versionDate != null) { + whereCond = whereCond.and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : + Timestamp.from(versionDate.toInstant()))); + } + + SelectConditionStep> query2 = dsl.select( + dateTimeCol, + valueCol, + qualityNormCol, + dataEntryDate + ) + .from(AV_TSV_DQU.AV_TSV_DQU) + .where(whereCond); + SelectConditionStep> query = dsl.select( dateTimeCol, @@ -391,59 +435,29 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (pageSize > 0) { query.limit(DSL.val(pageSize + 1)); + query2.limit(DSL.val(pageSize + 1)); } - SelectConditionStep> finalQuery = query; - logger.fine(() -> finalQuery.getSQL(ParamType.INLINED)); - if (includeEntryDate) { - SelectConditionStep> query2 = dsl.select( - dateTimeCol, - valueCol, - qualityNormCol, - dataEntryDate - ) - .from(AV_TSV_DQU.AV_TSV_DQU) - .where(dateTimeCol - .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.nvl(DSL.val(tsCursor == null ? null : - tsCursor.toInstant().toEpochMilli()), - DSL.val(beginTime.toInstant().toEpochMilli()))))) - .and(dateTimeCol - .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.val(endTime.toInstant().toEpochMilli()))) - .and(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID.equalIgnoreCase(names)) - .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)) - .and(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID.equalIgnoreCase(unit)) - .and(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE.eq(versionDate == null ? null : - Timestamp.from(versionDate.toInstant()))) - ); - - if (pageSize > 0) { - query2.limit(DSL.val(pageSize + 1)); - } - query2.forEach(tsRecord -> { - assert timeseries != null; - timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - tsRecord.getValue(dataEntryDate) - ); - }); + logger.fine(() -> query2.getSQL(ParamType.INLINED)); + final TimeSeriesWithDate timeSeries = new TimeSeriesWithDate(timeseries); + query2.forEach(tsRecord -> timeSeries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate) + )); + retVal = timeSeries; } else { - query.forEach(tsRecord -> { - assert timeseries != null; - timeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - null - ); - }); + logger.fine(() -> query.getSQL(ParamType.INLINED)); + final TimeSeries finalTimeseries = timeseries; + query.forEach(tsRecord -> finalTimeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue() + )); + retVal = finalTimeseries; } - - retVal = timeseries; } return retVal; @@ -525,7 +539,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar String cursorOffice = null; Catalog.CatalogPage catPage = null; if (page == null || page.isEmpty()) { - CommonTableExpression limiter = buildWithClause(inputParams, buildWhereConditions(inputParams), new ArrayList<>(), pageSize, true); + CommonTableExpression limiter = buildWithClause(inputParams, buildWhereConditions(inputParams), + new ArrayList<>(), pageSize, true); SelectJoinStep> totalQuery = dsl.with(limiter) .select(countDistinct(limiter.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.TS_CODE))) .from(limiter); @@ -574,7 +589,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .on(limiterCode .eq(AV_TS_EXTENTS_UTC.TS_CODE.coerce(limiterCode))); } - final SelectSeekStep2 overallQuery = tmpQuery.orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); + final SelectSeekStep2 overallQuery = tmpQuery.orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, + AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID); logger.info(() -> overallQuery.getSQL(ParamType.INLINED)); Result result = overallQuery.fetch(); @@ -610,8 +626,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar } }); - List entries = tsIdExtentMap.entrySet().stream() - .map(e -> e.getValue().build()) + List entries = tsIdExtentMap.values().stream() + .map(TimeseriesCatalogEntry.Builder::build) .collect(Collectors.toList()); return new Catalog(catPage != null ? catPage.toString() : null, @@ -737,20 +753,23 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar TableLike innerSelect = selectDistinct(selectFields) .from(fromTable) .where(whereConditions).and(DSL.and(pagingConditions)) - .orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID) + .orderBy(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID, + AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID) .asTable("limiterInner"); if (forCount) { return name("limiter").as( select(asterisk()) .from(innerSelect) - .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) + .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), + innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) ); } else { return name("limiter").as( select(asterisk()) .from(innerSelect) .where(field("rownum").lessOrEqual(pageSize)) - .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) + .orderBy(innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.DB_OFFICE_ID), + innerSelect.field(AV_CWMS_TS_ID.AV_CWMS_TS_ID.CWMS_TS_ID)) ); } } @@ -975,18 +994,18 @@ public List findMostRecentsInRange(List tsIds, Timestamp pa // Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp // and the generated sql refers to columns from the alias_??? table. Field[] queryFields = new Field[]{ - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), - unitField, - dateTimeField, - innerSelect.field(tsField) + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), + innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), + unitField, + dateTimeField, + innerSelect.field(tsField) }; SelectConditionStep query = dsl.select(queryFields) @@ -1073,18 +1092,18 @@ public List findRecentsInRange(String office, String categoryId, St Field unit = innerSelect.field(tsvView.UNIT_ID); Field[] queryFields = new Field[]{ - innerSelect.field(tsvView.OFFICE_ID), - innerSelect.field(tsvView.TS_CODE), - innerSelect.field(tsvView.VERSION_DATE), - innerSelect.field(tsvView.DATA_ENTRY_DATE), - innerSelect.field(tsvView.VALUE), - innerSelect.field(tsvView.QUALITY_CODE), - innerSelect.field(tsvView.START_DATE), - innerSelect.field(tsvView.END_DATE), - dateTime, - unit, - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID), - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE)}; + innerSelect.field(tsvView.OFFICE_ID), + innerSelect.field(tsvView.TS_CODE), + innerSelect.field(tsvView.VERSION_DATE), + innerSelect.field(tsvView.DATA_ENTRY_DATE), + innerSelect.field(tsvView.VALUE), + innerSelect.field(tsvView.QUALITY_CODE), + innerSelect.field(tsvView.START_DATE), + innerSelect.field(tsvView.END_DATE), + dateTime, + unit, + innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID), + innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE)}; return dsl.select(queryFields) .from(innerSelect) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 1fc5d53e0..050682adf 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -9,15 +9,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonRootName; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.annotations.FormattableWith; import cwms.cda.formatters.json.JsonV2; -import cwms.cda.formatters.json.adapters.TimeSeriesRecordDeserializer; import cwms.cda.formatters.xml.XMLv2; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; @@ -108,7 +105,7 @@ public class TimeSeries extends CwmsDTOPaginated { @SuppressWarnings("unused") // required so JAXB can initialize and marshal - private TimeSeries() {} + protected TimeSeries() {} public TimeSeries(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, ZonedDateTime end, String units, Duration interval) { this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, null, null, null); @@ -165,7 +162,6 @@ public ZonedDateTime getEnd() { // Use the array shape to optimize data transfer to client @JsonFormat(shape = JsonFormat.Shape.ARRAY) - @JsonDeserialize(contentUsing = TimeSeriesRecordDeserializer.class) public List getValues() { return values; } @@ -198,11 +194,10 @@ public VersionType getDateVersionType() { @JsonProperty(value = "value-columns") @Schema(name = "value-columns", accessMode = AccessMode.READ_ONLY) public List getValueColumnsJSON() { - return getColumnDescriptor((values != null && !values.isEmpty()) - && values.get(0) instanceof TimeSeriesRecordWithDate); + return getColumnDescriptor(); } - public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + public void addValue(Timestamp dateTime, Double value, int qualityCode) { // Set the current page, if not set if ((page == null || page.isEmpty()) && values.isEmpty()) { page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); @@ -210,34 +205,20 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam if (pageSize > 0 && values.size() == pageSize) { nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); } else { - if (dataEntryDate != null) { - values.add(new TimeSeriesRecordWithDate(dateTime, value, qualityCode, dataEntryDate)); - } else { - values.add(new Record(dateTime, value, qualityCode)); - } + values.add(new Record(dateTime, value, qualityCode)); } } - private List getColumnDescriptor(boolean includeDataEntryDate) { + private List getColumnDescriptor() { List columns = new ArrayList<>(); for (Field f: Record.class.getDeclaredFields()) { JsonProperty field = f.getAnnotation(JsonProperty.class); - if(field != null) { + if (field != null) { String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); int fieldIndex = field.index(); columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); } } - if (includeDataEntryDate) { - for (Field f: TimeSeriesRecordWithDate.class.getDeclaredFields()) { - JsonProperty field = f.getAnnotation(JsonProperty.class); - if(field != null) { - String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); - int fieldIndex = field.index(); - columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); - } - } - } return columns; } @@ -250,7 +231,7 @@ private List getColumnDescriptor(boolean includeDataEntryDate) { type = "array" ), arraySchema = @Schema( - type="array", + type = "array", example = "[1509654000000, 54.3, 0]", description = "Time is Milliseconds since the UNIX Epoch. Value is Double (for missing data you " + "can use null, or -Float.MAX_VALUE (-340282346638528859811704183484516925440), " @@ -260,14 +241,6 @@ private List getColumnDescriptor(boolean includeDataEntryDate) { ) ) - // This class is used to deserialize the time-series data JSON into an object - // Solves the issue of the deserializer getting stuck in a loop - // and throwing a StackOverflowError when trying to handle the Record class directly - @JsonDeserialize(using = JsonDeserializer.None.class) - public static final class RecordChild extends Record { - } - - @JsonDeserialize(using = TimeSeriesRecordDeserializer.class) @JsonIgnoreProperties(ignoreUnknown = true) public static class Record { // Explicitly set property order for array serialization @@ -283,9 +256,11 @@ public static class Record { int qualityCode; @SuppressWarnings("unused") // required so JAXB can initialize and marshal - private Record() {} + private Record() { + } public Record(Timestamp dateTime, Double value, int qualityCode) { + super(); this.dateTime = dateTime; this.value = value; this.qualityCode = qualityCode; @@ -339,7 +314,7 @@ public String toString() { } @Schema(hidden = true, name = "TimeSeries.Column", accessMode = Schema.AccessMode.READ_ONLY) - private static class Column { + protected static class Column { public final String name; public final int ordinal; public final Class datatype; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java new file mode 100644 index 000000000..c35c3ab21 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java @@ -0,0 +1,179 @@ +/* + * + * MIT License + * + * Copyright (c) 2024 Hydrologic Engineering Center + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE + * SOFTWARE. + */ + +package cwms.cda.data.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import cwms.cda.api.enums.VersionType; +import cwms.cda.formatters.Formats; +import cwms.cda.formatters.annotations.FormattableWith; +import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.xml.XMLv2; +import io.swagger.v3.oas.annotations.media.Schema; +import java.lang.reflect.Field; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) +@FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) +@FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) +public final class TimeSeriesWithDate extends TimeSeries { + + private List values; + + @Override + public List getValues() { + return values; + } + + TimeSeriesWithDate() { + super(); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(TimeSeries timeSeries) { + this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), timeSeries.getName(), + timeSeries.getOfficeId(), timeSeries.getBegin(), timeSeries.getEnd(), timeSeries.getUnits(), + timeSeries.getInterval(), timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), + timeSeries.getTimeZone(), timeSeries.getVersionDate(), timeSeries.getDateVersionType()); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, + ZonedDateTime begin, ZonedDateTime end, String units, Duration interval) { + this(page, pageSize, total, name, officeId, begin, end, units, interval, null, null, + null, null, null); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, + ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, ZonedDateTime versionDate, + VersionType dateVersionType) { + this(page, pageSize, total, name, officeId, begin, end, units, interval, info, null, + null, versionDate, dateVersionType); + values = new ArrayList<>(); + } + + public TimeSeriesWithDate(String page, int pageSize, Integer total, String name, String officeId, ZonedDateTime begin, + ZonedDateTime end, String units, Duration interval, VerticalDatumInfo info, Long intervalOffset, + String timeZone, ZonedDateTime versionDate, VersionType dateVersionType) { + super(page, pageSize, total, name, officeId, begin, end, units, interval, info, intervalOffset, + timeZone, versionDate, dateVersionType); + values = new ArrayList<>(); + } + + public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + // Set the current page, if not set + if ((page == null || page.isEmpty()) && (values == null || values.isEmpty())) { + page = encodeCursor(String.format("%d", dateTime.getTime()), pageSize, total); + } + if (pageSize > 0 && values.size() == pageSize) { + nextPage = encodeCursor(String.format("%d", dateTime.toInstant().toEpochMilli()), pageSize, total); + } else { + values.add(new Record(dateTime, value, qualityCode, dataEntryDate)); + } + } + + @Override + public List getValueColumnsJSON() { + return getColumnDescriptor(); + } + + private List getColumnDescriptor() { + List columns = new ArrayList<>(); + for (Field f: TimeSeries.Record.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + for (Field f: Record.class.getDeclaredFields()) { + JsonProperty field = f.getAnnotation(JsonProperty.class); + if (field != null) { + String fieldName = !field.value().isEmpty() ? field.value() : f.getName(); + int fieldIndex = field.index(); + columns.add(new TimeSeries.Column(fieldName, fieldIndex + 1, f.getType())); + } + } + + return columns; + } + + @JsonDeserialize(using = JsonDeserializer.None.class) + public static final class Record extends TimeSeries.Record { + @JsonProperty(value = "data-entry-date", index = 3) + @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + Timestamp dataEntryDate; + + // Default constructor for Jackson Deserialization + public Record() { + super(null, null, 0); + } + + public Record(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + super(dateTime, value, qualityCode); + this.dataEntryDate = dataEntryDate; + } + + public Timestamp getDataEntryDate() { + return dataEntryDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + Record that = (Record) o; + return Objects.equals(getDataEntryDate(), that.getDataEntryDate()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getDataEntryDate()); + } + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java b/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java deleted file mode 100644 index a838018b1..000000000 --- a/cwms-data-api/src/main/java/cwms/cda/formatters/json/adapters/TimeSeriesRecordDeserializer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * - * MIT License - * - * Copyright (c) 2024 Hydrologic Engineering Center - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE - * SOFTWARE. - */ - -package cwms.cda.formatters.json.adapters; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; -import java.io.IOException; -import java.sql.Timestamp; -import java.time.Instant; - -/** - * A time-series record deserializer for properly deserializing JSON data. - * Requires {@link cwms.cda.data.dto.TimeSeries.RecordChild} class to avoid - * a StackOverflowError when deserializing JSON data. This issue can be caused by the custom serializer - * getting stuck in a loop if the Record class is used directly. - * Allows for use of subclass with additional fields {@link TimeSeriesRecordWithDate}. - */ -public final class TimeSeriesRecordDeserializer extends JsonDeserializer { - @Override - public TimeSeries.Record deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.readValueAsTree(); - if (node.get("data-entry-date") != null) { - return jsonParser.getCodec().treeToValue(node, TimeSeriesRecordWithDate.class); - } - String nodeString = node.toString(); - if (nodeString.startsWith("[")) { - nodeString = nodeString.substring(1, nodeString.length() - 1); - String[] valList = nodeString.split(","); - if (valList.length == 4) { - Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); - double value = Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - Timestamp entryDate = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[3]))); - return new TimeSeriesRecordWithDate(dateTime, value, quality, entryDate); - } else if (valList.length == 3) { - Timestamp dateTime = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(valList[0]))); - if (valList[1].equalsIgnoreCase("null") || valList[1].isEmpty()) { - return new TimeSeries.Record(dateTime, null, Integer.parseInt(valList[2])); - } - double value = Double.parseDouble(valList[1]); - int quality = Integer.parseInt(valList[2]); - return new TimeSeries.Record(dateTime, value, quality); - } else { - throw new IOException("Invalid TimeSeries Record format"); - } - } - return jsonParser.getCodec().treeToValue(node, TimeSeries.RecordChild.class); - } -} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index cde473876..839ea8a93 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -14,7 +14,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.TimeSeriesRecordWithDate; +import cwms.cda.data.dto.TimeSeriesWithDate; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; @@ -134,21 +134,27 @@ private void assertSimilar(TimeSeries expected, TimeSeries actual) { assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); } + private void assertSimilarWithDate(TimeSeriesWithDate expected, TimeSeriesWithDate actual) + { + assertEquals(expected.getOfficeId(), actual.getOfficeId(), "offices did not match"); + assertEquals(expected.getName(), actual.getName(), "names did not match"); + assertDateRecordsMatch(expected.getValues(), actual.getValues()); + assertTrue(expected.getBegin().isEqual(actual.getBegin()), "begin dates not equal"); + assertTrue(expected.getEnd().isEqual(actual.getEnd()), "end dates not equal"); + } + private void assertRecordsMatch(List expected, List actual) { for (int i = 0; i < expected.size(); i++) { - if (expected.get(i) instanceof TimeSeriesRecordWithDate) { - if (!(actual.get(i) instanceof TimeSeriesRecordWithDate)) { - throw new AssertionError("Expected TimeSeriesRecordWithDate but got " + actual.get(i).getClass().getName()); - } - TimeSeriesRecordWithDate expectedRecord = new TimeSeriesRecordWithDate(expected.get(i).getDateTime(), - expected.get(i).getValue(), expected.get(i).getQualityCode(), - ((TimeSeriesRecordWithDate) expected.get(i)).getDataEntryDate()); - TimeSeriesRecordWithDate actualRecord = new TimeSeriesRecordWithDate(actual.get(i).getDateTime(), - actual.get(i).getValue(), actual.get(i).getQualityCode(), - ((TimeSeriesRecordWithDate) actual.get(i)).getDataEntryDate()); - assertEquals(expectedRecord.getDataEntryDate(), - actualRecord.getDataEntryDate(), "Entry dates did not match"); - } + assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); + assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); + assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); + } + } + + private void assertDateRecordsMatch(List expected, List actual) { + for (int i = 0; i < expected.size(); i++) { + assertEquals(((TimeSeriesWithDate.Record) expected.get(i)).getDataEntryDate(), + ((TimeSeriesWithDate.Record) actual.get(i)).getDataEntryDate(), "Entry dates did not match"); assertEquals(expected.get(i).getDateTime(), actual.get(i).getDateTime(), "Timestamps did not match"); assertEquals(expected.get(i).getValue(), actual.get(i).getValue(), "Values did not match"); assertEquals(expected.get(i).getQualityCode(), actual.get(i).getQualityCode(), "Quality codes did not match"); @@ -174,15 +180,15 @@ void testSerializeTimeSeries(String format) { void testSerializeTimeSeriesWithDataEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); assertEquals(4, fakeTs.getValueColumnsJSON().size()); - assertInstanceOf(TimeSeriesRecordWithDate.class, fakeTs.getValues().get(0)); - ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + assertInstanceOf(TimeSeriesWithDate.Record.class, fakeTs.getValues().get(0)); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); String formatted = Formats.format(contentType, fakeTs); assertNotNull(formatted); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + assertSimilarWithDate(fakeTs, ts2); } @@ -204,12 +210,12 @@ void testDeserializeTimeSeries(String format) { void testDeserializeTimeSeriesWithEntryDate(String format) { String officeId = "LRL"; String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeries fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); - ContentType contentType = Formats.parseHeader(format, TimeSeries.class); + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); String formatted = Formats.format(contentType, fakeTs); - TimeSeries ts2 = Formats.parseContent(contentType, formatted, TimeSeries.class); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); assertNotNull(ts2); - assertSimilar(fakeTs, ts2); + assertSimilarWithDate(fakeTs, ts2); } @Test @@ -287,14 +293,14 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plusMinutes(minutes); } return ts; } @NotNull - private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { + private TimeSeriesWithDate buildTimeSeriesWithEntryDate(String officeId, String tsId) { ZonedDateTime start = ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"); ZonedDateTime end = ZonedDateTime.parse("2021-06-21T09:00:00-07:00[PST8PDT]"); @@ -305,7 +311,7 @@ private TimeSeries buildTimeSeriesWithEntryDate(String officeId, String tsId) { int count = 60/15 ; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeries ts = new TimeSeries(null, + TimeSeriesWithDate ts = new TimeSeriesWithDate(null, -1, 0, tsId, diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index fe5d7f765..4de1262df 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -247,7 +247,7 @@ private TimeSeries buildTimeSeries(String officeId, String tsId) { ZonedDateTime next = START; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.from(next.toInstant()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plusMinutes(minutes); } return ts; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java index 5f9601ff3..dc68ee4f8 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesDaoTest.java @@ -13,6 +13,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import cwms.cda.data.dto.TimeSeriesWithDate; import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.impl.DSL; @@ -168,13 +169,13 @@ void testCreateWithData() throws Exception int count = 60 / 15; // do I need a +1? ie should this be 12 or 13? // Also, should end be the last point or the next interval? - TimeSeries ts = new TimeSeries(null, -1, 0, tsId, officeId, start, end, "m", Duration.ofMinutes(minutes)); + TimeSeriesWithDate ts = new TimeSeriesWithDate(null, -1, 0, tsId, officeId, start, end, "m", Duration.ofMinutes(minutes)); ZonedDateTime next = start; for(int i = 0; i < count; i++) { Timestamp dateTime = Timestamp.valueOf(next.toLocalDateTime()); - ts.addValue(dateTime, (double) i, 0, null); + ts.addValue(dateTime, (double) i, 0); next = next.plus(minutes, ChronoUnit.MINUTES); } @@ -253,6 +254,7 @@ private void createTs(CwmsDbTsJooq cwmsTsJdbc, Connection connection) throws SQL } catch(Exception e) { + LOGGER.log(Level.CONFIG, "Unable to create TimeSeries: " + e.getMessage()); } } diff --git a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java index d195bde15..3c50b8142 100644 --- a/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java +++ b/cwms-data-api/src/test/java/cwms/cda/formatters/TimeSeriesTestBase.java @@ -14,7 +14,7 @@ public abstract class TimeSeriesTestBase { protected TimeSeries getTimeSeries() { TimeSeries ts = new TimeSeries(null, -1, 0, "Test.Test.Elev.0.0.RAW", "SPK", ZonedDateTime.parse("2021-06-21T08:00:00-07:00[PST8PDT]"), ZonedDateTime.parse("2021-06-22T08:00:00-07:00[PST8PDT]"), null, Duration.ZERO); - ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0, null); + ts.addValue(Timestamp.from(ts.getBegin().toInstant()), 30.0, 0); return ts; } From 667248d98e23fe1fcae090415aca38a388332b02 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Tue, 5 Nov 2024 15:31:42 -0800 Subject: [PATCH 6/6] 634 TimeSeries Subclass update - added Mixin test --- .../cwms/cda/data/dto/TimeSeriesWithDate.java | 4 +--- .../cwms/cda/api/TimeSeriesControllerTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java index c35c3ab21..78ce3a57b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeriesWithDate.java @@ -28,9 +28,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import cwms.cda.api.enums.VersionType; import cwms.cda.formatters.Formats; @@ -55,6 +53,7 @@ public final class TimeSeriesWithDate extends TimeSeries { private List values; + // list of TimeSeriesWithDate.Record, uses raw to avoid typing errors @Override public List getValues() { return values; @@ -135,7 +134,6 @@ private List getColumnDescriptor() { return columns; } - @JsonDeserialize(using = JsonDeserializer.None.class) public static final class Record extends TimeSeries.Record { @JsonProperty(value = "data-entry-date", index = 3) @Schema(implementation = Long.class, description = "Milliseconds since 1970-01-01 (Unix Epoch), always UTC") diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 839ea8a93..58de111bc 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -218,6 +218,22 @@ void testDeserializeTimeSeriesWithEntryDate(String format) { assertSimilarWithDate(fakeTs, ts2); } + @Test + void testXMLSerializeDeserializeTimeSeries() + { + String format = Formats.XMLV2; + String officeId = "LRL"; + String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; + TimeSeriesWithDate fakeTs = buildTimeSeriesWithEntryDate(officeId, tsId); + ContentType contentType = Formats.parseHeader(format, TimeSeriesWithDate.class); + String formatted = Formats.format(contentType, fakeTs); + assertTrue(formatted.contains("quality-code")); + assertTrue(formatted.contains("data-entry-date")); + TimeSeriesWithDate ts2 = Formats.parseContent(contentType, formatted, TimeSeriesWithDate.class); + assertNotNull(ts2); + assertSimilarWithDate(fakeTs, ts2); + } + @Test void testDeserializeTimeSeriesXmlUTC() { TimeZone aDefault = TimeZone.getDefault();