Skip to content

Commit

Permalink
[FSTORE-1294] Set Java dateformat to 24h clock in Filter (#1733) (#1508)
Browse files Browse the repository at this point in the history 
Co-authored-by: davitbzh <44586065+davitbzh@users.noreply.github.com>
  • Loading branch information
@robzor92 @davitbzh
robzor92 and davitbzh authored Mar 18, 2024
1 parent e06af4e commit 57e0048
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 10 deletions.
8 changes: 4 additions & 4 deletions hopsworks-IT/src/test/ruby/spec/featureview_query_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,16 +567,16 @@
fs_query = get "#{ENV['HOPSWORKS_API']}/project/#{@project.id}/featurestores/#{featurestore_id}/featureview/#{feature_view_name}/version/#{feature_view_version}/query/batch?start_time=1234&end_time=4321"
fs_query_result = put "#{ENV['HOPSWORKS_API']}/project/#{@project.id}/featurestores/query", JSON.parse(fs_query)
parsed_query_result = JSON.parse(fs_query_result)
expect(parsed_query_result['query']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg1`.`h` `h`, `fg1`.`i` `i`\nFROM `#{featurestore_name}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{featurestore_name}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{featurestore_name}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 12:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 12:00:04.000'")
expect(parsed_query_result['queryOnline']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg1`.`h` `h`, `fg1`.`i` `i`\nFROM `#{project_name.downcase}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{project_name.downcase}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{project_name.downcase}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 12:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 12:00:04.000'")
expect(parsed_query_result['query']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg1`.`h` `h`, `fg1`.`i` `i`\nFROM `#{featurestore_name}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{featurestore_name}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{featurestore_name}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 00:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 00:00:04.000'")
expect(parsed_query_result['queryOnline']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg1`.`h` `h`, `fg1`.`i` `i`\nFROM `#{project_name.downcase}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{project_name.downcase}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{project_name.downcase}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 00:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 00:00:04.000'")
expect_status_details(200)

# with helper columns
fs_query = get "#{ENV['HOPSWORKS_API']}/project/#{@project.id}/featurestores/#{featurestore_id}/featureview/#{feature_view_name}/version/#{feature_view_version}/query/batch?start_time=1234&end_time=4321&with_primary_keys=true&with_event_time=true&inference_helper_columns&inference_helper_columns=true&training_helper_columns=true"
fs_query_result = put "#{ENV['HOPSWORKS_API']}/project/#{@project.id}/featurestores/query", JSON.parse(fs_query)
parsed_query_result = JSON.parse(fs_query_result)
expect(parsed_query_result['query']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg2`.`ts` `ts`, `fg2`.`pk` `pk`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg0`.`ts` `ts`, `fg1`.`g` `g`, `fg1`.`h` `h`, `fg1`.`i` `i`, `fg1`.`ts` `ts`\nFROM `#{featurestore_name}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{featurestore_name}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{featurestore_name}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 12:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 12:00:04.000'")
expect(parsed_query_result['queryOnline']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg2`.`ts` `ts`, `fg2`.`pk` `pk`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg0`.`ts` `ts`, `fg1`.`g` `g`, `fg1`.`h` `h`, `fg1`.`i` `i`, `fg1`.`ts` `ts`\nFROM `#{project_name.downcase}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{project_name.downcase}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{project_name.downcase}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 12:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 12:00:04.000'")
expect(parsed_query_result['query']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg2`.`ts` `ts`, `fg2`.`pk` `pk`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg0`.`ts` `ts`, `fg1`.`g` `g`, `fg1`.`h` `h`, `fg1`.`i` `i`, `fg1`.`ts` `ts`\nFROM `#{featurestore_name}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{featurestore_name}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{featurestore_name}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 00:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 00:00:04.000'")
expect(parsed_query_result['queryOnline']).to eql("SELECT `fg2`.`b` `b`, `fg2`.`c` `c`, `fg2`.`ts` `ts`, `fg2`.`pk` `pk`, `fg0`.`e` `e`, `fg0`.`f` `f`, `fg0`.`ts` `ts`, `fg1`.`g` `g`, `fg1`.`h` `h`, `fg1`.`i` `i`, `fg1`.`ts` `ts`\nFROM `#{project_name.downcase}`.`test_fg_1#{featuregroup_suffix}_1` `fg2`\nINNER JOIN `#{project_name.downcase}`.`test_fg_2#{featuregroup_suffix}_1` `fg0` ON `fg2`.`pk` = `fg0`.`pk`\nINNER JOIN `#{project_name.downcase}`.`test_fg_3#{featuregroup_suffix}_1` `fg1` ON `fg2`.`pk` = `fg1`.`pk`\nWHERE `fg2`.`ts` >= TIMESTAMP '1970-01-01 00:00:01.000' AND `fg2`.`ts` < TIMESTAMP '1970-01-01 00:00:04.000'")
expect_status_details(200)
end
end
Expand Down
4 changes: 3 additions & 1 deletion ...on/src/main/java/io/hops/hopsworks/common/featurestore/query/filter/FilterController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.TimeZone;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -73,7 +74,7 @@ public class FilterController {
private FeaturegroupController featuregroupController;
private ObjectMapper objectMapper = new ObjectMapper();
private DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
private DateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
private DateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

public FilterController() {
}
Expand Down Expand Up @@ -125,6 +126,7 @@ public FilterLogic convertFilterLogic(FilterLogicDTO filterLogicDTO,

public String convertToEventTimeFeatureValue(Feature feature, Date date) throws FeaturestoreException {
String timeType = feature.getType();
timestampFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
if ("date".equals(timeType)) {
return dateFormat.format(date);
} else if ("timestamp".equals(timeType)) {
Expand Down
21 changes: 16 additions & 5 deletions ...mon/src/test/io/hops/hopsworks/common/featurestore/query/filter/TestFilterController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,7 @@
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;

public class TestFilterController {

Expand Down Expand Up @@ -623,4 +619,19 @@ public void testGetSQLNodeOther() throws Exception {
result = node.toSqlString(new HiveSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql();
Assert.assertEquals(expected, result);
}

@Test
public void testConvertToEventTimeFeatureValue() throws Exception {
Feature feature = new Feature("fg1_pk", "fg1", "timestamp",true, null,
"prefix2_", fg1);
Date dateWithHandS = new Date(1710442079000L);
String expectedHandS = "2024-03-14 18:47:59";
String resultHandS = filterController.convertToEventTimeFeatureValue(feature, dateWithHandS);
Assert.assertEquals(resultHandS, expectedHandS);

Date date = new Date(1710374400000L);
String expected = "2024-03-14 00:00:00";
String result = filterController.convertToEventTimeFeatureValue(feature, date);
Assert.assertEquals(expected, result);
}
}

0 comments on commit 57e0048

Please sign in to comment.