From dea459edde6c98da4139a5a4e5af29554cfa05c6 Mon Sep 17 00:00:00 2001 From: Peng Huo <penghuo@gmail.com> Date: Mon, 15 Jul 2024 10:47:03 -0700 Subject: [PATCH 01/96] Welcome new maintainer LantaoJin (#2818) Signed-off-by: Peng Huo <penghuo@gmail.com> --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 913320e6b5..8e4af57c8a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 +* @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 0ee07757c6..95aaaa67fe 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -21,6 +21,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Sean Kao | [seankao-az](https://github.com/seankao-az) | Amazon | | Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | | Tomoyuki Morita | [ykmr1224](https://github.com/ykmr1224) | Amazon | +| Lantao Jin | [LantaoJin](https://github.com/LantaoJin) | Amazon | | Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | Improving | | Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | Improving | | Andrew Carbonetto | [acarbonetto](https://github.com/acarbonetto) | Improving | From 0c2e1da96a5b34337e7ffa971842fcce4a6e10f2 Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Tue, 16 Jul 2024 01:59:05 +0800 Subject: [PATCH 02/96] SQL pagination should work with the `pretty` parameter (#2759) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- .../org/opensearch/sql/jdbc/CursorIT.java | 20 +++++++++++ .../sql/sql/domain/SQLQueryRequest.java | 29 ++++++++------- .../sql/sql/domain/SQLQueryRequestTest.java | 36 +++++++++++++++++++ 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java index 325c81107f..e2b6287191 100644 --- a/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/jdbc/CursorIT.java @@ -22,6 +22,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.List; +import java.util.Map; import javax.annotation.Nullable; import lombok.SneakyThrows; import org.json.JSONObject; @@ -115,6 +116,8 @@ public void select_all_no_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -133,6 +136,8 @@ public void select_count_all_no_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -151,6 +156,8 @@ public void select_all_small_table_big_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -169,6 +176,8 @@ public void select_all_small_table_small_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -187,6 +196,8 @@ public void select_all_big_table_small_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -205,6 +216,8 @@ public void select_all_big_table_big_cursor() { var restResponse = executeRestQuery(query, null); assertEquals(rows, restResponse.getInt("total")); + var restPrettyResponse = executeRestQuery(query, null, Map.of("pretty", "true")); + assertEquals(rows, restPrettyResponse.getInt("total")); } } @@ -217,6 +230,12 @@ private static String getConnectionString() { @SneakyThrows protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size) { + return executeRestQuery(query, fetch_size, Map.of()); + } + + @SneakyThrows + protected JSONObject executeRestQuery( + String query, @Nullable Integer fetch_size, Map<String, String> params) { Request request = new Request("POST", QUERY_API_ENDPOINT); if (fetch_size != null) { request.setJsonEntity( @@ -224,6 +243,7 @@ protected JSONObject executeRestQuery(String query, @Nullable Integer fetch_size } else { request.setJsonEntity(String.format("{ \"query\": \"%s\" }", query)); } + request.addParameters(params); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); restOptionsBuilder.addHeader("Content-Type", "application/json"); diff --git a/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java index 79ad9bf317..3714b443f5 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java +++ b/sql/src/main/java/org/opensearch/sql/sql/domain/SQLQueryRequest.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Stream; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -85,19 +86,21 @@ public SQLQueryRequest( * @return true if supported. */ public boolean isSupported() { - var noCursor = !isCursor(); - var noQuery = query == null; - var noUnsupportedParams = - params.isEmpty() || (params.size() == 1 && params.containsKey(QUERY_PARAMS_FORMAT)); - var noContent = jsonContent == null || jsonContent.isEmpty(); - - return ((!noCursor - && noQuery - && noUnsupportedParams - && noContent) // if cursor is given, but other things - || (noCursor && !noQuery)) // or if cursor is not given, but query - && isOnlySupportedFieldInPayload() // and request has supported fields only - && isSupportedFormat(); // and request is in supported format + boolean hasCursor = isCursor(); + boolean hasQuery = query != null; + boolean hasContent = jsonContent != null && !jsonContent.isEmpty(); + + Predicate<String> supportedParams = Set.of(QUERY_PARAMS_FORMAT, QUERY_PARAMS_PRETTY)::contains; + boolean hasUnsupportedParams = + (!params.isEmpty()) + && params.keySet().stream().dropWhile(supportedParams).findAny().isPresent(); + + boolean validCursor = hasCursor && !hasQuery && !hasUnsupportedParams && !hasContent; + boolean validQuery = !hasCursor && hasQuery; + + return (validCursor || validQuery) // It's a valid cursor or a valid query + && isOnlySupportedFieldInPayload() // and request must contain supported fields only + && isSupportedFormat(); // and request must be a supported format } private boolean isCursor() { diff --git a/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java index 2b64b13b35..b569a89a2e 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/domain/SQLQueryRequestTest.java @@ -106,6 +106,42 @@ public void should_support_cursor_request() { () -> assertTrue(cursorRequest.isSupported())); } + @Test + public void should_support_cursor_request_with_supported_parameters() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request(null) + .cursor("abcdefgh...") + .params(Map.of("format", "csv", "pretty", "true")) + .build(); + + assertAll( + () -> assertTrue(fetchSizeRequest.isSupported()), + () -> assertTrue(cursorRequest.isSupported())); + } + + @Test + public void should_not_support_cursor_request_with_unsupported_parameters() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request(null) + .cursor("abcdefgh...") + .params(Map.of("one", "two")) + .build(); + + assertAll( + () -> assertTrue(fetchSizeRequest.isSupported()), + () -> assertFalse(cursorRequest.isSupported())); + } + @Test public void should_support_cursor_close_request() { SQLQueryRequest closeRequest = From 2c29a1a592f53be96bcab180f568a55f313a4c93 Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Wed, 17 Jul 2024 11:10:09 +0800 Subject: [PATCH 03/96] Add MacOS workflow back and fix the artifact not found issue (#2831) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- .github/workflows/integ-tests-with-security.yml | 2 +- .github/workflows/sql-test-and-build-workflow.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 58cbe831b1..751f3d01db 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -60,7 +60,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ windows-latest ] + os: [ windows-latest, macos-13 ] java: [21] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 4af0244dad..fd11716b51 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -105,6 +105,7 @@ jobs: matrix: entry: - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows } + - { os: macos-13, java: 21 } runs-on: ${{ matrix.entry.os }} steps: From a6394cd1f43423b55239529cd5586f6c39b602ac Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Thu, 18 Jul 2024 01:20:57 +0800 Subject: [PATCH 04/96] Update to Gradle 8.9 (#2835) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- gradle/wrapper/gradle-wrapper.jar | Bin 63721 -> 43504 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 6 ++++-- gradlew.bat | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7f93135c49b765f8051ef9d0a6055ff8e46073d8..2c3521197d7c4586c843d1d3e9090525f1898cde 100644 GIT binary patch literal 43504 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z<V--Q23O4&HBVn~<)q zmUaP7+TjluBM%#s1Ki#^GurGElkc7{cc6Skz+1nDVk%wAAQYx1^*wA%KSY>!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^e<cs4tSN~YA?c-d185$YFNA$Eq1&U{wh#b^OveuKoBPy0oYZ4 zAY2?B=x8yX9}pVM=cLrvugywt!e@Y3lH)i?7fvT*a`O;c)CJQ>O3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwA<BCEY82WDKJP< zB^CxjFxi=mg*OyI?K3GoDfk;?-K<Z#JoxhYNeEUf896)l%7gL``44}zn)7|Rf;)SC z_EfJr4I+3i(GiHN`R+vHqf}1wXtH?65<wKlxV1BU(#3XgtH<$Fir3S(7QeRA3)u89 zID&66K{&mq$DsB}s&o?H60{cskfh*hvn8hQW#~Q!qM04QtZvx3JEpqeKWE6|+OZW= z(LB7}flr|t7va%>yR<KG!FYzS$bs7qXcpM&wV@~>PZo2<wCq%CszVO$mosTTuv*Mz zOLoi?e^7B~xS22~QW8Rmnt{(AtL<HGi<_P9`0pH;3)@S9Eg`gt2X<om7C^q}pKX|* zTy3X{nOr-xyt4=Qx1IjrzGb!_SyAv^SZcf;air&-;Ua+)5k0z=#R7@UW%)3oEjGA| zZ#DE3px@h1k7w%|4rVIO=0Aid2A%?nBZrupg^_z5J-$$YKeDZ&q8+k7zccb<dc4D; zz}+UYkl_eUNL3PW+reZ6UUB}=sHp~$z%Q}gZ-#ow+ffQIj|A3`B9LO*6%t@)0PV!x ziJ=9fw_>Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1Ky<fW-rh4ehZ;%u960Gt5OF)<y$00S=6tVE=%Pt~( z!&BP&2I%`@>SGG#Wql>aL~k9tLrSO()LWn*q&YxHE<sT^`N@Q|)S3y<ZACaLXO56z zncP$~M5K!npWqz?)C50MMw=XqFtDO!3JHI*t-^8Ga&lGPHX2F0pIGdZ3w5ewE+{kf z-&Ygi?@-h(ADD|ljIBw%VHHf1xuQ~}IeIQ5JqlA4#*Nlvd`IfDYzFa?PB=RCcFpZ4 z|HFmPZM=;^DQ_z<IPz$$+yG(H4803QQAA7vQF7;_gv|AD1bH*R-CP3f<<utDpH)Ht zI@{uO12adp{;132YoKPx?C9{&;MtHdHb*0F0;Z~D42}#*l+WD2u?r>uzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(<VS*?#8Zt!w88FJrjasA1!6>!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA<eVn3dnmk^xq`=o2)~2c0ywsuTQsC?1WZZehsJYfK@LQ>*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^<IivRZw`Wa$`V6) zgX@^QL9j}-Od{q5<J*k0+1U=R5+PCYj(U}4VpX+BjfI~+dttS?HJ6uZSGH#H-twTo zaptG40+PAc$fs*zLFkOfGfc+xGs<T?rLGIA%SU7c%jh!E1SNN~*-`ccW8wo4gv2Sj zhify^C(ygi)uGwqXDLqVbH>Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+m<X+=`m<r!lO%3T zMp}MJd(WDoQ2&6(LClZxpv<vZPPM3Ngkye2VhB=i|B12g5ouw(%`gbWtRq8~sU|o* z$kQ8Jb~6&{ak;r$7@?#t*q9RfAOj=^uAf1z5Y8`N%M`oM@?!~VqN{g%-u$XR1u1Im zGE&AzFpIcER(5jtCPR%RZ)!+|*rU~jZBiOKdqYjO(%yK3Lz;{##(@QEVo>g&7$u!! z-^<eVk1WtrWdvAzoBMHoB$s2RXJCv}%muyVFFJ``?>+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)<T1$eOrb4-+U|WDC2BesgFRlgt`klbeQ^1S`7`r+uZ8 zH&U=geA}Si;CUcKvBA&^@<o1GQ7`{1Y(cCHZv|73JIJOvVwLOMZP%Q|)y@^j2e<+z zWVo=#FL!4XNKS~-_1`gw*qi$0j6P7ym_LTvG>us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;<s2pnue6O@?^QaAp;Ze6z9nX*w}4h7342+0lU$@;Knnve zqqY2Ci=`)@>KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{U<eziQYNZ-=4ReK3@^LFvNQI~(Pdvp+X@J@g#bd~m0wFc+sW3Xf5tyA3xKp;T3 zy14<o-`F}$ET-DQ;B;yNy?d>w%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+u<SJ)DEVF_yZnTw01M`(s#^BNx+c|MQ6ogb50Jjul0L;!#OmrYCs)iE)7(t z?%I~O!zVNt#Bf3#O2WXsGz!B}&s@MfyDeaoqqf=GELN3g$+DA`&&GKy(`Ya~A@6vK zn|WZ-+tB`DH^+SjI&K3KekF%-QIP%R{F)inWc~@cEO-=3Or<lm9g9}|`|ky#v{5*; zKA5d<ecC{<o9p<U4UUK$m|+q#@(>PsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2<b07B|^BQBjvq{FXx?kyJ);`+G*=&9PMD`1uf<{+pNnnsIQx~kaB?*5<-7a zqY)GyF_w$>d>_iO<o;tRi5=dcnU&wcur@4T5Z=-$xFUEsp-yX${|jSF|HMDPq3?MS zw;p9zjR`yYJOfJZsK~C-S=JQ?nX{z_y@06JFIpheAo-rOG|5&Gxv)%95gpu@ESfi| z7Auc&hjVL;&81Pc#L`^d9gJb`wEtLVH8q|h{>*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;s<dwKr_&w<X$Z*rmLmKUI3S>Iav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{X<DkOU(-L87#5hf4{m?aj!I6- zPEt$K07IXK8mI0TYf-jhke2QjQw3v?qN5h0-#Fel0)Krq1f)#^AFsfd|K$I={`Xs9 z{JIr8M>BdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<eS=8Og#NOG$&X&%|8sOyg zpZ6&%KPd&uh?v{hRMVvQjUL}gY3)Mk3{XQXF{><3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ib<ko|2T z<o~B%-$Y4Q9z_t97c`{g0veSfFt63Osbpe2Osn@<=nrAVk_JfMGt&lMGw9leshc#5 z*hkn0u>NBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV<T&F{)-N{)9$`9a!^D!-03RDN<TPH!aW46TC4L z>1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_<cF$~mH3zum`PN7rn^cr1XvcjzxFO{ms_482AyMFYi+#o7!*vecrNhft z48z<2q#fIw=ce!MXuptfT4+M8FP&|QfB3H@2)dceSR<*e5@hq<#7<$5tC^!RO8Zi< zd_Wl!>syQv5A2rj!Vbw8;|$@C!vfNmNV!yJ<MblqN@23-5g1<aeoul%Um5K((_QY} ze%_@BuNzay69}2PhmC<;m}2=FevDzrp!V!u4u|#h@B=rfKt+v!U`0k7>IWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6Q<xVqo{NJ3h9-a)s5XuYMqZ=Y{7{ z$O63J`)FM-y*mko#!-UBa!3~eYtX1hjRQY2jMxAx=q5uKNm#uaKIak>K=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%<xsJq4AotN+ zH6twFV=)FlAbs*F6vGws^==x5Tl0AIbcP{&2yxB=)*u+bvK^L6$Vp}U2{9nj{bK~d zee7tC)@DR<dI`D%cA(%7M9Ui3a)^iG?m=oJO0E^``<|5il2sf1fZHvy=D@e0<I)<l zI!|d{`X3u}lz2(4Vn>+clM1<yhZZgPANro5CwhUb>xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkS<W$zJN%xs9<lngf<utn=i|I;bCdr-Lr<EzK)tkE-pYh-fc0wqKz?&U8TTN zh_eAdl<>J3?zOH)OezMT{!YkCuSSn!<oaxO4?NS?VufjhPn>K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI<BVn6Upp<cc;cU|)&2W%nk!Ak8tXK8aT!m*5 z^9zmeeS|PCG$hgM&Uh}0wp+#$jK3YCwOT&nx$??=a@_oQemQ~hS6nx6fB5r~bFSPp z`alXuTYys4S5dCK)KDGR@7`I-JV^ewQ_BGM^o>@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7<FViITCBP{rA>m6ze=mZ<W0bN&bq-0D3>`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%<w%rbophph+BzYj>2i(Td=<hfIaF6Ll8+9!48Ti=xpXB{FgJbk;>tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&N<u ztispy>ykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWD<Q)gT}bxTg_YpJQ5s|m8}+B)KBN6 zYnlzh>qZ7J&~gAm1#~maIGJ<sH@F<m!Fuh_fvrMbcDJNJ5~Yg;LF}NFN}&Y&LL76S zv)~8W2?_rx`P;4LB-=JqsI{I~4U8DnSSIHWU2rHf%vWsA2-d=78An8z4q|lvgQ2iB zhUUI!H+|C+_qp(Tjzu5usOu}cEoivZK&XA==sh0cD|Eg7eERXx?KwHI=}A9S_rx8S zd)VLh_s!Juqi^!0xv7jH)UdSkEY~N|;QMWvs;HN`dMsdK=Dw2mtAHHcK8_+kS%a_V zGgeQoaMM>1sls^gxL9LLG_Nh<XXk<>U!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j<?~h)Y%y=zErI?{tl!(JWSDXxco7X8WI-6K;9Z-h&~kIv?$!6<k(g(xee? z53>0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|<j7k-g{75e!h)4SlFvEZ*AkqrJI;EWu$Zx+OwM zm{5Yk>iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho<sjDlFD=G`r<7$U?bJN+x5S z@0&tQ=-XO1uDq(HCa$X)-l<u1!s<!W`30F78UcZaZKc8)G0af1Dsh%OOWh5)q+Q+n zySBnE+3;9^#)U#Gq);&Cu=mtjNpsS~S0yjE@m4{Kq525G&cO_+b-_B$LeXWt_@XTq z`)(;=^RDS@oh5dPjKyGAP?-Dbh507E5zZ=D2_C*6s^HXiA)B3f=65_M+rC&rMIUP6 zi4@u>$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26<Ea z?or_^bK_`R)hBTfrBqA3Y^o7$K~Nzo)sh-vT%yWcc1I5wF1nkvk%!X_Vl_MK1IHC= zt}Dt+sOmg0sH-?}kqNB|M_}ZXui7H;?;?xCCSIPSHh8@h^K8WU5X(!3W|>Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<UD^T*M!yxMr=U!@&!rJfydk7CE7PGb<{)^=nM9Le#FQ=GkV~ z)_A$YPAn35??iNa@`g-wBX><4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5<wxn0{TP0tnD=JAzVUcIUoR85Xt>oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6N<sS-ys^qbJhGY7%0ZoC7dK=j7bGdau`J`{>oGqEkpJYJ?vc|B zOlwT3<tNmX!mXZdsEW2s2`|?DC8;N?2tT*Lfq)F*|4vf>t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&Fw<BqOnDKEdld8!Qk{Z zjI1+R_ciEqL3CLOv$+J~YVpzIy`S&V{koIi$Lj}ZFEMN=!rL1?_EjSryIV+OBiiJ- zIqT$oSMA>I=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#C<kI0i<ajCqQC!(pKlSsMl7M2N^mP%W`BGKb?hm zBK`pddcg5+WhE#$46+K<Z!1CW-hZdo7hAw13ZUVqwW*}&ujL=eh{m~phuOy=JiBMN z7FaCUn6boJ!M=6PtLN6%cveGkd12|1B{)kEYGTx#IiMN&re0`}NP-_{E-#FxOo3*P zkAXSt{et292KfgGN`AR|C`p{MRpxF-I?+`ZY1Vsv>GS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%Qi<evvBkNEkQkM%A>EWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e	yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76<bUr7Lsb65vEd}g z5JhMCmn#UeH#6Cew?bxogM)$x5ed{E)%2nWY5rb@Clvh$(JzQ#!CsQ(2I4QnhDDJ^ zYL%2bf8?`y)Ro=x{(dw<4^)(H^z7~3nfYFh-r7yBBb=l3V8dE-Dr&a%qs<OYcajo2 z(4Nw|k5_OQ@6zHmcIK%waj!yoZT(S1YlEFN?8-_lp9nf>PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M<cT6p|4(5fVa-WIh|@AphR|cJ1`?N>)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)H<F*kMvg%oJV~29ud_q>lo1euqTyM>^!HK*!Q2P;4UYry<i)yWXzKa zM^_qppY~vnIrhL_!;Z9msXMZTTwR{e`yH5t=HdD1Pni7?LqOpLoX}u5n5RfkGBvQ1 z@cdMeR4T6rp^S~>sje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT<gNU{ zn$Veg044#l=Z-&wsmEZhnw7IwT7Cd}hiZ%ke)-GzAR-Dt6)8Cb6>@Z<Y-SEE^OC5H z=$M0HjdWR5p?n;s9OTXrEa1eGt}G;Eu)ifSop!$z#6V<>zrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH<AWj}HgE@5&D9Ra@o(Km_Gm}5Zb61p%9mDz1% zya$Vd!_U~pDN*Y5%lo}-K~}4&F)rTjJ7uGyV@~kB-XNrIGRiB=UrNxJtX;JHb(EyQ z{!R%v{vC7m|L3bx6lCRb7!mP~Is!r!q&OXpE5nKnH3@l({o}PrL`o>~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVu<h{6ESg9k500(D<HXwz52OGq(JEKS2CJR}8N&E-#%vhhaRN zL#Q6%yUcel+!a#~g&e7w4$3s62d$Dv;SxCxhT}>xbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<<tS1{)`* zH!u#2_lf&B)x2)tE$?4|aMAYUFZ{|Se7->Ozh@Kw)<E~4fKYaJ{OS+>#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Q<Ww4SS<E23Sm*si$^C!!snD|AFym<+q$`*o0wokE?J{^g?f3>nd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OI<bVZt$VQ!oMxCu0 zbb7D5OIXV5Ynn@Y6)HLT=1`a=nh7{ee{vr<=$>C;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10<XTm*l1Jg2Z;UvGEN!6Wq%I@OP4p{k`RNRKlKFWPt_of11^Gr%_Mg*mVP3 zm?)&3I719~aYcs)TY&q^$zmQ=xoC++VJH@~YG6>+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+H<SF8|SM#pTc9|9|rf1w*m4Y0Vdj643qA#D| z!hJzb_-}IrrhkWr{zk_YC%(c-)UJl6Ma!mcbvj&~#yN-UhH?ZQ3TPq4hTVQ$(?eJ6 zNfJ_K+VJDBXN=l!7{2}lq?-$`fq|e&PEONfZDU<_SM+s2_3$vT_yqV<R&KG=K{zS} zKQF$?mYsg%vV|E_E=a*SL!`7*AeN6GMVDXC59yPgi$F2!7&8e}EyHVLwCm{i%<pN! zdc`SbZK}JQj7?6K&|261iHrsnVjdhxu_l_NKs&yy#;#^%8?Jlg`wcTlNZ3urUtEYd zsFE!K0}Eg39)z+J6mLW)#Kn<ok4*6AAE=n*vh*;TpgGnnM|npykFpO|a0`4#SjP^b z2<JG#Qk^#3FeFS`0eooK9|wEmCcvRKI*~6mamFTd^UW9Eg4!J4N9qz*C$3a#F;Sad zi#o9LaqNG5TsiT<`SDtY^`)zkYx$(C5;&K9#(Zj}HolT_st~#C`VS8q%#q1)HN+hT zz9IjVUdZNIp@;b88oR`~DvQL_zmsBy>Gi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGw<TLTZo~Zyx(+AKWvR~{L4S^5I;5+QT9bcQ-4cC{QnLfRBf&Pov~kv@`W6V zA|h{EGx|7msvR1t`a-jF$JZ>gH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n<jl%@&gd%^X|lsDQwDHEiKLCz}r`kC^h0t z(!vYS%C)Ku?w$ti5R##9jSkNC#5)Juc{8XfEhczdGQy8yNrZL6+d0~%V=N|iF{V)E zLT(gH!$j8Mf(1>{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&e<jP@@Q_fbXtVO&n9{e#)jg+D#~q=hoZ<9PIa)>P z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR<WSzBWU(MxAIA&4v~INVdLKA><BK zwCgTxJU0mM{;1UV<^ZRk0SQNNN(;SRZsH7^EDWVUu%^mFfvW{m5jOQuQWSy`f586I zTj}Z4e5WsvkNmBd`TJdfe=^>`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqA<e9rzV|ixGyk9uS=Vov2_ECA z^Sd0M$B)O&tv@%@UmTb%ngcl58ED9TyFp$y4JjFU+g+9EWUl?am<e#4uCGy9Tmt)z z2Y|kWUahugFHsF<J6o!<?X(Ncsy&Wg9<QLPD}g-`PWGHWDY5P6;<Y+5J1vz2Z|PSy zBN?Q^NkxnWq>OQq<EC8_d&#T2smn`YINd-HF@)Op)pBRHnx+Q|Hsv_BpWAPsT1>Lc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSch<f zIn>e7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm<g7T4Wx!m(zMlVE_2jX$1$$5DcfL6>7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2z<C?_X1)4xsl9%Z|w&L9k!F(V>J?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg<T-v~${38)1dqT{JCO5}Gk$$yZP*X!5)RaGFqqkZ zeHhqUgXb37$91~LS-3Zi29CKKki0sBTh7unqEK$%FG?oo$Sp>*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E<UbOmi3K%)5<dOJui+{^+b*shA_w8&X4_Icv*!}kT zW@BG{C%f{(K^kE?tjU`Led*kAj6wB_3f*UyIEV0T9TyMo4`NS;oA7Ec+71eFa;K|G zCyaKKi1bvX9fTLQ+uAgF*@ZR8fB%|JlT8A-jK$7FMyxW>$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuO<V3ijl7+~xmS#nUvH{qF0*%7G(r|}BSXsu}HwrFbXWzcYJouIY*34axA z(n@XsPrv%6;|GSbkH9Og>k559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV<Vu@5P52pgIa+J{M)H4nAC<>)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&d<S0a>RcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1<n2%>TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs2<i>6>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P<n- z??iM<JF!BTjD>{{s@<jPT1+pTPdk3<izB+}jAtjokIz)aPR$L&4%}45Et}?jz0w{( zC4G}+Nu0D*w=ay`v91hMo+V&V8q(a!`~K-2<yR0H)sK+mcY?TAaSS8F<Q+!pSc;`* z*c@5)+ZpT%-!K3O=Z0(hI8LH7KqK>sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9Kn<D3v{}Wpv2i&ghEZe;t&DmOA_QYc zM+NIUU}=*bkxOJsLKV3e^oGG8rufTpa8R~7Iki1y+fC(UT;;{l19@qfxO@0^!xMA? z#|<YBZ6;vAb>Y#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7Gb<mBTnJH7dKM2CB)0*o-AW2E4i5R+rHU%4A2BTVwOqj4zmJqsb|5^*{DT zv^HFARK6@^_1|vU{>voG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RH<y zF3MI;^J1vHI9U>mw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)<BWX>YsbHSz8!mG)WiJE| z2<APmuYD%tKwB@0u<C~CKyaC}XX{?mylzkDSuLMkAoj?zp*zFF7q515SrGD~s}ATn z`Ded41yk>f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z<h*hnP2Pol+z>~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc<a_3#EUXJj<z2jVv6VHGT zV^v1FiRwA!kPmt}m$qdr&9#-6{QeZqtM3|tRl$sws3Gy`no`Kj@X-)O(^sv>(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7y<P{h0$_I#EukRYag9%BMRXh|%Xl7C<>q$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV<Kqrcu9<z@R zSE>7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`lt<SmSV9vasBl&hE7ciOunD z?%e1Hl-5B3e+<+8CD{j5U*D3h89nV<zn^0g+t=uRKgZiGu)3h;vu#^y`HqWe_=jGm zW2p}*n<!QH%pQ2EV`&z|LD#BOpj0QS9R5#$q}3&-+@GL4F^wO-bcSo|J^I_{LATPF z2$`fUCOO=XxYVD!<7Yz4te$d-_>NebF46ZX_BbZNU}}ZOm{M2&nAN<H$fJIKS=j8q zwXlN!l^_4>L9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm<v)#bs=9p`s>34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{<m8xZ#>lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh<shPyABw|Ens8m6@ zIg($GO4)<g4x5icbki?U&2%56@tYd`zRs}Nk6R~4!AjVAihB3r8oDhQ8f)v^r}|(y z4B&Q<ARRqYXKQGAeJa_KHe`)04jUO~B=%q#SUlU@pU?apz0v{Al@s`Cvzo)u;2>6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`<?hW@{z#_gXtp%=2VbN+$~z+M($Vf(dl@)t-*82<$( zHi{FrD1wO9L~*Rc0{A2WU%f?ar(T9V1JpQ?M0Q|&{UES|#Z~k2-mj@z)8Rw^(XeYc zomT(B0EF!##4dQq_*NN<%Bo5)&+gCXSGZo`b>(M!j~B;#x?Ba<KDM~HJ!|Zzy=p2e z8;av`GLw{_*RgO(W|UK-<iDeT!t_x1c=M3%wGk|fDk<e0lLe8-5ga6apKYJD`*a3G zBl?Ps)hDb7X`7bW5S=IHr0Mm?fr|$zCf+gmZUrit$5n+)JZG>~&s6CopvO86oM?-? zOw#dIRc;6A<R&%m3DDJhF+|tb*0Yw8mV{a-bf^E~gh66MdsMHkog<r9`fVIVE+h@O zi)iM`rmA-Fs^c=>6T?B`Qp%^<<Dyu<%Kg0H=lq;E!p&UHzSpD1)q%^v)Y8yQkp>U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=D<O;$E>b!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz<KVOwgK<qq^3FEy1LAV}ep3|Zt z>&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vW>HF-ViB*%t0;Thq2} z+qP}n=Cp0wwr%5S+qN<7?r+``=l(h0z2`^8j;g2~Q4u?{cIL{JYY%l|iw&YH4FL(8 z1-*E#ANDHi+1f%<FEETVoPos>lMJbRfq*`nG)*#?EJEVoDH5XdfqwR-C{zmbQoh?E zhW!|TvYv~>R*OAnyZf@gC+=%}6N90yU@E;0b_OV#xL9B?GX(D&7BkujjFC@HVKFci zb_>I5e!yuHA1LC`xm&;wnn|3ht3h7|rDaOsh0ePhcg_^Wh8Bq|AGe`4t5Gk(9^F;M z8mFr{uCm{)Uq0Xa$Fw6+da`C4%)M_#jaX$xj;}&Lzc8wTc%r!Y#1akd|6FMf(a4I6 z`cQqS_{rm0iLnhMG~CfDZc96G3O=Tihnv8g;*w?)C4N4LE0m#H1?-P=4{KeC+o}8b zZX)x#(zEysFm$v9W8-4lkW%VJIjM~iQIVW)A*RCO{Oe_L;rQ3BmF*bhWa}!=wcu@# zaRWW{&7~V-e_$s)j!lJsa-J<VS<FyMLS`=mm2DyVkhoo@GUWHJSEvdPg7RxL3UHfy z*gs(_Q*VnSV{81NNpzoHc>?z;54!;KnU3vuhp~(9KRU2GKYfPj{qA?;#}H5f$Wv-_ zGrTb(EAnpR0*pKft3a}6$npzzq{}ApC&=C&9KoM3Ge@24D^8ZWJDiXq@r{hP=-02& z@Qrn-cbr2YFc$7XR0j7{jAyR;4LLBf_XNSrmd{dV3;ae;fsEjds*2DZ&@#e)Qcc}w zLgkfW=9Kz|eeM$E`-+=jQSt}*kAwbMBn7AZSAjkHUn4n||NBq*|2QPcKaceA6m)g5 z_}3?DX>90X|35eI7?n+>f9+hl5b>#q`2+`FXbOu9Q94UX-GWH;d*dpmSFd~7WM#H2 zvKNxjOtC)U_<K!UT4EWFJ`0%qla66gi)v;h2;>tx*0(J)eAI8xAD8SvhZ+VRUA?)| zeJjvg9)vi`Qx;;1QP!c_6<TaBw5yUTI?4(<(Ap023XRh}!HQ_3Fm-D8%`yp_x#8B! zN`-?}P}FCH(8@_>hJp1=J=*%!>ug}%O!CoSh-D_6LK0JyiY}rOaqSeja&jb#P|DR7 z_JannlfrFeaE$irfrRIiN|huXmQhQUN6VG*6`bzN4Z3!*G?FjN8!`ZTn6Wn4n=Ync z_|Sq=pO7+~{W2}599SfKz@umgRYj6LR9u0*BaHqdEw^i)dKo5HomT9zzB$I6w$r?6 zs2gu*wNOAMK`<Vwp^NK2&^tzKYH;5$ul;h083?OwlA%T%-S=_}dw4qW5+S+XO0ak! zevPUnu$QsnVyIYV{2~>+5yPBIxS<I@Dj|*;M}4yQ9m%YNUmuCMV>OJpL$@SN&iUaM zQ3%$EQt%zQBNd`+rl9R~utRDAH%7XP@2Z1s=)ks77I(>#FuwydE5>LzFx)8ye4ClM zb*e2i*E$Te%hTKh7`&rQXz;gvm4Dam(r-!FBEcw*b$U%Wo9DIPOwlC5Ywm3WRCM4{ zF42rnEbBzUP>o>MA){;KANhAW7=FKR<s?Q(b#6vUYUif4R6`g>=DKK&S1AqSxyP;k z;fp_GVuV}y6YqAd)5p=tJ~0KtaeRQv^nvO?*hZEK-qA;vuIo!}Xgec4QGW2ipf2HK z&G&ppF*1aC`C!FR9(j4&r|SHy74IiDky~3Ab)z@9r&vF+Bapx<{<G#FO>u~gb2?*J zSl{6YcZ$&m*X)X<A3nlHC|>?|8<2S}WDrWN3yhyY7wlf*q`n^z<p#C6Wwg0Lc^7yc z{k35LkK*(Ql>3LT4T$@$y``b{m953kfBBPpQ7hT;zs(Nme`Qw@{_p<uKf;<4btoSk zRm`706ZWQUJLyJS5NS(XVIk-1Qgl|?{JCQFalvu23h5}<#*fAfGk3F$41iz>UO0OG zfugi3N?l|jn-Du3Qn{Aa2#6w&qT+oof=YM!Zq~Xi`vlg<;^)Jreeb^x6_4HL-j}sU z1U^^;-WetwPLKMsdx4QZ$haq3)rA#ATpEh{NXto-tOXjCwO~nJ(Z9F%plZ{z(ZW!e zF>nv&4ViOTs58M+f+sGimF^9cB*9b(gAi<XFtFDIykkajehj!02m(2aL02PawRDy_ z*i~rdmXPeyi=)nB3g8Mx4Wi>zwyu5|--SLmBOP-uftqVnVBd$f7YrkJ8!jm*QQEQC zEQ+@T*AA1kV@SPF6H5sT%^$$6!e5;#N((^=OA5t}bqIdqf`PiMMFEDhnV#AQWSfLp zX=|ZEsbLt8Sk&wegQU0&kMC|cuY`&@<#r{t2*sq2$%epiTVpJxWm#OPC^wo_4p++U zU|%XFYs+ZCS4JHSRaVET)jV?lbYAd4ouXx0Ka6*wIFBRgvBgmg$kTNQEvs0=2s^sU z_909)3`Ut!m}}@sv<63E@aQx}-!qVdOjSOnAXTh~MKvr$0nr(1Fj-3uS{<m;+EJv+ z(b49OY^IhZ0b$bXM5E^i4)FWy3XukhWUP*i(QIB^`nJPbqZL6f!_ooB=w~NprhpQ4 zqPQCBat+sMcGd0B$rBNY{D(W1XU2RrsQ<DV;I7-88%iIY>U6-T9NG1Y(Ua)Nc}Mi< zOBQz^&^v*$BqmTIO^;r@kpaq3n!BI?L{#bw)pdFV&M?D0HKqC*YBxa;QD_4(RlawI z5wBK;7T^4dT7zt%%P<*-M~m?Et;S^tdNgQSn?4$mFvIHHL!`-@K~_Ar4vBnhy{xuy zigp!>UAwPyl!@~(bkOY;un&B~Evy@5#Y&cEmzGm+)L~4o4~|g0uu&9bh8N0`&{B2b zDj2>biRE1`iw}lv!rl$Smn(4Ob>j<{4dT^TfLe-`cm#S!w_9f;U)@aXWSU4}90LuR zVcbw;`2|6ra88#Cjf#u62xq?J)}I)_y{`@hzES(@mX~}cPWI8}SRoH-H;o~`>JWU$ zhLudK3ug%iS=xjv9tnmOdTXcq_?&o30O;(+VmC&p+%+pd_`V}RY4ibQM<T~=yiutj z8PY3n0uEtXb9>NE&N5O+hb3bQ8bxk^33Fu4DB2*~t1909gqoutQHx^plq~;@g<Tn< zGIy~gC#ZKTD#!!uxjrLk#?Tj4SA(prkyQk5LV8nE+cuzXv_@|$a5Y|l6wO*uhLx)e z4a4f>$d_+rzS0`2;}2UR2h#?p35B=B*f0BZS4ysiWC!kw?4B-dM%m6_BfRbey1Wh? zT1!@>-y=U}^fxH0A`u1)Mz90G6-<4aW^a@l_9L6Y;cd$3<#xIrhup)XLkFi$W&Ohu z8_j~-VeVXDf9b&6aGelt$g*BzEHgzh)KDgII_Y<!yt~^$ZjIYrUsiH!TWfQi@+75> zb$fcY8?XI6-GEGTZVWW%O;njZld)29a_&1QvNYJ@OpFrUH{er@mnh*}326TYAK7_Z zA={KnK_o3QLk|%m@bx3U#^tCChLxjPxMesOc5D4G+&mvp@Clicz^=kQlWp1|+z|V7 zkU#7l61m@^#`1`<I>{+m2L{sZC#j?#>0)2z4}}kqGhB{NX%~+3{5jOyij!e$5-OAs zDvq+>I2(XsY9%NNhNvKiF<%!6t^7&k{L7~FLdkP9!h%=2Kt$bUt(Zwp*&xq_+nco5 zK#5RCM_@b4WBK*~$CsWj!N!3sF>ijS=~$}_iw@vbKaSp5Jfg89?peR@51M5}xwcHW z(@1TK_kq$c4lmyb=aX3-JORe+JmuNkPP=bM*B?};c=_;h2gT-nt#qbriPkpaqoF@q z<)!80iKvTu`T-B3VT%qKO^lfPQ#m5Ei6Y%Fs@%Pt!8yX&C#tL$=|Ma8i?*^9;}Fk> zyzdQQC5YTBO&gx6kB~yhUUT&%q3a3o+zueh>5D7tdByYVcMz@>j<TXu&?x1p)Dnh4 zskNHSw3{&FN-50Bq2nx!SyUG4E$vH!z7_A#oMR)rO?_v?C8{JTwDkINn0II$m85#c z=Oz^^KRrLn$uotBRrtGqFhJj!f{G=XOZ**42DuGuDV6Abhb?6wWQOK{N<$^)G9JVq zt~tbaQV!u?NZzl7@GbKqp0ciBI=QTB*7T*H^m0C+Pgw-+N?f7g%v34xjXqwU!THzz zwM!z}e-39`DuQ;0;)(52iX5VSp8CeKy2w+ekh}=S)Tpqyj)`%7J^n-rd%;J%K3Gi} z%J>!C@Iyg{N1)veYl`SPshuH6Rk=O6pvVrI71rI5*%uU3u8<J<JbTb96qe0Z){uD! z(?jD_k*1=Jk;r`o{F1rd>1DpD%qmXsbKWMFR@2m4vO_^l6MMbO9a()DcWmYT&?0B_ zuY~tDiQ6*X7;9B*5pj?;xy_B}*{G}LjW*<O@op=u{qXpBL4WR|o`e}#yS2mt=`0;g zk*qCDShkd<mE}6<cHkR<36dq_>qU&%*QAyt30@-@O&NQTARZ+%VScr>`s^KX;M!p; z?8)|}P}L_CbOn!u(A{c5?g{s31Kn#7i)U@+_KNU-ZyVD$H7rtOjSht8%N(ST-)%r` z63;Hyp^KIm-?D;E-EnpAAWgz2#z{fawTx_;MR7)O6X~*jm*VUkam7>ueT^@+Gb3-Y zN3@wZls8ibbpaoR2xH=$b3x1Ng5Tai=LT2@_P&4JuBQ!r#Py3ew!ZVH4~T!^TcdyC ze#^@k4a(<cMT=zWd{@BY-TQaG5^m)U6v^8CU<UZ_h<X&S{e&<ef!K*nDS^u)sz!oF zv3GpwPwiz<Z+%$2K{duN*{|U`y!Fv~<V~KMYNpb%jaA|d%{RrZuC9RL9x}h~2mfw1 z&(5bE{Pi$T`3c	}kCcXotNM^kKsx=!bNtXw@bPnpdnxv8U{@5jwl4iOuy*Ulv$E zRr*O)*pGI|*{;DjKY6EYGY$UA(CyZ^Mb{|X{&ULt(gt&t##t?F&^Ke41N>nNe~G+y zI~yXK@1HHWU4pj{gWT6v@$c(x){cLq*KlFeKy?f$_u##)hDu0X_mwL6uKei~oPd9( zRaF_k&<bAvtIGrfv>w(J3J8b_`F~?0(Ei_pH}U^c&r$uSYawB8Ybs-JZ|&;vKLWX! z|HFZ%-uBDaP*hMcQKf*|j5!b%H40SPD*#{A`kj|~esk@1?q}-O7WyAm3mD@-vHzw( zTSOlO(K9>GW;@?@xSwpk%<BKz8jI}jIy`hcjM+(B2?y5RU8)J_t<^5T@5q(eVqo~3 zBO68aBr!giDu{-&mRO;X`_kCOk)SVi<#mG97Yofg&i@{QaPtr9sJxfvm*9;H@SuqC z#w2yv`XkX1Fm#r#GTnk<zr=n4H&oj2trmnx;X0_I*?j3mQjbM7=91jZ%cM_@DOx<; zBe@nMc^$^rGU$*j_K!qihH$KrnG1{anC~y4rqtiuKPczy-nHp9ziDlM+nm{BSGIn0 zjn88Zt)ktBnnU9JS3+R3k%!$OX$-hyV?#RNtxr;3X|+ibRRJk|C)0tNBhCqmxt%~k zW?jp7e1}Enb4c}+LT;RXFHvB<;HU$-BkxGUpu}xuVnee$aH?Ht5a*=Bfi1cpjp1AR zW0AS6J<>X3Ui4<EIxkjj2rf49E}4o!{lom8bZats3e|!F%@Mv0>_Psm;c*HF<uwRu zX2v}!DHM6Kd-vD^RoMp^t<T_OS$tZpDzDxl@Nzk;J6quA5BF_A1kKnbiWC2T)kFlH z(V^D=L0ygiE7V2xzZuVegW~^zIRA&`lxRZvU>~RW+q+C#RO_VT5(x!5B#On-W`T|u z>>=t)W{=B-8wWZejxMaBC9sHzBZGv5uz_uu281kxHg2cll_sZBC&1AKD`CYh2vKeW zm#|MMdC}6A&^DX=>_(etx8f}9o}`(G?Y``M?D+aTPJbZqONmSs>y>WSbvs>7PE~cb zjO+1Y)<CO?dR1Cg5s4M@qpdPA5@SkWDB@mH^43~g;%513aUq^B5_>PMi*!=06^$%< z*{b^66BIl{7zKvz^jut7ylDQBt)ba_F*$UkDgJ2gSNfHB6+`OEiz@xs$Tcrl>X4?o zu9~~b&Xl0?w(7lJXu8-9Yh6V|A3f?)1|~+u-q&6#YV`U2i?XIqUw*lc-QTXwuf@8d zSjMe1BhBKY`Mo{$s%Ce~Hv(^B{K%w{yndEtvyYjjbvFY^rn<KGkLBo^m9vg?nW<6+ zPmh<G^yK&-NDUiC8Z6ast0*z8xPQ8$(*j>2>C1Lbi!3RV7F>&;zlSDSk}R>{twI}V zA~NK%T!z=^!qbw(OEgsmSj?#?GR&A$0&K>^(?^4iphc3rN_(xXA%joi)k~DmRLEXl zaWmwMolK%@YiyI|HvX{X$*Ei7y+zJ%m{b}$?N7_SN&p+FpeT%4Z_2`0CP=}Y3D-*@ zL|4W4ja#8*%SfkZzn5sfVknpJv&>glRk^oUqykedE8yCgIwCV)fC1iVwMr4hc#<Z- zL-+W<jR>KcV!|M-r_N|nQWw@`j+0(Ywct~kLXQ)Qyncmi{Q4`Ur7A{Ep)n`zCtm8D zVX`kxa8Syc`g$6$($Qc-(_|LtQKWZXDrTir5s*pSVmGhk#dKJzCYT?vqA9}N9DGv> zw}N$byrt?Mk*ZZbN5&zb>pv;rU}EH@Rp54)vhZ=330bLvrKPEPu!WqR%yeM3LB!(E zw|J05Y!tajnZ9Ml*-aX&5T8YtuWDq@on)_*FMhz-?m|>RT0~e3OHllrEMthVY(KwQ zu>ijTc4>Xz-q1(g!ESjaZ+C+Zk5FgmF)rFX29_RmU!`7Pw+0}>8xK^=pOxtUDV)ok zw-=p=OvEH&VO3wToRdI!hPHc`qX+_{T_mj!NxcA&<p<h=eBG647W~srCOJ%YnAZc= z_Nfs1f}E6YV&pASfTpH}9tbu)4{#Ba2MsPj-56upCP@MeA3PUmV6EQ*1z#*N^TEue zy@dbDn?Ia-I`54^Z~pl7(zVZ_pG(9*?1GC3@YUj}QeUNX!ITTXIG3PGU6uEwG;a^& zqj~LaL>xOgkEuvz`-Aa`ZlNv>qnD0`YT1T3USO0ec!%{KE~UOGPJX%I5_rZDGx@|w zVIMsRPP+}^Xxa&{x!q{hY1wat8jDO7YP0(8xHWeEdr<P*6`9ax>d79lUj<UfD%PK; z4Isv2N9v5&5n-oxF81zDR@cW8ecaSm>B8%)v{X1pQu|1dr*y9M&a(J`038}4>lK&K zIM~6wnX{XA?pFHz{hOmEq{oYBnB@56twXqEcFrFqvCy)sH9B{pQ`G50o{W^t&onwY z-l{ur<Ka?a_Vj#)=^e+nK%m4zzm%RZtDxs>4#8ylPV5YRLD%%j^d0&_WI>0nmfZ8! zaZ&vo@7D`!=?215+Vk181*U@^{U>VyoXh2F&ZNzZx5tDDtlLc)gi2=|o=GC`uaH;< zFuuF?Q9Q`>S#c(~2p|s49RA`3242`2P+)F)t2N!CIrcl^0#gN@MLRDQ2W4S#MXZJO z8<(9P>MvW;rf2qZ$6sHxCVIr0B-gP?G{5jEDn%W#{T#2_&eIjvlVqm8J$*8A#n`5r zs6PuC!JuZJ@<8cFbbP{cRnIZs>B`?`rPWWL*A?1C3QqGEG?*&!*S0|DgB~`vo_xIo z&n_Sa(>6<$P7%Py{R<>n6Jy?3W|mYYoxe5h^b6C#+UoKJ(zl?^WcBn#|7wMI5=?S# zRgk8l-J`oM%GV&jFc)9&h#9mAyowg^v%Fc-7_^ou5$*YvELa!1q>4tHfX7&PCGqW* zu8In~5`Q5qQvMdToE$w+RP^_cIS2xJjghjCTp6Z(za_D<$S;0Xjt?mAE8~Ym{)zfb zV62v9|59XOvR}wEpm~Cnhyr`=JfC$*o15k?T`3s-ZqF6Gy;Gm+_6H$%oJPywWA^Wl zzn$L=N%{VT8DkQba0|2LqGR#O2Pw!b%LV4#Ojcx<DkATQzZt2m)Z{XGHSSJVB)dEu z$k5uf0bru0;$A03ffmj4X4+w-#Ex!3zY<g3vy{Q-;>5`?Cm;+aLpkyZ=!r1z@E}V= z$2v6v%Ai)MMd`@IM&UD!%%(63VH8+m0Ebk<5Du#0=WeK(E<2~3@>8TceT$wy5F52n zRFtY>G9Gp~h#&R92{G{jLruZSNJ4)gN<n8RjXM9L<b|8XyoMl0^H+4(zB;w~SJ!kD zrVI|u0t_5@!H0jw+sQtegKy~|{2WS+${5Oz(JgLw!@`G#G?5)|&o4Ab&mtkFW6R;I zbIl_@SvfNAX(Na{b+@Q70a`Sk3lQIt*FDI<(O)!c#B&A(Q`95=aPuOdvOYD_Tbcm% z%D-4n{DMJ=hdY}r!%6RohhWRUg~J@(vFV$}5IJdXuC=~`w%u&<vo@A4ca>K+zg*$P zW@~Hf>_Do)tvfEAAMKE1nQ=8coTgog&S;wj(s?Xa0!r?UU5#2>18V#|tKvay1Ka53 zl$RxpMqrkv`Sv&#!_u8$8PMken`QL0_sD2)r&dZziefzSlAdKNKroVU;gRJE#o*}w zP_bO{F4g;|t!iroy^xf~(Q5qc8a3<+vBW%V<d`nS{lSj$4Nm}P5a}hnc(p%WzfvlG zT;H!5Z;n`VUd)l*N1r5XgtUc;QC!pff=V3Yc^2yBHlkn@&9nqcoHdJKJ=v^;^yV-f z<}BU(Bvrs_fCG)axqSM5flf2Ba{5fV)L3cPzWA?Q6se0a396m5Gr#_YQP8|uF0kH< zVB|z<#l1YCwIqxwN0n0`BRlP5TH;3I`~`BmZR!I&q>IOQ1!??d;yEn1at1wpt}*n- z0iQtfu}Isw4ZfH~8p~#RQU<wR$wNwXfz=BXCSL%flpWfABL6v7O5I*uK2pS}NYHvt zi`MA*@0*}k0)LtGV=asdz5UAs6ox*tjq2V>Kwf<$XeqUr-5?8TSqokdHL7tY|47R; z#d+4NS%Cqp>LQbvvAMIhcCX@|HozKXl)%*5o>P2ZegGuOerV&_MeA}|+o-3L!ZNJd z#1xB^(r!IfE~i>*5r{u;pIfCjhY^Oev$Y1MT16w8pJ0<E9%+ZY3k8rPYxm8wOPTy9 z@nZU~c*2|DKwp!jN;Um`8W><e2p}A{tG<hJX)A5!!YpV2>?9@&FH*`d;hS=c#F6fq z{mqsHd*xa;>Hg?j80MwZ%}anqc@&s&2v{vHQS68fueNi5Z(VD2eH>jmv4uvE|HEQm z^=b&?1R9?<@=kjtUfm*I!wPf5Xnma(4*DfPk}Es*H$%NGCIM1qt(LSvbl7&tV>e2$ zUqvZOTiwQyxDoxL(mn?n_x%Tre?L&!FYCOy0>o}#DTC3uSPnyGBv*}!<Y7tbCS^J_ z6*X7wjnZ%Q0s4^$g0<Mf0kC^>*Yv5IV)Bg_t%V+UrTXfr!Q8+eX}ANR*YLzwme7Rl z@q_*fP7wP2AZ(3WG*)4Z(q@)~c{Je&7?w^?&Wy3)v0{TvNQRGle9mIG>$M2TtQ(Vf z3*PV@1mX)}beRTPjoG#&&IO#Mn(DLGp}mn)_0e=9kXDewC8Pk@yo<8@XZjFP-_zic z{mocvT9Eo)H4Oj$>1->^#DbbiJn^M4?v7XbK>co+v=7g$hE{#HoG6ZEat!s~I<^_s zlFee93KDSbJKlv_+GPfC6P8b>(;dlJ5r9&Pc4kC2uR(0{Kjf+SMeUktef``iXD}8` zGufkM9*Sx4>+5WcK#Vqm$g#5z1DUhc_#gLGe4_icSzN5GKr|J&eB)LS;jTXWA$?(k zy?*%U9Q#Y88(blIlxrtKp6^jksNF>-K1?8=pmYAPj?qq}yO5L>_s8CAv=LQMe3J6? zOfWD>Kx_5A4jRoIU}&aICTgdYMqC|45}St;@0~7>Af+uK3vps9D!9qD)1;Y6Fz>4^ zR1X$s{QNZl7l%}Zwo2wXP+Cj-K|^wqZW?)s1WUw_APZLhH55g{wNW3liInD)WHh${ zOz&K>sB*4inVY3m)3z8w!yUz+CKF%_-s2KVr7DpwTUuZjPS9k-em^;>H4*?*B0Bg7 zLy2nfU=ac5N}x1+Tlq^lkNmB~Dj+t&l#fO&%|7~2iw*N!*xBy+ZBQ>#g_;I*+J{W* z=@*<AAG@ThQqLTS8LoW9hl8e8slgs|pH%Z@1|LXbEv=yEDaqvFtkgO0fm!oT;UD4I zgw!e(vKorcHj2>15><)Bh9f>>dgQrE<e-x!^l8zejAkt^^^`zlF;a1n;X-XLc~Fs% z=F|#T8rRX07v}A5%C@Bo>hkrr2FEJ;R2rH%`kda8sD-FY6e#7S-<)V*zQA>)Ps)L- zgUuu@5;Ych#jX_KZ+;qEJJbu{_Z9WSsLSo#XqLpCK$gFidk}gddW(9$v}iyGm_OoH ztn$pv81zROq686_7@avq2heXZnkRi4n(3{5jTDO?9iP%u8S4KEqGL?^uBeg(-ws#1 z9!!Y_2Q~D?gCL3MQZO!n$+Wy(Twr5AS3{F7ak2f)Bu0iG^k^x??0}b6l!>Vjp{e*F z8r*(Y?3ZDDoS1G?lz#J4`d9jAEc9YGq1LbpYoFl!W!(j8-33Ey)@yx+BVpDIVyvpZ zq5QgKy>P}LlV?Bgy@I)JvefCG)I69H1;q@{8E8Ytw^s-rC7m5>Q>ZO(`<rJD^N3)b zrY@XQ=`M~mXZ#s{XEHo_{Wul}Fs^7CdvbCX?LZ+OHC5%T;xFOXVwOJR8Lq}00$$<s z0m&M?Y3<bP7OrI_ENundw|clc`k>$`9@`49s2)q#{2eN0A?~qS8%wxh%P*99h*Sv` zW_z3<=iRZBQKaDsKw^TfN;6`mRck|6Yt&e$R~tMA0ix;qgw$n~fe=62aG2v0S`7mU zI}gR#W)f+Gn=e3mm*F^r^tcv&S`Rym`X`6K`i8g-a0!p|#69@Bl!*&)QJ9(E7ycxz z)5-m9v`~$N1zszFi^=m%vw}Y{ZyYub!-6^KIY@mwF|W+|t~bZ%@rifEZ-28I@s$C` z>E+k~R1JC-M>8iC_GR>V9f9+uL2wPRATL9bC(sxd;AMJ>v6c#PcG|Xx1N5^1>ISd0 z4%vf-SNOw+1%yQq1YP`>iqq>5Q590_pr?OxS|HbLjx=9~Y)QO37RihG%JrJ^=Nj>g zPTc<Z%(meLC3t3DA3mXspnPQXL*D!Ta65mfL8n5cTVeIXz1M&EvRvf~9Y&HGRe34j zC{Z<XTx+@;8#G)UDzJa=zNkJ)ZThO~AbC95=lEA9s93CKH~q`T0Lo^V<~2qE_P`rr zT8g<d&liKY6OZ@w#*&(eHNxz~{nnk+F?=TJnWu+d?Cct!$0t}FWu$tmDwPn{x}W3D z(#hW-J#P+zSREZaB(H<>O$6r{jdE_096b&L;W<UT0WXd-7EcYUKC#!}=R=zh^fQ!w zw^!4tduWEBa!eB6VD%8*Zm;lJUaOv3q&ftLM>m8vcxUVxF0mA%W`aZz4n6XtvOi($ zaL!{WUCh&{5ar=>u)!m<SP1r*2Q!B7!r%>it|&EkGY$|YG<_)ZD)I32uEIWwu`R-_ z<l-F2@T;+Rfp>`FVeKyrx3>8Ep#2~%VVrQ%u#exo!anPe`bc)-M=^IP1n1?L2UQ@# zpNjoq-0+XCfqXS!LwMgFvG$PkX}5^6yxW)6%`S8{r~BA2-c%-u5SE#%mQ~5JQ=o$c z%+qa0udVq9`|=2n=0k#M=yiEh_vp?(tB|{J{EhVLPM^<GU_9IW&W5r0EZfqfvqKJa zj$A!0V@^%^G*@LfCxnQBRsf!SFI>S@f-O*Lgb390BvwK7{wfdMKqUc0uIXKj5>g^z z#2`5^)>T73Eci+=E4n&jl42E@VYF2*UDiWLUOgF#p9`E4&-A#MJLUa&^hB@g7KL+n zr_bz+kfCcLIlAevILckIq~RCwh6dc5@%yN@#f3lh<g!DD;$~9!F?Umj0oq+-L;Pq3 zWBjyxUsn|F4v2<-tDya8_xD#3tf+Btl6xMKmQwU{uG^pCHy8%(g8{EzO=viuWMlpe zxaYI`vE-I~M7S(FL%W(!O9O@x5R%(RDcs<mY)UE+SKgn!vO!l5lVfKknb{^`9|ACm zFc>HIx4fZ_yT~o0#3@h#!HCN(rHHC6#0$+1AMq?bY~(3nn{o5g8{*e_#4RhW)xPmK zTYBEntuYd)`?`bzDksI9*MG$=^w!iiIcWg1lD&kM1NF@q<Uw8p<g_Dpnj9vN66UU6 z#Z3*p$ka?~sV+=@HS4)aShwh2%xUcFRzNG=CK_7wDr^y+Hym1XjM0Y)8#0I8xk*BJ z6~Y?ps>Kha0fDVz^W7JCam^!AQFxY@7*`a3tfBwN0uK_~YBQ18@^i%=YB}K0Iq(Q3 z=7hNZ#!N@YErE7{T|{kjVFZ+f9Hn($zih;f&q^wO)PJSF`K)|LdT>!^JLf=zXG>>G z15TmM=X`1%Ynk&dvu$Vic!XyFC(c=qM33v&SIl|p+z6Ah9(XQ0CWE^N-LgE#WF6Z+ zb_v`7^Rz8%KKg_@B>5*s-q*TVwu~MCRiXvVx&_3#r1<P_Vsg0r#|~)TS-_d=4M~cE z+&T48ska5zw)EmTXQm+ioQ#?!TAFgl6YGNnf^vvUGF8c#WveY-j?v>h&L+{rM&-H6 zrc<W1b?Kswwnc7D+{tF5OXqdbQIoE?`I?`AYOA|Nz_u97H4|;)Y(B#Jlx4%Vyq`C% z>gH@I>0eY8WBX#Qj}Vml+fpv?;EQXBbD0lx%L?E4)b-nvrmMQS^}p_CI3M24IK(f| zV?tWzkaJXH87MBz^HyVKT&oHB;A4DRhZy;fIC-TlvECK)nu4-3s7qJfF<RMe%2ki1 zTI6+Q&ku0Tf_}LhN9vYEpKLB63Apcx<A44C!(?Ku{@lPv1p=C*{NKQPqW?SoDrx8J zVrXOY|6`QKsKfYZo1*^sOJ;d{XYp>-ZZGt7+6C3xZt!ZX4`M{eN|q!y*d^B+cF5W- zc9C|FzL;$bAfh56fg&y0j!PF8mjBV!qA=z$=~r-orU-{0AcQUt4<urmeM_viOanZ> zNYC=_9(MOWe$Br9_50i#0z!*a1>U6ZvH>JYS9U$kkrCt7!mEUJR$W#Jt5vT?U&LCD zd@)kn%y|rkV|CijnZ((B2=j_rB;`b}F9+E1T46sg_aOPp+&*W~44r9t3AI}z)yUFJ z+}z5E6|oq+oPC3Jli)EPh9)o^B4KUYkk~AU9!g`OvC`a!#Q>JmDiMLTx>96<qgBn8 z5Yx>_iDD9h@nW%Je4%>URwY<khi%2<#7=1x6bKj`i+VNfqc83y4%c<DW8#bYi5YZb z$#9uBFd7dEOo$AJb4JZJO5H`_z^H7us>M%5YU1&Dcdulvv3IH3GSrA4$)QjlGwUt6 zsR6+PnyJ$1x{|R=ogz<jrS1DHwc3k9SyCRDme*{=6T#8t{oN}(u`b1y&NoUMw~k&i z-ZMT<e7mL5FWq(-nCv;nz!91AZl{NhC%CdljGG*{I`mQsTERle-US8eePrJIvi@(~ z7-9FHtjkeQ>Err~U|X!+b+F8=6y?Yi`E$yjWXsdmxZa^hIqa)YV9ubUqOj&IGY}bk zH4*DEn({py@MG5LQCI;J#6+98GaZYGW-K-&C`(r5#?R0Z){DlY8ZZk}lIi$xG}Q@2 z0LJhzuus-7dLAEpG1Lf+KOxn&NSwO{wn_~e0<P6hRP=HtV6GZO4BWR+u?s}P=mtR? z^s-q@iNL0$t&{z^w<8l8Gfidj9i;%JZjA&yE?EO$&aT3#)<VfKjT&xVy=ob6_fCQ1 z1ZANC@-v}}f_Ot^v6-^$=p`K1pzl0%@pT?@;VeuG3OCx<;!tLX(*FCgcF6%v6I-u+ zOtolW2R3pOY_+(+h`%+h#=|Jp1>=}dovX)T(|WRMTqacoW8;A>8tTDr+0yRa+U!LW z!H#Gnf^iCy$tTk3kBBC=r@xhskjf1}NOkEEM4*r+A4`yNAIjz`_JMUI#xTf$+{UA7 zpBO_aJkKz)iaKqRA{8a6AtpdUwtc#Y-hxtZnWz~i(sfj<Z|BkJ?5yNRyVr*CY(ck` zYBr4&*s>Mk`lq|kGea=`62V6y)TMPZw8q<PMu}{|F!lI=W@pt{Brl{lZ$Kvki;7jR zu5U<jQ;cdN982L5LwwYW4w)&epo>}tFDDHrW_n(Z84ZxWvRrntcw;F|Mv4ff9iaM% z4IM{=*zw}vIpbg=9%w&v`sA+a3UV@Rpn<6`c&5h+8a7izP>E@7CSsCv*AAvd-izwU z!sGJQ?fpCbt+LK`6m2Z3&cKtgcElAl){*m0b^0U#n<7?`8ktdIe#ytZTvaZy728o6 z3GDmw=vhh*U#hCo0gb9s#V5(IILXkw>(6a?BFdIb0%3~Y*5FiMh&JWHd2n(|y@?F8 zL$%!)uFu&n+1(6)oW6Hx*?{<Fve1i68O0|}VY@-CnZ2#qMDX4_TH5{a14Z!yY5Jyy zx~A7-X<-PE0WM<Tm|Nvc^ZEwz8GrqOWa`Z1*RZ}hIr1a1?laD~;emg18pAJ#9>d~y zBeR)N*Z{7*gMlhMOad#k4gf`37OzEJ&pH?h!Z4#mNNCfnDI@LbiU~&2Gd^q7<CQs0 z&asYoA!Bw(%<&`9zGMPFrBS@o$PG#iIU>ix8~Y6$a=B9bK(BaTEO0$Oh=VCkBPwt0 zf#QuB25&2!m7MWY5xV_~sf(0|Y*#Wf8+FQI(sl2wgdM5H7V{aH6|ntE+OcLsTC`u; zeyrlkJgzdIb5=n#SCH)+kjN)rYW7=rppN3Eb;q_^8Zi}6jtL@eZ2XO^w{mCwX(q!t ztM^`%`ndZ5c+2@?p>R*dDNeVk#v>rsn>vEo;cP2Ecp=@E>A#n0!jZACKZ1=D0`f|{ zZnF;Ocp;$j86m}Gt~N+Ch6CJo7+Wz<khC+niM3qr-iS&UBcHui{9XrK>v|nlsXBvm z?St-5Ke&6hbGAWoO!Z2Rd8ARJhOY|a1rm*sOif%Th`*<gwKLW?(Zk1#*ZOXQ`cojG zD13Ehmfi1VRcjC6^(x$Mr?4CrwnTiR=R5g=ptRC8<8eNz%nut#PS8c6u#;@$&2ey+ zdY<bNMhywpCa2S@h7Q3wbgCGuX_x#vX-yqzvPsMWe)TLqZo9uM5`ZDjRM95Ep@ZY3 z3!vbipe75C=xz+QTQ^*cq&P%8D{YFL0(rKe)l;OSVquh&DYX(RJ=S6F9Om4%x%qm8 z&MQBXRTf8OLdIIe4a>=^jlgWo%e9`3sS51n*>+Mh(9C7g@*mE|r%h*3k6I_uo;C!N z7CVMIX4kbA#gPZf_0%m18+BVeS4?D;U$QC`TT;X<fDeDKHK{v9^#caNvL$%M`~|e> zP#H}tMsa=zS6N7n#BA$Fy8#R7vOesiCLM@d1UO6Tsnwv^gb}Q9I}ZQLI?--C8ok&S z9Idy06+V(_aj?M78-*v<PSvKN>YBu|AaJ9mlEJpFEIP}{tRwm?G{ag>6u(ReBKAAx zDR6qe!3G88NQP$i99DZ~CW9lzz}iGynvGA4!yL}_9t`l*SZbEL-%N{n$%JgpDHJRn zvh<{AqR7z@ylV`kXdk+uEu-WWAt^=A4n(J=A1e8DpeLzAd;Nl#qlmp#KcHU!8`YJY zvBZy@>WiBZpx*wQ8JzKw?@k}8l99Wo&H>__<oxE+DvXhRO^f>vCFL}>m~MTmGvae% zPTn9?iR=@7NJ)?e+n-4kx$V#qS4tLpVUX*Je0@`f5LICdxLnph&Vjbxd*|+PbzS(l zBqqMlUeNoo8wL&_HKnM^8{iDI3IdzJAt32UupSr6XXh9KH2LjWD)Pz+`cmps%eHeD zU%i1SbPuSddp6?th;;DfUlxYnjRpd~i7vQ4V`cD%4+a9*!{+#QRBr5^Q$5Ec?gpju zv@dk9;G>d7QNEdRy}fgeA?i=~KFeibDtYffy)^OP?Ro~-X!onDpm+uGpe&6)*f@xJ zE1I3Qh}`1<7aFB@TS#}ee={<#9%1wOL%cuvOd($y4MC2?`1Nin=pVLXPkknn*0kx> z!9XHW${hYEV;r6F#iz7W=fg|a@GY0UG5>>9>$3Bj5@!N{nWDD`;JOdz_ZaZVVIUgH zo+<=+n8VGL*U%M|J$A~#ll__<`y+jL>bv;TpC!&|d=q%E2B|5p=)b-Q+ZrFO%+D_u z4%rc8BmOAO6{n(i(802yZW93?U;K^ZZlo0Gvs7B+<%}R;$%O}pe*Gi;!xP-M73W`k zXLv473Ex_VPcM-M^JO|H>KD;!sEGJ<HTW@*Usun3M)y1SZy-}aD>|E}Qepen;yNG2 zXqgD5sjQUDI(XLM+^8ZX1s_(X+PeyQ$Q5RukRt|Kwr-FSnW!^9?OG64UYX1^bU9d8 zJ}8K&UEYG+Je^cThf8W*^RqG07nSCm<CEe(KJqi=yo4Zu+%xm@?EsCwAe&T3j;YZ= z`fzz&jda9suA?ug-<)Lt|J!;C=m%2_6!M#s+o$=|h!UsUZze~j6B|oBnDcXoBo6s( znVOI7#2w)n`o&*71n0&}PmJ49=A^!uAEdr;5G1}y8xW=VO4p=PtZGRvI4Sg{DMqQq z%SIK&<Wa1Nqat-4s@X(z$m)__E$wOxmm~0!_GrS1d?bG*JLP%k;er00_>p*o5Z;#F zS?jochDWX@p+%CZ%dOKUl}q{9)^U@}qkQtA3zBF)`I&zyIKgb{mv)KtZ}?_h{r#VZ z%C+hwv&nB?we0^H+H`OKGw-&8FaF;=ei!tAclS5Q?qH9J$nt+YxdKkbRFLnWvn7GH zezC6<{mK0dd763JlLFqy&Oe|7UXII;K&2pye~yG4jldY~N;M9&rX}m76NsP=R#FEw zt(9h+=m9^zfl=6pH*D;JP~OVgbJkXh(+2MO_^;%F{V@pc2nGn~=U)Qx|JEV-e=vXk zPxA2J<9~IH{}29#X~KW$(1reJv}lc4_1JF31gdev>!CddVhf_62nsr6%w)?IWxz}{ z(}~~@w>c07!r=FZA<wFaiohRbKb~^?3q5?_zk?fqI0MHaAe;iY+em}3rG(ta;5O`r z;1)QPwN9vHyE)(@T!k}>Nq4R!F2Qi2?QGavZ{)PCq~X}3x;4ylsd&m;dQe;0GFSn5 zZ*J<=Xg1<rmtct^RD?`a5gOMC4L)d>fEGYYDZ0{Z4}Jh*xlXa}@412nlKSM#@wjMM z*0(k>Gfd1Mj)smUuX}EM6m)811%n5zzr}T?$ZzH~*3b`3q3gHSpA<3cbzTeRDi`SA zT{O)l3%bH(CN<IN>0EEF9ph1(Osw5y$SJolG&Db~uL!I3U{X`h(h%^KsL71`2B1Yn z7(xI+Fk?|xS_Y5)x?oqk$xmjG@_+JdErI(q95~UBTvOXTQaJs?lgrC6Wa@d0%O0cC zzvslIeWMo0|C0(<f+k5>{iEWX{=5F)t4Z*`rh@-t0ZTMse3VaJ`5`1zeUK0~F^KRY zj2z-gr%sR<(u0@SNEp%Lj38AB2v-+cd<8pKdtRU&8t3eYH#h7qH%bvKup4cnnrN>l z!5fve)~Y5_U9US`uXDFoOtx2gI&Z!t&VPIoqiv>&H(&1;J9b}kZhcO<PyO-o^uK1x z>X7EiW*Bujy#MaCl52%NO-l|<wz{;LglcbSGr9RqPoL*nYEN|Murl0Ya0<J_(ZpWZ zF)>@2$aRKvZ!YjwpXwC#nA(tJtd1p?jx&U|?&jcb!0MT6oBlWurVRyiSCX?sN3j}d zh3==XK$^*8#zr+U^wk(UkF}bta4bKVgr`elH^az{w(m}3%23;y7dsEnH*pp{HW$Uk zV9J^I9ea7vp_A}0F8qF{>|rj`CeHZ?lf%HImvEJF<@7cgc1Tw%vAUA47{Qe(sP^5M zT=z<~l%*ZjJvObcWtlN?0$b%NdAj&l`Cr|x((dFs-njsj9%IIqoN|Q?tYtJYlRNIu zY(LtC-F14)Og*_V@gjGH^tLV4uN?f^#=dscCFV~a`r8_o?$gj3HrSk=YK2k^UW)sJ z&=a&&JkMkWshp0sto$c6j8f$J!Bsn*MT<h8dS6cWTC2@)2)r~Q6P<?JMO=z=v6=h0 z{z0O{ZSMEdl||RciL2=bIRol$EQX%GoV#vmsfKn64QmInOxq%^Q3!={`}-O}OF!W# z=Hj!CX?@sGbF3Dtmb5z~EA>jC`3cv@l@7cINa!}fNcu(0XF7ZCAYbX|WJIL$iGx8l zGFFQsw}x|i!jOZIaP{@sw0BrV5Z5u!TGe@JGTzvH$}55Gf<;rieZlz+6E1}z_o3m2 z(t;Cp^Geen7iSt)ZVtC`+tzuv^<6--M`^5JXBeeLXV)>2;f7=l%(-4?+<5~;@<fp) z1Lui?ARVsm?nP~!ne9-FRx3JSEwwC5O{HygKy6`yWh?gVhS?YcW4+v5JWTeVDAmjv zCOtdeHV1%x2dOX@vq0y;J^p<!(r|O)vi|(<nvMWYjpI;<j^T#vi*<7po`Kd6FA;^r zmyETseWV1c90=BR0ZP&?D?U4l#rsfm+~24-c!J7PR{eD=x@cqZ#~<}g+~EX)E|2a$ z+gi~<mvHzfsze~C6NT^{GLC>=Th{1#>rK3+rLn(44TAFS@u(}dunUSYu}~))W*fr` zkBL}3k_@a4pXJ#u*_N|e#1gTqxE&WPsfDa=`@LL?PRR()9^HxG?~^SNmeO#^-5tMw zeGEW&CuX(Uz#-wZOEt8MmF}hQc%14L)0=ebo`e$$G6nVrb)afh!>+Nfa5P<oZ7>;N zCCOQ^NRel#saUVt$Ds0rGd%gkKP2LsQRxq6)g*`-r(FGM!Q51c|9lk!ha8Um3ys1{ zWpT7XDWYshQ{_F!8D8@3hvXhQDw;GlkUOzni&T1>^uD){WH3wRONgjh$u4u7?+$(Y zqTXEF>1aPNZCXP0nJ;z<N&&~t`L3v?GV)58KGTHHGL!swTJXxH{~NMLx(L72U7AvS zcP}RNT->s6_%6;+D&J_|ugcih**y(4ApT`RKAi5>SZe0Bz|+l7z>P14>0ljIH*LhK z@}2O#{?1RNa&!~sEPBvIkm-u<y&MCPHyI+!uRA$A&rl2a>It^Pt#%JnsbJ`-T0%pb ze}d;dzJFu7oQ=i`VHNt%Sv@?7$*oO`Rt*bRNhXh{FArB`9#f%ksG%q?Z`_<19;dBW z5pIoIo-JIK9N$IE1)g8@+4}_`sE7;Lus&WNAJ^H&=4rGjeAJP%Dw!tn*koQ&PrNZw zY88=H7qpHz11f}oTD!0lWO>pMI;i4sa<gbCHb*?!TW%nQ=Crc6PM8jDTh%vfNUoN9 z<Dyk7#UrltDltZCanZAJp~$Uj3z%^}HyR>uS`%_!zM!n@91sLH#rz1~iEAu#1b%LA zhB}7{<u6opdJcy2km#)#Y#n?t1euj>1(8{1{V8+SEs=*f=FcR<FV0_Ap!<#zSZ}~< zN7Q;-uIX)e5`ftQ>E^;`6Pxm$Hie~|aD~W1BYy#@Y$C?pxJh*cC!T@8C9{xx*T*8P zhbkRk3*6)Zbk%}u>^?ItOhxdmX$j9KyoxxN>NrYGKMkLF4*fLsL_PRjHNNHCyaUHN z7W8yEhf&ag07fc9FD>B{t0#Civsoy0hvVepDREX(NK1LbK0n*>UJp&1FygZMg7T^G z(02BS)g#qMOI{RJIh7}pGNS8WhSH@kG+4n=(8j<+gVfTur)s*hYus70A<L}puU}Lb zu200ys|`>HUBS2bN6Zp_GOHYxsbg{-Rcet{@0gzE`t$M0_!ZIqSAIW53j+Ln7N~8J zLZ0DOUjp^j`MvX#hq5dFixo^1szoQ=FTqa|@m>9F@%>7OuF9&_C_MDco&<yl?^CH) z_{m)+n(GUp{jz&DgTPmM1Ie32W;fO2o&4S0WHZRaK}sd;OrQE#_U&x0+&FI~40PuQ ze#9WlNf2()A6n%*=)0;D=cW$&=;s%|mjn+596d9azJX@ws^_agA9S@SqCb%5)Pt1$ zTuDQ#W)9rV*dI`OL>-{wfLKNrDMEN4pRUS8-SD6@GP`>_7$;r>dJo>KbeXm>GfQS? zjFS+Y6^%pDCaI0?9(z^ELsAE1`WhbhNv5DJ$Y<V<y8)GDQRyg3oc8D}R=a~^=C+bR zA8dliBl?zMD&C-Y1@r5^4sc0o&y-Oly%>}~r;>FynHjmjmA{bfDbseZXsKUv<S~ds z>`%Fekv)1@f%7ti;B5hhs}5db1dP+P0${1DgKtb(DvN}6H6;0*LP6blg*rpr;Z(7? zrve>M`x6ZI(wtQc4%lO?v5vr{0iTPl&JT!@k-7qUN8b$O9YuItu7zrQ*$?xJIN#~b z#@z|*5z&D7g5>!o(^v+3N?JnJns5O2W4EkF>re*q1uVjgT#6ROP5>Ho)XTJoHDNRC zuLC(Cd_ZM?FAFPoMw;3FM4Ln0=!+vgTYBx2TdXpM@EhDCorzTS6@2`swp4J^9C0)U zq?)H8)=D;i+H`EVYge>kPy8d*AxKl};iumYu^UeM+e_3>O+LY`D4?pD%;Vextj!(; zomJ(u+dR(0m>+-61HTV7!>03vqozyo@uY@Zh^KrW`w7^ENCYh86_P2VC|4}(ilMBe zwa&B|1a7%Qkd>d14}2*_yYr@8-N}^&?LfSwr)C~UUHr)ydENu=?ZHkvoLS~xTiBH= zD%A=OdoC+10l7@rXif~Z#^AvW+4M-(KQBj=Nhgts)>xmA--IJf1jSZF6>@Ns&nmv} zXRk`|`@P5_9W4O-SI|f^DCZ-n*yX@2gf6N)epc~lRWl7QgCyXdx|zr^gy>q`Vwn^y z&r3_zS}N=HmrVtTZhAQS`3$kBmVZDqr4+o(oNok?tqel9kn3;uUerFRti=k+&W{bb zT{ZtEf51Qf+|Jc*@(nyn#U+nr1SFpu4(I7<1a=)M_yPUAcKVF+(vK!|DTL2;P)yG~ zrI*7V)wN_92cM)j`PtAOFz_dO)jIfTeawh2{d@x0nd^#?pDkBTBzr0Oxgmvjt`U^$ zcTPl=iwuen=;7ExMVh7LLFSKUrTi<KY<!-61)q_{3{un*!3+ioAJ}ikykUEszqLUz z2y?(ZGd?cRqh9pYpRi`+24AatY4$_e?EUKUn3Jh7T4ApDTj(45gg+&t`;?Q{T~cbD zn|bNQZR|89v%#GpNPQYtE?(B#%uDIfpgsUsQ7<Fll~M$jRoZq6vmY&{1Nwz^qA#Go zc|+flUXyLCb2t)u-mzSXfB*oT%i$GD5BYWsfIs$wF8Vdij$+{>PJpMB&*Ml32>wl} zYn(H0N4+>MCrm2BC4p{meYPafDEX<uu8*GYOdb||1L86<D25;RK8@zd2kY<K`lVFV zWxW7%O)+izx>d4yf$i%ylWpC|9%R4XZBUQih<i`-uBppkqY;Qs*4ob$&^Iv0MXueR zs^9GZt9u)s2Cv^Z+@fi83jHq!9bj``t`2%W4X`>a(x%w<wBlpV>gQ5iJ?K_wQBRfw z+pYuKoIameAWV7Ex4$PCd>bYD7)A9J`ri&bwTRN*w~7DR0EeLXW|I2()Zkl6vxiw? zFBX){0zT@w_4YUT4~@TXa;nPb^Tu$DJ=vluc~9)mZ}uHd#4*V_eS7)^eZ9oI%Wws_ z`;97^W|?_Z6xHSsE!3EKHPN<3IZ^jTJW=Il{rMmlnR#OuoE6dqOO1KOMpW84Z<G}M zeQ$nWk@-Y@mRFFq;S1oSSbG(l@Q(aR3<%_F<)_uOg^z>tDHNn)(pYvs=frO`$X}sY zKY0At$G85&2>B|-{*+B*aqQn&Mqjt*DVH2kdwE<Y_7iHmg2Cm#(Sh<4J6@f<d1{0_ z2^3BA&qWOWztYY+Dypt+<A6#?mxDA)NDd$fk`6d@DK+ANbl1=*APv$f-5{MJAl-w2 zf`Uj$ONmlaBIq~c^Smzu4-fLL?_O&bi}~Ze&OUq2K4;(i?CV$8RrU!hsiuTc?&as1 z$PLg}c#@!R^fMP&Bo&)b(J0Hj)VzLLO?2JCM;a9lb|SKw_bcf&<Y?auTw*?8j}mKc zh;zbPoG-a*lTJF;cYTd>m5f}~Xwn9+tPt?EPwh8=8=VWA8rjt*bHEs1FJ92QohQ)Y z4sQH~AzB5!Pisyf?pVa0?L4gthx2;SKlrr?XRU`?Y>RJgUeJn!az#sNF7oDbzksrD zw8)f=f1t*UK&$}_ktf!yf4Rjt{56ffTA{A=9n})E7~iXaQkE+%GW4zqbmlYF(|hE@ z421q9`UQf$uA5y<hO3<CDipt9*4<dYT6T>DLx67`=EnS<JRni<euaZulyr%vyhnlr z#?Eu%Cbt&$Qh2GVuhims!t(sZtTn6DH+khMm(wy>TxdEaG!6C%9_obpb?;u-^QFX% zU1wQ}Li{PeT^fS;&S<Y5`Q3*lDsv9CPMdV}m~xen5jZ}thFa&>k2#$ZM#<B;v8g#V zrswx2FBXZ(;R=Gq-stwS$#c?AI-?_}zv5!~CV2?X(<s$x(_gxir@sm|Cja!_@{6`W zQl|YUi5DeUC0^oOBKvUvK18FLFqV>Zpxrn7jsd<@qhfWy*H)cw9q!I9!fDOCw~4zg zbW`EHsTp9IQUCETUse)!ZmuRICx}0Oe1KVoqdK+u>67A8v`*X*!*_i5`_<c9{Noqv zpU!p<<G8iqQns&0K%<D)4OON5-!m?A%hk(FPQ+Q?ws_ccLu7=PI@ag2GOfVp`$hQj z3;P~3wbX%$?}k-KCY{_e``4|)XjLi70dV={`^!vOAo$+G9c(WLP8o?{9YHP}4$KUz z(G1HPLl11DLU0Dcu^>qTzYRkbYXg#4vT5~A{lK#bA}Oc4ePu5hr-@;i%Z!4Y;-(yR z(1rHYTc7i1h1aipP4DaIY3g2kF#MX{XW7g&zL!39ohO98=eo5nZtq+nz}2E$OZpxx z&OFaOM1O;?mxq+`%k>YS<Yu?md-1GMIyZf&AQWvPmo6CZVe@@DU-|T6jR$5=yrKb4 zb&a8G6}6XioJBsL5hN?i*Dbi)&%CK4?m};2MCujIboR+QjHGiTDJfq5RGO0R9CX)u z0zU4#8R^dZ{jzr_AyfBOUorP{&S*A_6&UYnFI3&tm>!-=H7BB&WhqSTUC{S!x*k9E zcB;u0I!h%3nEchQwu1GnNkaQxuWnW0D@Xq5j@5WE@E(<QNfqn|w8mRlTQ7sX*U5L5 zX1c+-14vB<S>WlgDU;FLsT*eV|Bh)aH0;~@^yygFj<=+Vu3p)LlF%1AA%y5z-Oh`2 z$RDKk_6r+f#I`8fQ%y#Wx%~de1qkWL2(q^~veLKwht-dIcpt(@lc>`~@mISRIPKPm zD!Za&aX@7dy*CT!&Z7JC1jP2@8+ro8SmlH>_gzRte%ojgiwfd?TR+%Ny0`sp`QRLy zl5TiQkFh<S=sKi}DbZK?W%4)lIIs>IC!2aaJ&=Ua`c9UuOk9GkSFZ}!IGeMZ5MXrL zGtMj`m{(<sAcQ4XR^(Yx=+F5D_F|kGh4WcL`ge_pig{Vz@@q-k5QqEE>X9+l%=d|L zW2OY?8!_pyhvJ1@O!Chsf6}@3HmKq@)x;CFItPMpkSr@npO&8zMc_O?*|sqkuL^U? zV9+x3vbr|6;Ft0J^J>IH_xpa<{S5K?u-sQWC7FB9YFMwoCKK3<ooKkM@8CB4tWkc& zbMUGaZ4+o`OF+ekbFK0Bi13*;T8t4<3_}*h{=Fc^ee#H)>WZ*gvO-wAApF`K%#7@1 z^sEj4*%hH`f0@sRDGI|#Dl20o$Z*gttP$q(_?#~2!H9(!d=)I93-3)?e<kCJHws!1 z?TUS>%@$1^*F=t9t&OQ9!p84Z`+y<$yQ9wlamK~Hz2CRpS8dWJfBl@(M2qX!9d_F= zd|4A&U~8dX^M25wyC7$Swa22$G61V;fl{%Q4Lh!t_#=SP(sr_pvQ=wqOi`R)do~QX zk*_gsy75$xoi5XE&h7;-xVECk;DLoO0lJ3|6(Ba~ezi73_SYdCZPItS5MKaGE_1My zdQpx?h&RuoQ7I=UY{2Qf<WTgMg!V?k)r-qf)mjply2B4{@!80S6L@j%-&~1;m<Mv> ziGQ-FpR%piffR_4X{74~>Q!=i`)J@T415!{8e`AXy`J#ZK)5WWm3oH?x1PVvcAqE@ zWI|DEUgxyN({@Y99vCJVwiGyx@9)y2jNg`R{$s2o;`4!^6nDX_pb~fTuzf>ZoPV@X zXKe1ehcZ+3dxCB+vikgKz8pvH?>ZzlOEObd{(-aWY;F0XIbuIjSA+!%TNy87a>BoX zsae$}Fcw&+)z@n{Fvzo;SkAw0U*}?unSO)^-+sbpNRjD8&qyfp%GNH;YKdHlz^)4( z;n%`#2Pw&DPA8tc)R9FW7EBR3?GDWhf@0(u3G4ijQV;{qp3B)`Fd}kMV}gB2U%4Sy z3x>YU&`V^PU$xWc4J!OG{Jglti@E3rdYo62K31iu!BU&pdo}S66Ctq{NB<88P92Y9 zTOqX$h6HH_8fKH(I>MEJZl1_2GB~xI+!|BLvN;CnQrjHuh?grzUO7h;1AbzLi|_O= z2S=(0tX#nBjN92gRsv;7`rDCATA!o(ZA}6)+;g;T#+1~HXGFD1@3D#|Ky9!E@)u=h z3@zg3Us0BCYmq(pB`^QTp|RB9!lX*{;7r|Z(^>J+av(0-oUmIdR78c4(q%hP#=R@W ze{;yy$T^8kXr(oC*#NQMZSQlgU)aa=BrZDwpLUk5tm&(AkNt&Gel`=ydcL*<@Ypx{ z2uOxl>2vSY2g3%Si&JU<9D5#{_z{9PzJh=miNH;STk^;5#%8iMRfPe#G~T>^U_zt? zgSE)`UQhb!G$at%yCf5MU)<&(L73(hY3*%qqPbX;`%QDHed3ZaWw^k)8Vjd#ePg<i zcp5jVnE)b@51=ikxwz)bMOpsO;};uPB3dwwQb3K)6V1vE)dkj`sik^TzenJqS*D^y zgw2x@0s;B&1HiwmZ~VP26=BY3Y-ZwUr-w9aJXRieNAMmI&U<i^jQ+df?=BH#v)Ki@ zd*03PoC8I$#Gdb{^!AE~u&QgIg{~&nS;+EjfcEhk``bI)0Ul`LF|aD<+#FHMXau3{ zmyMIN@y>@;I&pMe+A18k+S+bou|QX?8eQ`{P-0vrm=uR;Y(bHV>d>Gen4LHILqcm_ z3peDMRE3JMA8wWgPkSthI^K<|8aal38qvIcEgL<!9d48_9(GR4^MhM(vJ|52p_y_O z-SJ$7ByWZitf#aD(-IW>jHAFB0P#IfqP2y}L>=8eBR}Fm^V*mw2Q4+o=exP@*#=Zs zIqHh@neG)Vy%v4cB1!L}w9J>IqAo}CsqbFPrUVc@;~Ld7t_2IIG=15mT7Itrjq#2~ zq<N^#v=@%=&M%|x1ULBCT7f|hQVQ#o?%1XMnY$$#3{&nv&0>X*&nwZP>vso$6W!#` z-YZ}jhBwQku-Qc>TIMpn%_z~`^u4v3Skyf)<h26Xm+@qv*!BTf&uZux3OH{&_Yv6F zjXaOTDs^(_X7i-8-U1PZ5HC8*b?lhzPL-f}lh*@pHE+;jbL+ZhqJgkss~Ko|6rU!p zVw}Ug?pzYfMOLRH+{j9|33K;uPIu6@A=kZj$(emVeVUmz%+Y*SIwik2sA{3iSbD7T z@z|AHO^y?MK_OJPT@E6|VWSHIT38zGTAg7KA0jE8>KA}V{`dr!Q;3x<!|BZNGP06J z!})YPd1l2)l)w1qu3OGp_}mnU13VCHq@N(!#BL%~WFM}YtU@0n7`~J=SvmZ$612JY zY}y%D+N@WuUy2kC&&{3GV|2?!k6dfO46(QIeiWTyT86Hgig~*U<Ch;jCQKMaQM*Ri zpA>K1TuGYdl}$sKF^9X!*a-R*Oq1#tLq!W)gO}{q`1HM;oh1-k4FU@8W(qe>P05$+ z`ud2&;4IW4vq8#2yA{G>OH=G+pS_jctJ*BqD$j-MI#avR+<>m-`H1@{3VgKYn2_Ih z0`2_1qUMRuzgj_V^*;5Ax_0s{_3tYR>|$i#c!F7)#`oVGmsD*M2?%930cBSI4Mj>P zTm&JmUrvDXlB%zeA_7$&ogjGK3>SOlV$ct{4)P0k)Kua%*fx9?)_fkvz<(G=F`KCp zE`0j*=FzH$^Y@iUI}MM2Hf#Yr@oQdlJMB5xe0$aGNk%tgex;0)NEuVYtLEvOt{}ti zL`o$K9HnnUnl*;DTGTNiwr&ydfDp@3Y)g5$pcY9l1-9g;yn6SBr_S9MV8Xl+RWgwb zXL%kZLE4#4rUO(Pj484!=`jy74tQxD0Zg>99vvQ}R$7~GW)-0DVJR@$5}drsp3IQG zlrJL}M{+SdWbrO@+g2BY^a}0VdQtuoml`jJ2s6GsG5D@(<cgPph;^-s;xpb_sl;c! z`2I{ZRZ2rKu`+#w`%eDfFL1(_+@w{9uIMXJzK1C)cC-5XhmFjdJ$J0AeVu{!qN~7C ztXoC%fqAJLH9pp5w$D@q4|(WtMQt<uBX$;F`+;c(GdppC^RGeVPhF-vsn=ih;u6jW zm6GF4%m_juV9>^$5pMi3$27psEIOe^n=<va#F%To3t%){5yLqKe^;S6k~-{X7FZ5` z8C8uLS+^b!l95lWbWj$NdkSG>*Nj|Ug7VXN0OrwMrRq&@sR&vdnsRlI%*$vfmJ~)s z^?lstAT$Ked`b&UZ@A6I<(uCHGZ9pLqNhD_g-kj*Sa#0%(=8j}4zd;@!o;#vJ+Bsd z4&K4RIP>6It9Ir)ey?M6Gi6@JzKNg;=jM=$)gs2#u_WhvuTRwm1x2^*!e%l&j02xz zYInQgI$_V7Epzf3*BU~gos}|EurFj8l}hsI(!5yX!~ECL%cnYMS-e<`AKDL%(G)62 zPU;uF1(~(YbH2444JGh58c<L!e)x87c-46Lp>oXT>(*CdEwaFuyvB|%CULgVQesH$ znB`vk3BMP<-QauWOZ0W6xB5y7?tE5cisG|V;bhY^8+*BH1T0ZLbn&gi12|a9Oa%;I zxvaxX_xe3<zJ5_;M)%eUc$b1qh*#3cy)NFgC88^2+a(k+Yl_LMWaIPd<}RY|!D0d{ z$uOrm$<_CFd@uE5uyxO5sHA5PAtZehPrhIp9e0Nsutu7#eGO!z!OxmFo&`&VCBFZL zmvMu$ka54jcn_SlzEGRg)E{&pt1bUyd5FSM(K?xHX-(;lwoBxm=G}pTsCynZ)GP5G z>@ng%;4C?zPHQ1v%dbhjA6Sl7w<*)Nr#F{Ahzj}%n9c&!g5HVrlvUO&R2C)_$x6M9 zahficAbeHL2%jILO>Pq&RPPxl;i{K5#O*Yt15AORTCvkjNfJ)LrN4K{sY7>tGuTQ@ z^?N*+xssG&sfp0c$^vV*H)U1O!fTHk8;Q7@42MT@z6UTd^&DKSxVcC-1OLjl7m63& zBb&goU!hes(GF^yc!107bkV6Pr%;A-WWd@DK2;&=zyiK*0i^0@f?fh2c)4&DRSjrI zk!W^=<E?~bQ>l^JKlPW9US{*yo?_XT@T2Bx+Cm^+r{*5LVcKVw*ll3+)lkebA-4)o z8f5xHWOx0!FDSs4nv@o@>mxTQrOeKzj@5uL`d>mXSp|#{FE54EE_!KtQNq>-G(&5) ztz?xkqPU16A-8@-quJ|SU^ClZ?bJ2kCJPB|6L>NTDYBprw$Wcw<Y;;ldx=&gJa}4H z@Wwakn*Q^&RVtHa47by$I4sxs2Xr6hO|}|b8g;8P+cdPU{P00GD^=yOpI@GzPh+l` zi(s|O;0{OV%*#9PT6u8IaQp6YbA~_!T`w03A?yq_T&YSRnqY&$9SIkVQol~v{2NC( zPEQSJOOmN%HRmJS_NUT3hVmC!EL}-VEM1@e^gDltyerz9=+0e`-3a5G4k0SZ7Fa!1 z#ag*v`7$GXC1L{lPUrk^`@3^20bjuUD_37Kxt!~92XMGK0w&T|F$%LSOP_R^>CH{B z5qlJ6wK_9sT@Kl6G|Q&$gsl@WT>hE;nDAbH#%f1ZwuOkvWLj{qV$m3LF423&l!^iV zhym*>R>Yyens++~6F5+uZQTCz9t~PEW+e?w)XF2g!^^%6k?@Jcu;MG0FG9!T+Gx{Z z<gaMF{iLoiR26S<6~t-k;EsdU7e_zysFg2SI^5;A;*}V9MU-Fs%r-t{&pmIK5IGAc zzNfdSinv>K;31y@(J{!-$k4E{5#Sv(2DGy3EZQY}G_*z*G&CZ_J?m&Fg4IBrvPx1w z1zAb3k}6nT?E)HNCi%}aR^?)%w-DcpBR*tD(r_c{QU6V&2vU-j0;{TVDN6los%YJZ z5C(*ZE#kv-Bvl<Em5rSV(B!w_4$#2T42Z%3LIBAftq?Y(2w!<4+Y_EtkGH%njwHbv zaUXw<csfQ_AP(a;(P=s-f<2BWH_U9H*Fq$(LKHrb*ong+Bt0$pUwAi0kTJsL`Y$2- zV-MWNm4at^033)WXd{RvNQ(Pm3?N=<`uEfIw~~aBv5g(Vg5S>GLDf<J@6-7QNmQgf zlDvHwc9+i}DSsA3<MC#qa=?!^vxEK&nrA`RAD2YsB_D03@6|I%{?&b;({ba+HBphg zND}j5^f8}V^RKQ&c%?fS{h=pOQLIOQrc>9>EO#RH_jtolA)iRJ>tSfJpF!#DO+tk% zBAKCwVZwO^p)(Rhk2en$XLfWjQQ`ix>K}Ru6-sn8Ih6k&$$y`zQ}}4dj~o@9gX9_= z#<!b)PJX7r5EUA7^jB6Od_>~EkchJqd5$**l}~~6mOl(q#GMIcFg&XCKO;$w>!K14 zko1egAORiG{r|8qj*FsN>?7d`han?*MD#xe^)sOqj;o;hgdaVnBH$BM{_73?znS+R z*G2VHM!Jw6#<<Msx_|T}KQ4vp{)aSZJ`7HUQ>FfJ-J%-9AuDW$@mc-Eyk~F{Jbvt` zn;(%DbBDnKIYr~|I>ZTvbH@cxUyw%bp*)OSs}lwO^HTJ2M#u5QsPF0?Jv*OVPfdKv z+t$Z5P!~jzZ~Y!d#iP?S{?M_g%Ua0Q)WawbIx+2uYpcf(7Im%W=rAu4dSceo7RZh# zN38=RmwOJQE$qbPXIuO^E`wSeJKCx3Q76irp~QS#19dusEVCWPrKhK9{7cbIMg9U} TZiJi*F`$tkW<p)b$p8Kiq>Ln) literal 63721 zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0 zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0<b<>oE!`Zf6fM>C<V>R?!y@zU(cL8NsKk`a z6tx5mAk<liamrzlCS@BsX~|&`RS-HU8cGq`t>djD;J=LcJ;;Aw8p!v#ouk>mUDZF@ zK>yvw%+bKu+T{N<MgC_~H%9||dlSch>k@LZ;zkYy0HBKw06_IWcM<!q!PNfx0T}}e zTRJ0a11G0!b#QO&CEQP4n)k!|A)#qSG|8;N24)yY|3OH|n9Ef#Qn-}F#h?W3i%43c z)2szbS#t|1^laxjK<9Y@_Ix3>Ho*0HKpTsEFZhn<oTMi&w}vW*Ra?K_!_)1rk7w^0 zcz%y-9{{$<M=0I0eaFor!J){*JHz%a;XWx9WpR5@-ICoSDBGt4RNpQ|Al>5qCHH9j z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|? z;(ggJt>9l?C|zoO@5)tu?<y?2z)*Z;1quxvz;2Wr$0J)*8MlO}_`_m{D`H0p@e^(M z$iAC}1xU~1A4L(ddtDLl_Pr6{Hx8(|zsON}%665gG;b|X+4q@!y;YHT4o3!{_{jPB z>EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#< zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&<gtMWk{spc zdf)IQ%Esm7d(=p(gnM#+I?kJb&Lb5@<u)2i>z2x+*>o2FwPFjWpeaL=!tzv#JOW#( z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$# zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK% zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|( zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm<fczTCmKxlM?(K(U1MI{ zH7CsNY39I?b2iW&6QmO%p`j~N;r{IyO6k!4?8&O+JtB37&qWiLadcH|I)VEU4X_&G z2Ff&+i<CegULNGA;Rqo82kZIenL`+2Sql2b=gdn7-sCd|*$dW%v3%JHM_jz-$G*h( z2Zhp66mL0wC-)~7Dg~G4tT5AYOjgU1YJq|3QV~+nZV4#kZI8ONh{_OOoJD(-6)TQN zW~`V4m}~|uOv@KO-ydJ-Zh&N$+FY<g0*jjHHbxmR>@jt!EPoLA6>r)?@DIobIZ5Sx zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW( z8DSlG<ofVg|3LaJ-=P0V))`RIkE+lq+qJ{`jPVl(|5Va1SGyg3MTkZ3a5}sBX&c*A ztSA~lX)HnZl`w$}6pc6PwMwbP#y;;c$+yzscOU^s%U^DuE7fk%u_oE_r4U9DlvWzf zB&zu$+)hR%%$d2sDve|n{X>N>Ts|n*xj+%If~+E_B<wY#42?CSY)Lp9f}%hd8G2BV zYz5{>xK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK zr5<Xf@p4l8GDDqq9Ca=)oYjzaL8(efxR9(LX_X`~m?l6@BVW;!8T+Uyif6_AH70S3 z1&@6Mj!ceBBfVQ`!9u3Zf{wVpf0C(t(aC=1R{{IVv+00AMAS(+n-xbkL&B4F#nFqZ ztPDBX@p?jtnq^cL=+U%~e$BEw+yxI)sZyO4JhtoSg*p=}x1?BGLbn0LMY>Sr^g+LC zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4 zRP122_dCJl>hZc~?58w~><cp^hJ_ZV5fj>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{<k$0*s;%bL~F z;<2lwblZB7HV^gmjYfnL$(oz3yiwuVBNdItsJS*edKT4)myLcW$QJUM;r94>YSsn$ z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@ zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb zl+b<?C%WXJNEX{LzvH<usRTrr8A!~O>Ty7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZ<Hcy3%$%v9V&0Kee&a zKV!#7+J;9|6x<uodo*>B=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M? zS(W?K4nOtoBe4tvBXs@@`i<!ZAMR~@rqD5uMjjRS2DHf>?4G$S2W&;$z8VBSM;Mn9 zxcaEiQ9=v<fz_-ouh|d?S}QqE(0qsl4bj8_Oo|Bt%%h=cX$JXg7?LJ&<ET0ro2;%t z1-1gS%DOru2m?YDj1D3wKn1f3YvfFowv0`#z<)Q7Eu0mY1aD;Bp(<0D*dvtLJmX0i zAI;vT=wP6!8*)iKc4+k{>S|bIJ>*tf9AH~m&U%2+Dim<)E=<ebn)#5-YItTnbgMqQ zt=Y>}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD zk1Utyc<n(<e1-Rj>5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my zIs+pE0Y^`q<Mo02TIpLU)qh7WQ?@rM6U&!Z;VW(ggXdTSsIrFphI+dpATgu%)3IHs z@+Xj*BwLyZ=r@R=;kqR6lCAdnF<CWp%t6J7J#JoVQ!Q2LM-rk#T_J^oRW%%3=2#t< zlx+Py@hbE`=>JTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1 zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQc<ZbLh@dc$w|qk{r@1?l>a`S zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w z<G-5=7&<vS8W=eX+1c0_*cwY)*qR90*}8t;u!-Ye>^fOl#|}vVg%=n)@_e1BrP)`A zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EH<oD|H%>mK>A~Q5o73yM z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM zyeRT9)oMt97Agvc4sEKUEy%M<cvw&A<E0smes0HD4KT3M{WaOQQ;89w>pXr2vz*lb zh*L}}iG>-pqDRw7ud{=FvTD?}<cu3Tk)KD+t)3BG<`{7$)a-(xssJsd76$3kXZ|K+ zux7WJ7n@%B)`GvtXt2vB^wx3Cq%hbMjsz#YIUp6%4@(wA$XOe1;039$$Hc7QvIpbU zLS8D9A2JK!>xjD)w{`KzjNom-$jS^;iw0+7n<H%EYFc(Oy%QoCT$qyKODp6T3?dHk z!A)db&e@dFRGDaEZ1f6Uhkq#S5W3t3q@<p|gafXRD$(FZPe|Di#nme60l3B9fVDQI z7v|md<AFmDM_>XSnt1R@G|VqoRh<xeTuMLt3A;kkH;cO*#r^ytgz}n?7coM19}rK` z);^|be>E%12<OWj>nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ zSCs$nfcxK_vRYM34O<1}Q<iD$7sC+}q<B7R-C|JDpp;azgo0#wbVy`Lz$zBEbO-~2 z>HZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrC<wq zp!=`znU^{;qwI4(IwPTBbeO&*i}Y=rKz<}0qd2sSuD;n+Mp~oxu2u^U_@*f$2SH4& zl?ba0Bgc;6q%NBUleYBwY{7zE^Vfp-*+^4E--SmUnP*kpPGgQ7i#F(|?Htpi_BGIr zb@Gxu5=<~CQJoX};=}ZoAqI?aQ`aURT7|_bL85cc5*32Ec(l3BkkWJ!;u(fz)0rjQ zM$>w&)@s^Dc~^)#<wKTL<XKhL5mN0<#J&V|X6YXFoTAHh1v3e?`1qzOHJxAfrJ&Ia z>HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn z3Q)2VjU^ti1myod<e*tXJ``;<uRxWgHj9DL9reuUk;3Yai3$ZVxc$Lik!QkFkY_p- zQ0zfew$syu%`J??OaA$e3yhROoyi??MOs)*6Y|FoRjo5YG8>v+tjhSZp%D978m~p& z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^ zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxM<DMa${DuB@U~n2TvsN>gUah3$@C zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(<S5im8rZJ+hXaeF4?)$Bxlh$wotZoqo;9C zoV$lbzz-{=kDm#H-UEvsDLF6Q650PW_m^)`Fnl2k7$|cT<4p%DP+QQ%Pm<dng5Tlm zJY!}F9=K|jqEaN6rLUGWH!wZ0U(TSP7a_ulN!0{(D76G_DCCMYc~&1UiwGoGVvr{o zKJ*3=vWF1EmCF?ngO(A#QHO73AHw39x4#7x|2*uRJP=H^WAL{ITtOufKQPefaR-dw z;G$VL`0sIgw-zB%E@}zvSQFs9-~l@jy~i@_j=aHRkdEWSDI;LT_XD0ffYvUye7<si z2Fb>OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3 z(lfnB`-Xf<Eah_TH0`p#iJ%bwIruYbtv4=iev1%ADQ5<nqfk9TiY>*L<pJwX*6 z6rk6>fC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B<Nc z0Y@_z8xvb+5qBdKduI!~zgMP`<EJEn8Bv1e-k1xUTQqH`&-$;LRKPb?p@^XRcl%SW z7A(?4O_9bX%W97*cKg9^@&`$1Rhl479TL49uifNE-$%}|e=@U3QRq(u*`T|i!vY;= zLFYU{oP~b!`V{F3i<~?v4T-GsVj-c>9i<^E`_Qf0pv9(P%_<ZnXV3#<!Itln<wgcO z_ag@&>s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$<HtTbQGA?-M`Fa zN~3r+{;f4I^wTt)Y$;V0A?b}t39$3`u-!SmQRz2~BgR0Y22AtoiqyTce$gRQ#;)xn z(H=h1rzHb3B0D>~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd z3jz9vS@{<x6RjX4HShz$XJL7Gv9^MIhKL19l!vXDKtut8g2a8N<h+4&Yt&WgZG-0p z_>aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7 zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnx<UAAx@pFpd`WS-_yK7SJSHbCM zJ+sycj{FkEU&9Ysa-wV2!;2(ImdDdIZgJ}`9j;jTiYPXEDq*CO`T4-t*|XS#9~HNC zu96BV=Ry2qi)VUChoa}C_CB44h;*&oc0EWPU$hYH8{zPphs-sTrb;$I`Tk25Ef6wI z)-7g@DMK6f){DP<6&$RnaJ4vp86eii6XT#?kKzCG^Hnm1S^@(5e!g%30A&B?^OgGt zSI<_}azj?Z*h(zPW=Yo#YqH4KJ|wab#BOfNtKQV48`7O!MvH)0FqQ@{NoPp6N4$3X z1K#yg(se^X=dYqMag+$(^NRillP<Mw#+WO8vuGkT>pt75r?GeAQ}*|>pYJE=uZb73 zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2<a!(dEKJOdD7OJ~`mJ#&3lVWo z2(|vK+K6Dp{tAw<@IDkF-OU~{Fey=i5LyAY`xe{ZP)J-QHDxPH%5%%ni&>_zlIGb9 zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5 zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiU<dJw*iNTYgDXXO3%H4$mrD2+2if zR#sZlF&7^<X^ey&*l`pd(b870Yl;d^q~$DJ4j>si<i1L1H7=S6VPERSA>gnxXNaR3 zm_}4iWU$gt2Mw5NvZ5(Vp<B5%4ml4%u2XX{cb%`vs{9^lq|NV~2Us}ADnGUgJZqX- zvwS;i%5bY0rx<UeBWyPSiTAfxZ8Te<Y^2=Q6Uyjb@`B9@uPO^RqSGRQ8L=vx?~S*{ zt!O7dY09tk+Q(K@^7dsqbRFj3D?R)D=uSPhZfFr)&^PL7B^!(GLR_d(Kw!yNd&IP$ znV)B>F`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT= zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCS<xwK zC7(yN8jDThv(|6XTqj5k)nXJHl?i2Q&>Em)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<} z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@ z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P} zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=<i2xqPYPe_t`z^~U4bI&mS zeK8h(VJQzW*&0F;1J5rkP14OFRVV|<ULvN%7sx(;Rti9xZLhau-~!P2{WfUAn2q*` zd|=*_Vb!;8;KGMfl41$VF7fE>2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR z?&EP8#t&m0B=?aJeuz~lH<ij*LuuHi5!4Rd8ZU2wg>jAzRBX>&x=A;gIvb>MD{XEV zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>= zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou<DyN^`2@H+<{3q_pZ|fCRGf^h zvtT4FGJj|vS-l9;nX`=;6AMdLY77qfRlAH(xzJbez);$Wc|j0JS86%Riccga7l&Q^ z7DDh5jhBvJ0eBnJZoBnclA)#bn$D1A`JT3aY&tu3wlfU}!It+X%B_(|pGP1-6at%6 z9G;Q{hFp?BH`-HYKrn-(5-7%bIR8)}bl%^bc}8y}>3kHCAD7EYkw@l$8TN#LO9jC( z1B<i{*|v`>eFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$<r<rS z?gfFH3ULExuxO;h09`>A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw<j-Y9ZSgmH9DO&6{}V;z z4IG_J97!1eDmMg22|)ETAc%aKH#bAM9(9CS1?uKgKtu$Phh55R&4VPI?P<FMz>}7= zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9 zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_ zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U<yhPFxA*<jTKd}k{c~z90FpaZKIj}7mLZZR zVlskQe<0xI9>!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4 z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI z%7}WB+TRlMIrEK<z8Y-G_4JTi0dxbex2YwD(&eIklPGFZaWLB&GD=ZnUD^~B#;k{< zjP^KiL#JbSns`pE$?*&<=bFPwu*}^i6&=HjW3#5UHflvIkmn+HmO8$)V)qRxk*3l@ zOO9ib60_+Zpll9hiP2eYZBRUKjvXd)MdN}}smA0!UK^qy;<^pk_jf6elpJ`B)>#s0 z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD( z83s4GmQ^Wf*0Bd04f#0H<rzc{Zw2|AZqo(GiNDwicoG{misd0-Mku7fEh(b%bV@{& zro_rCgoAMr<vEX067x&DjEdA&lB?SNTC@l2#eL4j&Fx~(S<U2Qj$}%g_p>X@ua_d8 z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8 zEOwW6L*ufvK;TN{=S&R@pzV^U=Q<n5LbrjaQ=f5@7_~`mTQ9mj1lTX|puGXCkhc-% zDqQ!ov(P;Fh`r;zNT#tw6ShQ_Wb=wsd)-t85jQ<PT~cSb(KG~zb^;j9%nmc1=u1J= znM6vCx;p+afnlGOK^Z(FtJX%b2Laq9%EC)v3-}QHS=dL;;3Z|eP=v~{8Igl4x<in+ zs+~^lyBk3)zB{QIT=g<UC4Dvc@uY$A(I$Qm(r%M)rb;eRGv~cUyVvsbhIKxiBZOdE z&4GjvPs^czPS?~yx=kmokn<?z^Iu|gjSZTs;5%U;1OKb!`@bg*{}`ix5nQLgVzWB= zOBPuGVWiiKw%d`msf^&W1_DTJ7XVcxDttFKPMJj@Q@p^`V#d*Pi+Mxn7SS91D^8en zZty#+i)vgc%xXIPl}6Ud+}N0#zLvf5`S$Ta{!?R<CC_N_2bR$mN%T1dmbl^kFBBTw z1ujzzCe&Kp;{r{`peY9BJL9Pe30)VP%6+b7BRXtX7mFD)e?pfD#2CL!17n(PpRUO` z?Yjz)8OniiQ=hFAxt9*96eH5w{w=1|n0X<i`5k*Km^JQNZ!JF0bM74Zvj&6~ZAXtn zgG9^ASe$T762j2Jcgl0`Y(Mo@H6OZ<lK6bTs)vl;gOmx8Db2@XVoT@)D;P*RE2{Pu zkq|sNVFWHy%(o0jW-8h@<pE7_$58cKk*4~kS1*H4Ot~s|!`6&RJxm(c3R*s0mV_8; zn2FG~1NRH*S@8AcCakja5ctO6KJ`a7lcApLvK|cc`*cNA(|7?@ShdEJko*Iz0dJa* za(Cjc=YoMHSiGhp!^zXOoFUjrW@)@~Kuoe2$0wLZF+12d?*qAHgQiPS_>Nk^Ec}5H z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8 zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_ zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T> z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z<Zw*_46AZG)LPZf(N{7;dl4f;=ChNJ&((0HR z>_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQR<iP3ONA)u3T^sr$hJu) z!;4BKQQ3Ke*v_=`LpX`s9^P!rY`x6m`OB{xdu)~9l-PrYu_eSHL`$3Jn77r}HXM<V zt(63|ZNf?J_G2$D@(>S38V2F__7MW~sgh!a>98Q2%lUNFO=^x<M$kfz5r<ep4<gy2 zqj#v58_1DTD9KgRF!$T?T|hpgQFqS{y9=z}$c192Is9kheW211%6d$Lv95L2Dj6<Y zb#^>U52|?D=IK#QjwBky-C>zO<IBf}7iok}7&M@d#5zo!yo-N`Xt*z8pz#W=a}edt zZG-pIp+-`FRdYN5kGiz~5pI5v{UL@+ll7;0f%sl_#Rz#s#c=hclIZch%vfdNL6kTn zr*OojJocIA;LUP<Ni!nCp8^IKsKOF;%d~A3Zec1Z)onv06oLt;&zK*gg)?kMAG~wt z$?%9ZDd#C*i@r2pEnc3pbg`;ZSgHhk8hnJ=MZQDKa})LR^M<VYo_<&7>WlsiiM&1n z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj* zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^ zL(V_5JK`XT?OH<!?|M@&0-Z{-IE8Y%j&9{KOrqhAFsdE>Vk|{_$vQ|oNEpab*BO4F zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^<aLnRI zIQFl^{&&$2$-fmuufLTX(f?#w5i)Qxk+5|#v30U=ws193a(1+^HT!10f0I0&?f$MZ z7Axt<A%ClkjrclcTIHY>1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2 zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{<zp2fNOj3=!d#J-DZOZZGDsQytEg`t+g z3s*%5CrGI0LC#hm)1QTr3)Q~mP=>Lkh6u8J<B0%4?J^Iw+=WHCe(yhjohQDag#Q-y zuxp6&sgx+NVPxq!=P=H(FOhwRd2_*m=|$Mcn3vF&f(Fz>`eK%u0WtKh6B>GW_)PVc zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J zQKa-dPt<oy^t}rwUk4E{A=4M9sOFfr7Ds9yI!q0r@t|+qU_|sPwWo~)0N*{XeSJ2j zt||$@K&w$2s%KuI4R}Av!VN4FKqSw8V4~H!Grt8-#>~M~E}V?PhW0R26xdA%<ogsP zN|DDB6`HT`^}UE=1A}Thaak~Emv(0YF@z$Cl+aE_pyty?6u<ojo%}e0uS=bI+~z8+ zN@qxB>1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq znJ!W<Tk7rKF4_TgJoZCW5Z^!*fTJ-Zk)y;)2fnZbAE(sksf_kg&-X&Eg#6@NPIv)e zPk4oG=#<n+u~!?2Waj@D(b4RIBp>FR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5 zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3* zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n z(v~rW+(5L96L{vBb<M(xcH!jFDY91P;>^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF<TNlh3Zu<wDObc>24O~u zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{ z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+ zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbE<yUx>aoFIDr~y;@r&I>FMW{ z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^ z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8k<h1nlJpEW7DYjzCm^(#wnSe&>hR)6B(--!9Q zubo}h+1T)>a@c)H^i``@<^j?|r4*{<Pxu7~B&!$dLmL9Ys=1M8gJeDpJrYz<2V9nr zTvmS4mM<@Hxmfnk<1RI6FD*TtRhU+s9@>;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#<J1rUcxa8T8CP^@o- z9LUqXz6y%L)dYBeo?hHD`m4vf;Ko!GhH#gWIg{IB<GEyusoLX^qjW_iBFR#|P|H$t z1PVdv4xB@M4hvXb%aZB@j?)`0DTujf<&yn?+Ww^h>Sw0EJ*edYcuOtO#~Cx^(M7w5 z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9 zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk> zuqsld#G(b$G8tus=M!<V>N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8 zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1<zRinbW1<P#|UQotg93waYH2w1*h*jWPj6 z^zMSH#dD)R!pp{n89hOchS-cu?Yi0ISv7!)1hAHH{1T%@r!ud3;HLw5<Vb67p@OE@ zP1TDxu7L{<vN=z>*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6 z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m| zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_ z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B; z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1 zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL) zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$ zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_<Vgo zXaA`|b1qI$OB^nkP*k8;>Re>6lPyDCjxr*R(+HE%c&QN+b^tbT<D}=?S8O%T8tX+C zcq<H{RI|G2B9m%Rvp=d?PUf-1CZ*M)qUmtvLRWz*mFB-QNI1$Q(ryDe(K%5UcMcw> zXBJk?p)zhJj#<NYmRaIG#%{yKso~Jl);(QZ{BXnYyStuZE8Z;ST(ba;;Cf6`eaY~e zro&T_@|5eV&LV-Vdlye+OlA8HSGS?PEP0tn!gk^nHiROTHF+mL6DBZ*<LBy0zzNW0 z?K`Dd6_+8u^4Tx%YqyDRx3oaO`51F%4~hz9FS=g55uAbZ9<uEXv*DeH(f5XqNd{@0 zp<n?$CoIPaZ{u=WOkQK=>I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkf<vz98R4^QE*0+j0Yt5Bq_8W~pIEsdB>q>am z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V> zN<mGWxPciFI#UK&1{)&R+tKETKKJppI5eS&4QOQDb2gC@9#amL!Z4`6_?sgG50#TO zUE{XN@zEh;wAcp{%1UraASTco@__6>B1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9 z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQ<d^ZC}lYirx z)hZjd3qKHeGus^Y+enhww8u%4lE|(|Z6qnX?I}@3Q1b~uMX2mD2SFAFYnI`H<@TW6 z_W((t!X&)`@PpH2wi2iW=uqjmv(p=oqs&Y%b9;Nf0OWslW9*Nb&oWZEt+0AVS&R~u z_Wf#$fP|$gQ9fiHWGF1iGfW}Wb;*#iU5QAsVTwY*RhU@;t!16`ZZ<f|b==EnUgA$9 z4GDXmcVu4RoyBc?5+IzLNU`y7!@x5Q-0Qn*X9{dMTclVEa<*>DKvm*7NCxu&i;zub zAJ<l&f69)KP9=Oa-e|<}G6{v&t3Gsy-GA$_$uw;4(^(<e6x2^i-<E!;86)%4VjM%- z<_j^P(DEMAyY~*<c=R4aPSMdD=QV?HQb>h#11%?w>E2rf2e~C4+rAb-&$^vsdACs7 z@|<k_fU+PhC9U|KK^m>Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5< zBVBI_xkBAc<(p<um#S5<Wtzr=ckJ6;=I>cb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~ zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1( z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG zP##3>LDC3jZPE4PH32<r6$U<HX=R_L2D`8>AxrqPk|yIIrq~`aL-=}`okhNu9aT%q z1b<WnlSbT*V4<ymTCe0MD#5~7Whz|=;>)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2 z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q z?zAM8j1l4jrq|5X+V!8S*2<l)&qsCusW$QhU{vI`y!|$e;d+{bf(RH@<)nJ517c7s zDim)TBb^~cK*c)35Gg|q1$&KAYNncsQOHJ|nu#)~+T76>Wl@=7*pPgciTVK6kS1Ge zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj zl@Z47h$){PlEL<j^JXcV*jDSt#inW`1Gn}CBm-N=8TRh<K)!JpM`mq>JgYZCIHHL= z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{<b??kNj&VqPGa79d@(m$2L68%nd! z{_JjeX;XQdwTu-^XN2zhv1k+fQHndX43k}u83t_z&A52ZBFI*?6D8Q{uBSWGwQ#nR zSnaCVwkuW{YP1TLJ@kJx_A>Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5<s3FN;{WX^H?!5dAC7_ToT& z049v%;5~bT4K%9GqC;d+sXL`9S9+;oZpLE#7knG~OKbHmpUA?yU6n&99^Z!ZI?xYz z{BGz0U$>E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^ zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V( zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=<c9QK{JSBnxbTl~S4d3F zZhX~`iBZf``YC!s;R3x;^$k-@k$#)|!OTN1novAwmG;&P2)PD!FyXg4r(4gHh?ySe z7<%V?@*d*j&#%W*MAHm*B6GjS8B_74D&Msq691~iVscJ{Zps}t4Abjxm(LlYOjt<z z0=dj7A`k(bp)&5Hm__k9JT(e0PyOe3?@>+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5 zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80| z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42 zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++ zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*<T<d*k5U>T}UhWNN`I)RRS8za? z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_ zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI zP!1uAl}r=_871(G?y`i&)-7{u=%nxk<TF^9;4ekh$qpT49mIUTo!QB3IxD^Xo8$2N z0jCW$vUnPpla#O+%Oja`23x?g8&sst<rR<!i-fJAT!pW4qQWcl7>7CZ_Qh#!|ITec zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8? zA<Q7k-wX5MjEY4*mC%%h!I%wsiVeW4+`u*(F}G%hLu4v_No<?+)c0h3mM0wvOcTzb z6mLpttd1pgh;LPDfUyEl9?m3fSr8}4WkXDbERL<t;7gDrg+uTPTiBati$g>@$?}Sc z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i# z{DV9W$)>34<d17om!!)9&{`T5uHYJXV$G;?A+9420_wrs59Q5xU_?zk<S-MZw#YF! z)5TK_({W1E10+oWfpH?yLthllM<4BAQSN*>wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3| z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P z<ID{omUfJz++3+6M6A3|He)%RuG>qQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%# zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_ z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S<H?5z{)#QL{V zr*a_q>;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)<uN zPvr+1FFDIM@A4#y;a<?S=xTH`XYK!+^->**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0 z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P z3Ka2l<H7(tNv_ycRF43oWEx{?2ychJaXYk!nafC$;QPt)Dhl%pHY&kHF=u*jMxFxR z8hs|e%r39oE}ts%zE`x;@(JZF%`*hV&TQs;k7&^PC%_|U=Z^yM6XVPX3WWp6H|w9J z45Q%jM?z_P7Kg74lHMQ_-ZVr_NbHvg3EMK85#qK7Y#v#ov_?2?Wu|DHo1}82@XlNo z*7dav>Ro52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^ zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS* zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`& z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G) zO3-_Xh%aG}fHWe*==58zBwp<ewOv=l7F;`(EW(2I^P`O~+>%&`mg<U+`XEp_tI#7M zlHXq!JFASK8=TxtsItown-vYtx@G%cb7t%FpgERtlejdO-?8O0py|EQiBWNJ#diHb z8h^Y>e<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5 z#ZK4VeQEXNPc4r$K00F<wRh*06<c<Cw!0P4***aS+tgLJ2H$n=v+UM@P`jG9HJHW0 zeqBot9>g>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si<jiK`FCX|r(xrQ!0U zNF-2!m|??dd%b!3w5!;b;@Y>{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G z3+d_5y<nkN(o+zEYZBPE7qE4X4RTq~xP<0UuT}eq);wA`P~5mS&}Ni6sX$kQ!#Y14 zw<=&7Ca^#oAVnvbz-T-b@50=C)>@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{ zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7<U!}O-C%Wp93|!6 z_maftKQVDsmc-oSX^Wf5%~c*ohRjQuXO7WUoJr4mKQ1O_S_G_rS=b~3S&JLLEOKDV z{3LUpGl1vt1pkI#r_j=V*y<p!g}(O0?op0V5}i0e<PEWYB8x5J4<%9#9y9X9M*bUx zicdfReJ#lhlVUp=<yC<tza847t{vKR$R*ci-p`+Gl}ts*E09J&7%h92yemiDYmk~} zTm@z(hV@tXo`0Xdz4V~IJu7i|)?jOn6Nt~0dUv^z2PHbxR-jssAHfY`%7{ql=vzVB zs_4Na@y!!YjTV?bkxAK(=I=549YO&vP5F{dsX_6-68_n)LiIx2IwV=E#EmY<Yodu6 z9z7+@5zk>Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^ z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E zNuQjg|G7iwU(N8pI@_6==0CL;lRh<Er=!!1>1dQF#wePhmu@hADFd3B5KIH#dx(2A zp~K&;Xw}F_N6CU~0)<woWpm{<(lL~%8t8hZ?O>QpQk7s$a+LcTOj1%=WXI(U=Dv!6 z{#<#-)2+gCyyv=Jw?Ab#PV<kuPJ4UouiQa>kxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+ zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*<Z21jxz28!z4 z)+Qw^>>Cd<OYjfTeBDpnxINod_ps<v{(V~~r+~z4oEIjOYCK|$#(JnCT7;!!Y}f=y zFNsGLXIqR_Lc24Epk`I91kx04#KNLQB=sN}`>Pkt(m150rCue?FVdL0nFL$V%5y6N z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED} z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc} zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6 z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+} zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o<TP7Iv6D<p-`geitU42O0|5*#cI2& zD#rHm-Vc3+vP+a)$@(=16*wM+J-aXvee~$jPN+y)1J;9JyU$}nCH;?MkiD^?^{YXd z()hszm1?+Vl-hdUHRO1qu43l7<f-+JFAVi;sMe)sVUTGeF5*U18gVs~b3a#HICwn? zOc3D$MdBtKd+A-2`!?hp+dNtbrf)b+gMTQ^qA%J;1l0Bf$I4T>9f9}vKJy9NDBjBW zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p# zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4 zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl zf<JQN-rLl3?(b@3I(eFE1!7*!5QjuNkUcQB*{SCO?Dz6>mK9MNnIX9v{?%xdw7568 zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~ zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^u<JFRGA?x);o z`Z~}R6`4W#%h(d)aQ>Nh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#<S` z2%`mn3A&KL|FUYx1DDYTb2GMnO1WEyfJ-eVoSqd~=r!kw@InSGE<>FTjy?h&X3TnH zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU zha?^AQTDpYcuN!B+<Y^i3NB&h=8l2*^8*j@dwN-$SX`JG4?X#EO)n_d+Mm*qNtLvW z1{=j+)$5S8QKKaCKpO>#1dE*X{<#!M%zfUQbj=zL<r~1!%Q56_&?RVt*j39RdBbdU zvt>E{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O zI4vh;g7824l}*`_p@MT4+d`JZ<NgM|d-wED*_BDR=X2CYG0s+@tH~}mHA<v@@*LLa zcTk2OQd|qCF;Irq7ZT2t<bCnFzjKHMYi_FEX5uA1sMB~b=-gExnk*fx9Jolk@GBaP zo2{A-B!6SMvS~u~??*1DY*%B^{i&5Xid$7&jHLv;Csgpyh12H&Wr+sb8jR356U$OH z#keINf2882?;$z(=9b`_o!xWZsvxb)AId~zQ-ypi#22f~snWv+_Q$md&MYLZH1*5& zgU2`BbMmltaER}JO!m5!`^u~)I>2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*= zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU z);*ldgwwCmzM3ug<bwv^{e8k-I_Ia))Ca<<K85KO7s<Z8_qINV*w7o<JN><pez`8$ z*U(_%(Oddx;Dy@<By6!p<ae@SHe5;+DISJZbTAq-U`Q`A1)YLa`3xqvnU#=JMDwvc zT=fd_B(g|SbuM?{hEp2{k!4hh2k1}qTl{TSl*cD|duWT^U1%zqX9UbNuTdGS)?ic- zFWu0OzODT7)oL^9a3Iy*#7Rk@72_$KGruLmz}W@8{rhO(Lndv7z61c>lr}!Z2G+?& zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3 zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ<G&YKu_KEA~r2_|MY6U!vEc zYq^WKw2*I=^(R7(!~~v`>=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo- zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H& zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2 z3wix|aL<TqR7Y;}gRV7Q6u)-qpm%oMjSmV6D=p0OrNXwr5;y^b5cF7C7&Mp&D`?Ob z8ESq3ScyN7w@J>BV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5 zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH<Ny`yVx$sah_BnMO|Vl_4M%y|BVBOcD(&Tf zIi%w5mBkQA-m8WhIS+m)@HEq^i=}^RPX#BvtKJYieRDhM9CpMXBxjmn?hoV<pKsfM zQ3`)(<)?1Do&LG^9T4w<TIx#Djhk>9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9<cxOL&fF^435YAz<*2lIsx>#! z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>Vs<fWSaAAk=E0a4xz;CoE+n zvV|`k(cS-gI#<~znD&6(Dyi8%>J4W7Kv{<|#4f-qDE$D-W>gWT%<wM^e7+vR+ZVxu zJA%k!wV1jm=-?CPfHci1I%oS6_$rRC_i%Dy1_C}}(R>z-!qXnDHhOvLk=?^a1*|0j z{pW{M0{#1VcR5;F!!fIl<yJC4LQf<m+NFcvrhH-9Oq`TslF!sxh9CTya<1|Z@Sf8S z#)!cL{VHJYkWIKNj^M2D@K4#yCJQKlT2}zO7tRTvNED*cmVv~6G8g$V6W>LVNh_Gj zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZa<R$b|!F4rBVu<@_&`m0` zvC-aJ+X!p>Xy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4% zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4 znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq ze?zK^<n_C~sSO$T&zHJ&gqMJm2ooswNa9fe;pI&q8BGtsLvsv{E`UcDopP-qDeO>K zA%sj<KGR#nku^U`P7U%dm^(-)^vJ8a7zEx#hISA%f9a1Ybx@Dr&>F64*Wufad%H<) z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r zO=aBJ;meX1^{O%q;kqdw*5k<?!FYD}X~SRg{bAptI6CT~WZcECii<d{!~H9SptJA{ z(IMO5d`_qI=h*DGo=n0v@_q*TN1Rb~B&ITpk8DJlRXa*ROudIg-K94et8*W|ahl(A z2RLvW1}v%VuO9`Ef9t?PKUXTW2f&D}3vrtNJ87$D?Y9if%$|t@;m`i#_BSRV)jj(n z<)v7wOK4@tW$UKKWR0fsc^c<~vm5M`u2vnP<@`C7-1h}V)vH?GIQ6kut>!Y7%t_30 zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+k<d z?qR|O?7_P?Q6<qQfw{t}<omxO-y4wzz13f^3s3)!sA+n!=}0n9^jL6z6UFtDDE}Mp z!BsppJn%YvdkxcpH0L^o3RScOB1lQrKBaMr+6<a345~U+wL$oOCB}VtK17ZH;V1Ue zdj4e4ug6v=rEE8!n9=MxEf@easRS`JT*=LVd(P_|@2%P^bJF0dJak~<TFSD+7ZH;0 zk0}Z5&3UxSBxu~{Rz*8BKG13&fcf@9dyUN>vQ89KWA0T~L<vZ;GW*aTR}HF1-jedY z#-MsfGyBFAcQAdcd`Yx-bcvlJbsoc$%4C5;EuOf%FC1$=NHxR@P$wjlROtPJ?~8@j z{p*zq)|ri!j1uu-Pw*x?ETPT6(jkMz{tB)03YK+l%8c_vwo>j$aCcfW#nD5bt&Y_< z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$ zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2 zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44 z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9# zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZ<ObsOT=LjG!@YPQ+Y%TP2q;%&e6bD0<;#D zn1mKO23ndd^;;2ec_vb`0m}1R5{A-e6@I<Gaf6P$y+!C|ZytXPFNr}*6sJD;{rbI+ zRwg~nC(gjZ<H0G$h5d*YGLSy%7k#zcB&IGLVazVInCg9bgF6KKKSeDarF+`2IFlWj z%#J~%w5&}@qlKlZ9XX8WVB)+9_$hOD{jg_1meULyOnTAY(X(RGDVFl%IWWYYn*#Gf zs5wy97!DZR>O@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^e<e%~cuMqeXJnyNEIpgKcP&BSGZ3-V#G zgnyc%fsRb47ImAxDT<IihCipil@oTF)0wHSZ%ab4RM$MVn<5Pqo=k;p=s1{wkr4K3 zPn=Vq->i4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmN<Z(NFM5`sDgUq1$L^2{X$m zQ<@5WTkRGfqF>K_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+ z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY<e6n(3tLO%M0YHj0diWfkmtmb0?9X&BDU zs6Tl5CHz~g>6(?+R#B?W3hY_a*)hnr4PA|v<bL3cOEhhyT5k=(qGlmAL3eDw%3aS) z+mMt2HXmQhxUE+tIpq`0N<C;yWn7~iBEGnslqQO0EY^z~!M$i!^_m>J<6p`K3Z5Hy z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~v<S1*NrZ8z<H*IRys?O`Oqz|}FcV<W%7zks=K zjhgy^OR8^T{XcG5A)q#Ixd7)hWJZgOgp%s-!KiICJ<7)nHG%9IeF-=|(W+fGO(~eL zUZW|&`a8*ZnEW_JOqB-k$82w~>Z&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy z`6<o)fI|_!EcP+Bw?+2|y3t!@@OewwE@am14Pw<&ymmicd%g=pQX-4Dl6iXM5Mi<r zx3XG;>Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI| zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U zEoT67<jXapz33SQGdVleW*J=p6l-iq5HGgGZNX3gcnDtqDqT6Lxf&Cy#Ap6*__@(N zI9l&1+PZ(2&t7;rjbr<N3;GXPQmR+Jb=l@GOikZr!v7KeKwAmIz6g%ucSA?oWgocV zk;d7L7JWl9$p0E+2q4zN?=f=Aph6SF??-{0uw#y<{6dXC$UJvvFots-eZ*CWmiR<m z^L^@#CZ2{u<>R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g? z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`<G0?@o|1&5 z++RB?zrCfT7`A&iZHeO}uiAn(9Y;?H+|QxaD&F$T&wYPv^@%QzORq4V7h)QaA?IS+ zFB!z6yTuA4F{2w*bnO?o`KM7U>B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh z);73T!>3@{ZobPwRv*W?7<u$fR#FIIDQ|?d;-=4~nL_DQNR_L(s2gGB|Bi%bXbI)! ztn@6?NW&ORN!+hVmb|xpUMHeP-Y*-8YoeQ$=7K(Sd>m0Ml9GmJBCJd&6E?hdj9lV= z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SN<fFWRb2=9964Bd}-9L0# za&bfyJOi!#T%8p@)e{#M*HC~_RDcB;kF@?XCf0o$;#0Tz<%{^IxQ6b()`kCzGw`pn zvVV06qCVS*7lZ)(o7Gp6ekOkuzzxs?O%%&Yp1+cWnn|fKYX<Z-8@MV#wqF)qqV5*o zSnWk1rIdh6FPQo!-hS8qsY!O5JD<AlIn95(ae6Jd?fWIAFM+L&sw_o{)ZY*mza^{a zFgSda?4Av^CAF7cj@(}R6JgOpyFKsFTD;d44h4&<Jz-_1F5J{G+LW?&6IhxYcs@xb zl@w&aT)k@tK}w%(8c;t<C(7s<T@#tgO(Q)4E9>S6n-_Ge3Q;TGE;EQvZg86%wZ`MB zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo><o$qrN{Uy#{EYZAYV zhkQ~i`~tBVbX%VYXh`B$bAR$*>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@J<s@rA3iE(48`CbnWn(=&3DF#J=}a z4FTXYopd?mZI}}1ZlC8~VV`QLNWV;~TVEZ~uaDc-&}W%0xomagCDhH<gooe&naXnn z<tUH-X!YNVZ5{|0$(F_l%}+PcEZ$(<;b4&E&i=){rcO+Hkk=zdTK^Ffd=*LS^6eUA zUbs7^NXMxkl#^HyJ=dcJiSCooRo<stw%BQ{lj}<*D~N?4*Cut%z#~2r#dSIApv9<Q zjHtNwb#+f5agUh9@-@^){GvYFJ5KG7zCVqj%Df~m{m3x+{p&MW3!dR+pvL^5ZRmC0 zC309}MqA9m^d9B_Vk}86VV~U#6&}A3g}?sHV+qTd>jimqjvmd!3E7+QnL>|(^3!R} zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF<h<Cj<zZh5#4y5>)8V zyXXN=!*bpyRg9#~Bg1+U<pnWYhrolu{FPCsV0iobLA4JkV_p&4r@K1M;NHG>DYCt0 ztp4&?t1X0q>uz;an<Pmca*5{xy^4kc>n$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8* z!?UTG*Zi_baA1^0<wK9e#G~fPDt@KdN$SZ|%nA9j-17!`BH9vUH0o`1P&6J*h<;ef zVW;53QYa7AXJdrlTA-n?%wp6d3?_b6<x05IZ{WEejqFp)B0lVPV-bRe>eda8S|@&F z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3> zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+ zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~ zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2 zR)Ad1z{zh`=sDs<zGHk8(=f(sFR?;R<HJ%pYo-KSa+@gOo;(hTO4p7NJfbujY~Kee zGHQPHC}zX0H$dR?nrR`jLKzUvcA{-a5@SQ^UbQXYN=CS}aw?OAqkUt?H8F&>^&V}J z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx& zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb<Z@pezuQ&fWzt;cz#SUWI zcqV2XJ90lftQ?~%HDz)~)GJXKcKN}4st=)aT3chrg_EA{?3LcT)!JaRZ}=sv*>3n# z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5 zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0 z6TPZj<t(m7DtKHBH@$**2Zo{c79USiF>(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&<d zeHwJP(&NSri;iaCB`Kw+7PmoLhp%WLBbMtYj3S7->(YQrsY&uGM{O~}(oM@YWmb*F zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr# z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2 z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_<LSzDTRe;Ie+2@ ztS!_=d%s5$G(s@--l3N6rQ)s8U_l;8K^1JNd@9$&rF%j>mG0ISmyhnQV33Z$#&yd{ zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa<Q)$L85GR#EfdqV+R= zf3l>^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f z*<txU<LU~pP7SiW&ptI|Ssyh1rN!%<&LcD}5;B1lVw_<Fz9i;*7Icnh+iR{wkbxFv zHQ>A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~<b2tOP~ z47Nm=L%=a2H5%-;PoTI9Zf8Q{gX)6FgL99Sq~HCCHEEWym2icXnIe}8P_;ArgG0CO zf`4Rr(ciS_AIGt|OsCGh)=l12V2IHdqu&-WW<-O!NRu$)_PXwf_YA1=lIto-SBdBp zc;mWJN=QE`GD!ww=QztESZcD3U_Jx*+2x>p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=<O-p8_PlVR?L zh4I(^CY;zbW$TKH9!Xrci9D`~7Tt_MnPoyJiw3E%9=)V8`no!#&SD?Z3i8z<VyM}0 zABD#E*H)<&R|%HW;*e1VGvE8RGjpVF<el}tkQ-ZH>bQJMcH+_S`zVtf;!0kt*(zwJ zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH= zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcg<XB+ozL*{rR!_$M~K9Ao~6H!H^=h zwoacr(!lN?6COXY0RI?8^Y3)nD5dew#%KWle2X)4QJ|3aSbkvB3|XvJ4T7PtDp@RC zL=FRTdKkZak;Ble+c&|%U<4_;=Pv@V_7`H`L@;$HHik1Cov%9Y?v|ejzhoH-_ORGg z?z#NpZ8<kuALb{N_e(NeGkem>S+dB6b_;PY1FsrdE8(2K6<T$5h-ID+y%Pgc&YiJj zQSx)n1qnTmVVNMYY68M{^SPS)&CE>FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA z<TH9~Xay1oO$zQ#3a3<TF-+#*^SkQT;5{#&5>qW`YaSUBWg_v^Y5tPJfWLcLpsA8T zG)<v7DAcVy#7Wn65S?qZDrUX35PlA`Aoi+&wVKj813Fm5$JK2HOhK_!#&jYh-6-UE z?+!tn`6hPZXOsq_tvt7v))8yvK|C^QB-3YDBn&zFRg<%>!x>pKMpt!lv3&KV!-um= zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18 zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9 z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3 zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;<k-Rm zCOX3iwRBMS%2HbhB&55bKxXVrjksHaE!$yhFQVOkB9&b;RXWYu12Q{oZxRnIOVr=+ zV;u%|w58=ulh(mY=94oS*pT{cO%ppm(zvJWm<qAqWJ+tsD$mc#zQ-$!O_aUVS(xv& zlic&3<NOINl%vfa(YE-09OenqqI00N?`6YZ&y5jRWu1$*;ND0xgkIT8GGJ<pZ_BqS zfzf6E9oArEF5xqlU<TZaFS?_~jIlVR?u(-sg0I7Ln4&`lfjy*Sukl^_TcEsDm~(l} z+s}VbwTMn&y0~O&Noc7}EK>}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW% z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_<AT&sf>W z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9 zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY<Y65Bs! z6}9J&T3Jmx%Jeo0UA1RUWfC4^3~`bOw#&=(RdYfoBOtonl9x?Q%%ud_8xGfac}V3S z_t?8@P9D{AMY+HQj#M?0#<Loprwk<zPs3>(^UWXgvthH52zLy&T+B)Pw;5>4D6>74 zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#<Ofl>>g+o&Ys<k|1Ag^|(Lcq#YS zJ3m+ShjMEKKXbb?#zc6~_{8T^k&I4kx<j?2OLnw(e9?2R%ab*KIh|OcYkV*-ppCU+ z&<4=VL>b>dX9EC8q?D$pJH!MTA<fJX`-o>qa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?<g~DII^Bcx&poTYi zLVuwObActGKn>OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wM<y{<R6>c=JV{XiO+^ z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9 z?DQAh5z%q;UkP48jgMFH<isTC5e=i>Tf#m<K<awZyB<dC!4ZWVVj?0l^>j?#z|=w= z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5 ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@ zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H<xfnQ6>?;tWYdM_kHPubA%vy47i=9>Bq) zRQ&0UwLQHeswmB1yP)+Bi<gYuF=;5%P5fb($ua)vN{<dTS}pc$-noYj&~P(-vz}$v zu3jpy02nRu2m5&K8!o)7k}0wq<;X00@Xx7A<mINk!<+VpN#`s2^mC5Ofe4$`&tl#* z>R;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO<y)w_Q~B}%2$aGK(6Ummn<`kl%; z`ex#21X!9UBm3Vc-U~%seDuubCS=6He};CM149#aL;PRFuT<U<#xD2*e6V!HB9TS@ z-1@gJDr5k3GW(TMRP`M0C}32Sp)@iN+m7Q7FVkR-!Q93t0T{<wiGM)0t%$WJc@v;> z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7<xlD85HX*0-hkMZF zkjr5ZvT-#;Mth(cu<tj#atu~`yerH!&_Rq;m6S6;E=E^t8E;_e)$<aCEe^z9S}-Gm zR}d6r*AN)OHk0vR$wldVl&!rNQoW*!$WBpDRw`=6+$-g54384TvT>$WGqNy=%fqB+ zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R<H!tySet(EP)5Lj`0&gFCZ@coqtHFV zq;TC+Ud)isOY`?v*vYVa0u2u<OuzeQhQx-th#lEeb|E85m0u7j#xz<Q(27O50YNZG z_X!_ZeKj36XsF4fNkWJ-X&&`{PiP|d5I{)ntoL7!_lyc3!C3?s@K?fxx2uj~W~|6a z@37XYDSt)e<cy5Hg6yFElSnilL`l_9#54dfOVI=v!_cyZ8P0}j&eAUCbFyagF03t) ziN>4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR; zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W? z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*` zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wG<xJ7RkURX zL?-%U*5SbEvYh>tK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8Np<qwhY!r zPGi5Uv36_KU&P;Ysny7Yn7Aq*w^YJk+j;r5!{54y2KTRW=n4P9Q$RrUV@IzXwTG1s zzW93%6>QW_*<cOtWE@YNsI)L#ntpV!%&tkEA2b<G(kTCfol|w3coc&So$?P2SFmWk zBWp7V`+IlUb`e$B*<dRPJ-ehhZQ$M{k7#onRlEE~dw{L=-9ElX-V5A(;-ZBE+Yf{^ z{+-Mf7yUbWUdboXPA1sh*z~xCt^6m6c{6_JCok9HQbYRhr51?*l;sOM;AIxuJ4Q$p z`CgAJ+<9O64PUJ^k}^%FQclq}Wk!;S2TZ^{;?u15<et5AgcPi>a&kD&ANjedxT0Ar z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT& z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eE<F$7REzl$4Vg!h~1-QZGdPJ^bJ@K4OS*c zP-fR4iXy^QA;#1=y2VI*LRPhjykLV({@6o-twx$xfBE}QnTt6vqFElm=UM-(NfZCi z=lx&9)JZKEFO|hbLCVw#&(sbpFfqulk`VBkNi?$lD5(B0WM5ff*mCA1f5%740p~O| ztQOb8UFr=BBea^EKn!z+v}nk*YvS7NtKQ8K+R4>RSLKsWf=`{R@yv7AuRh&ALRTAy z8=g&nxsSJCe!QLchJ=}6|LshnX<g_y(YX??7D<ya@4%V<9h+|Ic_N&p+^6cL1%rRw zI`_<$r7i-QeU%G7oxJ6b$}<GVE+7g7j;H#VPbD7FMNK~{pfkqz8k&Qk100>IK)SNd zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b z`*8dVBlXTUanlh6n)!EHf2<Z;<pqvCOZ=A2H>&PDG8sXNAt6~u-_1EjPI1|<=33T8 zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P zd2F9<3cXS<T6BO6%Wff4)w8RK^!PZ}fJ&%B?D(O0phg;sxkMq1g;&s2yVy|dlo7$& zut6p%Pt5t7R%%o@lpz7mamy48(tCjGd57eFCrjgxV_IjQOo{Eq=LdqW@am;MINbXP zIQr$cxXwNaQ_H6v`p4(aUBXrmz}**&%<Zzfbtj+pDbBMu#7x_{KfkOxH1}OCyx<aM zu@SXrn_{seG?|N7mo)o<BmjNPRWwBLiQLKA5vhgn!8ZTe76=gv#-hh7?ex$Xtz9>} znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G# zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&% zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1} z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np# zvCZqY%o258CI=SGb+A3yJe~J<XAm<h+@x%=dk!>H^i{uU`#U#fvSC~rWTq+K`E%J@ zasU07&pB6A4w3b?d?q<ZUdN`d9`jSQ6}@hE_t>}2=0rA#SA7D`X+zg@&zm^iA*HVi z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+ zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID<T*k)hrSs$ZMYU#eY?f%J<TYM`ZV5WD$80 zG6we^sim<v0ffzyR3HV@k#_?T5yB1?r-E~gnJ!jx=#l~KWX=|*TsGqUMbMy7&Dlm$ zkg`~ug9}A)NTCqAmF?i{?tn%$xnbe}sXgn0Ns#1Tz9yxnXess97ti7|9c?mVx|E|M zsw|@!R#sWRksSuwE0qkq{xVpVBsF7Y5`nBxFc=Vb@S*=tu3_rkiM*j^3$OH{X3IB$ zsB0-+Fdr>0bTH-jCL&Xk8b&<MC!nk<L_l*|?aq$M0{!H9`RQ21GGD+c^BcX*2|SYI zugFZ`gqDwwr<(%$82O_6nOWDm!eA(RGpBfa<+B;MKzgz`W@E7E&XQR*%r|=RjJ(AQ z1GcBC!v)wKu_J~RfHh}+ZjQp_Xx>;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1 z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5<mj|3vYvi%XA!0DTe17}~aG2h7eq*~r z9|l2u19&ExDJOsRZ$@8=z;;HhZw$_SviSm}uAwc_ow2rT=iX2K1>EKUwIxjPbsr$% zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@ z;v5(9_2%eca(9iu@J@aqaMS6*$TM<otkyEUfDAtMmVO5=3I}?EG6T26ue>w!S>H(b z4(*B!|H|8&EuB%mITr~O?vV<E-8Z6S)k5lzkIaVLJC566nZk>Ef%(Gr)6E=>H~1VR z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE) ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn| zIf7$)RlK^pW-$87U;431;<klhG5_ER>Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV! z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8<Yusf zN5l6kO$1L;ws<m1`Zh#?`63unV{T(2V21gcCYV&FhZqe6Nass1MK|Z4KyXbkzmY5} zZ?_?UexvC&zoC0o1N+Vde1)9v5Lgfss-n~Y`wrFPCpU&f2y~X5VS8!)_$jxG?x+Q> zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3 z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3# zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli zg%r)}?<O#LIQ<x^r>%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^? z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2 z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=t<I5W)Q1 zkAGZ6mVbAZdwNx+cM^(`G%H7T29zgao{aSHP_2`opHKbyoo@LZV-ND6*ct<1X->Gp z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+T<R{A2|{q4h1HZkio;YA1t6>L5eah)K^Tn zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8 zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM<?RV=J&{dty%0kuvcV zvDY*i(P?K=6~(~xZAsj_c^PMb&#Z`Y|Lurd8fSVQU$p5WH?x?Xpj)rsBdv*QxJ##A zM=~|M*Y>=()T()Ii#+*$y@lTZBkmMMda><ZXF}$122DAC4inciHEXN*L_tT(?E|%+ zt7Py*&-=;dUzJ7C=EUZz8ph9x9vB|ACLKw@s&c^<^F0YA3k;p}2F-(+6L*|cO&Kx| zGV=>7s#O(1YZR+zTG@&<R7(G2<E$<wduz9!FnD8M=}U%cAl$d*6}Sq3Sm>}!EXFG{ zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67 z-v-Fa=Fl^u3lnGY^o5v)Bux}<gD3`?Qhv)PX)U=^-kffCXQ$p2csKZNM5x8!RMUOX zrn9?t`{WKqC0A+hrI3CS?R@viwL~gPCU^BRur-hw0CEwaK7f#%S}_w7_H%2lZVcgk zQL;J=ry;kmi}ZUr-!!m`rCH2ERpwJ6Q|G`5r-Xy0r?6;<g{<5%zr{9CmP^vf>bNZ~ z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^<Z>g0kZjg(b0b<J|) z&kyXHVzP24v$IxevEHN?k6>Jvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_ z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}m<cS(iD4<A0N$g+SqA zl%O;ERC__&2rvP%TMdpBGey$+i_(aSuJSoOVG@%hWSWo?*uZz-46eyt;fOY4@j*cs z(Nlf$ie?ANAv6K4_-a2q*kk+$A3)y-+s^rqkRC|T=Yj2fF?eZ}!9uZg3msy525K;C zG$r&@M9n%7`Sgm&aIl}13Vz%ip9hGWt!og5bdx1qTG)j2nL>Tpb7&ofF3u&9&wPS< zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T<Fhnjc^O*n1^SI<&BNNFb%X zHxbuJadh!4YtYH|wpEljX5ubnIb*m`KO@(XQ|K!ErMf$l_=~Nst8I^_;~LFMY;jPd z>=~#E<K9VH{%))?p1uEh9GO_D_^6?&!kOhc5(&300G7)A4!13Ozm(lvh<tIpVpv;w z$Zq4R)GbGst(09X1uL^1TcWYOO$_QS5|?mAC8?<QY1GusEAbgtUMGPRN?~sU>McB| z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$ zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2! zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC<By>9*7<! z3r5ih2IB%6&?*r6Z<C9~bdv;$Jz)vwe<Nu0(L5l%QJ@HdxjvyfFnvb*!a}9oGNK6E zqHxM7juWe=SSnY{v&Q7EoMOb}E}wD)CsC?77@`e0AQsRgp-@%+t|BGqN=<`p&67Ay z|1Ca)Xw?J?`0+tpDJa4VbOI+nCoMRhc94J)*YTm*Cvm6NKcmS0S8lEaP@BDlDO2j! z3B#9mbQbob%QSF`NGB1uJE%GFPC5TQVb=AS_#@8Xn3od@{x#f5jU7spPqbI@;gM2n zy<$hk+Hy??zt_V4Zq9&R;7&^l#vS@`iD+}{y4jLIDKXTJNC<QY9H?-HD>Je<e@N~V zM8{whHc_Y)oJRTaE@}6XBK;XqJh6DOZ%b~};oF1$Ja8>h)`;eec}1`nkRP(%iv-`N zZ@ip-g|7l6Hz%j%gcAM}6<zu1Fy2Y69l>-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ z1vNbya&OVjK`<C+)|Y=S6qD0g+yj}rJu+*{Nv+8EH`6C*w=QvJZy_0aJA=(RR*FuE z=Ve*%{0>2pdRr<aX@8G=KRVz8TtomBSpcq@r(_ajX~o2yoaeZ}oez2h)8-QBk)?}3 z07=L4P3BU4*%bpPu*ZY*FM)E8NlN*R9eF#VQ}7$t%LL}o3#|L+gi2ok?oW7%M=|~p zC5<%;sq@8S<pakq(618P52%D<(#5rYl@k)nhPsY1k)aFy(uH>M?LuK6BgrLN7H_3m z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^ z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{ zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O<sMJO3^;cmzln7W^zYPW<c) z|Nn)@|5@a8iRp(7<VO~{rdqT_5uSV!nd9F~6^REIQGA!cD-9=NGWybr;?0kXWZrN^ z3+v>_{*OfMfxe)V0=e{|N?J#fgE>j9jA<EEh|%C%>ajze$iN!*yeF%jJU#G1c@@rm zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cH<tNdh?t1Gk+qA{Pne*ng|&%*k<pK?D`Q}5 zVD>LrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+ z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}<H!J~9x8ns3P(-h{dr(SdVxo7mkj}AsjC5HV zo6g6-m3quL?mvNQgld&;Wk&NDE-R7EZ)*~rtG<Lq_zyu{lXW&{xOyIFvsws8eo>F! z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%* zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(<nvsf&9hvsWr%#7CsQ**Fe-0<7veWn* zbd^GxM~>6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{ zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`a<X-1W{<+F6zY3jg>P)pc~b<Vo8 zn2YAtCd;B0Avz@7p^Po{xMRCQYS{JVW8Z`GH$zG=#SS&KZ2A$u^x!Isx6mLPi?<ZN z*{kt-YdG0}`9!9hbnjn<=b=7lTPuWEpF+k^SZAjar2B<DQb{uEO;vTv3FqIDI3!LU zvasv6BD^}y#db_7<6NwPQSk6X)=~uy$Zd95xS~u)s@;O!EALmaQ@kYB`EY75*h2)s z-R#8r)o{~&P%kZgz*(Kw!pn_O3rshJwHWRYI|!$r!a4#|kLw{Kz&k3CZ#RtrYB!Yu z*A++a?kRokM)%Uo3N_uT!~ugsw#&4oIID7K+!k+)I;<)Si^E{(i)cD@HTao5;+q!0 zbwB*KzCL0ZC~g-PH7MbBVgTO07?^K#9=bcG8FQEIE=(6id^)U|^hS5OYQT5$J-!Sa zBvfO%E+b9eID~Xvvo@#oSJO3E?jiXTQ<upuXRYN+dqAs$<{%yP2cnwB9G5^{RErN2 z5a`n%B*&Qd&SoW|&P~{87+q;P_bbKnMD-j91aHnUm-Ol<>E~mM!i1mi!~LTf>1Wp< zuG+ah<cN%^mGc@zm->p^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO& z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w; z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6 z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ zyE<&ep<?&Ja!<vf;^Rc_Ext&<l)$GW^vhyI+JCe2O3`LUd|)s%0qsYi2%EvJz-tM3 zKY=mZW?7k^N!wTSw*{;yb$3mRD9vNKYL6QIU4KGF{JZXpqeFF?UNh<Hsu=#~nZ?*` z8?`-sY#3wWljoYqahkg_LR+fxC=Ok@srcz_lf5JG(Aw?<nC(WNHZ^iNeqvbZ784f| zUF|zFa%ZSkIT}iZS*J5%>$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD zpA4tV2*0E^OUimSsV#?Tg#oiQ<Gc63Xnq8{B?9(QSV36W&7QYB_fY=P6DiK#CwX}S zgr8jyAKT1k<~JoNlpe8K#uDWD6A;w{rqu&jtmBFgY_BM6uK=HK6yOqhbD)4G&d4Zx zgx}5JZQcjx2iiz|qys{K@f>>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X} z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21 zT8@B&a0lJH<Is)g$je?v)zrGnVnN*a?`1A_B7jSFYFL!G^YdU54T8BJ`&=Grt0wbK zm=GTh#mq0;L&QL~1)M*45{rzjOV&&Ibr2i!Ltb})&Q3g>Vn1I$I3I1I{W9fJAYc+8 zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsV<K&5XXRr5uQ}LF z+0CB-DJWvs=zyhUDM(~V3gV_A(2WHskwSfbLhWS!Vr~&q4bY$lqS1mvz2zv7a&eyv zq27v0&hua?e7Hjc)2G9WDUS0kzHi?zAo?IsP=#m-cTywmevo}cL`cE(<Xi1(J>j`P z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5 zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9 zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U zuq;KSv9VMboRvs<muUdEos4J@28PT^*5txvGZbKDaszyz$QdH{051B8A}wS50ihLR z^}*xLvwA_M{Vi%9v2EHPcjW2dClsn~)MPlr{(wj9a}gF+Q-M)HC}0<xaSL~ZQN-$F z7pVo2NF$PZ!~7c5`YO~K2nWlk*2_#MS>9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC) z*HS`*2+VtJO{|e$mM^|q<Nl<aNnR+M;uGuLoy^|5_yMTrUl*Jc;J-xFCNFUlNS9`1 z>v1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{ z&d1}<wI2Yxf4k?!umu<cm>D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM- zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY zHezEj8V+7m!knn7)W!-5QI3=IvC<r`G1r;-#=KH#^bDsbD^<>^as5+TW1@Ern@yX| z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW zxo+tnvICK=lImjV$<vu==Ri_*yyu6*srp=+w%VMI++>Z|O_c<d`u5Z43azu#^&ypv z2XQf<KjK;)X-?wB*Sewpf;vW`6)cGyqY0^Kmt<sX6AgDavh{KfTqndPyGjf#w5CPH zd5wvsmzb)a>Yj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$ zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailh<R zTW0JnWEDqjPC~D*L>g_|0`g!E&GZJEr?bh#T<kIti^!4|N+ecge$#dzW2oYeVA6W^ zTf=2ts!v<D6N@P&^Yk^QSZQdun$SpPGW!lb8i(GI;CLFfP&>pb8siR=JxWKc{#w7g zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5 zv8<xjOE`)Xl$&nfKK_(a*GxWq=IP<8*TsiQ<?73@Fo<n&f=T#zZE61~c1WExC#Z*C zBR?0KIU<_708sM|Ni}WhSK@G4$2@TsRGI0k6P~Edb~#f7+1%73{4^^R;XGADxxf+k zSKP@Qw?N!!n%j~Ps{Av<y>d>dK9x8C@Qoh01u@3h0X_`SZ<H1Jt*l+r%vhj38gu7( zi%Y(Yu(yCRu7j&QPtzY&|NIVK<{aX%!5|>luTb@5o;{4{{eF!-4405x8X7hewZWpz z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz zk8p^OV2KY&?5MUwGrB<d@yk90?8kfe*7#9+-#&)Q0>Oo?ki`Sxo#?-Q4gw*Sh0k`@ zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&<HGFJsB^*5Aw|=c;4ki9<qCK7e%Vj(VHL~6 z&DTbldm(&0B0fTx|5re(!MJSE^QU1#`&m2qUm7Nf|9k}h=kY)0pa1#ZNlE^hgpfxc z$@}d>MTN8bF+!J2VT6x^XBci6O)Q#JfW{<sb5)HkTF~_Sa_M^vi<UXTocVSE>YMz) zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi<J46HHYSLL&OeFRm%u#)=VN9PH zxaTYq?JHLwz2fTT`H!RbdHX@6n6p6?mn|3kIU%%1k}C2(3hi^IDh(udokZ1xF-p+u z0u<3zht=l5wn_zAAWK?U0XT-L6xs5k3ZJ=V1H`#dpB4>*Iz-8uOcCcsX0L>ZXjLqk zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4<N?Fq) zn=ndP#eVyN4)AOmF>A(RsYN@CyXNrC&hxGmW)u5m35Om<gLIl43|ftfxD;4@0U%W{ zfCt<dp}9hk8dzvf-#}P{$rRcsI2FbF%>WwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6` zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm? zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo& z+~_?7rA<BY)cqU~+}^_%&5(AuHg02IxD?AG?j^Zhn%1`rT{GFYPZA>+(Um&o@Tddl zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPV<ryWQl0P?MiPwBL z+vex2kKClH;6k1Et=IFG!&SyN$8+S#_UnR?aFa6Eq|~rBS)8W_vwKAj+o^X)(9027 z*WrGQ$Ej`dD5*y_K^!R^&+N3W?cTJmXL9S<fpm^mH*?0O@w{qI>v;Ah=k<*u!Zq^7 z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a zZeFjrJ*$IwX$f-Rz<J3Ld3QGgbbgt9L|A^RC+}TLgII?Lz8a4l887}}cuTMGGhsX* z9&mmFqP?djyuRYNLb|y#gPeE?&x5*{yL4yV`z1bx{$hup<=h=EzEhKN_egi{3sPw} z!?<6KVR?6VYA;nDAIyA2AKd4Ab>r_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~ zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3 zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5<W5z;@6p;6;|O z%1xS<PrHS@9bs_>@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*f<w%m`;Qd>dpa*n z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;% zPw>t^kb<Y!37E>W9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE5<?Y(}PPRapoJOytLr zH%=!UQ}Y5J_(3KVTFf%D7DXvmTFStSsvMk1_bhZw*QC=UXI9MplG;au>4t~UBu9VZ zl_I1tBB~>jm@bw<SOr`xU7_Kg$PU){Os_I8cve<UKbUg-U~fB$8f2Y<)c>0Aljz8! zXBB6ATG6i<ky6{p$)@{!N}M!yKr)m#;X?<H(Z75&7#=qg5yAe!nNXMBxO$uuu4{+; zB;Z$SC9Hkye>ByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV z7mlZZJFMACm>h9v^2J9+^_zc1<NQ`E;}bmano=+KqF=1iDw+>=JjL#qM5ZHaThH&n zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*<pdMvH^(qd~4b z&U$~Fo(WzrnMy~ykcI{stgLy~unZF&M1>Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec<Vnys z#0P7P+6h@<uM?dbofaPCeGw4^GKj)BhZ;UWJ+<6Nx^ge1;*1yP2rFzRz&wW{MTEr2 zHXxRizPbhoG%+`mqNb$aK0~}2_bn~FMY2@vFZ0AA!pFio4f|r;(+@Q1=`h#RqX!CO zrKiCBy`_GlRuCrdEk+*L2qw)Xi3a$4Yu;T-ek#YzAVQMsU=A4R@x`B#O+Rf$w;qdW z?}xS=&C)dEt1bY5wPQ*Qhbfh3qM{iKuWW?ZRgK1yH>4~D2~X6vo;ghep-@&yOivYP zC19L0D`jjKy1Yi-SGPAn94(768<MS&a!S%v@?~BDz5em7uiJCVng8mCX4kKzoQ6PZ z2Tk0a6O=C#;z%H(u6zVb=|H2_?Mkm8Gc%N0k^Pp7o_nH69Yyq@mT_v(ZVS($NBa&F z6xwW+#+_X3s)pC4l=DX;IIvOLHG0qBsgo?lu%3&9euMN`&SyK73Bo<x@&AHA*=am& z1@no;r9Z{@*~p)rGlS`fyJCBB`|!&7#=qvn{2=@K-S4-@S0u|BDv=7_-i!Ic_QmBs zjXb>Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S<uFZ_;?KwDx~9UUr?%y@ex}8 z_9H~!Xc3m^qNrtT@3y|;1c?=J#VjGm2#~m8gbETU8K{z_hDYEnF$+2Q2Oc9Mp&ga4 z#Mhq}0`Jk@q}^00F79AOKffu=y|_%9nA^(yl2kj(9G$y+k?BKg2^S**>@DyR$UKQk zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{ zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W> zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7<pG4gt)B4$6wsrn;hWv_Oig17Y?jJ&7E4<tmZn zOCP%0<fYi!jPBWOw9w}#K7$n=jY!?ZAO-w*mO9|G92xj4(OrW0?2j);##sDv4x>?- zP8L|Q0RM<y7372ia7WA3T&finv`tA1B}OSEw3SiAZho>~sB0w1w53f&Kd*y}ofx@c z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS zE|<<io8>HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76 z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6 zB&SQreAAMqwp}rjy`HsG<WwCuHn5_unq_y9e#cRc8<%lnA}KbA9;x1=pR7&&N!F-G zjdr@AW->({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0 zj1!<ZReHVMm7dlC$9b#yLY{IPu{3a%?*mdb?Ln7M9kK-6dElH{UJFdM&-U|~tV)|A zbcx=;RI5^&eB34O5VoIsgM>K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd<IbwTtn<2Y5Nlu3=6HqY@ID|;XJ`> zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w? zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!<TEFe=Er$+pf$t`J5iiJseKY6 zOKcLAm!-S>AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM<p_$AQsg0LlIt|lbiT;}c3 zw^mT}aw%3C?rkh$t46uo%m3)>@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D zr5?A*C{eJ`z9Ee;E$8)ME<J+@MZh2qBdMetA0Ap@SWv(BczRx8QLfufLwM}8P0mCM zqL)NPc0jmFO8S}^g)dV@LX%jnUO0Nfp9J$lfwZiNA(bY@QLPrgG(eM{>CqatHkbHH z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-Q<p|>DbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpi<UUDp%7N@Va%&%d{hBCU%M}b3(4mHNaXl%^x zy!Jj<1JK)G8qrZwHKQaxXMHgDJ>Gy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb<P1JL_~J>|J{IvD*l6IG8WUgDJ*UGz z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5 zM#v*vtT}*Gm1<v7k$Q8)e+>^)Bv!s7<Pmunc}KvQIoTurYA$VFfn68@NS2!fMC<J7 zU5k+UNB7w2H5=t<P?=shL{5Ib0NG240C`@MTdpWV847hZ=*V+;L(?h<*^25EoClgz zyD7a#m?)n<0d;}sfn7Zdh>2T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83 zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|da<c-!XQ(Z6e^nFF6CW&kh!QD zJBO*^4S)F;_i)EMF6B*oE&=d8{@I29Wcz|o39n|gU$iV{0mmf~bE{AKN63AsId2PB zg*_mqkTRn*8K3S&kIzHn;I0dhO)-->Dly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q<l{uKGQ*rm=eJZ(-a1nFbLMjzjPyq^Zl*<ly16J->>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&<Y@4%3KJjIC&Ci<sWjP2Dfw= zuyvM4SPH14EA{O`2rzE>jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~<Lp)MD3iR}ejbGt7Rtt!H zbM>a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9<p~_g)xTTw=X;R>jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN<cEfx!)Rv#OxA~Op%fRB@%V*lRYTee=t2@R^;Yw?#mEg+C=Q= zojy#mB^&>=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3ME<NoNRheH>by zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7V<ce%+9mXv^Yaa3Jhd!0 zVk?95!P$`MAquqor+9D#c(4q*0U*gkAX5okY6G!AZg^p?IbWVd!~IAXj%<mibcw^E z0M3HMVXuXDmhDXxE;Vjmfy%F+0X%+{&lK~`;T7cS`cLHThTCpHZupIqw|&Aqppk3r z-&np;eZoZM2&XOVoZ;>I5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm<tGIMBC*Y4;VwmbLfiAj7y{1&1Jb>!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5><v|;Qz(!vWawg1_tKdRVrHI_W5DOUncaXPaZDhIYYS>U2fU7V*h;%n`8 zN95Qh<STRuU64jVjbmKjdqZSJY%8zw4$Tfj@Wvru9*X0H8*3HTGDxdEq_neggWPUu z?xft@*y1f4!?2gYA)>fD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z<T?`@;|?};a~ft^83ljDfAQ3!94d~hNv5n{)4AwKJATa`zA71ee}fN!jaN8Z#4EjL z-f$?vQg}|9Xo{%n!=1Mx&dq=e%YrtN37{wyLgfqAgdwo({~~Q__5WR5p+@9NOz51F zn`CL;9K(yDAz%jg^1P_Hc`Bi##Ur7V7Egt@oIK-E#LjT)$}{W%w45C#V=-fOXxI=t z^7_J*h#~Pik#Js#zy5ys39=Vz5a6#V82w{&CtitoXAwOl8~H^TQXd+vIaX<UxmEZ+ zVA?E;a0GBuKV>%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3<vj3F!Lf~Ms<;i zbhaGgzo{F~=cvK)<7sgUCyNidX;S2p_?WD^=yZOmYaGxQQ-~@!!w5=kxBhe-MppxS zf%rYJdpz6$k?w>j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0<h|8_^}%F!#OlpyrbcO<dw!GUDNyxENZ?*z zXa;m>I#KP48tiAPYY!T<suq7MlJU4p2C@*^!}?Z}Bs+ffZEdwIV~rhP4EI#LVSE#E zKAv&oq@=|P;@PtQ@(J{wnUWAbS17e!I7@`^bq|U&TSNB%V@)WhqU`zr*SYoRa15`t zq@?ryhS#AfQ#j8*PNC?6U-qg<^sQ_jI^Oh8HLIjwIVAJWx<QIp$*(EZ9$rxFs<o(f z&qU=|nW^RP$x<u*s5N|2WXF}vN|`Y{xjc0I<5vRQ@T4U1{aj`*N<gWmUnEJdAd=LM zS<sS>dW(o|KtVI|EUB9V`CBBN<Jb6P#IAaNVgj&r>aBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQ<D_NGuAv_X}p9rcKdkJ-CY72 zY;M@t?NA)MG(kc+%pPv#HrW@CR&+GK&%ZfgLEA^}da`PjtxP`2he{N2tMkOQBzSdU zqbmpW3;gHqz$`asO$luCw6=w|wgSS|BN5dhU^m;NTBn4zBEe1bKmQJTcIjeq=_4;- z<%Lk#6J$}CB#|%0(w^Q++N0xCR7{uhNVvN{SCZ__oxC7+<4L39n=PVh0~(Pz$Z2kz zsD-eU@YACA>z!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLi<P3fz)VD6=F(UIjNi&4<Piytn-i3OvEhi7O_1<!4Bb=elUisJ5+JC zX3<Z{mUVV<bu6}1{$m+MIUKl?IZ(DneHsoYb;kLtxDj9+Q`hKxkGfO!F3`9no26-x zqe3&#LrJ-tqEc7B*%q?Pp5?r>Z_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%<gf6*u=C<iEed;KBwJxLRtV%EsYYZE^_I&am&FwOQIrs&rNv zsJkfwCitU8wV%PAV4*8&C!5{qUgyYY#T}|<>zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^<z6SRx&;ge+MnfR=LP4lbl*GlbJMl=;M7>c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZ<bg~PJE<rY>tJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=<qwp0>}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK<Q62tM^ zi3!pz7{^fEGTr8{(f#W&Re=O*i3#l1dMP2CM@R9Dz7(}LH_=oT4s2({F6)NqjLkfn zWyb9}W9)`Eu!Z8~Tk)-$ftsw64V9oQ!&L>~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e<hIoTpEPx(5QYd%^TYj)|S%=LSr!JTs zz!;*wB%3?qZoTm|(U@GJ+#WuaK@<xquK@r4JW{I|?LhkR#^Q(lZ<%iY8sqr4Yy6*A zRP{^`YZZkjC>6m_ozRF&zux2mlK=v_(_s^R6b5l<OklUMWKH5m9CK)GA(}BNrnAhY z)mdhvSSE{gE9<H_hJ>u?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*<Bq@X!wy{g3X=$ULK*~ws9t@GTdzNv=8$q3p#^9{iAshe2<iFXWu|*=#t*^ zHiMowYBuE7!#m)z7kwaYz#Uc(_5ibIU;zg~@927U0DWlH<0UlQF4CPh*e--2*e>h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2<C3EzZ8 zhJn#!of$=`Z!hW|DTzs`N(xHpzG~5_y<$0J`1RsK1TZ1v{7!~LCAC@6G637W!~rk| zLx+h*<9F&&V-5Ie2REOa1?BIx`oEMIjt$k+8IcS%|EII70E=pQ-+-W`(g+CBps;jE zhtk~$2oej@U6M;lmo$jPN{1kgAl)GCrAtadSp-2s|A(twE-P34{pWf1IZMp@&U{nf zIWy<|G__isvfkx&jFyQpKg<vGx_3kCalUAl^%NntK#6(2lW1e<U)CGbFQUPSU&Hih zN~SsTvwJ7vUK%oWe<^FW<$VJlS91FR+Ang7i@BCO7_QPS9$yRY9L+a`GWU$_BS$My z%KO<mCE#1|s27xKI4jobQyI#G#shn67pXsz&cAj4?7POazD<?>EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|<HR3fdmQIEk)2Sh}0yO~96C1H<% z?QzVeMOyeG53X0w(8tYWACBt{TjL#WIIj{m`My#tMB;vJ{a~L+FxMlWMgTKK^KR0o zJ;X}EooSp!+2+G$e0wWSL-uZ;icHwY>xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSf<lJl-iWu&1UwGnK(pvra?!Ge}y|x}g-8E3%rQnB2p@S*aS0l>j z?#HY$y=O<nlXsK9%dUnfsdN1u-Swe<-W_b&r&^uHFD7qA9zLa8;*KhGP$Wd1TRU3& ztEwlUTD>~F|2pZs22pu|_&Ajd<gZcA$Jk5{2YpKm>+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Y<zvl5N}16U~{2_GuG6q#_|JfMPqxfGtP5?Y4Mg| zyuvVEU!;&g*bD4Uw&V&*(NB>d6C<gbrz78@@`&t*Y)2+zhDQ!r1~__bZ$!MIY{hdK zBvhOU_=wf9=WDc_17is97>Hn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j<aFr1R~xS}s~%p?Tvbh^pTwhKxj)!tQV8+Fl)bzUj`73ZqG~f+ zWSqF|`Gl$F?L)li$PwFjgAh#|DN8z!3tX5Sk_}`_4}-ZspoOseDU{j7Ti8%*vj}b6 z!IL&Z(PW4r+7(`f(&N9n^0NAMH0{AUeVSnnjZI%3h9AB_P13b({Q?6h@ewu{vztxi zi*&5Pl_mU)Ysqz^vmyhLIWWvN)Hw=`Cx$Vrwrqq?{hpCEd*k5Ph0hP@wEBG#s4-aR zr>8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3<VugKu#}L&f$tPP-i|8Q%zt8Y0zy98R@0)>FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc<T<>0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$<DWdXQ3R#5&V?~bks;#i}6WLoWplVhIsF#fA%#X=}+{bA9=wr zM^>d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjO<JEcz!71PwCy&w1EL#RH@3db^Ta;FM00H0|Mp0R zTpn{$THw|Y^1IL=kOtj&J`-2ec-4zB&kyxvlF+QM;i6t6y+_@6dbj@S^owU3pQ_HH z9V4D!R*q77p6}D`1gWi#=H#L`<?!BK(-r>o5K<lY0`DON8bR;qp%j?!ikO3v{CO;C zr0hoVGpgX&#zAv@v~t0WKK3UWhFI@!4za_cZypgom@RlHWXjhb*zkH>qvn|`FLizX zB+@<Q$;$QqzF2GLY>-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wp<LcU26iy~^KE9SueUhM9hSw5kFoch;KoAlT!eZTy6~rH?gNli?L&L}beKU`4T0 z({~FKWDwz-RkBOo`jHu$QZ~kQ5hqg(17SNo!H8^sKoQ@sO@9*??XWU5dC7fKZfAM^ z2&bW;u5NQsiN78LoRoCLadg3L1V;2%Am2iq%z(Q$=@X7Rr39Bjk?oA-7B^0IaaHoZ z3%j;&y}ZdKT?VLFIiCAzK=`ZWp2^>DCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8z<ixvefPSW(R``v-ey(C3!C1-J|HQHNtMWDk@k-%_AI{727f2fp)E3nzP2zne83t z(KHQ3Yp(UnUc1B>I4XFQys}q)<X?lz9;PIDH}W2ejg(xq4^Dln*?prNo2!$AYL(&x zQ^n*upjhh><`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuX<l$E^Sy6fzZeBWpXwtZtJ*+^5CvK zm5o@)Vb`rsjDy{;ty&^h;yuf(Qr~Ixfg8~=YlY3#vkGe-He?J$+-qAU_lqZP;hegA zk*AEh4ihR~5Jo#08klD@qHx90U57vSVf9#s#`LJAUz(qMmmvGKi`3#h#kmAJjG9h6 zQ)B)8jyj(255#*8@2i<tO5B5CO)}P|yt3DVt)qJDZd`sE+o{(_Ii^q?>rHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^Yc<Qa{ zc=f(3k0FXE4g8yHjnttlIF~$;u0A21@bchm@(lE%*(K8|UX&n%>PpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s<ATc7q8AYRlt3+=;0YO5Kh=m6 zcJQSxR8f1@n@G{FQOYA>0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs<BCtUSSVUPQs2-L)cP6#hn*j_NRz!rNALK#^L&c9rXdhSPT_+NmK*)Z5xF$xiJ=3 z8+Mc<!WJ?e3I&g_14W8h)QxMzSB66MIj*RP^*N)W9*O3{I9!<#lwG5$#G;69c_Ici z`@Ou~cYuw(AE3NQtoUi`%GvRnV9k1A0~o8pga`->=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$<j`_j%GZX3(oc_SxO~-G*=c4ZAbD8%BzP zd)Pi)VtGNDLDj>NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<<lO{*K50r$dT8<<B_m+L}7)kTP?9nuT8`~rXnwmPpNu& z_<Hj7^*-8j&~7D0OWBmB6J|2NeYzm{G=39Rh<aW*6|DbSdXI^GaeRfwgIs@eF%-$X z>JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+p<xU1eMV2Dp)d&jL}1T-4yJBks?w&E4)Bl#ayf5z?e)=tGbQS;h3(gA$e=k z6KY(=q)^im5{@d1i=Ix4={^<VU*;0T*7?%g*~`NlJ)S&FWzzYph<0?QcO=pLP8?e$ z*FaH6Y-^O^gO|=h>s46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=<b6daWfefB)Aj< zmFAkvPGmuOH|UXs98|Izd34i4mjd%!t1Qfh2QG#C`?Q72>+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*<SW1ss4Nyr-0;)>qjCVa?OIBj!fW zm|g?>CVfG<LKn)qVK&7vOUVLdB{HCz?|MJr^jJ<qMG*a<=xVmP)}?vXFtiYfxJQ@S z#a>XNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9E<z?vk<wVk?axOB(w*1mgxPEOC4Q_bKOg8aZS|<Jy7e5qZiZy zi_|O~v^<Tb8jU^hW(}=ov(p@txag^UbHjwTdt5JHG|siOz_cTtZE_L!P;3VdkA;2O zLo^GkBN2HNqX9mQqP?;}M=Weu;f8w@!h2(}g9(ObqEVL=EgisIBV{7fu4ome13nCE zdwW~^rkG^r1c_Jwl>S>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{<p15xt>abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9S<WqRBxZXWrLfD3v(g<)+j7~DBJ!D1`Yc*cLPVUN6i!o z<WHr{5RdXyi7LHb^NKbKUg}tH*PPH!`V-!@Sd!s(E-vzsn6!5N&eAaqoX%HxTMM{P zIJeCXkWwFSIEY&)8%SeRPki$2eJ?TcIF$5@j`=f}32hbX8_y10W!UQZL+acoY3HWL z^6U9-tleDkD!6JsRBj>UlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5Q<X)1 z%$Q)YQ_xs_u@fVbNBxjMBTpgM&=XlBlp}w#R&41H+3NH%W*0}&d|5rv#al8&><AlF z8HoEcIWb_>D7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0<u*j8bI(fE4R+~6W9t@%ANP1F@ zp$eJoBlfrurBBjoT#vaVVYUuEOoWffDHI4t?-V4p5pZZ3DkD`0^rkb1n)as8WP>Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8f<yhFr_4W{*sa2b)64y#l+N#ywviYKA zHs^$|vSmQ9h25pfc|F}ni`8kid}w)OD+3UEhP%9Dlwy33NMgJ*oN!k%?qfbTx#r!J zY-dPtmtNT~GY1Wt*spshItZ8i4R+jh2L%btp^)9t=mbb(EpeeR#tnR}gtoxxvFD|m zhb}`&kCesZUqA7FVifG~RXu8NP->uX!I}#8g+(wxzQwUT#Xb2(t<I|Vdp4<hf#R}E zO$m~GhlyfnIij1YM1#7L8*;8lzFA>bY1+EUhG<XbGlkEvuzFKy+tnOPm$r=L#YRzV zwl2E?nbHJTp_m!TZ`;N8RAqq@pSx_H9#jNyRdy^@&BAm<aBE+@>KoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}<Hop~&Xqn<t%=;~lIDY-u92ye&Z%8~1AbQ5!`jhH%qI zx(5`}Haa*r;yTwa;Iry8BUl6}+9-EGitaF$Aq_C?<%Xbp{$*-bSPHf$IdY92oxi$9 zcggG$CBdW=NKdNvJ@yJUY`u+>v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF<nu9$;OS+a41|h`ofOZC21#Y@_xv+k3^Ljg zEP^3r*gL6M9RW3lIubK}wzMJ3Vq#cPO<cz7xXnc!R;|M8d|v`nL0G&xB&n;gb?&kd z#fHl)FBAfL#Kd5aQG;&jsz|6*t@OWoiXQ1Qs^e+x9z47f=xOXd8X%iV!R%6NpDAQ2 zTW=KVlc2NZatQ2fBO>8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f<nT83VyE*=trSIMwSK+ z4z|GFEwin?jV;*T(G2VW4|oi4V$|b?I7v{*;m?4!2KEM4VBj1m$Qrmh{2}a>6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x<u`s6FSLLTv<|cn{|Pp5g+jgo+oN!0JAnt({C-&Q&xt&X`rRaf=9UHOa<(4N zfldWS^e<RZds8PXAUYACtOvF|eLw<Vj~BBG`@{geDFA=A=_Ck#1^*lKsG6XUA_1@M zoBr4<KCuuKk`3G@{&%Sre^J!KjgXRO0MHWfIlj?6Nl?i8wO?T>8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq<h|F<+{0IyH zi8D-XK*RiZ>;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~<a5^&n0jI1r5GnDy`M#F|h(qiMKHrd*pVsPffa+}n9r&yvC)xjiO5V)D0jSV- zGGG|~f~m<FCo&&kY6Y0iR%(jt514*XxER=je_N@~Rw;NXK<|(S7GRz;_L~O|?EJRP zzEl0Kks34-UgZF@NftnKd<^I$K_P>HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#v<R9RxTU-1O0|)yvA4xba;H&`N3f&14aD_soWPVR}ep`@)E(vu3 zg+~Bz&teka`w8=Ja~S`ahL2bA^D*6KQ5+`#qlg3T%XFrkbl~4(ejf_wBkMZ1i+KP8 z00S5Rd}okl9{h}KZ(|NNa{T{z1?aDD2_Ewt0=3{h!$WTV6A%3M@xSczn`QhM8DRK3 zVgI-y{Oy6kEY8q4IhtAi<boY%ILQqtp!`V34lt$V&$-R4ftA$S;AfcY{Zw*wfIWkO zN%HJ)*Zw68;IpdP8#sgQ9Ski076v-mF^6ATl(pNMM1g`517i@FK>kTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|<!Bd8Mzh5$(Z*X{#@KZSM#9zVz<vk}ZGJI*_HMvjW>5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O<nja==1v+{2K<RX!qLBMf>;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a1f138116b..8d1ea75dcd 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae +distributionSha256Sum=d725d707bfabd4dfdc958c624003b3c80accc03f7037b5122c4b1d0ef15cecab diff --git a/gradlew b/gradlew index 1aa94a4269..02fae2493a 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 6689b85bee..0ebb4c6c76 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## From 607354c90cad2308fc1d6278684534f152305276 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 17 Jul 2024 15:49:12 -0700 Subject: [PATCH 05/96] Fix SparkSubmitParameterModifier issue (#2837) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../sql/spark/parameter/SparkSubmitParametersBuilder.java | 4 +++- .../spark/parameter/SparkSubmitParametersBuilderTest.java | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java index 01a665a485..3fe7d99373 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java @@ -154,7 +154,9 @@ public SparkSubmitParametersBuilder sessionExecution(String sessionId, String da } public SparkSubmitParametersBuilder acceptModifier(SparkSubmitParameterModifier modifier) { - modifier.modifyParameters(this); + if (modifier != null) { + modifier.modifyParameters(this); + } return this; } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java index 3f4bea02f2..8947cb61f7 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java @@ -152,6 +152,11 @@ public void testAcceptModifier() { verify(sparkSubmitParameterModifier).modifyParameters(sparkSubmitParametersBuilder); } + @Test + public void testAcceptNullModifier() { + sparkSubmitParametersBuilder.acceptModifier(null); + } + @Test public void testDataSource() { when(sparkParameterComposerCollection.isComposerRegistered(DataSourceType.S3GLUE)) From 956ec15c27ab1d5bac379a2469a318de55791a3b Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Thu, 18 Jul 2024 23:09:11 +0800 Subject: [PATCH 06/96] Boolean function in PPL should be case insensitive (#2758) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- .../org/opensearch/sql/ppl/parser/AstExpressionBuilder.java | 2 +- .../opensearch/sql/ppl/parser/AstExpressionBuilderTest.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 47db10c99b..f36765d3d7 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -187,7 +187,7 @@ public UnresolvedExpression visitTakeAggFunctionCall( /** Eval function. */ @Override public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext ctx) { - final String functionName = ctx.conditionFunctionBase().getText(); + final String functionName = ctx.conditionFunctionBase().getText().toLowerCase(); return buildFunction( FUNCTION_NAME_MAPPING.getOrDefault(functionName, functionName), ctx.functionArgs().functionArg()); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 7bcb87d193..de230a1fee 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -107,12 +107,15 @@ public void testLogicalLikeExpr() { @Test public void testBooleanIsNullFunction() { assertEqual("source=t isnull(a)", filter(relation("t"), function("is null", field("a")))); + assertEqual("source=t ISNULL(a)", filter(relation("t"), function("is null", field("a")))); } @Test public void testBooleanIsNotNullFunction() { assertEqual( "source=t isnotnull(a)", filter(relation("t"), function("is not null", field("a")))); + assertEqual( + "source=t ISNOTNULL(a)", filter(relation("t"), function("is not null", field("a")))); } /** Todo. search operator should not include functionCall, need to change antlr. */ From 0fad56db4b3e8983e2e7fafcf9fb80e592d97ddb Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Fri, 19 Jul 2024 10:40:13 -0700 Subject: [PATCH 07/96] Add support for custom date format and openSearch date format for date fields as part of Lucene query (#2762) Github Issue - https://github.com/opensearch-project/sql/issues/2700 Signed-off-by: Manasvini B S <manasvis@amazon.com> --- docs/user/general/datatypes.rst | 42 +++ .../data/type/OpenSearchDataType.java | 19 +- .../data/type/OpenSearchDateType.java | 63 ++++ .../value/OpenSearchExprValueFactory.java | 7 +- .../dsl/BucketAggregationBuilder.java | 6 +- .../script/filter/lucene/LuceneQuery.java | 307 +++++++++++------- .../script/filter/lucene/RangeQuery.java | 11 +- .../script/filter/lucene/TermQuery.java | 11 +- .../client/OpenSearchNodeClientTest.java | 2 +- .../client/OpenSearchRestClientTest.java | 2 +- .../data/type/OpenSearchDataTypeTest.java | 22 +- .../data/type/OpenSearchDateTypeTest.java | 229 +++++++++++-- .../storage/OpenSearchIndexTest.java | 2 +- .../dsl/BucketAggregationBuilderTest.java | 34 ++ .../script/filter/FilterQueryBuilderTest.java | 6 +- .../script/filter/lucene/LuceneQueryTest.java | 76 ++++- .../script/filter/lucene/RangeQueryTest.java | 67 +++- .../script/filter/lucene/TermQueryTest.java | 82 +++++ 18 files changed, 791 insertions(+), 197 deletions(-) create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index c423bd7b10..042e97396e 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -400,6 +400,48 @@ Querying such index will provide a response with ``schema`` block as shown below "status": 200 } +If the sql query contains an `IndexDateField` and a literal value with an operator (such as a term query or a range query), then the literal value can be in the `IndexDateField` format. + +.. code-block:: json + + { + "mappings" : { + "properties" : { + "release_date" : { + "type" : "date", + "format": "dd-MMM-yy" + } + } + } + } + +Querying such an `IndexDateField` (``release_date``) will provide a response with ``schema`` and ``datarows`` blocks as shown below. + +.. code-block:: json + + { + "query" : "SELECT release_date FROM test_index WHERE release_date = \"03-Jan-21\"" + } + +.. code-block:: json + + { + "schema": [ + { + "name": "release_date", + "type": "date" + } + ], + "datarows": [ + [ + "2021-01-03" + ] + ], + "total": 1, + "size": 1, + "status": 200 + } + String Data Types ================= diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index ddbba61260..c35eacfc72 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -62,19 +62,23 @@ public String toString() { @EqualsAndHashCode.Exclude @Getter protected MappingType mappingType; // resolved ExprCoreType - protected ExprCoreType exprCoreType; + @Getter protected ExprCoreType exprCoreType; /** * Get a simplified type {@link ExprCoreType} if possible. To avoid returning `UNKNOWN` for - * `OpenSearch*Type`s, e.g. for IP, returns itself. + * `OpenSearch*Type`s, e.g. for IP, returns itself. If the `exprCoreType` is {@link + * ExprCoreType#DATE}, {@link ExprCoreType#TIMESTAMP}, {@link ExprCoreType#TIME}, or {@link + * ExprCoreType#UNKNOWN}, it returns the current instance; otherwise, it returns `exprCoreType`. * * @return An {@link ExprType}. */ public ExprType getExprType() { - if (exprCoreType != ExprCoreType.UNKNOWN) { - return exprCoreType; - } - return this; + return (exprCoreType == ExprCoreType.DATE + || exprCoreType == ExprCoreType.TIMESTAMP + || exprCoreType == ExprCoreType.TIME + || exprCoreType == ExprCoreType.UNKNOWN) + ? this + : exprCoreType; } /** @@ -230,6 +234,9 @@ public String legacyTypeName() { if (mappingType == null) { return exprCoreType.typeName(); } + if (mappingType.toString().equalsIgnoreCase("DATE")) { + return exprCoreType.typeName(); + } return mappingType.toString().toUpperCase(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 7e6bee77c2..5ffce655d0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,11 +11,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAccessor; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -137,6 +142,11 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + private static final List<DateFormatter> OPENSEARCH_DEFAULT_FORMATTERS = + Stream.of("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis") + .map(DateFormatter::forPattern) + .toList(); + @EqualsAndHashCode.Exclude private final List<String> formats; private OpenSearchDateType() { @@ -235,6 +245,59 @@ public List<DateFormatter> getAllCustomFormatters() { .collect(Collectors.toList()); } + /** + * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user, and + * attempts to parse the given date/time string using these formatters. + * + * @param dateTime The date/time string to parse. + * @return A ZonedDateTime representing the parsed date/time in UTC, or null if parsing fails. + */ + public ZonedDateTime getParsedDateTime(String dateTime) { + List<DateFormatter> dateFormatters = + Stream.concat(this.getAllNamedFormatters().stream(), this.getAllCustomFormatters().stream()) + .collect(Collectors.toList()); + ZonedDateTime zonedDateTime = null; + + // check if dateFormatters are empty, then set default ones + if (dateFormatters.isEmpty()) { + dateFormatters = OPENSEARCH_DEFAULT_FORMATTERS; + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(dateTime); + zonedDateTime = DateFormatters.from(accessor).withZoneSameLocal(ZoneOffset.UTC); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + return zonedDateTime; + } + + /** + * Returns a formatted date string using the internal formatter, if available. + * + * @param accessor The TemporalAccessor object containing the date/time information. + * @return A formatted date string if a formatter is available, otherwise null. + */ + public String getFormattedDate(TemporalAccessor accessor) { + if (hasNoFormatter()) { + return OPENSEARCH_DEFAULT_FORMATTERS.get(0).format(accessor); + } + // Use the first available format string to create the formatter + return DateFormatter.forPattern(this.formats.get(0)).format(accessor); + } + + /** + * Checks if the formatter is not initialized. + * + * @return True if the formatter is not set, otherwise false. + */ + public boolean hasNoFormatter() { + return this.formats.isEmpty(); + } + /** * Retrieves a list of named formatters that format for dates. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3341e01ab2..3cb182de5b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -230,7 +230,7 @@ private Optional<ExprType> type(String field) { private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { List<DateFormatter> formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); - ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); + ExprCoreType returnFormat = dataType.getExprCoreType(); for (DateFormatter formatter : formatters) { try { @@ -273,8 +273,7 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da private static ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; - ExprType returnFormat = dt.getExprType(); - + ExprCoreType returnFormat = dt.getExprCoreType(); if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0 || !dt.hasFormats()) { @@ -287,7 +286,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) epochMillis = value.longValue(); } Instant instant = Instant.ofEpochMilli(epochMillis); - switch ((ExprCoreType) returnFormat) { + switch (returnFormat) { case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(ZoneOffset.UTC))); case DATE: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index ff66ec425a..4488128b97 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.span.SpanExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; /** Bucket Aggregation Builder. */ @@ -65,7 +66,10 @@ private CompositeValuesSourceBuilder<?> buildCompositeValuesSourceBuilder( .missingOrder(missingOrder) .order(sortOrder); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { + if ((expr.getDelegated().type() instanceof OpenSearchDateType + && List.of(TIMESTAMP, TIME, DATE) + .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) + || List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { sourceBuilder.userValuetypeHint(ValueType.LONG); } return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 11533c754e..c9ef5bcca5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,8 +8,9 @@ import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; import com.google.common.collect.ImmutableMap; +import java.time.ZonedDateTime; import java.util.Map; -import java.util.function.Function; +import java.util.function.BiFunction; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; @@ -32,6 +33,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query abstraction that builds Lucene query from function expression. */ public abstract class LuceneQuery { @@ -105,135 +107,164 @@ public QueryBuilder build(FunctionExpression func) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); Expression expr = func.getArguments().get(1); ExprValue literalValue = - expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr); + expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr, ref); return doBuild(ref.getAttr(), ref.type(), literalValue); } - private ExprValue cast(FunctionExpression castFunction) { + private ExprValue cast(FunctionExpression castFunction, ReferenceExpression ref) { return castMap .get(castFunction.getFunctionName()) - .apply((LiteralExpression) castFunction.getArguments().get(0)); + .apply((LiteralExpression) castFunction.getArguments().get(0), ref); } /** Type converting map. */ - private final Map<FunctionName, Function<LiteralExpression, ExprValue>> castMap = - ImmutableMap.<FunctionName, Function<LiteralExpression, ExprValue>>builder() - .put( - BuiltinFunctionName.CAST_TO_STRING.getName(), - expr -> { - if (!expr.type().equals(ExprCoreType.STRING)) { - return new ExprStringValue(String.valueOf(expr.valueOf().value())); - } else { - return expr.valueOf(); - } - }) - .put( - BuiltinFunctionName.CAST_TO_BYTE.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprByteValue(expr.valueOf().byteValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_SHORT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprShortValue(expr.valueOf().shortValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_INT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprIntegerValue(expr.valueOf().integerValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_LONG.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprLongValue(expr.valueOf().longValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_FLOAT.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprFloatValue(expr.valueOf().floatValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_DOUBLE.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprDoubleValue(expr.valueOf().doubleValue()); - } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); - } else { - return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); - } - }) - .put( - BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), - expr -> { - if (ExprCoreType.numberTypes().contains(expr.type())) { - return expr.valueOf().doubleValue() != 0 - ? ExprBooleanValue.of(true) - : ExprBooleanValue.of(false); - } else if (expr.type().equals(ExprCoreType.STRING)) { - return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); - } else { - return expr.valueOf(); - } - }) - .put( - BuiltinFunctionName.CAST_TO_DATE.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDateValue(expr.valueOf().stringValue()); - } else { - return new ExprDateValue(expr.valueOf().dateValue()); - } - }) - .put( - BuiltinFunctionName.CAST_TO_TIME.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimeValue(expr.valueOf().stringValue()); - } else { - return new ExprTimeValue(expr.valueOf().timeValue()); - } - }) - .put( - BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), - expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimestampValue(expr.valueOf().stringValue()); - } else { - return new ExprTimestampValue(expr.valueOf().timestampValue()); - } - }) - .build(); + private final Map<FunctionName, BiFunction<LiteralExpression, ReferenceExpression, ExprValue>> + castMap = + ImmutableMap + .<FunctionName, BiFunction<LiteralExpression, ReferenceExpression, ExprValue>> + builder() + .put( + BuiltinFunctionName.CAST_TO_STRING.getName(), + (expr, ref) -> { + if (!expr.type().equals(ExprCoreType.STRING)) { + return new ExprStringValue(String.valueOf(expr.valueOf().value())); + } else { + return expr.valueOf(); + } + }) + .put( + BuiltinFunctionName.CAST_TO_BYTE.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprByteValue(expr.valueOf().byteValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_SHORT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprShortValue(expr.valueOf().shortValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_INT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprIntegerValue(expr.valueOf().integerValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_LONG.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprLongValue(expr.valueOf().longValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_FLOAT.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprFloatValue(expr.valueOf().floatValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_DOUBLE.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return new ExprDoubleValue(expr.valueOf().doubleValue()); + } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { + return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); + } else { + return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); + } + }) + .put( + BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), + (expr, ref) -> { + if (ExprCoreType.numberTypes().contains(expr.type())) { + return expr.valueOf().doubleValue() != 0 + ? ExprBooleanValue.of(true) + : ExprBooleanValue.of(false); + } else if (expr.type().equals(ExprCoreType.STRING)) { + return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); + } else { + return expr.valueOf(); + } + }) + .put( + BuiltinFunctionName.CAST_TO_DATE.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprDateValue(zonedDateTime.toLocalDate()); + } + return new ExprDateValue(expr.valueOf().stringValue()); + } else { + return new ExprDateValue(expr.valueOf().dateValue()); + } + }) + .put( + BuiltinFunctionName.CAST_TO_TIME.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprTimeValue(zonedDateTime.toLocalTime()); + } + return new ExprTimeValue(expr.valueOf().stringValue()); + } else { + return new ExprTimeValue(expr.valueOf().timeValue()); + } + }) + .put( + BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), + (expr, ref) -> { + if (expr.type().equals(ExprCoreType.STRING)) { + ZonedDateTime zonedDateTime = getParsedDateTime(expr, ref); + if (zonedDateTime != null) { + return new ExprTimestampValue(zonedDateTime.toInstant()); + } + return new ExprTimestampValue(expr.valueOf().stringValue()); + } else { + return new ExprTimestampValue(expr.valueOf().timestampValue()); + } + }) + .build(); + + /** + * Parses the date/time from the given expression if the reference type is an instance of + * OpenSearchDateType. + * + * @param expr The expression to parse. + * @return The parsed ZonedDateTime or null if the conditions are not met. + */ + private ZonedDateTime getParsedDateTime(LiteralExpression expr, ReferenceExpression ref) { + if (ref.type() instanceof OpenSearchDateType) { + return ((OpenSearchDateType) ref.type()).getParsedDateTime(expr.valueOf().stringValue()); + } + return null; + } /** * Build method that subclass implements by default which is to build query from reference and @@ -248,4 +279,36 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l throw new UnsupportedOperationException( "Subclass doesn't implement this and build method either"); } + + /** + * Converts a literal value to a formatted date or time value based on the specified field type. + * + * <p>If the field type is an instance of {@link OpenSearchDateType}, this method checks the type + * of the literal value and converts it to a formatted date or time if necessary. The formatting + * is applied if the {@link OpenSearchDateType} has a formatter. Otherwise, the raw value is + * returned. + * + * @param literal the literal value to be converted + * @param fieldType the field type to determine the conversion logic + * @return the formatted date or time value if the field type requires it, otherwise the raw value + */ + protected Object value(ExprValue literal, ExprType fieldType) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + return openSearchDateType.hasNoFormatter() + ? literal.timestampValue().toEpochMilli() + : openSearchDateType.getFormattedDate(literal.timestampValue()); + } else if (literal.type().equals(ExprCoreType.DATE)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.dateValue()); + } else if (literal.type().equals(ExprCoreType.TIME)) { + return openSearchDateType.hasNoFormatter() + ? literal.value() + : openSearchDateType.getFormattedDate(literal.timeValue()); + } + } + return literal.value(); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 2e33e3cc7c..e9a38b6ee3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -10,7 +10,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; /** Lucene query that builds range query for non-quality comparison. */ @@ -30,7 +29,7 @@ public enum Comparison { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { - Object value = value(literal); + Object value = this.value(literal, fieldType); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); switch (comparison) { @@ -46,12 +45,4 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l throw new IllegalStateException("Comparison is supported by range query: " + comparison); } } - - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); - } - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index cd506898d7..f8988b3cd9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -8,7 +8,6 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; @@ -18,14 +17,6 @@ public class TermQuery extends LuceneQuery { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); - return QueryBuilders.termQuery(fieldName, value(literal)); - } - - private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { - return literal.timestampValue().toEpochMilli(); - } else { - return literal.value(); - } + return QueryBuilders.termQuery(fieldName, this.value(literal, fieldType)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 040b7d2759..9da6e05e92 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -169,7 +169,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Double), parsedTypes.get("balance")), () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), parsedTypes.get("city")), - () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals("TIMESTAMP", mapping.get("birthday").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Date), parsedTypes.get("birthday")), () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), () -> diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 99201aae4f..b83313de07 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -169,7 +169,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Double), parsedTypes.get("balance")), () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), parsedTypes.get("city")), - () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals("TIMESTAMP", mapping.get("birthday").legacyTypeName()), () -> assertEquals(OpenSearchTextType.of(MappingType.Date), parsedTypes.get("birthday")), () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), () -> diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 82e6222dc4..76fbbd6e65 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -70,7 +70,7 @@ public void typeName() { assertEquals("STRING", textType.typeName()); assertEquals("STRING", textKeywordType.typeName()); assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).typeName()); - assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).typeName()); + assertEquals("TIMESTAMP", OpenSearchDataType.of(MappingType.Date).typeName()); assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); } @@ -80,7 +80,7 @@ public void legacyTypeName() { assertEquals("TEXT", textType.legacyTypeName()); assertEquals("TEXT", textKeywordType.legacyTypeName()); assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).legacyTypeName()); - assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).legacyTypeName()); + assertEquals("TIMESTAMP", OpenSearchDataType.of(MappingType.Date).legacyTypeName()); assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).legacyTypeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).legacyTypeName()); } @@ -104,8 +104,8 @@ private static Stream<Arguments> getTestDataWithType() { Arguments.of(MappingType.ScaledFloat, "scaled_float", DOUBLE), Arguments.of(MappingType.Double, "double", DOUBLE), Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), - Arguments.of(MappingType.Date, "date", TIMESTAMP), - Arguments.of(MappingType.DateNanos, "date", TIMESTAMP), + Arguments.of(MappingType.Date, "timestamp", TIMESTAMP), + Arguments.of(MappingType.DateNanos, "timestamp", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), Arguments.of(MappingType.GeoPoint, "geo_point", OpenSearchGeoPointType.of()), @@ -124,7 +124,15 @@ public void of_MappingType(MappingType mappingType, String name, ExprType dataTy assertAll( () -> assertEquals(nameForPPL, type.typeName()), () -> assertEquals(nameForSQL, type.legacyTypeName()), - () -> assertEquals(dataType, type.getExprType())); + () -> { + if (dataType == ExprCoreType.TIMESTAMP + || dataType == ExprCoreType.DATE + || dataType == ExprCoreType.TIME) { + assertEquals(dataType, type.getExprCoreType()); + } else { + assertEquals(dataType, type.getExprType()); + } + }); } @ParameterizedTest(name = "{0}") @@ -133,7 +141,7 @@ public void of_ExprCoreType(ExprCoreType coreType) { assumeFalse(coreType == UNKNOWN); var type = OpenSearchDataType.of(coreType); if (type instanceof OpenSearchDateType) { - assertEquals(coreType, type.getExprType()); + assertEquals(coreType, type.getExprCoreType()); } else { assertEquals(coreType.toString(), type.typeName()); assertEquals(coreType.toString(), type.legacyTypeName()); @@ -416,7 +424,7 @@ public void test_getExprType() { assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); - assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index c6885c8ffe..3c1cf1bf0f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -5,12 +5,7 @@ package org.opensearch.sql.opensearch.data.type; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -22,6 +17,9 @@ import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; import com.google.common.collect.Lists; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.stream.Stream; @@ -48,8 +46,6 @@ class OpenSearchDateTypeTest { OpenSearchDateType.of(defaultFormatString); private static final OpenSearchDateType dateDateType = OpenSearchDateType.of(dateFormatString); private static final OpenSearchDateType timeDateType = OpenSearchDateType.of(timeFormatString); - private static final OpenSearchDateType datetimeDateType = - OpenSearchDateType.of(timestampFormatString); @Test public void isCompatible() { @@ -76,8 +72,8 @@ public void isCompatible() { public void check_typeName() { assertAll( // always use the MappingType of "DATE" - () -> assertEquals("DATE", defaultDateType.typeName()), - () -> assertEquals("DATE", timeDateType.typeName()), + () -> assertEquals("TIMESTAMP", defaultDateType.typeName()), + () -> assertEquals("TIME", timeDateType.typeName()), () -> assertEquals("DATE", dateDateType.typeName())); } @@ -85,8 +81,8 @@ public void check_typeName() { public void check_legacyTypeName() { assertAll( // always use the legacy "DATE" type - () -> assertEquals("DATE", defaultDateType.legacyTypeName()), - () -> assertEquals("DATE", timeDateType.legacyTypeName()), + () -> assertEquals("TIMESTAMP", defaultDateType.legacyTypeName()), + () -> assertEquals("TIME", timeDateType.legacyTypeName()), () -> assertEquals("DATE", dateDateType.legacyTypeName())); } @@ -94,9 +90,9 @@ public void check_legacyTypeName() { public void check_exprTypeName() { assertAll( // exprType changes based on type (no datetime): - () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), - () -> assertEquals(TIME, timeDateType.getExprType()), - () -> assertEquals(DATE, dateDateType.getExprType())); + () -> assertEquals(TIMESTAMP, defaultDateType.getExprCoreType()), + () -> assertEquals(TIME, timeDateType.getExprCoreType()), + () -> assertEquals(DATE, dateDateType.getExprCoreType())); } private static Stream<Arguments> getAllSupportedFormats() { @@ -129,22 +125,22 @@ public void check_datetime_format_names(FormatNames datetimeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, camelCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } String snakeCaseName = datetimeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, snakeCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } else { fail(); } @@ -161,18 +157,22 @@ public void check_date_format_names(FormatNames dateFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - camelCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = dateFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - snakeCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -189,18 +189,22 @@ public void check_time_format_names(FormatNames timeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - camelCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = timeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - snakeCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -244,9 +248,9 @@ private static Stream<Arguments> get_format_combinations_for_test() { @MethodSource("get_format_combinations_for_test") public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( ExprCoreType expected, List<String> formats, String testName) { - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); formats = Lists.reverse(formats); - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); } @Test @@ -259,4 +263,171 @@ public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); } + + @Test + void test_valid_timestamp_with_custom_format() { + String timestamp = "2021-11-08T17:00:00Z"; + String format = "strict_date_time_no_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_valid_timestamp_with_multiple_formats() { + String timestamp = "2021-11-08T17:00:00Z"; + String timestamp2 = "2021/11/08T17:00:00Z"; + + List<String> formats = Arrays.asList("strict_date_time_no_millis", "yyyy/MM/dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + + // Testing with the first timestamp + ZonedDateTime zonedDateTime1 = dateType.getParsedDateTime(timestamp); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime1.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime1.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + + // Testing with the second timestamp + ZonedDateTime zonedDateTime2 = dateType.getParsedDateTime(timestamp2); + + assertEquals("2021-11-08T17:00:00Z", dateType.getFormattedDate(zonedDateTime2.toInstant())); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime2.toLocalDate()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_openSearch_datetime_named_formatter() { + String timestamp = "2019-03-23T21:34:46"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + + assertEquals("2019-03-23T21:34:46", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_openSearch_datetime_with_default_formatter() { + String timestamp = "2019-03-23T21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of(TIMESTAMP); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + // formatted using OpenSearch default formatter + assertEquals("2019-03-23T21:34:46Z", dateType.getFormattedDate(zonedDateTime.toInstant())); + assertEquals(LocalDate.parse("2019-03-23"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("21:34:46"), zonedDateTime.toLocalTime()); + assertTrue(dateType.hasNoFormatter()); + } + + @Test + void test_invalid_date_with_named_formatter() { + // Incorrect date + String timestamp = "2019-23-23"; + String format = "strict_date_hour_minute_second"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(zonedDateTime); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_invalid_time_with_custom_formatter() { + String timestamp = "invalid-timestamp"; + List<String> formats = Arrays.asList("yyyy/MM/dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX"); + OpenSearchDateType dateType = OpenSearchDateType.of(String.join(" || ", formats)); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + assertNull(zonedDateTime); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_epoch_datetime_formatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + + assertEquals(Long.toString(epochTimestamp), dateType.getFormattedDate(zonedDateTime)); + assertEquals(LocalDate.parse("2021-11-08"), zonedDateTime.toLocalDate()); + assertEquals(LocalTime.parse("17:00:00"), zonedDateTime.toLocalTime()); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_timeStamp_format_with_default_formatters() { + String timestamp = "2021-11-08 17:00:00"; + String format = "strict_date_time_no_millis || epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertNull(dateType.getParsedDateTime(timestamp)); + } + + @Test + void test_valid_date_with_custom_formatter() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021-11-08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void test_valid_date_string_with_custom_formatter() { + String dateString = "03-Jan-21"; + String format = "dd-MMM-yy"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(LocalDate.parse("2021-01-03"), parsedDate); + assertEquals("03-Jan-21", dateType.getFormattedDate(parsedDate)); + assertFalse(dateType.hasNoFormatter()); + } + + @Test + void test_valid_date_with_multiple_formatters() { + String dateString = "2021-11-08"; + String format = "yyyy/MM/dd || yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + + assertEquals(expectedDate, parsedDate); + assertEquals("2021/11/08", dateType.getFormattedDate(parsedDate)); + } + + @Test + void test_valid_time_with_custom_formatter() { + String timeString = "12:10:30.000"; + String format = "HH:mm:ss.SSS"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern(format)); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } + + @Test + void test_valid_time_with_multiple_formatters() { + String timeString = "12:10:30"; + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + LocalTime parsedTime = dateType.getParsedDateTime(timeString).toLocalTime(); + + assertEquals(expectedTime, parsedTime); + assertEquals("12:10:30.000", dateType.getFormattedDate(parsedTime)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ddb07d86a..3ca566fac6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -148,7 +148,7 @@ void getFieldTypes() { hasEntry("gender", ExprCoreType.BOOLEAN), hasEntry("family", ExprCoreType.ARRAY), hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("birthday", (ExprType) OpenSearchDataType.of(MappingType.Date)), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 4250b3297f..08c4017f1d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -40,6 +40,7 @@ import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.parse.ParseExpression; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -134,6 +135,39 @@ void should_build_bucket_with_parse_expression() { buildQuery(Arrays.asList(asc(named("name", parseExpression))))); } + @Test + void terms_bucket_for_opensearchdate_type_uses_long() { + OpenSearchDateType dataType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"value_type\" : \"long\",\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + + @Test + void terms_bucket_for_opensearchdate_type_uses_long_false() { + OpenSearchDateType dataType = OpenSearchDateType.of(STRING); + + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"field\" : \"date\",\n" + + " \"missing_bucket\" : true,\n" + + " \"missing_order\" : \"first\",\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery(Arrays.asList(asc(named("date", ref("date", dataType)))))); + } + @ParameterizedTest(name = "{0}") @EnumSource( value = ExprCoreType.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 90b982e017..bd2a9901ed 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1772,9 +1772,9 @@ void cast_to_date_in_filter() { + " }\n" + " }\n" + "}"; - assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); + assertJsonEquals( json, buildQuery( @@ -1821,7 +1821,7 @@ void cast_to_timestamp_in_filter() { "{\n" + " \"term\" : {\n" + " \"timestamp_value\" : {\n" - + " \"value\" : 1636390800000,\n" + + " \"value\" : \"2021-11-08 17:00:00\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1847,7 +1847,7 @@ void cast_in_range_query() { "{\n" + " \"range\" : {\n" + " \"timestamp_value\" : {\n" - + " \"from\" : 1636390800000,\n" + + " \"from\" : \"2021-11-08 17:00:00\",\n" + " \"to\" : null," + " \"include_lower\" : false," + " \"include_upper\" : true," diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java index df3a730bad..1713d1dd1b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQueryTest.java @@ -8,18 +8,22 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.expression.DSL.literal; +import static org.opensearch.sql.expression.DSL.ref; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class LuceneQueryTest { @Test void should_not_support_single_argument_by_default() { - assertFalse(new LuceneQuery() {}.canSupport(DSL.abs(DSL.ref("age", INTEGER)))); + assertFalse(new LuceneQuery() {}.canSupport(DSL.abs(ref("age", INTEGER)))); } @Test @@ -27,4 +31,74 @@ void should_throw_exception_if_not_implemented() { assertThrows( UnsupportedOperationException.class, () -> new LuceneQuery() {}.doBuild(null, null, null)); } + + @Test + void should_cast_to_time_with_format() { + String format = "HH:mm:ss.SSS || HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("time_value", dateType), DSL.castTime(literal("17:00:00"))))); + } + + @Test + void should_cast_to_time_with_no_format() { + String format = "HH:mm"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("time_value", dateType), DSL.castTime(literal("17:00:00"))))); + } + + @Test + void should_cast_to_date_with_format() { + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("date_value", dateType), DSL.castDate(literal("2017-01-02"))))); + } + + @Test + void should_cast_to_date_with_no_format() { + String format = "yyyy/MM/dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal(ref("date_value", dateType), DSL.castDate(literal("2017-01-02"))))); + } + + @Test + void should_cast_to_timestamp_with_format() { + String format = "yyyy-MM-dd HH:mm:ss"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + UnsupportedOperationException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal( + ref("timestamp_value", dateType), + DSL.castTimestamp(literal("2021-11-08 17:00:00"))))); + } + + @Test + void should_cast_to_timestamp_with_no_format() { + String format = "2021/11/08T17:00:00Z"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + assertThrows( + SemanticCheckException.class, + () -> + new LuceneQuery() {}.build( + DSL.equal( + ref("timestamp_value", dateType), + DSL.castTimestamp(literal("2021-11-08 17:00:00 "))))); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java index ca87f42900..2f5482171d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java @@ -5,13 +5,17 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene; +import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import java.time.*; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.model.*; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery.Comparison; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -26,4 +30,65 @@ void should_throw_exception_for_unsupported_comparison() { new RangeQuery(Comparison.BETWEEN) .doBuild("name", STRING, ExprValueUtils.stringValue("John"))); } + + @Test + void test_timestamp_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("time", openSearchDateType, new ExprTimestampValue("2021-11-08 17:00:00"))); + } + + @Test + void test_timestamp_has_format() { + String timestamp = "2019-03-23 21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of("yyyy-MM-dd HH:mm:ss"); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + ExprValue literal = ExprValueUtils.timestampValue(zonedDateTime.toInstant()); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("time_stamp", dateType, literal)); + } + + @Test + void test_time_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIME); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("time", openSearchDateType, new ExprTimeValue("17:00:00"))); + } + + @Test + void test_time_has_format() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + ExprValue literal = ExprValueUtils.timeValue(zonedDateTime.toLocalTime()); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("time", dateType, literal)); + } + + @Test + void test_date_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.DATE); + assertNotNull( + new RangeQuery(Comparison.LT) + .doBuild("date", openSearchDateType, new ExprDateValue("2021-11-08"))); + } + + @Test + void test_date_has_format() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + ExprValue literal = ExprValueUtils.dateValue(parsedDate); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("date", dateType, literal)); + } + + @Test + void test_non_date_field_type() { + String dateString = "2021-11-08"; + OpenSearchDateType dateType = OpenSearchDateType.of(STRING); + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new RangeQuery(Comparison.LT).doBuild("string_value", dateType, literal)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java new file mode 100644 index 0000000000..def9fafba3 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java @@ -0,0 +1,82 @@ +package org.opensearch.sql.opensearch.storage.script.filter.lucene; + +import static org.junit.Assert.*; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.time.*; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.*; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class TermQueryTest { + + @Test + void test_timestamp_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); + assertNotNull( + new TermQuery() + .doBuild("time", openSearchDateType, new ExprTimestampValue("2021-11-08 17:00:00"))); + } + + @Test + void test_timestamp_has_format() { + String timestamp = "2019-03-23 21:34:46"; + OpenSearchDateType dateType = OpenSearchDateType.of("yyyy-MM-dd HH:mm:ss"); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(timestamp); + ExprValue literal = ExprValueUtils.timestampValue(zonedDateTime.toInstant()); + assertNotNull(new TermQuery().doBuild("time_stamp", dateType, literal)); + } + + @Test + void test_time_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.TIME); + assertNotNull( + new TermQuery().doBuild("time", openSearchDateType, new ExprTimeValue("17:00:00"))); + } + + @Test + void test_time_has_format() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + String format = "epoch_millis"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + ZonedDateTime zonedDateTime = dateType.getParsedDateTime(String.valueOf(epochTimestamp)); + ExprValue literal = ExprValueUtils.timeValue(zonedDateTime.toLocalTime()); + assertNotNull(new TermQuery().doBuild("time", dateType, literal)); + } + + @Test + void test_date_with_no_format() { + OpenSearchDateType openSearchDateType = OpenSearchDateType.of(ExprCoreType.DATE); + assertNotNull( + new TermQuery().doBuild("date", openSearchDateType, new ExprDateValue("2021-11-08"))); + } + + @Test + void test_date_has_format() { + String dateString = "2021-11-08"; + String format = "yyyy-MM-dd"; + OpenSearchDateType dateType = OpenSearchDateType.of(format); + LocalDate parsedDate = dateType.getParsedDateTime(dateString).toLocalDate(); + ExprValue literal = ExprValueUtils.dateValue(parsedDate); + assertNotNull(new TermQuery().doBuild("date", dateType, literal)); + } + + @Test + void test_invalid_date_field_type() { + String dateString = "2021-11-08"; + OpenSearchDateType dateType = OpenSearchDateType.of(STRING); + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new TermQuery().doBuild("string_value", dateType, literal)); + } + + @Test + void test_string_field_type() { + String dateString = "2021-11-08"; + ExprValue literal = ExprValueUtils.stringValue(dateString); + assertNotNull(new TermQuery().doBuild("string_value", STRING, literal)); + } +} From db2bd6619453a320eadd741df8ca318a905f3fa4 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar <reddyvam@amazon.com> Date: Fri, 19 Jul 2024 13:31:27 -0700 Subject: [PATCH 08/96] Restrict UDF functions (#2790) Signed-off-by: Vamsi Manohar <reddyvam@amazon.com> --- .../src/main/antlr/SqlBaseLexer.g4 | 1 + .../src/main/antlr/SqlBaseParser.g4 | 52 +++++++++---- .../dispatcher/SparkQueryDispatcher.java | 62 +++++++++++----- .../sql/spark/utils/SQLQueryUtils.java | 28 +++++++ .../dispatcher/SparkQueryDispatcherTest.java | 73 +++++++++++++++++++ .../sql/spark/utils/SQLQueryUtilsTest.java | 19 +++++ docs/user/interfaces/asyncqueryinterface.rst | 2 + 7 files changed, 206 insertions(+), 31 deletions(-) diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index 85a4633e80..bde298c23e 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -316,6 +316,7 @@ NANOSECOND: 'NANOSECOND'; NANOSECONDS: 'NANOSECONDS'; NATURAL: 'NATURAL'; NO: 'NO'; +NONE: 'NONE'; NOT: 'NOT'; NULL: 'NULL'; NULLS: 'NULLS'; diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index 54eff14b6d..a50051715e 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -52,7 +52,7 @@ singleCompoundStatement ; beginEndCompoundBlock - : BEGIN compoundBody END + : beginLabel? BEGIN compoundBody END endLabel? ; compoundBody @@ -61,11 +61,26 @@ compoundBody compoundStatement : statement + | setStatementWithOptionalVarKeyword | beginEndCompoundBlock ; +setStatementWithOptionalVarKeyword + : SET (VARIABLE | VAR)? assignmentList #setVariableWithOptionalKeyword + | SET (VARIABLE | VAR)? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ + LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword + ; + singleStatement - : statement SEMICOLON* EOF + : (statement|setResetStatement) SEMICOLON* EOF + ; + +beginLabel + : multipartIdentifier COLON + ; + +endLabel + : multipartIdentifier ; singleExpression @@ -174,6 +189,8 @@ statement | ALTER TABLE identifierReference (partitionSpec)? SET locationSpec #setTableLocation | ALTER TABLE identifierReference RECOVER PARTITIONS #recoverPartitions + | ALTER TABLE identifierReference + (clusterBySpec | CLUSTER BY NONE) #alterClusterBy | DROP TABLE (IF EXISTS)? identifierReference PURGE? #dropTable | DROP VIEW (IF EXISTS)? identifierReference #dropView | CREATE (OR REPLACE)? (GLOBAL? TEMPORARY)? @@ -202,7 +219,7 @@ statement identifierReference dataType? variableDefaultExpression? #createVariable | DROP TEMPORARY VARIABLE (IF EXISTS)? identifierReference #dropVariable | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? - statement #explain + (statement|setResetStatement) #explain | SHOW TABLES ((FROM | IN) identifierReference)? (LIKE? pattern=stringLit)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? @@ -241,26 +258,29 @@ statement | (MSCK)? REPAIR TABLE identifierReference (option=(ADD|DROP|SYNC) PARTITIONS)? #repairTable | op=(ADD | LIST) identifier .*? #manageResource - | SET COLLATION collationName=identifier #setCollation - | SET ROLE .*? #failNativeCommand + | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? + identifierReference (USING indexType=identifier)? + LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN + (OPTIONS options=propertyList)? #createIndex + | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex + | unsupportedHiveNativeCommands .*? #failNativeCommand + ; + +setResetStatement + : SET COLLATION collationName=identifier #setCollation + | SET ROLE .*? #failSetRole | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone | SET (VARIABLE | VAR) assignmentList #setVariable | SET (VARIABLE | VAR) LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariable + LEFT_PAREN query RIGHT_PAREN #setVariable | SET configKey EQ configValue #setQuotedConfiguration | SET configKey (EQ .*?)? #setConfiguration | SET .*? EQ configValue #setQuotedConfiguration | SET .*? #setConfiguration | RESET configKey #resetQuotedConfiguration | RESET .*? #resetConfiguration - | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? - identifierReference (USING indexType=identifier)? - LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN - (OPTIONS options=propertyList)? #createIndex - | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex - | unsupportedHiveNativeCommands .*? #failNativeCommand ; executeImmediate @@ -853,13 +873,17 @@ identifierComment relationPrimary : identifierReference temporalClause? - sample? tableAlias #tableName + optionsClause? sample? tableAlias #tableName | LEFT_PAREN query RIGHT_PAREN sample? tableAlias #aliasedQuery | LEFT_PAREN relation RIGHT_PAREN sample? tableAlias #aliasedRelation | inlineTable #inlineTableDefault2 | functionTable #tableValuedFunction ; +optionsClause + : WITH options=propertyList + ; + inlineTable : VALUES expression (COMMA expression)* tableAlias ; @@ -1572,6 +1596,7 @@ ansiNonReserved | NANOSECOND | NANOSECONDS | NO + | NONE | NULLS | NUMERIC | OF @@ -1920,6 +1945,7 @@ nonReserved | NANOSECOND | NANOSECONDS | NO + | NONE | NOT | NULL | NULLS diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 3366e21894..0e871f9ddc 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -6,6 +6,7 @@ package org.opensearch.sql.spark.dispatcher; import java.util.HashMap; +import java.util.List; import java.util.Map; import lombok.AllArgsConstructor; import org.jetbrains.annotations.NotNull; @@ -45,25 +46,50 @@ public DispatchQueryResponse dispatch( this.dataSourceService.verifyDataSourceAccessAndGetRawMetadata( dispatchQueryRequest.getDatasource()); - if (LangType.SQL.equals(dispatchQueryRequest.getLangType()) - && SQLQueryUtils.isFlintExtensionQuery(dispatchQueryRequest.getQuery())) { - IndexQueryDetails indexQueryDetails = getIndexQueryDetails(dispatchQueryRequest); - DispatchQueryContext context = - getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) - .indexQueryDetails(indexQueryDetails) - .asyncQueryRequestContext(asyncQueryRequestContext) - .build(); - - return getQueryHandlerForFlintExtensionQuery(dispatchQueryRequest, indexQueryDetails) - .submit(dispatchQueryRequest, context); - } else { - DispatchQueryContext context = - getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) - .asyncQueryRequestContext(asyncQueryRequestContext) - .build(); - return getDefaultAsyncQueryHandler(dispatchQueryRequest.getAccountId()) - .submit(dispatchQueryRequest, context); + if (LangType.SQL.equals(dispatchQueryRequest.getLangType())) { + String query = dispatchQueryRequest.getQuery(); + + if (SQLQueryUtils.isFlintExtensionQuery(query)) { + return handleFlintExtensionQuery( + dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); + } + + List<String> validationErrors = SQLQueryUtils.validateSparkSqlQuery(query); + if (!validationErrors.isEmpty()) { + throw new IllegalArgumentException( + "Query is not allowed: " + String.join(", ", validationErrors)); + } } + return handleDefaultQuery(dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); + } + + private DispatchQueryResponse handleFlintExtensionQuery( + DispatchQueryRequest dispatchQueryRequest, + AsyncQueryRequestContext asyncQueryRequestContext, + DataSourceMetadata dataSourceMetadata) { + IndexQueryDetails indexQueryDetails = getIndexQueryDetails(dispatchQueryRequest); + DispatchQueryContext context = + getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) + .indexQueryDetails(indexQueryDetails) + .asyncQueryRequestContext(asyncQueryRequestContext) + .build(); + + return getQueryHandlerForFlintExtensionQuery(dispatchQueryRequest, indexQueryDetails) + .submit(dispatchQueryRequest, context); + } + + private DispatchQueryResponse handleDefaultQuery( + DispatchQueryRequest dispatchQueryRequest, + AsyncQueryRequestContext asyncQueryRequestContext, + DataSourceMetadata dataSourceMetadata) { + + DispatchQueryContext context = + getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) + .asyncQueryRequestContext(asyncQueryRequestContext) + .build(); + + return getDefaultAsyncQueryHandler(dispatchQueryRequest.getAccountId()) + .submit(dispatchQueryRequest, context); } private DispatchQueryContext.DispatchQueryContextBuilder getDefaultDispatchContextBuilder( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index a96e203cea..0bb9cb4b85 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -5,6 +5,8 @@ package org.opensearch.sql.spark.utils; +import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -73,6 +75,32 @@ public static boolean isFlintExtensionQuery(String sqlQuery) { } } + public static List<String> validateSparkSqlQuery(String sqlQuery) { + SparkSqlValidatorVisitor sparkSqlValidatorVisitor = new SparkSqlValidatorVisitor(); + SqlBaseParser sqlBaseParser = + new SqlBaseParser( + new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery)))); + sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); + try { + SqlBaseParser.StatementContext statement = sqlBaseParser.statement(); + sparkSqlValidatorVisitor.visit(statement); + return sparkSqlValidatorVisitor.getValidationErrors(); + } catch (SyntaxCheckException syntaxCheckException) { + return Collections.emptyList(); + } + } + + private static class SparkSqlValidatorVisitor extends SqlBaseParserBaseVisitor<Void> { + + @Getter private final List<String> validationErrors = new ArrayList<>(); + + @Override + public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { + validationErrors.add("Creating user-defined functions is not allowed"); + return super.visitCreateFunction(ctx); + } + } + public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor<Void> { @Getter private List<FullyQualifiedTableName> fullyQualifiedTableNames = new LinkedList<>(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 5582de332c..f9a83ef9f6 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -41,9 +41,11 @@ import com.amazonaws.services.emrserverless.model.GetJobRunResult; import com.amazonaws.services.emrserverless.model.JobRun; import com.amazonaws.services.emrserverless.model.JobRunState; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -438,6 +440,77 @@ void testDispatchWithPPLQuery() { verifyNoInteractions(flintIndexMetadataService); } + @Test + void testDispatchWithSparkUDFQuery() { + List<String> udfQueries = new ArrayList<>(); + udfQueries.add( + "CREATE FUNCTION celsius_to_fahrenheit AS 'org.apache.spark.sql.functions.expr(\"(celsius *" + + " 9/5) + 32\")'"); + udfQueries.add( + "CREATE TEMPORARY FUNCTION square AS 'org.apache.spark.sql.functions.expr(\"num * num\")'"); + for (String query : udfQueries) { + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + .thenReturn(dataSourceMetadata); + + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + sparkQueryDispatcher.dispatch( + getBaseDispatchQueryRequestBuilder(query).langType(LangType.SQL).build(), + asyncQueryRequestContext)); + Assertions.assertEquals( + "Query is not allowed: Creating user-defined functions is not allowed", + illegalArgumentException.getMessage()); + verifyNoInteractions(emrServerlessClient); + verifyNoInteractions(flintIndexMetadataService); + } + } + + @Test + void testInvalidSQLQueryDispatchToSpark() { + when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + HashMap<String, String> tags = new HashMap<>(); + tags.put(DATASOURCE_TAG_KEY, MY_GLUE); + tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); + tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); + String query = "myselect 1"; + String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + StartJobRequest expected = + new StartJobRequest( + "TEST_CLUSTER:batch", + null, + EMRS_APPLICATION_ID, + EMRS_EXECUTION_ROLE, + sparkSubmitParameters, + tags, + false, + "query_execution_result_my_glue"); + when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + .thenReturn(dataSourceMetadata); + + DispatchQueryResponse dispatchQueryResponse = + sparkQueryDispatcher.dispatch( + DispatchQueryRequest.builder() + .applicationId(EMRS_APPLICATION_ID) + .query(query) + .datasource(MY_GLUE) + .langType(LangType.SQL) + .executionRoleARN(EMRS_EXECUTION_ROLE) + .clusterName(TEST_CLUSTER_NAME) + .sparkSubmitParameterModifier(sparkSubmitParameterModifier) + .build(), + asyncQueryRequestContext); + + verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); + Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + verifyNoInteractions(flintIndexMetadataService); + } + @Test void testDispatchQueryWithoutATableAndDataSourceName() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index 0d7c43fc0d..bf6fe9e5db 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -390,6 +390,25 @@ void testAutoRefresh() { .autoRefresh()); } + @Test + void testValidateSparkSqlQuery_ValidQuery() { + String validQuery = "SELECT * FROM users WHERE age > 18"; + List<String> errors = SQLQueryUtils.validateSparkSqlQuery(validQuery); + assertTrue(errors.isEmpty(), "Valid query should not produce any errors"); + } + + @Test + void testValidateSparkSqlQuery_InvalidQuery() { + String invalidQuery = "CREATE FUNCTION myUDF AS 'com.example.UDF'"; + List<String> errors = SQLQueryUtils.validateSparkSqlQuery(invalidQuery); + assertFalse(errors.isEmpty(), "Invalid query should produce errors"); + assertEquals(1, errors.size(), "Should have one error"); + assertEquals( + "Creating user-defined functions is not allowed", + errors.get(0), + "Error message should match"); + } + @Getter protected static class IndexQuery { private String query; diff --git a/docs/user/interfaces/asyncqueryinterface.rst b/docs/user/interfaces/asyncqueryinterface.rst index af49a59838..9b889f7f97 100644 --- a/docs/user/interfaces/asyncqueryinterface.rst +++ b/docs/user/interfaces/asyncqueryinterface.rst @@ -68,6 +68,8 @@ Async Query Creation API ====================================== If security plugin is enabled, this API can only be invoked by users with permission ``cluster:admin/opensearch/ql/async_query/create``. +Limitation: Spark SQL queries that create User-Defined Functions (UDFs) are not allowed. + HTTP URI: ``_plugins/_async_query`` HTTP VERB: ``POST`` From 2117650358eb071070895f0863a5e3a2b831473a Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 23 Jul 2024 16:32:00 -0700 Subject: [PATCH 09/96] Fix SparkExecutionEngineConfigClusterSetting deserialize issue (#2838) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../config/SparkExecutionEngineConfigClusterSetting.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java index 5c1328bf91..adaaa57d31 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java @@ -6,9 +6,9 @@ package org.opensearch.sql.spark.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; import lombok.Builder; import lombok.Data; -import org.opensearch.sql.utils.SerializeUtils; /** * This POJO is just for reading stringified json in `plugins.query.executionengine.spark.config` @@ -29,7 +29,6 @@ public class SparkExecutionEngineConfigClusterSetting { public static SparkExecutionEngineConfigClusterSetting toSparkExecutionEngineConfig( String jsonString) { - return SerializeUtils.buildGson() - .fromJson(jsonString, SparkExecutionEngineConfigClusterSetting.class); + return new Gson().fromJson(jsonString, SparkExecutionEngineConfigClusterSetting.class); } } From 593ffabbbdc6d6a9529ef3d6c233785e29fedb07 Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Thu, 25 Jul 2024 00:10:09 +0800 Subject: [PATCH 10/96] IF function should support complex predicates in PPL (#2756) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- docs/user/ppl/functions/condition.rst | 11 ++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 16 ++- .../opensearch/sql/ppl/parser/AstBuilder.java | 3 +- .../sql/ppl/parser/AstExpressionBuilder.java | 2 +- .../ppl/parser/AstExpressionBuilderTest.java | 114 ++++++++++++++++++ .../sql/sql/antlr/SQLSyntaxParserTest.java | 16 +++ 6 files changed, 154 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index fea76bedda..e48d4cb75c 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -181,3 +181,14 @@ Example:: | Bates | Nanette | Bates | | Adams | Dale | Adams | +----------+-------------+------------+ + + os> source=accounts | eval is_vip = if(age > 30 AND isnotnull(employer), true, false) | fields is_vip, firstname, lastname + fetched rows / total rows = 4/4 + +----------+-------------+------------+ + | is_vip | firstname | lastname | + |----------+-------------+------------| + | True | Amber | Duke | + | True | Hattie | Bond | + | False | Nanette | Bates | + | False | Dale | Adams | + +----------+-------------+------------+ diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 39fb7f53a6..4dc223b028 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -255,6 +255,7 @@ expression | valueExpression ; +// predicates logicalExpression : comparisonExpression # comparsion | NOT logicalExpression # logicalNot @@ -362,7 +363,7 @@ dataTypeFunctionCall // boolean functions booleanFunctionCall - : conditionFunctionBase LT_PRTHS functionArgs RT_PRTHS + : conditionFunctionName LT_PRTHS functionArgs RT_PRTHS ; convertedDataType @@ -382,7 +383,8 @@ evalFunctionName : mathematicalFunctionName | dateTimeFunctionName | textFunctionName - | conditionFunctionBase + | conditionFunctionName + | flowControlFunctionName | systemFunctionName | positionFunctionName ; @@ -392,7 +394,7 @@ functionArgs ; functionArg - : (ident EQUAL)? valueExpression + : (ident EQUAL)? expression ; relevanceArg @@ -623,11 +625,15 @@ timestampFunctionName ; // condition function return boolean value -conditionFunctionBase +conditionFunctionName : LIKE - | IF | ISNULL | ISNOTNULL + ; + +// flow control function return non-boolean value +flowControlFunctionName + : IF | IFNULL | NULLIF ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3c693fa0bd..78fe28b49e 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -332,8 +332,7 @@ public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) { arg -> { String argName = (arg.ident() != null) ? arg.ident().getText() : null; builder.add( - new UnresolvedArgument( - argName, this.internalVisitExpression(arg.valueExpression()))); + new UnresolvedArgument(argName, this.internalVisitExpression(arg.expression()))); }); return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f36765d3d7..aec22ac231 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -187,7 +187,7 @@ public UnresolvedExpression visitTakeAggFunctionCall( /** Eval function. */ @Override public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext ctx) { - final String functionName = ctx.conditionFunctionBase().getText().toLowerCase(); + final String functionName = ctx.conditionFunctionName().getText().toLowerCase(); return buildFunction( FUNCTION_NAME_MAPPING.getOrDefault(functionName, functionName), ctx.functionArgs().functionArg()); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index de230a1fee..fbb25549ab 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -138,6 +138,120 @@ public void testEvalFunctionExprNoArgs() { assertEqual("source=t | eval f=PI()", eval(relation("t"), let(field("f"), function("PI")))); } + @Test + public void testEvalIfFunctionExpr() { + assertEqual( + "source=t | eval f=if(true, 1, 0)", + eval( + relation("t"), + let(field("f"), function("if", booleanLiteral(true), intLiteral(1), intLiteral(0))))); + assertEqual( + "source=t | eval f=if(1>2, 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + compare(">", intLiteral(1), intLiteral(2)), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(1<=2, 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + compare("<=", intLiteral(1), intLiteral(2)), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(1=2, 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + compare("=", intLiteral(1), intLiteral(2)), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(1!=2, 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + compare("!=", intLiteral(1), intLiteral(2)), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(isnull(a), 1, 0)", + eval( + relation("t"), + let( + field("f"), + function("if", function("is null", field("a")), intLiteral(1), intLiteral(0))))); + assertEqual( + "source=t | eval f=if(isnotnull(a), 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", function("is not null", field("a")), intLiteral(1), intLiteral(0))))); + assertEqual( + "source=t | eval f=if(not 1>2, 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + not(compare(">", intLiteral(1), intLiteral(2))), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(not a in (0, 1), 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + not(in(field("a"), intLiteral(0), intLiteral(1))), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(not a in (0, 1) OR isnull(a), 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + or( + not(in(field("a"), intLiteral(0), intLiteral(1))), + function("is null", field("a"))), + intLiteral(1), + intLiteral(0))))); + assertEqual( + "source=t | eval f=if(like(a, '_a%b%c_d_'), 1, 0)", + eval( + relation("t"), + let( + field("f"), + function( + "if", + function("like", field("a"), stringLiteral("_a%b%c_d_")), + intLiteral(1), + intLiteral(0))))); + } + @Test public void testPositionFunctionExpr() { assertEqual( diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index f68c27deea..c43044508b 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -719,6 +719,22 @@ public void canParseMultiMatchAlternateSyntax() { assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multimatch(\"query\")")); } + @Test + public void canParseIfFunction() { + assertNotNull(parser.parse("SELECT IF(1 > 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 < 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 >= 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 <= 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 <> 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 != 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 = 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(true, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(1 IS NOT NULL, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(NOT 1 > 2, 1, 0)")); + assertNotNull(parser.parse("SELECT IF(NOT 1 IN (0, 1), 1, 0)")); + assertNotNull(parser.parse("SELECT IF(NOT 1 IN (0, 1) OR 1 IS NOT NULL, 1, 0)")); + } + private static Stream<String> matchPhraseQueryComplexQueries() { return Stream.of( "SELECT * FROM t WHERE matchphrasequery(c, 3)", From a5ede6436b20a63b30da02598ee6f864bf985bf6 Mon Sep 17 00:00:00 2001 From: Chen Dai <daichen@amazon.com> Date: Thu, 25 Jul 2024 11:28:35 -0700 Subject: [PATCH 11/96] Add release notes for v2.16.0 (#2854) * Add release notes for 2.16 Signed-off-by: Chen Dai <daichen@amazon.com> * Update with latest PRs Signed-off-by: Chen Dai <daichen@amazon.com> * Update with latest PRs Signed-off-by: Chen Dai <daichen@amazon.com> --------- Signed-off-by: Chen Dai <daichen@amazon.com> --- .../opensearch-sql.release-notes-2.16.0.0.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.16.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.16.0.0.md b/release-notes/opensearch-sql.release-notes-2.16.0.0.md new file mode 100644 index 0000000000..607d6e14dd --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.16.0.0.md @@ -0,0 +1,39 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.16.0 + +### Enhancements +* Added Setting to Toggle Data Source Management Code Paths ([#2811](https://github.com/opensearch-project/sql/pull/2811)) +* Span in PPL statsByClause could be specified after fields ([#2810](https://github.com/opensearch-project/sql/pull/2810)) +* Updating Grammer changes same as main branch ([#2850](https://github.com/opensearch-project/sql/pull/2850)) + +### Bug Fixes +* Temp use of older nodejs version before moving to Almalinux8 ([#2816](https://github.com/opensearch-project/sql/pull/2816)) +* Fix yaml errors causing checks not to be run ([#2823](https://github.com/opensearch-project/sql/pull/2823)) +* Well format the raw response when query parameter "pretty" enabled ([#2829](https://github.com/opensearch-project/sql/pull/2829)) +* Add support for custom date format and openSearch date format for date fields as part of Lucene query ([#2762](https://github.com/opensearch-project/sql/pull/2762)) +* Fix SparkExecutionEngineConfigClusterSetting deserialize issue ([#2838](https://github.com/opensearch-project/sql/pull/2838)) +* Fix SparkSubmitParameterModifier issue ([#2837](https://github.com/opensearch-project/sql/pull/2837)) + +### Infrastructure +* Increment version to 2.16.0-SNAPSHOT ([#2743](https://github.com/opensearch-project/sql/pull/2743)) +* Fix checkout action failure ([#2819](https://github.com/opensearch-project/sql/pull/2819)) +* Fix MacOS workflow failure ([#2831](https://github.com/opensearch-project/sql/pull/2831)) + +### Refactoring +* Change DataSourceType from enum to class ([#2746](https://github.com/opensearch-project/sql/pull/2746)) +* Fix code style issue ([#2745](https://github.com/opensearch-project/sql/pull/2745)) +* Scaffold async-query-core and async-query module ([#2751](https://github.com/opensearch-project/sql/pull/2751)) +* Move classes from spark to async-query-core and async-query ([#2750](https://github.com/opensearch-project/sql/pull/2750)) +* Exclude integ-test, doctest and download task when built offline ([#2763](https://github.com/opensearch-project/sql/pull/2763)) +* Abstract metrics to reduce dependency to legacy ([#2768](https://github.com/opensearch-project/sql/pull/2768)) +* Remove AsyncQueryId ([#2769](https://github.com/opensearch-project/sql/pull/2769)) +* Add README to async-query-core ([#2770](https://github.com/opensearch-project/sql/pull/2770)) +* Separate build and validateAndBuild method in DataSourceMetadata ([#2752](https://github.com/opensearch-project/sql/pull/2752)) +* Abstract FlintIndex client ([#2771](https://github.com/opensearch-project/sql/pull/2771)) +* Fix statement to store requested langType ([#2779](https://github.com/opensearch-project/sql/pull/2779)) +* Push down OpenSearch specific exception handling ([#2782](https://github.com/opensearch-project/sql/pull/2782)) +* Implement integration test for async-query-core ([#2785](https://github.com/opensearch-project/sql/pull/2785)) +* Fix SQLQueryUtils to extract multiple tables ([#2791](https://github.com/opensearch-project/sql/pull/2791)) +* Eliminate dependency from async-query-core to legacy ([#2792](https://github.com/opensearch-project/sql/pull/2792)) +* Pass accountId to EMRServerlessClientFactory.getClient ([#2822](https://github.com/opensearch-project/sql/pull/2822)) +* Register system index descriptors through SystemIndexPlugin.getSystemIndexDescriptors ([#2817](https://github.com/opensearch-project/sql/pull/2817)) +* Introduce SparkParameterComposerCollection ([#2824](https://github.com/opensearch-project/sql/pull/2824)) From 41100be087b016971250b686dff0d14aafa34b9f Mon Sep 17 00:00:00 2001 From: Vamsi Manohar <reddyvam@amazon.com> Date: Mon, 29 Jul 2024 13:44:56 -0700 Subject: [PATCH 12/96] Fix Integ test for datasource enabled setting with security plugin (#2865) Signed-off-by: Vamsi Manohar <reddyvam@amazon.com> --- .../sql/datasource/DataSourceEnabledIT.java | 39 +++++-------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java index 480a6dc563..9c522134a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java @@ -6,7 +6,6 @@ package org.opensearch.sql.datasource; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; -import static org.opensearch.sql.legacy.TestsConstants.DATASOURCES; import lombok.SneakyThrows; import org.json.JSONObject; @@ -25,31 +24,25 @@ protected boolean preserveClusterUponCompletion() { } @Test - public void testDataSourceIndexIsCreatedByDefault() { - assertDataSourceCount(0); - assertSelectFromDataSourceReturnsDoesNotExist(); - assertDataSourceIndexCreated(true); - } - - @Test - public void testDataSourceIndexIsCreatedIfSettingIsEnabled() { - setDataSourcesEnabled("transient", true); + public void testAsyncQueryAPIFailureIfSettingIsDisabled() { + setDataSourcesEnabled("transient", false); assertDataSourceCount(0); assertSelectFromDataSourceReturnsDoesNotExist(); - assertDataSourceIndexCreated(true); + assertAsyncQueryApiDisabled(); } @Test - public void testDataSourceIndexIsNotCreatedIfSettingIsDisabled() { - setDataSourcesEnabled("transient", false); - assertDataSourceCount(0); - assertSelectFromDataSourceReturnsDoesNotExist(); - assertDataSourceIndexCreated(false); - assertAsyncQueryApiDisabled(); + public void testDataSourceCreationWithDefaultSettings() { + createOpenSearchDataSource(); + createIndex(); + assertDataSourceCount(1); + assertSelectFromDataSourceReturnsSuccess(); + assertSelectFromDummyIndexInValidDataSourceDataSourceReturnsDoesNotExist(); } @Test public void testAfterPreviousEnable() { + setDataSourcesEnabled("transient", true); createOpenSearchDataSource(); createIndex(); assertDataSourceCount(1); @@ -141,18 +134,6 @@ private void assertDataSourceCount(int expected) { Assert.assertEquals(expected, jsonBody.getJSONArray("datarows").length()); } - @SneakyThrows - private void assertDataSourceIndexCreated(boolean expected) { - Request request = new Request("GET", "/" + DATASOURCES); - Response response = performRequest(request); - String responseBody = getResponseBody(response); - boolean indexDoesExist = - response.getStatusLine().getStatusCode() == 200 - && responseBody.contains(DATASOURCES) - && responseBody.contains("mappings"); - Assert.assertEquals(expected, indexDoesExist); - } - @SneakyThrows private Response performRequest(Request request) { try { From 6b8ee3da41908d9b0e8987feb19d02790f546158 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" <dblock@amazon.com> Date: Mon, 29 Jul 2024 16:51:32 -0500 Subject: [PATCH 13/96] Update PULL_REQUEST_TEMPLATE to include an API spec change in the checklist. (#2808) * Update PULL_REQUEST_TEMPLATE to include an API spec change in the checklist. Signed-off-by: dblock <dblock@amazon.com> * Re-added sections. Signed-off-by: dblock <dblock@amazon.com> --------- Signed-off-by: dblock <dblock@amazon.com> --- .github/PULL_REQUEST_TEMPLATE.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2e325678fe..c84ed5b13a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,16 +1,18 @@ ### Description [Describe what this change achieves] - -### Issues Resolved -[List any issues this PR will resolve] - + +### Related Issues +Resolves #[Issue number to be closed when this PR is merged] +<!-- List any other related issues here --> + ### Check List - [ ] New functionality includes testing. - - [ ] All tests pass, including unit test, integration test and doctest - [ ] New functionality has been documented. - - [ ] New functionality has javadoc added - - [ ] New functionality has user manual doc added -- [ ] Commits are signed per the DCO using --signoff + - [ ] New functionality has javadoc added. + - [ ] New functionality has a user manual doc added. +- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md). +- [ ] Commits are signed per the DCO using `--signoff`. +- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose). By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. -For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). \ No newline at end of file +For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/sql/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). From ba82e1255b301d92eee9e1ad36e44e07afdb3839 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 30 Jul 2024 13:51:35 -0700 Subject: [PATCH 14/96] Add RequestContext parameter to verifyDataSourceAccessAndGetRawMetada method (#2866) * Add RequestContext parameter to verifyDataSourceAccessAndGetRawMetadata method Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add comments Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix style Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../model/AsyncQueryRequestContext.java | 6 +- .../dispatcher/SparkQueryDispatcher.java | 2 +- .../asyncquery/AsyncQueryCoreIntegTest.java | 3 +- .../dispatcher/SparkQueryDispatcherTest.java | 72 ++++++++++++------- .../sql/datasource/DataSourceService.java | 5 +- .../sql/datasource/RequestContext.java | 15 ++++ .../sql/analysis/AnalyzerTestBase.java | 4 +- .../service/DataSourceServiceImpl.java | 4 +- .../service/DataSourceServiceImplTest.java | 8 ++- 9 files changed, 84 insertions(+), 35 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/datasource/RequestContext.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java index 56176faefb..d5a478d592 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java @@ -5,7 +5,7 @@ package org.opensearch.sql.spark.asyncquery.model; +import org.opensearch.sql.datasource.RequestContext; + /** Context interface to provide additional request related information */ -public interface AsyncQueryRequestContext { - Object getAttribute(String name); -} +public interface AsyncQueryRequestContext extends RequestContext {} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 0e871f9ddc..0061ea7179 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -44,7 +44,7 @@ public DispatchQueryResponse dispatch( AsyncQueryRequestContext asyncQueryRequestContext) { DataSourceMetadata dataSourceMetadata = this.dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - dispatchQueryRequest.getDatasource()); + dispatchQueryRequest.getDatasource(), asyncQueryRequestContext); if (LangType.SQL.equals(dispatchQueryRequest.getLangType())) { String query = dispatchQueryRequest.getQuery(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 99d4cc722e..34ededc74d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -512,7 +512,8 @@ private void givenFlintIndexMetadataExists(String indexName) { } private void givenValidDataSourceMetadataExist() { - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(DATASOURCE_NAME)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn( new DataSourceMetadata.Builder() .setName(DATASOURCE_NAME) diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index f9a83ef9f6..a7a79c758e 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -180,7 +180,8 @@ void testDispatchSelectQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -223,7 +224,8 @@ void testDispatchSelectQueryWithLakeFormation() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadataWithLakeFormation(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -255,7 +257,8 @@ void testDispatchSelectQueryWithBasicAuthIndexStoreDatasource() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadataWithBasicAuth(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -278,7 +281,8 @@ void testDispatchSelectQueryCreateNewSession() { doReturn(new StatementId(MOCK_STATEMENT_ID)).when(session).submit(any(), any()); when(session.getSessionModel().getJobId()).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -304,7 +308,8 @@ void testDispatchSelectQueryReuseSession() { when(session.getSessionModel().getJobId()).thenReturn(EMR_JOB_ID); when(session.isOperationalForDataSource(any())).thenReturn(true); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -324,7 +329,8 @@ void testDispatchSelectQueryFailedCreateSession() { doReturn(true).when(sessionManager).isEnabled(); doThrow(RuntimeException.class).when(sessionManager).createSession(any(), any()); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); Assertions.assertThrows( @@ -358,7 +364,8 @@ void testDispatchCreateAutoRefreshIndexQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -393,7 +400,8 @@ void testDispatchCreateManualRefreshIndexQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_glue")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_glue", asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -426,7 +434,8 @@ void testDispatchWithPPLQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -450,7 +459,8 @@ void testDispatchWithSparkUDFQuery() { "CREATE TEMPORARY FUNCTION square AS 'org.apache.spark.sql.functions.expr(\"num * num\")'"); for (String query : udfQueries) { DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); IllegalArgumentException illegalArgumentException = @@ -489,7 +499,8 @@ void testInvalidSQLQueryDispatchToSpark() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -532,7 +543,8 @@ void testDispatchQueryWithoutATableAndDataSourceName() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -568,7 +580,8 @@ void testDispatchIndexQueryWithoutADatasourceName() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -589,8 +602,7 @@ void testDispatchMaterializedViewQuery() { tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.STREAMING.getText()); String query = - "CREATE MATERIALIZED VIEW mv_1 AS query=select * from my_glue.default.logs WITH" - + " (auto_refresh = true)"; + "CREATE MATERIALIZED VIEW mv_1 AS select * from logs WITH" + " (auto_refresh = true)"; String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query, "streaming"); StartJobRequest expected = new StartJobRequest( @@ -604,7 +616,8 @@ void testDispatchMaterializedViewQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -637,7 +650,8 @@ void testDispatchShowMVQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -670,7 +684,8 @@ void testRefreshIndexQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -703,7 +718,8 @@ void testDispatchDescribeIndexQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -739,7 +755,8 @@ void testDispatchAlterToAutoRefreshIndexQuery() { "query_execution_result_my_glue"); when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_glue")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_glue", asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); DispatchQueryResponse dispatchQueryResponse = @@ -762,7 +779,8 @@ void testDispatchAlterToManualRefreshIndexQuery() { "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = false)"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_glue")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_glue", asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); when(queryHandlerFactory.getIndexDMLHandler()) .thenReturn( @@ -785,7 +803,8 @@ void testDispatchDropIndexQuery() { String query = "DROP INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_glue")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_glue", asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); when(queryHandlerFactory.getIndexDMLHandler()) .thenReturn( @@ -808,7 +827,8 @@ void testDispatchVacuumIndexQuery() { String query = "VACUUM INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_glue")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_glue", asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); when(queryHandlerFactory.getIndexDMLHandler()) .thenReturn( @@ -824,7 +844,8 @@ void testDispatchVacuumIndexQuery() { @Test void testDispatchWithUnSupportedDataSourceType() { - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata("my_prometheus")) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "my_prometheus", asyncQueryRequestContext)) .thenReturn(constructPrometheusDataSourceType()); String query = "select * from my_prometheus.default.http_logs"; @@ -1018,7 +1039,8 @@ void testGetQueryResponseWithSuccess() { void testDispatchQueryWithExtraSparkSubmitParameters() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata(MY_GLUE)) + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) .thenReturn(dataSourceMetadata); String extraParameters = "--conf spark.dynamicAllocation.enabled=false"; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java index 6af5d19e5c..a8caa4719a 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java +++ b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java @@ -82,6 +82,9 @@ public interface DataSourceService { * Specifically for addressing use cases in SparkQueryDispatcher. * * @param dataSourceName of the {@link DataSource} + * @param context request context used by the implementation. It is passed by async-query-core. + * refer {@link RequestContext} */ - DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata(String dataSourceName); + DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata( + String dataSourceName, RequestContext context); } diff --git a/core/src/main/java/org/opensearch/sql/datasource/RequestContext.java b/core/src/main/java/org/opensearch/sql/datasource/RequestContext.java new file mode 100644 index 0000000000..199930d340 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/datasource/RequestContext.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.datasource; + +/** + * Context interface to provide additional request related information. It is introduced to allow + * async-query-core library user to pass request context information to implementations of data + * accessors. + */ +public interface RequestContext { + Object getAttribute(String name); +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index b35cfbb5e1..0bf959a1b7 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -28,6 +28,7 @@ import org.opensearch.sql.config.TestConfig; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.RequestContext; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; @@ -236,7 +237,8 @@ public Boolean dataSourceExists(String dataSourceName) { } @Override - public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata(String dataSourceName) { + public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata( + String dataSourceName, RequestContext requestContext) { return null; } } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java index 61f3c8cd5d..81b6432891 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java @@ -11,6 +11,7 @@ import java.util.*; import java.util.stream.Collectors; import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.RequestContext; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceStatus; @@ -122,7 +123,8 @@ public Boolean dataSourceExists(String dataSourceName) { } @Override - public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata(String dataSourceName) { + public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata( + String dataSourceName, RequestContext requestContext) { DataSourceMetadata dataSourceMetadata = getRawDataSourceMetadata(dataSourceName); verifyDataSourceAccess(dataSourceMetadata); return dataSourceMetadata; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java index 5a94945e5b..9a1022706f 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java @@ -36,6 +36,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.RequestContext; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceStatus; @@ -52,6 +53,7 @@ class DataSourceServiceImplTest { @Mock private DataSourceFactory dataSourceFactory; @Mock private StorageEngine storageEngine; @Mock private DataSourceMetadataStorage dataSourceMetadataStorage; + @Mock private RequestContext requestContext; @Mock private DataSourceUserAuthorizationHelper dataSourceUserAuthorizationHelper; @@ -461,7 +463,9 @@ void testVerifyDataSourceAccessAndGetRawDataSourceMetadataWithDisabledData() { DatasourceDisabledException datasourceDisabledException = Assertions.assertThrows( DatasourceDisabledException.class, - () -> dataSourceService.verifyDataSourceAccessAndGetRawMetadata("testDS")); + () -> + dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + "testDS", requestContext)); Assertions.assertEquals( "Datasource testDS is disabled.", datasourceDisabledException.getMessage()); } @@ -484,7 +488,7 @@ void testVerifyDataSourceAccessAndGetRawDataSourceMetadata() { when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")) .thenReturn(Optional.of(dataSourceMetadata)); DataSourceMetadata dataSourceMetadata1 = - dataSourceService.verifyDataSourceAccessAndGetRawMetadata("testDS"); + dataSourceService.verifyDataSourceAccessAndGetRawMetadata("testDS", requestContext); assertTrue(dataSourceMetadata1.getProperties().containsKey("prometheus.uri")); assertTrue(dataSourceMetadata1.getProperties().containsKey("prometheus.auth.type")); assertTrue(dataSourceMetadata1.getProperties().containsKey("prometheus.auth.username")); From 103c4160ae2a129284ff5be70bac40951e0c6a18 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar <reddyvam@amazon.com> Date: Tue, 30 Jul 2024 15:20:19 -0700 Subject: [PATCH 15/96] Fixed 2.16 integ test failures (#2871) Signed-off-by: Vamsi Manohar <reddyvam@amazon.com> --- .../opensearch/sql/datasource/DataSourceEnabledIT.java | 10 ++++++++++ .../sql/legacy/OpenSearchSQLRestTestCase.java | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java index 9c522134a4..b0bc87a0c6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java @@ -38,6 +38,7 @@ public void testDataSourceCreationWithDefaultSettings() { assertDataSourceCount(1); assertSelectFromDataSourceReturnsSuccess(); assertSelectFromDummyIndexInValidDataSourceDataSourceReturnsDoesNotExist(); + deleteSelfDataSourceCreated(); } @Test @@ -52,6 +53,8 @@ public void testAfterPreviousEnable() { assertDataSourceCount(0); assertSelectFromDataSourceReturnsDoesNotExist(); assertAsyncQueryApiDisabled(); + setDataSourcesEnabled("transient", true); + deleteSelfDataSourceCreated(); } @SneakyThrows @@ -142,4 +145,11 @@ private Response performRequest(Request request) { return e.getResponse(); } } + + @SneakyThrows + private void deleteSelfDataSourceCreated() { + Request deleteRequest = getDeleteDataSourceRequest("self"); + Response deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index d73e3468d4..ced69d54a0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -195,7 +195,9 @@ protected static void wipeAllOpenSearchIndices(RestClient client) throws IOExcep try { // System index, mostly named .opensearch-xxx or .opendistro-xxx, are not allowed to // delete - if (!indexName.startsWith(".opensearch") && !indexName.startsWith(".opendistro")) { + if (!indexName.startsWith(".opensearch") + && !indexName.startsWith(".opendistro") + && !indexName.startsWith(".ql")) { client.performRequest(new Request("DELETE", "/" + indexName)); } } catch (Exception e) { From aa7a6902a8d03647eecf45564c488c936c53ee3f Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Wed, 31 Jul 2024 17:05:28 +0800 Subject: [PATCH 16/96] Change the default value of plugins.query.size_limit to MAX_RESULT_WINDOW (10000) (#2860) * Change the default value of plugins.query.size_limit to MAX_RESULT_WINDOW (10000) Signed-off-by: Lantao Jin <ltjin@amazon.com> * fix ut Signed-off-by: Lantao Jin <ltjin@amazon.com> * fix spotless Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com> --- docs/user/admin/settings.rst | 2 +- docs/user/optimization/optimization.rst | 22 +++++++++---------- docs/user/ppl/admin/settings.rst | 4 ++-- docs/user/ppl/interfaces/endpoint.rst | 2 +- .../org/opensearch/sql/legacy/ExplainIT.java | 2 +- .../setting/OpenSearchSettings.java | 3 ++- .../setting/OpenSearchSettingsTest.java | 6 ++--- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 662d882745..6b24e41f87 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -202,7 +202,7 @@ plugins.query.size_limit Description ----------- -The new engine fetches a default size of index from OpenSearch set by this setting, the default value is 200. You can change the value to any value not greater than the max result window value in index level (10000 by default), here is an example:: +The new engine fetches a default size of index from OpenSearch set by this setting, the default value equals to max result window in index level (10000 by default). You can change the value to any value not greater than the max result window value in index level (`index.max_result_window`), here is an example:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ "transient" : { diff --git a/docs/user/optimization/optimization.rst b/docs/user/optimization/optimization.rst index 8ab998309d..835fe96eba 100644 --- a/docs/user/optimization/optimization.rst +++ b/docs/user/optimization/optimization.rst @@ -44,7 +44,7 @@ The consecutive Filter operator will be merged as one Filter operator:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)" }, "children": [] } @@ -71,7 +71,7 @@ The Filter operator should be push down under Sort operator:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)" }, "children": [] } @@ -102,7 +102,7 @@ The Project list will push down to Query DSL to `filter the source <https://www. { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)" }, "children": [] } @@ -128,7 +128,7 @@ The Filter operator will merge into OpenSearch Query DSL:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)" }, "children": [] } @@ -154,7 +154,7 @@ The Sort operator will merge into OpenSearch Query DSL:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)" }, "children": [] } @@ -188,7 +188,7 @@ Because the OpenSearch Script Based Sorting can't handle NULL/MISSING value, the { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\"}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)" }, "children": [] } @@ -257,7 +257,7 @@ If sort that includes expression, which cannot be merged into query DSL, also ex { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\"}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)" }, "children": [] } @@ -287,7 +287,7 @@ The Aggregation operator will merge into OpenSearch Aggregation:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" }, "children": [] } @@ -313,7 +313,7 @@ The Sort operator will merge into OpenSearch Aggregation.:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" }, "children": [] } @@ -348,7 +348,7 @@ Because the OpenSearch Composite Aggregation doesn't support order by metrics fi { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)" }, "children": [] } @@ -373,4 +373,4 @@ At the moment there is no optimization to merge similar sort operators to avoid Sort Push Down -------------- -Without sort push down optimization, the sort operator will sort the result from child operator. By default, only 200 docs will extracted from the source index, `you can change this value by using size_limit setting <../admin/settings.rst#opensearch-query-size-limit>`_. +Without sort push down optimization, the sort operator will sort the result from child operator. By default, only 10000 docs will extracted from the source index, `you can change this value by using size_limit setting <../admin/settings.rst#opensearch-query-size-limit>`_. diff --git a/docs/user/ppl/admin/settings.rst b/docs/user/ppl/admin/settings.rst index ad56408693..28e6897d3d 100644 --- a/docs/user/ppl/admin/settings.rst +++ b/docs/user/ppl/admin/settings.rst @@ -125,9 +125,9 @@ plugins.query.size_limit Description ----------- -The size configure the maximum amount of documents to be pull from OpenSearch. The default value is: 200 +The size configure the maximum amount of documents to be pull from OpenSearch. The default value is: 10000 -Notes: This setting will impact the correctness of the aggregation operation, for example, there are 1000 docs in the index, by default, only 200 docs will be extract from index and do aggregation. +Notes: This setting will impact the correctness of the aggregation operation, for example, there are 1000 docs in the index, if you change the value to 200, only 200 docs will be extract from index and do aggregation. Example ------- diff --git a/docs/user/ppl/interfaces/endpoint.rst b/docs/user/ppl/interfaces/endpoint.rst index 793b94eb8d..fb931fb0ba 100644 --- a/docs/user/ppl/interfaces/endpoint.rst +++ b/docs/user/ppl/interfaces/endpoint.rst @@ -91,7 +91,7 @@ The following PPL query demonstrated that where and stats command were pushed do { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)" }, "children": [] } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ExplainIT.java index b42e9f84f4..27f8eca3ef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ExplainIT.java @@ -185,7 +185,7 @@ public void orderByOnNestedFieldTest() throws Exception { Assert.assertThat( result.replaceAll("\\s+", ""), equalTo( - "{\"from\":0,\"size\":200,\"sort\":[{\"message.info\":" + "{\"from\":0,\"size\":10000,\"sort\":[{\"message.info\":" + "{\"order\":\"asc\",\"nested\":{\"path\":\"message\"}}}]}")); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index b4ce82a828..475a584623 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -28,6 +28,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.IndexSettings; import org.opensearch.sql.common.setting.LegacySettings; import org.opensearch.sql.common.setting.Settings; @@ -90,7 +91,7 @@ public class OpenSearchSettings extends Settings { public static final Setting<?> QUERY_SIZE_LIMIT_SETTING = Setting.intSetting( Key.QUERY_SIZE_LIMIT.getKeyValue(), - LegacyOpenDistroSettings.QUERY_SIZE_LIMIT_SETTING, + IndexSettings.MAX_RESULT_WINDOW_SETTING, 0, Setting.Property.NodeScope, Setting.Property.Dynamic); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index e99e5b360a..84fb705ae0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -34,6 +34,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.index.IndexSettings; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.sql.common.setting.LegacySettings; import org.opensearch.sql.common.setting.Settings; @@ -132,8 +133,7 @@ void settingsFallback() { org.opensearch.common.settings.Settings.EMPTY)); assertEquals( settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT), - LegacyOpenDistroSettings.QUERY_SIZE_LIMIT_SETTING.get( - org.opensearch.common.settings.Settings.EMPTY)); + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(org.opensearch.common.settings.Settings.EMPTY)); assertEquals( settings.getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW), LegacyOpenDistroSettings.METRICS_ROLLING_WINDOW_SETTING.get( @@ -165,7 +165,7 @@ public void updateLegacySettingsFallback() { assertEquals( QUERY_MEMORY_LIMIT_SETTING.get(settings), new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2))); - assertEquals(QUERY_SIZE_LIMIT_SETTING.get(settings), 100); + assertEquals(QUERY_SIZE_LIMIT_SETTING.get(settings), 10000); assertEquals(METRICS_ROLLING_WINDOW_SETTING.get(settings), 2000L); assertEquals(METRICS_ROLLING_INTERVAL_SETTING.get(settings), 100L); } From 53bfeba8ffa0a79027c06fbb6157fa740333d5df Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 31 Jul 2024 15:08:35 -0700 Subject: [PATCH 17/96] Add AsyncQueryRequestContext to QueryIdProvider parameter (#2870) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../DatasourceEmbeddedQueryIdProvider.java | 5 ++- .../sql/spark/dispatcher/QueryIdProvider.java | 4 ++- .../dispatcher/SparkQueryDispatcher.java | 12 ++++--- .../asyncquery/AsyncQueryCoreIntegTest.java | 17 ++++----- ...DatasourceEmbeddedQueryIdProviderTest.java | 35 +++++++++++++++++++ 5 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java index c170040718..3564fa9552 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.dispatcher; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; import org.opensearch.sql.spark.utils.IDUtils; @@ -12,7 +13,9 @@ public class DatasourceEmbeddedQueryIdProvider implements QueryIdProvider { @Override - public String getQueryId(DispatchQueryRequest dispatchQueryRequest) { + public String getQueryId( + DispatchQueryRequest dispatchQueryRequest, + AsyncQueryRequestContext asyncQueryRequestContext) { return IDUtils.encode(dispatchQueryRequest.getDatasource()); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java index 2167eb6b7a..a108ca1209 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java @@ -5,9 +5,11 @@ package org.opensearch.sql.spark.dispatcher; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; /** Interface for extension point to specify queryId. Called when new query is executed. */ public interface QueryIdProvider { - String getQueryId(DispatchQueryRequest dispatchQueryRequest); + String getQueryId( + DispatchQueryRequest dispatchQueryRequest, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 0061ea7179..a424db4c34 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -69,7 +69,8 @@ private DispatchQueryResponse handleFlintExtensionQuery( DataSourceMetadata dataSourceMetadata) { IndexQueryDetails indexQueryDetails = getIndexQueryDetails(dispatchQueryRequest); DispatchQueryContext context = - getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) + getDefaultDispatchContextBuilder( + dispatchQueryRequest, dataSourceMetadata, asyncQueryRequestContext) .indexQueryDetails(indexQueryDetails) .asyncQueryRequestContext(asyncQueryRequestContext) .build(); @@ -84,7 +85,8 @@ private DispatchQueryResponse handleDefaultQuery( DataSourceMetadata dataSourceMetadata) { DispatchQueryContext context = - getDefaultDispatchContextBuilder(dispatchQueryRequest, dataSourceMetadata) + getDefaultDispatchContextBuilder( + dispatchQueryRequest, dataSourceMetadata, asyncQueryRequestContext) .asyncQueryRequestContext(asyncQueryRequestContext) .build(); @@ -93,11 +95,13 @@ private DispatchQueryResponse handleDefaultQuery( } private DispatchQueryContext.DispatchQueryContextBuilder getDefaultDispatchContextBuilder( - DispatchQueryRequest dispatchQueryRequest, DataSourceMetadata dataSourceMetadata) { + DispatchQueryRequest dispatchQueryRequest, + DataSourceMetadata dataSourceMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { return DispatchQueryContext.builder() .dataSourceMetadata(dataSourceMetadata) .tags(getDefaultTagsForJobSubmission(dispatchQueryRequest)) - .queryId(queryIdProvider.getQueryId(dispatchQueryRequest)); + .queryId(queryIdProvider.getQueryId(dispatchQueryRequest, asyncQueryRequestContext)); } private AsyncQueryHandler getQueryHandlerForFlintExtensionQuery( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 34ededc74d..d82d3bdab7 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -185,7 +185,7 @@ public void setUp() { public void createDropIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); String indexName = "flint_datasource_name_table_name_index_name_index"; givenFlintIndexMetadataExists(indexName); givenCancelJobRunSucceed(); @@ -209,7 +209,7 @@ public void createDropIndexQuery() { public void createVacuumIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); String indexName = "flint_datasource_name_table_name_index_name_index"; givenFlintIndexMetadataExists(indexName); @@ -231,7 +231,7 @@ public void createVacuumIndexQuery() { public void createAlterIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); String indexName = "flint_datasource_name_table_name_index_name_index"; givenFlintIndexMetadataExists(indexName); givenCancelJobRunSucceed(); @@ -261,7 +261,7 @@ public void createAlterIndexQuery() { public void createStreamingQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); when(awsemrServerless.startJobRun(any())) .thenReturn(new StartJobRunResult().withApplicationId(APPLICATION_ID).withJobRunId(JOB_ID)); @@ -297,7 +297,7 @@ private void verifyStartJobRunCalled() { public void createCreateIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); when(awsemrServerless.startJobRun(any())) .thenReturn(new StartJobRunResult().withApplicationId(APPLICATION_ID).withJobRunId(JOB_ID)); @@ -321,7 +321,7 @@ public void createCreateIndexQuery() { public void createRefreshQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); when(awsemrServerless.startJobRun(any())) .thenReturn(new StartJobRunResult().withApplicationId(APPLICATION_ID).withJobRunId(JOB_ID)); @@ -344,7 +344,7 @@ public void createInteractiveQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); givenSessionExists(); - when(queryIdProvider.getQueryId(any())).thenReturn(QUERY_ID); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); when(sessionIdProvider.getSessionId(any())).thenReturn(SESSION_ID); givenSessionExists(); // called twice when(awsemrServerless.startJobRun(any())) @@ -538,7 +538,8 @@ private void givenGetJobRunReturnJobRunWithState(String state) { } private void verifyGetQueryIdCalled() { - verify(queryIdProvider).getQueryId(dispatchQueryRequestArgumentCaptor.capture()); + verify(queryIdProvider) + .getQueryId(dispatchQueryRequestArgumentCaptor.capture(), eq(asyncQueryRequestContext)); DispatchQueryRequest dispatchQueryRequest = dispatchQueryRequestArgumentCaptor.getValue(); assertEquals(ACCOUNT_ID, dispatchQueryRequest.getAccountId()); assertEquals(APPLICATION_ID, dispatchQueryRequest.getApplicationId()); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java new file mode 100644 index 0000000000..7f1c92dff3 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.dispatcher; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.verifyNoInteractions; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; + +@ExtendWith(MockitoExtension.class) +class DatasourceEmbeddedQueryIdProviderTest { + @Mock AsyncQueryRequestContext asyncQueryRequestContext; + + DatasourceEmbeddedQueryIdProvider datasourceEmbeddedQueryIdProvider = + new DatasourceEmbeddedQueryIdProvider(); + + @Test + public void test() { + String queryId = + datasourceEmbeddedQueryIdProvider.getQueryId( + DispatchQueryRequest.builder().datasource("DATASOURCE").build(), + asyncQueryRequestContext); + + assertNotNull(queryId); + verifyNoInteractions(asyncQueryRequestContext); + } +} From 1b17520d79ee6e90e3994298e3998adc263b02b0 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar <reddyvam@amazon.com> Date: Wed, 31 Jul 2024 15:59:37 -0700 Subject: [PATCH 18/96] Fixed integ test delete myindex issue and wipe All indices with security enabled domain (#2878) Signed-off-by: Vamsi Manohar <reddyvam@amazon.com> --- .../sql/datasource/DataSourceEnabledIT.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java index b0bc87a0c6..a53c04d871 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceEnabledIT.java @@ -7,8 +7,10 @@ import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import java.io.IOException; import lombok.SneakyThrows; import org.json.JSONObject; +import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.opensearch.client.Request; @@ -18,9 +20,9 @@ public class DataSourceEnabledIT extends PPLIntegTestCase { - @Override - protected boolean preserveClusterUponCompletion() { - return false; + @After + public void cleanUp() throws IOException { + wipeAllClusterSettings(); } @Test @@ -39,6 +41,7 @@ public void testDataSourceCreationWithDefaultSettings() { assertSelectFromDataSourceReturnsSuccess(); assertSelectFromDummyIndexInValidDataSourceDataSourceReturnsDoesNotExist(); deleteSelfDataSourceCreated(); + deleteIndex(); } @Test @@ -55,6 +58,7 @@ public void testAfterPreviousEnable() { assertAsyncQueryApiDisabled(); setDataSourcesEnabled("transient", true); deleteSelfDataSourceCreated(); + deleteIndex(); } @SneakyThrows @@ -98,6 +102,12 @@ private void createIndex() { Assert.assertEquals(200, response.getStatusLine().getStatusCode()); } + private void deleteIndex() { + Request request = new Request("DELETE", "/myindex"); + Response response = performRequest(request); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + } + private void createOpenSearchDataSource() { Request request = new Request("POST", "/_plugins/_query/_datasources"); request.setJsonEntity( From 3daf64fbce5a8d29e846669689b3a7b12c5c7f07 Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Wed, 31 Jul 2024 16:10:13 -0700 Subject: [PATCH 19/96] [Feature] Flint query scheduler part1 - integrate job scheduler plugin (#2834) * [Feature] Flint query scheduler part1 - integrate job scheduler plugin Signed-off-by: Louis Chu <clingzhi@amazon.com> * Add comments Signed-off-by: Louis Chu <clingzhi@amazon.com> * Add unit test Signed-off-by: Louis Chu <clingzhi@amazon.com> * Remove test rest API Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix doc test Signed-off-by: Louis Chu <clingzhi@amazon.com> * Add more tests Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix IT Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix IT with security Signed-off-by: Louis Chu <clingzhi@amazon.com> * Improve test coverage Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix integTest cluster Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix UT Signed-off-by: Louis Chu <clingzhi@amazon.com> * Update UT Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix bwc test Signed-off-by: Louis Chu <clingzhi@amazon.com> * Resolve comments Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix bwc test Signed-off-by: Louis Chu <clingzhi@amazon.com> * clean up doc test Signed-off-by: Louis Chu <clingzhi@amazon.com> * Resolve comments Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix UT Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .gitignore | 1 + .../src/main/antlr/SqlBaseParser.g4 | 17 +- async-query/build.gradle | 3 + .../OpenSearchAsyncQueryScheduler.java | 197 ++++++++ ...penSearchRefreshIndexJobRequestParser.java | 71 +++ .../job/OpenSearchRefreshIndexJob.java | 93 ++++ .../OpenSearchRefreshIndexJobRequest.java | 108 +++++ .../async-query-scheduler-index-mapping.yml | 41 ++ .../async-query-scheduler-index-settings.yml | 11 + .../OpenSearchAsyncQuerySchedulerTest.java | 434 ++++++++++++++++++ .../job/OpenSearchRefreshIndexJobTest.java | 145 ++++++ .../OpenSearchRefreshIndexJobRequestTest.java | 81 ++++ build.gradle | 3 +- common/build.gradle | 4 +- core/build.gradle | 2 +- doctest/build.gradle | 53 +++ integ-test/build.gradle | 65 ++- legacy/build.gradle | 2 +- plugin/build.gradle | 11 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 32 +- ...rch.jobscheduler.spi.JobSchedulerExtension | 6 + ppl/build.gradle | 2 +- protocol/build.gradle | 2 +- sql/build.gradle | 2 +- 24 files changed, 1357 insertions(+), 29 deletions(-) create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java create mode 100644 async-query/src/main/resources/async-query-scheduler-index-mapping.yml create mode 100644 async-query/src/main/resources/async-query-scheduler-index-settings.yml create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java create mode 100644 plugin/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension diff --git a/.gitignore b/.gitignore index 1b892036dd..b9775dea04 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,5 @@ gen .worktrees http-client.env.json /doctest/sql-cli/ +/doctest/opensearch-job-scheduler/ .factorypath diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index a50051715e..c7aa56cf92 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -66,8 +66,8 @@ compoundStatement ; setStatementWithOptionalVarKeyword - : SET (VARIABLE | VAR)? assignmentList #setVariableWithOptionalKeyword - | SET (VARIABLE | VAR)? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ + : SET variable? assignmentList #setVariableWithOptionalKeyword + | SET variable? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword ; @@ -215,9 +215,9 @@ statement routineCharacteristics RETURN (query | expression) #createUserDefinedFunction | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction - | DECLARE (OR REPLACE)? VARIABLE? + | DECLARE (OR REPLACE)? variable? identifierReference dataType? variableDefaultExpression? #createVariable - | DROP TEMPORARY VARIABLE (IF EXISTS)? identifierReference #dropVariable + | DROP TEMPORARY variable (IF EXISTS)? identifierReference #dropVariable | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? (statement|setResetStatement) #explain | SHOW TABLES ((FROM | IN) identifierReference)? @@ -272,8 +272,8 @@ setResetStatement | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone - | SET (VARIABLE | VAR) assignmentList #setVariable - | SET (VARIABLE | VAR) LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ + | SET variable assignmentList #setVariable + | SET variable LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ LEFT_PAREN query RIGHT_PAREN #setVariable | SET configKey EQ configValue #setQuotedConfiguration | SET configKey (EQ .*?)? #setConfiguration @@ -438,6 +438,11 @@ namespaces | SCHEMAS ; +variable + : VARIABLE + | VAR + ; + describeFuncName : identifierReference | stringLit diff --git a/async-query/build.gradle b/async-query/build.gradle index 5a4a0d729d..abda6161d3 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -16,6 +16,8 @@ repositories { dependencies { + implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" + api project(':core') api project(':async-query-core') implementation project(':protocol') @@ -97,6 +99,7 @@ jacocoTestCoverageVerification { // ignore because XContext IOException 'org.opensearch.sql.spark.execution.statestore.StateStore', 'org.opensearch.sql.spark.rest.*', + 'org.opensearch.sql.spark.scheduler.OpenSearchRefreshIndexJobRequestParser', 'org.opensearch.sql.spark.transport.model.*' ] limit { diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java new file mode 100644 index 0000000000..c7a66fc6be --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java @@ -0,0 +1,197 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler; + +import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import org.apache.commons.io.IOUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.index.engine.DocumentMissingException; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.scheduler.job.OpenSearchRefreshIndexJob; +import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; +import org.opensearch.threadpool.ThreadPool; + +/** Scheduler class for managing asynchronous query jobs. */ +public class OpenSearchAsyncQueryScheduler { + public static final String SCHEDULER_INDEX_NAME = ".async-query-scheduler"; + public static final String SCHEDULER_PLUGIN_JOB_TYPE = "async-query-scheduler"; + private static final String SCHEDULER_INDEX_MAPPING_FILE_NAME = + "async-query-scheduler-index-mapping.yml"; + private static final String SCHEDULER_INDEX_SETTINGS_FILE_NAME = + "async-query-scheduler-index-settings.yml"; + private static final Logger LOG = LogManager.getLogger(); + + private Client client; + private ClusterService clusterService; + + /** Loads job resources, setting up required services and job runner instance. */ + public void loadJobResource(Client client, ClusterService clusterService, ThreadPool threadPool) { + this.client = client; + this.clusterService = clusterService; + OpenSearchRefreshIndexJob openSearchRefreshIndexJob = + OpenSearchRefreshIndexJob.getJobRunnerInstance(); + openSearchRefreshIndexJob.setClusterService(clusterService); + openSearchRefreshIndexJob.setThreadPool(threadPool); + openSearchRefreshIndexJob.setClient(client); + } + + /** Schedules a new job by indexing it into the job index. */ + public void scheduleJob(OpenSearchRefreshIndexJobRequest request) { + if (!this.clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) { + createAsyncQuerySchedulerIndex(); + } + IndexRequest indexRequest = new IndexRequest(SCHEDULER_INDEX_NAME); + indexRequest.id(request.getName()); + indexRequest.opType(DocWriteRequest.OpType.CREATE); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponse; + try { + indexRequest.source(request.toXContent(JsonXContent.contentBuilder(), EMPTY_PARAMS)); + ActionFuture<IndexResponse> indexResponseActionFuture = client.index(indexRequest); + indexResponse = indexResponseActionFuture.actionGet(); + } catch (VersionConflictEngineException exception) { + throw new IllegalArgumentException("A job already exists with name: " + request.getName()); + } catch (Throwable e) { + LOG.error("Failed to schedule job : {}", request.getName(), e); + throw new RuntimeException(e); + } + + if (indexResponse.getResult().equals(DocWriteResponse.Result.CREATED)) { + LOG.debug("Job : {} successfully created", request.getName()); + } else { + throw new RuntimeException( + "Schedule job failed with result : " + indexResponse.getResult().getLowercase()); + } + } + + /** Unschedules a job by marking it as disabled and updating its last update time. */ + public void unscheduleJob(String jobId) throws IOException { + assertIndexExists(); + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(jobId) + .enabled(false) + .lastUpdateTime(Instant.now()) + .build(); + updateJob(request); + } + + /** Updates an existing job with new parameters. */ + public void updateJob(OpenSearchRefreshIndexJobRequest request) throws IOException { + assertIndexExists(); + UpdateRequest updateRequest = new UpdateRequest(SCHEDULER_INDEX_NAME, request.getName()); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + updateRequest.doc(request.toXContent(JsonXContent.contentBuilder(), EMPTY_PARAMS)); + UpdateResponse updateResponse; + try { + ActionFuture<UpdateResponse> updateResponseActionFuture = client.update(updateRequest); + updateResponse = updateResponseActionFuture.actionGet(); + } catch (DocumentMissingException exception) { + throw new IllegalArgumentException("Job: " + request.getName() + " doesn't exist"); + } catch (Throwable e) { + LOG.error("Failed to update job : {}", request.getName(), e); + throw new RuntimeException(e); + } + + if (updateResponse.getResult().equals(DocWriteResponse.Result.UPDATED) + || updateResponse.getResult().equals(DocWriteResponse.Result.NOOP)) { + LOG.debug("Job : {} successfully updated", request.getName()); + } else { + throw new RuntimeException( + "Update job failed with result : " + updateResponse.getResult().getLowercase()); + } + } + + /** Removes a job by deleting its document from the index. */ + public void removeJob(String jobId) { + assertIndexExists(); + DeleteRequest deleteRequest = new DeleteRequest(SCHEDULER_INDEX_NAME, jobId); + deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + ActionFuture<DeleteResponse> deleteResponseActionFuture = client.delete(deleteRequest); + DeleteResponse deleteResponse = deleteResponseActionFuture.actionGet(); + + if (deleteResponse.getResult().equals(DocWriteResponse.Result.DELETED)) { + LOG.debug("Job : {} successfully deleted", jobId); + } else if (deleteResponse.getResult().equals(DocWriteResponse.Result.NOT_FOUND)) { + throw new IllegalArgumentException("Job : " + jobId + " doesn't exist"); + } else { + throw new RuntimeException( + "Remove job failed with result : " + deleteResponse.getResult().getLowercase()); + } + } + + /** Creates the async query scheduler index with specified mappings and settings. */ + @VisibleForTesting + void createAsyncQuerySchedulerIndex() { + try { + InputStream mappingFileStream = + OpenSearchAsyncQueryScheduler.class + .getClassLoader() + .getResourceAsStream(SCHEDULER_INDEX_MAPPING_FILE_NAME); + InputStream settingsFileStream = + OpenSearchAsyncQueryScheduler.class + .getClassLoader() + .getResourceAsStream(SCHEDULER_INDEX_SETTINGS_FILE_NAME); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(SCHEDULER_INDEX_NAME); + createIndexRequest.mapping( + IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), XContentType.YAML); + createIndexRequest.settings( + IOUtils.toString(settingsFileStream, StandardCharsets.UTF_8), XContentType.YAML); + ActionFuture<CreateIndexResponse> createIndexResponseActionFuture = + client.admin().indices().create(createIndexRequest); + CreateIndexResponse createIndexResponse = createIndexResponseActionFuture.actionGet(); + + if (createIndexResponse.isAcknowledged()) { + LOG.debug("Index: {} creation Acknowledged", SCHEDULER_INDEX_NAME); + } else { + throw new RuntimeException("Index creation is not acknowledged."); + } + } catch (Throwable e) { + LOG.error("Error creating index: {}", SCHEDULER_INDEX_NAME, e); + throw new RuntimeException( + "Internal server error while creating " + + SCHEDULER_INDEX_NAME + + " index: " + + e.getMessage(), + e); + } + } + + private void assertIndexExists() { + if (!this.clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) { + throw new IllegalStateException("Job index does not exist."); + } + } + + /** Returns the job runner instance for the scheduler. */ + public static ScheduledJobRunner getJobRunner() { + return OpenSearchRefreshIndexJob.getJobRunnerInstance(); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java new file mode 100644 index 0000000000..0422e7c015 --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler; + +import java.io.IOException; +import java.time.Instant; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; + +public class OpenSearchRefreshIndexJobRequestParser { + + private static Instant parseInstantValue(XContentParser parser) throws IOException { + if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { + return null; + } + if (parser.currentToken().isValue()) { + return Instant.ofEpochMilli(parser.longValue()); + } + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + return null; + } + + public static ScheduledJobParser getJobParser() { + return (parser, id, jobDocVersion) -> { + OpenSearchRefreshIndexJobRequest.OpenSearchRefreshIndexJobRequestBuilder builder = + OpenSearchRefreshIndexJobRequest.builder(); + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case OpenSearchRefreshIndexJobRequest.JOB_NAME_FIELD: + builder.jobName(parser.text()); + break; + case OpenSearchRefreshIndexJobRequest.JOB_TYPE_FIELD: + builder.jobType(parser.text()); + break; + case OpenSearchRefreshIndexJobRequest.ENABLED_FIELD: + builder.enabled(parser.booleanValue()); + break; + case OpenSearchRefreshIndexJobRequest.ENABLED_TIME_FIELD: + builder.enabledTime(parseInstantValue(parser)); + break; + case OpenSearchRefreshIndexJobRequest.LAST_UPDATE_TIME_FIELD: + builder.lastUpdateTime(parseInstantValue(parser)); + break; + case OpenSearchRefreshIndexJobRequest.SCHEDULE_FIELD: + builder.schedule(ScheduleParser.parse(parser)); + break; + case OpenSearchRefreshIndexJobRequest.LOCK_DURATION_SECONDS: + builder.lockDurationSeconds(parser.longValue()); + break; + case OpenSearchRefreshIndexJobRequest.JITTER: + builder.jitter(parser.doubleValue()); + break; + default: + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + } + } + return builder.build(); + }; + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java new file mode 100644 index 0000000000..e465a8790f --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.job; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.plugins.Plugin; +import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; +import org.opensearch.threadpool.ThreadPool; + +/** + * The job runner class for scheduling refresh index query. + * + * <p>The job runner should be a singleton class if it uses OpenSearch client or other objects + * passed from OpenSearch. Because when registering the job runner to JobScheduler plugin, + * OpenSearch has not invoked plugins' createComponents() method. That is saying the plugin is not + * completely initialized, and the OpenSearch {@link org.opensearch.client.Client}, {@link + * ClusterService} and other objects are not available to plugin and this job runner. + * + * <p>So we have to move this job runner initialization to {@link Plugin} createComponents() method, + * and using singleton job runner to ensure we register a usable job runner instance to JobScheduler + * plugin. + */ +public class OpenSearchRefreshIndexJob implements ScheduledJobRunner { + + private static final Logger log = LogManager.getLogger(OpenSearchRefreshIndexJob.class); + + public static OpenSearchRefreshIndexJob INSTANCE = new OpenSearchRefreshIndexJob(); + + public static OpenSearchRefreshIndexJob getJobRunnerInstance() { + return INSTANCE; + } + + private ClusterService clusterService; + private ThreadPool threadPool; + private Client client; + + private OpenSearchRefreshIndexJob() { + // Singleton class, use getJobRunnerInstance method instead of constructor + } + + public void setClusterService(ClusterService clusterService) { + this.clusterService = clusterService; + } + + public void setThreadPool(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void setClient(Client client) { + this.client = client; + } + + @Override + public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { + if (!(jobParameter instanceof OpenSearchRefreshIndexJobRequest)) { + throw new IllegalStateException( + "Job parameter is not instance of OpenSearchRefreshIndexJobRequest, type: " + + jobParameter.getClass().getCanonicalName()); + } + + if (this.clusterService == null) { + throw new IllegalStateException("ClusterService is not initialized."); + } + + if (this.threadPool == null) { + throw new IllegalStateException("ThreadPool is not initialized."); + } + + if (this.client == null) { + throw new IllegalStateException("Client is not initialized."); + } + + Runnable runnable = + () -> { + doRefresh(jobParameter.getName()); + }; + threadPool.generic().submit(runnable); + } + + void doRefresh(String refreshIndex) { + // TODO: add logic to refresh index + log.info("Scheduled refresh index job on : " + refreshIndex); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java new file mode 100644 index 0000000000..7eaa4e2d29 --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.model; + +import java.io.IOException; +import java.time.Instant; +import lombok.Builder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +/** Represents a job request to refresh index. */ +@Builder +public class OpenSearchRefreshIndexJobRequest implements ScheduledJobParameter { + // Constant fields for JSON serialization + public static final String JOB_NAME_FIELD = "jobName"; + public static final String JOB_TYPE_FIELD = "jobType"; + public static final String LAST_UPDATE_TIME_FIELD = "lastUpdateTime"; + public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FIELD = "enabledTime"; + public static final String ENABLED_TIME_FIELD_READABLE = "enabled_time_field"; + public static final String LOCK_DURATION_SECONDS = "lockDurationSeconds"; + public static final String JITTER = "jitter"; + public static final String ENABLED_FIELD = "enabled"; + + // name is doc id + private final String jobName; + private final String jobType; + private final Schedule schedule; + private final boolean enabled; + private final Instant lastUpdateTime; + private final Instant enabledTime; + private final Long lockDurationSeconds; + private final Double jitter; + + @Override + public String getName() { + return jobName; + } + + public String getJobType() { + return jobType; + } + + @Override + public Schedule getSchedule() { + return schedule; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public Instant getLastUpdateTime() { + return lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return enabledTime; + } + + @Override + public Long getLockDurationSeconds() { + return lockDurationSeconds; + } + + @Override + public Double getJitter() { + return jitter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) + throws IOException { + builder.startObject(); + builder.field(JOB_NAME_FIELD, getName()).field(ENABLED_FIELD, isEnabled()); + if (getSchedule() != null) { + builder.field(SCHEDULE_FIELD, getSchedule()); + } + if (getJobType() != null) { + builder.field(JOB_TYPE_FIELD, getJobType()); + } + if (getEnabledTime() != null) { + builder.timeField( + ENABLED_TIME_FIELD, ENABLED_TIME_FIELD_READABLE, getEnabledTime().toEpochMilli()); + } + builder.timeField( + LAST_UPDATE_TIME_FIELD, + LAST_UPDATE_TIME_FIELD_READABLE, + getLastUpdateTime().toEpochMilli()); + if (this.lockDurationSeconds != null) { + builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); + } + if (this.jitter != null) { + builder.field(JITTER, this.jitter); + } + builder.endObject(); + return builder; + } +} diff --git a/async-query/src/main/resources/async-query-scheduler-index-mapping.yml b/async-query/src/main/resources/async-query-scheduler-index-mapping.yml new file mode 100644 index 0000000000..36bd1b873e --- /dev/null +++ b/async-query/src/main/resources/async-query-scheduler-index-mapping.yml @@ -0,0 +1,41 @@ +--- +## +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +## + +# Schema file for the .async-query-scheduler index +# Also "dynamic" is set to "false" so that other fields cannot be added. +dynamic: false +properties: + name: + type: keyword + jobType: + type: keyword + lastUpdateTime: + type: date + format: epoch_millis + enabledTime: + type: date + format: epoch_millis + schedule: + properties: + initialDelay: + type: long + interval: + properties: + start_time: + type: date + format: "strict_date_time||epoch_millis" + period: + type: integer + unit: + type: keyword + enabled: + type: boolean + lockDurationSeconds: + type: long + null_value: -1 + jitter: + type: double + null_value: 0.0 \ No newline at end of file diff --git a/async-query/src/main/resources/async-query-scheduler-index-settings.yml b/async-query/src/main/resources/async-query-scheduler-index-settings.yml new file mode 100644 index 0000000000..386f1f4f34 --- /dev/null +++ b/async-query/src/main/resources/async-query-scheduler-index-settings.yml @@ -0,0 +1,11 @@ +--- +## +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +## + +# Settings file for the .async-query-scheduler index +index: + number_of_shards: "1" + auto_expand_replicas: "0-2" + number_of_replicas: "0" \ No newline at end of file diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java new file mode 100644 index 0000000000..de86f111f3 --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java @@ -0,0 +1,434 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME; + +import java.io.IOException; +import java.time.Instant; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.index.engine.DocumentMissingException; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; +import org.opensearch.threadpool.ThreadPool; + +public class OpenSearchAsyncQuerySchedulerTest { + + private static final String TEST_SCHEDULER_INDEX_NAME = "testQS"; + + private static final String TEST_JOB_ID = "testJob"; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Client client; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ClusterService clusterService; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ThreadPool threadPool; + + @Mock private ActionFuture<IndexResponse> indexResponseActionFuture; + + @Mock private ActionFuture<UpdateResponse> updateResponseActionFuture; + + @Mock private ActionFuture<DeleteResponse> deleteResponseActionFuture; + + @Mock private ActionFuture<CreateIndexResponse> createIndexResponseActionFuture; + + @Mock private IndexResponse indexResponse; + + @Mock private UpdateResponse updateResponse; + + private OpenSearchAsyncQueryScheduler scheduler; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + scheduler = new OpenSearchAsyncQueryScheduler(); + scheduler.loadJobResource(client, clusterService, threadPool); + } + + @Test + public void testScheduleJob() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + when(client.admin().indices().create(any(CreateIndexRequest.class))) + .thenReturn(createIndexResponseActionFuture); + when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, TEST_SCHEDULER_INDEX_NAME)); + when(client.index(any(IndexRequest.class))).thenReturn(indexResponseActionFuture); + + // Test the if case + when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); + + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + scheduler.scheduleJob(request); + + // Verify index created + verify(client.admin().indices(), times(1)).create(ArgumentMatchers.any()); + + // Verify doc indexed + ArgumentCaptor<IndexRequest> captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(client, times(1)).index(captor.capture()); + IndexRequest capturedRequest = captor.getValue(); + assertEquals(request.getName(), capturedRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); + } + + @Test + public void testScheduleJobWithExistingJob() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) + .thenReturn(Boolean.TRUE); + + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + when(client.index(any(IndexRequest.class))).thenThrow(VersionConflictEngineException.class); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + scheduler.scheduleJob(request); + }); + + verify(client, times(1)).index(ArgumentCaptor.forClass(IndexRequest.class).capture()); + assertEquals("A job already exists with name: testJob", exception.getMessage()); + } + + @Test + public void testScheduleJobWithExceptions() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + when(client.admin().indices().create(any(CreateIndexRequest.class))) + .thenReturn(createIndexResponseActionFuture); + when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, TEST_SCHEDULER_INDEX_NAME)); + when(client.index(any(IndexRequest.class))).thenThrow(new RuntimeException("Test exception")); + + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + + when(client.index(any(IndexRequest.class))).thenReturn(indexResponseActionFuture); + when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + + RuntimeException exception = + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + assertEquals("Schedule job failed with result : not_found", exception.getMessage()); + } + + @Test + public void testUnscheduleJob() throws IOException { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.UPDATED); + + when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); + + scheduler.unscheduleJob(TEST_JOB_ID); + + ArgumentCaptor<UpdateRequest> captor = ArgumentCaptor.forClass(UpdateRequest.class); + verify(client).update(captor.capture()); + + UpdateRequest capturedRequest = captor.getValue(); + assertEquals(TEST_JOB_ID, capturedRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); + + // Reset the captor for the next verification + captor = ArgumentCaptor.forClass(UpdateRequest.class); + + when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); + scheduler.unscheduleJob(TEST_JOB_ID); + + verify(client, times(2)).update(captor.capture()); + capturedRequest = captor.getValue(); + assertEquals(TEST_JOB_ID, capturedRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); + } + + @Test + public void testUnscheduleJobWithIndexNotFound() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); + + assertThrows(IllegalStateException.class, () -> scheduler.unscheduleJob(TEST_JOB_ID)); + } + + @Test + public void testUpdateJob() throws IOException { + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.UPDATED); + + when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); + + scheduler.updateJob(request); + + ArgumentCaptor<UpdateRequest> captor = ArgumentCaptor.forClass(UpdateRequest.class); + verify(client).update(captor.capture()); + + UpdateRequest capturedRequest = captor.getValue(); + assertEquals(request.getName(), capturedRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); + } + + @Test + public void testUpdateJobWithIndexNotFound() { + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); + + assertThrows(IllegalStateException.class, () -> scheduler.updateJob(request)); + } + + @Test + public void testUpdateJobWithExceptions() { + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + when(client.update(any(UpdateRequest.class))) + .thenThrow(new DocumentMissingException(null, null)); + + IllegalArgumentException exception1 = + assertThrows( + IllegalArgumentException.class, + () -> { + scheduler.updateJob(request); + }); + + assertEquals("Job: testJob doesn't exist", exception1.getMessage()); + + when(client.update(any(UpdateRequest.class))).thenThrow(new RuntimeException("Test exception")); + + RuntimeException exception2 = + assertThrows( + RuntimeException.class, + () -> { + scheduler.updateJob(request); + }); + + assertEquals("java.lang.RuntimeException: Test exception", exception2.getMessage()); + + when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); + when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + + RuntimeException exception = + assertThrows(RuntimeException.class, () -> scheduler.updateJob(request)); + assertEquals("Update job failed with result : not_found", exception.getMessage()); + } + + @Test + public void testRemoveJob() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + DeleteResponse deleteResponse = mock(DeleteResponse.class); + when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.DELETED); + + when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); + + scheduler.removeJob(TEST_JOB_ID); + + ArgumentCaptor<DeleteRequest> captor = ArgumentCaptor.forClass(DeleteRequest.class); + verify(client).delete(captor.capture()); + + DeleteRequest capturedRequest = captor.getValue(); + assertEquals(TEST_JOB_ID, capturedRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); + } + + @Test + public void testRemoveJobWithIndexNotFound() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); + + assertThrows(IllegalStateException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + } + + @Test + public void testCreateAsyncQuerySchedulerIndex() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); + + CreateIndexResponse createIndexResponse = mock(CreateIndexResponse.class); + when(createIndexResponseActionFuture.actionGet()).thenReturn(createIndexResponse); + when(createIndexResponse.isAcknowledged()).thenReturn(true); + + when(client.admin().indices().create(any(CreateIndexRequest.class))) + .thenReturn(createIndexResponseActionFuture); + + scheduler.createAsyncQuerySchedulerIndex(); + + ArgumentCaptor<CreateIndexRequest> captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + verify(client.admin().indices()).create(captor.capture()); + + CreateIndexRequest capturedRequest = captor.getValue(); + assertEquals(SCHEDULER_INDEX_NAME, capturedRequest.index()); + } + + @Test + public void testCreateAsyncQuerySchedulerIndexFailure() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); + + when(client.admin().indices().create(any(CreateIndexRequest.class))) + .thenThrow(new RuntimeException("Error creating index")); + + RuntimeException exception = + assertThrows( + RuntimeException.class, + () -> { + scheduler.createAsyncQuerySchedulerIndex(); + }); + + assertEquals( + "Internal server error while creating .async-query-scheduler index: Error creating index", + exception.getMessage()); + + when(client.admin().indices().create(any(CreateIndexRequest.class))) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(false, false, SCHEDULER_INDEX_NAME)); + + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + RuntimeException runtimeException = + Assertions.assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + Assertions.assertEquals( + "Internal server error while creating .async-query-scheduler index: Index creation is not" + + " acknowledged.", + runtimeException.getMessage()); + } + + @Test + public void testUpdateJobNotFound() { + OpenSearchRefreshIndexJobRequest request = + OpenSearchRefreshIndexJobRequest.builder() + .jobName(TEST_JOB_ID) + .lastUpdateTime(Instant.now()) + .build(); + + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + when(client.update(any(UpdateRequest.class))) + .thenThrow(new DocumentMissingException(null, null)); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + scheduler.updateJob(request); + }); + + assertEquals("Job: testJob doesn't exist", exception.getMessage()); + } + + @Test + public void testRemoveJobNotFound() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + DeleteResponse deleteResponse = mock(DeleteResponse.class); + when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + + when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + scheduler.removeJob(TEST_JOB_ID); + }); + + assertEquals("Job : testJob doesn't exist", exception.getMessage()); + } + + @Test + public void testRemoveJobWithExceptions() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + when(client.delete(any(DeleteRequest.class))).thenThrow(new RuntimeException("Test exception")); + + assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + + DeleteResponse deleteResponse = mock(DeleteResponse.class); + when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); + when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); + + RuntimeException runtimeException = + Assertions.assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + Assertions.assertEquals("Remove job failed with result : noop", runtimeException.getMessage()); + } + + @Test + public void testGetJobRunner() { + ScheduledJobRunner jobRunner = OpenSearchAsyncQueryScheduler.getJobRunner(); + assertNotNull(jobRunner); + } +} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java new file mode 100644 index 0000000000..cbf137997e --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.job; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; +import org.opensearch.threadpool.ThreadPool; + +public class OpenSearchRefreshIndexJobTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ClusterService clusterService; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ThreadPool threadPool; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Client client; + + @Mock private JobExecutionContext context; + + private OpenSearchRefreshIndexJob jobRunner; + + private OpenSearchRefreshIndexJob spyJobRunner; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + jobRunner = OpenSearchRefreshIndexJob.getJobRunnerInstance(); + jobRunner.setClient(null); + jobRunner.setClusterService(null); + jobRunner.setThreadPool(null); + } + + @Test + public void testRunJobWithCorrectParameter() { + spyJobRunner = spy(jobRunner); + spyJobRunner.setClusterService(clusterService); + spyJobRunner.setThreadPool(threadPool); + spyJobRunner.setClient(client); + + OpenSearchRefreshIndexJobRequest jobParameter = + OpenSearchRefreshIndexJobRequest.builder() + .jobName("testJob") + .lastUpdateTime(Instant.now()) + .lockDurationSeconds(10L) + .build(); + + spyJobRunner.runJob(jobParameter, context); + + ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class); + verify(threadPool.generic()).submit(captor.capture()); + + Runnable runnable = captor.getValue(); + runnable.run(); + + verify(spyJobRunner).doRefresh(eq(jobParameter.getName())); + } + + @Test + public void testRunJobWithIncorrectParameter() { + jobRunner = OpenSearchRefreshIndexJob.getJobRunnerInstance(); + jobRunner.setClusterService(clusterService); + jobRunner.setThreadPool(threadPool); + jobRunner.setClient(client); + + ScheduledJobParameter wrongParameter = mock(ScheduledJobParameter.class); + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(wrongParameter, context), + "Expected IllegalStateException but no exception was thrown"); + + assertEquals( + "Job parameter is not instance of OpenSearchRefreshIndexJobRequest, type: " + + wrongParameter.getClass().getCanonicalName(), + exception.getMessage()); + } + + @Test + public void testRunJobWithUninitializedServices() { + OpenSearchRefreshIndexJobRequest jobParameter = + OpenSearchRefreshIndexJobRequest.builder() + .jobName("testJob") + .lastUpdateTime(Instant.now()) + .build(); + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("ClusterService is not initialized.", exception.getMessage()); + + jobRunner.setClusterService(clusterService); + + exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("ThreadPool is not initialized.", exception.getMessage()); + + jobRunner.setThreadPool(threadPool); + + exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("Client is not initialized.", exception.getMessage()); + } + + @Test + public void testGetJobRunnerInstanceMultipleCalls() { + OpenSearchRefreshIndexJob instance1 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); + OpenSearchRefreshIndexJob instance2 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); + OpenSearchRefreshIndexJob instance3 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); + + assertSame(instance1, instance2); + assertSame(instance2, instance3); + } +} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java new file mode 100644 index 0000000000..108f1acfd5 --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Test; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; + +public class OpenSearchRefreshIndexJobRequestTest { + + @Test + public void testBuilderAndGetterMethods() { + Instant now = Instant.now(); + IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); + + OpenSearchRefreshIndexJobRequest jobRequest = + OpenSearchRefreshIndexJobRequest.builder() + .jobName("testJob") + .jobType("testType") + .schedule(schedule) + .enabled(true) + .lastUpdateTime(now) + .enabledTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + assertEquals("testJob", jobRequest.getName()); + assertEquals("testType", jobRequest.getJobType()); + assertEquals(schedule, jobRequest.getSchedule()); + assertTrue(jobRequest.isEnabled()); + assertEquals(now, jobRequest.getLastUpdateTime()); + assertEquals(now, jobRequest.getEnabledTime()); + assertEquals(60L, jobRequest.getLockDurationSeconds()); + assertEquals(0.1, jobRequest.getJitter()); + } + + @Test + public void testToXContent() throws IOException { + Instant now = Instant.now(); + IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); + + OpenSearchRefreshIndexJobRequest jobRequest = + OpenSearchRefreshIndexJobRequest.builder() + .jobName("testJob") + .jobType("testType") + .schedule(schedule) + .enabled(true) + .lastUpdateTime(now) + .enabledTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + jobRequest.toXContent(builder, EMPTY_PARAMS); + String jsonString = builder.toString(); + + assertTrue(jsonString.contains("\"jobName\" : \"testJob\"")); + assertTrue(jsonString.contains("\"jobType\" : \"testType\"")); + assertTrue(jsonString.contains("\"start_time\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"period\" : 1")); + assertTrue(jsonString.contains("\"unit\" : \"Minutes\"")); + assertTrue(jsonString.contains("\"enabled\" : true")); + assertTrue(jsonString.contains("\"lastUpdateTime\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"enabledTime\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"lockDurationSeconds\" : 60")); + assertTrue(jsonString.contains("\"jitter\" : 0.1")); + } +} diff --git a/build.gradle b/build.gradle index b3e09d7b50..702d6f478a 100644 --- a/build.gradle +++ b/build.gradle @@ -50,6 +50,7 @@ buildscript { return "https://github.com/prometheus/prometheus/releases/download/v${prometheus_binary_version}/prometheus-${prometheus_binary_version}."+ getOSFamilyType() + "-" + getArchType() + ".tar.gz" } aws_java_sdk_version = "1.12.651" + guava_version = "32.1.3-jre" } repositories { @@ -192,7 +193,7 @@ configurations.all { exclude group: "commons-logging", module: "commons-logging" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' - resolutionStrategy.force 'com.google.guava:guava:32.0.1-jre' + resolutionStrategy.force "com.google.guava:guava:${guava_version}" } // updateVersion: Task to auto increment to the next development iteration diff --git a/common/build.gradle b/common/build.gradle index b4ee98a5b7..15c48dd6b3 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -34,7 +34,7 @@ repositories { dependencies { api "org.antlr:antlr4-runtime:4.7.1" - api group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + api group: 'com.google.guava', name: 'guava', version: "${guava_version}" api group: 'org.apache.logging.log4j', name: 'log4j-core', version:"${versions.log4j}" api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' api group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' @@ -46,7 +46,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1' - testImplementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + testImplementation group: 'com.google.guava', name: 'guava', version: "${guava_version}" testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.7.0' diff --git a/core/build.gradle b/core/build.gradle index 655e7d92c2..f36777030c 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -46,7 +46,7 @@ pitest { } dependencies { - api group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + api group: 'com.google.guava', name: 'guava', version: "${guava_version}" api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' api group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' diff --git a/doctest/build.gradle b/doctest/build.gradle index ec5a26b52b..a125a4f336 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -5,6 +5,8 @@ import org.opensearch.gradle.testclusters.RunTask +import java.util.concurrent.Callable + plugins { id 'base' id 'com.wiredforcode.spawn' @@ -109,6 +111,10 @@ if (version_tokens.length > 1) { String mlCommonsRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-' + opensearch_build + '.zip' String mlCommonsPlugin = 'opensearch-ml' +String bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/' + + 'opensearch/plugins/opensearch-job-scheduler-' + opensearch_build + '.zip' +String jsPlugin = 'opensearch-job-scheduler' + testClusters { docTestCluster { // Disable loading of `ML-commons` plugin, because it might be unavailable (not released yet). @@ -133,6 +139,7 @@ testClusters { } })) */ + plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload)) plugin ':opensearch-sql-plugin' testDistribution = 'archive' } @@ -159,3 +166,49 @@ spotless { googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } + +def getJobSchedulerPlugin(String jsPlugin, String bwcOpenSearchJSDownload) { + return provider(new Callable<RegularFile>() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + // Use absolute paths + String basePath = new File('.').getCanonicalPath() + File dir = new File(basePath + File.separator + 'doctest' + File.separator + jsPlugin) + + // Log the directory path for debugging + println("Creating directory: " + dir.getAbsolutePath()) + + // Create directory if it doesn't exist + if (!dir.exists()) { + if (!dir.mkdirs()) { + throw new IOException("Failed to create directory: " + dir.getAbsolutePath()) + } + } + + // Define the file path + File f = new File(dir, jsPlugin + '-' + opensearch_build + '.zip') + + // Download file if it doesn't exist + if (!f.exists()) { + println("Downloading file from: " + bwcOpenSearchJSDownload) + println("Saving to file: " + f.getAbsolutePath()) + + new URL(bwcOpenSearchJSDownload).withInputStream { ins -> + f.withOutputStream { it << ins } + } + } + + // Check if the file was created successfully + if (!f.exists()) { + throw new FileNotFoundException("File was not created: " + f.getAbsolutePath()) + } + + return fileTree(f.getParent()).matching { include f.getName() }.singleFile + } + } + } + }) +} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 93153cf737..1acacdb4a5 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -80,7 +80,6 @@ ext { var projectAbsPath = projectDir.getAbsolutePath() File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile() - configureSecurityPlugin = { OpenSearchCluster cluster -> cluster.getNodes().forEach { node -> @@ -138,6 +137,10 @@ ext { cluster.plugin provider((Callable<RegularFile>) (() -> (RegularFile) (() -> downloadedSecurityPlugin))) } + + bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + baseVersion + '/latest/linux/x64/tar/builds/' + + 'opensearch/plugins/opensearch-job-scheduler-' + bwcVersion + '.zip' + bwcJobSchedulerPath = bwcFilePath + "job-scheduler/" } tasks.withType(licenseHeaders.class) { @@ -153,7 +156,6 @@ configurations.all { resolutionStrategy.force "commons-logging:commons-logging:1.2" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' - resolutionStrategy.force 'com.google.guava:guava:32.0.1-jre' resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" @@ -166,6 +168,7 @@ configurations.all { resolutionStrategy.force "joda-time:joda-time:2.10.12" resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" resolutionStrategy.force "com.amazonaws:aws-java-sdk-core:${aws_java_sdk_version}" + resolutionStrategy.force "com.google.guava:guava:${guava_version}" } configurations { @@ -191,6 +194,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } @@ -219,22 +223,42 @@ testClusters.all { } } +def getJobSchedulerPlugin() { + provider(new Callable<RegularFile>() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + } + }) +} + testClusters { integTest { testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } remoteCluster { testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" } integTestWithSecurity { testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" } remoteIntegTestWithSecurity { testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" } } @@ -502,6 +526,24 @@ task comparisonTest(type: RestIntegTestTask) { testDistribution = "ARCHIVE" versions = [baseVersion, opensearch_version] numberOfNodes = 3 + plugin(provider(new Callable<RegularFile>(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { + project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) + } + project.mkdir bwcJobSchedulerPath + bwcVersion + ant.get(src: bwcOpenSearchJSDownload, + dest: bwcJobSchedulerPath + bwcVersion, + httpusecaches: false) + return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() + } + } + } + })) plugin(provider(new Callable<RegularFile>(){ @Override RegularFile call() throws Exception { @@ -522,17 +564,18 @@ task comparisonTest(type: RestIntegTestTask) { } List<Provider<RegularFile>> plugins = [ - provider(new Callable<RegularFile>() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return fileTree(bwcFilePath + project.version).getSingleFile() + getJobSchedulerPlugin(), + provider(new Callable<RegularFile>() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + project.version).getSingleFile() + } } } - } - }) + }) ] // Creates 2 test clusters with 3 nodes of the old version. diff --git a/legacy/build.gradle b/legacy/build.gradle index 0467db183d..e3ddf27066 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -107,7 +107,7 @@ dependencies { because 'https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379' } } - implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + implementation group: 'com.google.guava', name: 'guava', version: "${guava_version}" implementation group: 'org.json', name: 'json', version:'20231013' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' diff --git a/plugin/build.gradle b/plugin/build.gradle index 710d81ed0a..7ebd0ad2d9 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -48,6 +48,7 @@ opensearchplugin { name 'opensearch-sql' description 'OpenSearch SQL' classname 'org.opensearch.sql.plugin.SQLPlugin' + extendedPlugins = ['opensearch-job-scheduler'] licenseFile rootProject.file("LICENSE.txt") noticeFile rootProject.file("NOTICE") } @@ -98,7 +99,8 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' - resolutionStrategy.force 'com.google.guava:guava:32.0.1-jre' + resolutionStrategy.force "com.google.guava:guava:${guava_version}" + resolutionStrategy.force 'com.google.guava:failureaccess:1.0.2' resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" @@ -139,6 +141,10 @@ spotless { } dependencies { + compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" + compileOnly "com.google.guava:guava:${guava_version}" + compileOnly 'com.google.guava:failureaccess:1.0.2' + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" @@ -204,11 +210,10 @@ dependencyLicenses.enabled = false // enable testingConventions check will cause errors like: "Classes ending with [Tests] must subclass [LuceneTestCase]" testingConventions.enabled = false -// TODO: need to verify the thirdPartyAudi +// TODO: need to verify the thirdPartyAudit // currently it complains missing classes like ibatis, mysql etc, should not be a problem thirdPartyAudit.enabled = false - apply plugin: 'com.netflix.nebula.ospackage' validateNebulaPom.enabled = false diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index b86ab9218a..a1b1e32955 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -42,6 +42,9 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ScriptPlugin; @@ -91,6 +94,9 @@ import org.opensearch.sql.spark.flint.FlintIndexMetadataServiceImpl; import org.opensearch.sql.spark.flint.operation.FlintIndexOpFactory; import org.opensearch.sql.spark.rest.RestAsyncQueryManagementAction; +import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; +import org.opensearch.sql.spark.scheduler.OpenSearchRefreshIndexJobRequestParser; +import org.opensearch.sql.spark.scheduler.job.OpenSearchRefreshIndexJob; import org.opensearch.sql.spark.storage.SparkStorageFactory; import org.opensearch.sql.spark.transport.TransportCancelAsyncQueryRequestAction; import org.opensearch.sql.spark.transport.TransportCreateAsyncQueryRequestAction; @@ -105,7 +111,8 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, SystemIndexPlugin { +public class SQLPlugin extends Plugin + implements ActionPlugin, ScriptPlugin, SystemIndexPlugin, JobSchedulerExtension { private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class); @@ -116,6 +123,7 @@ public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, Sys private NodeClient client; private DataSourceServiceImpl dataSourceService; + private OpenSearchAsyncQueryScheduler asyncQueryScheduler; private Injector injector; public String name() { @@ -208,6 +216,8 @@ public Collection<Object> createComponents( this.client = (NodeClient) client; this.dataSourceService = createDataSourceService(); dataSourceService.createDataSource(defaultOpenSearchDataSourceMetadata()); + this.asyncQueryScheduler = new OpenSearchAsyncQueryScheduler(); + this.asyncQueryScheduler.loadJobResource(client, clusterService, threadPool); LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); LocalClusterState.state().setClient(client); @@ -243,6 +253,26 @@ public Collection<Object> createComponents( pluginSettings); } + @Override + public String getJobType() { + return OpenSearchAsyncQueryScheduler.SCHEDULER_PLUGIN_JOB_TYPE; + } + + @Override + public String getJobIndex() { + return OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + return OpenSearchRefreshIndexJob.getJobRunnerInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + return OpenSearchRefreshIndexJobRequestParser.getJobParser(); + } + @Override public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) { return singletonList( diff --git a/plugin/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension b/plugin/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension new file mode 100644 index 0000000000..5337857c15 --- /dev/null +++ b/plugin/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension @@ -0,0 +1,6 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +org.opensearch.sql.plugin.SQLPlugin \ No newline at end of file diff --git a/ppl/build.gradle b/ppl/build.gradle index d58882d5e8..2a3d6bdbf9 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -48,7 +48,7 @@ dependencies { runtimeOnly group: 'org.reflections', name: 'reflections', version: '0.9.12' implementation "org.antlr:antlr4-runtime:4.7.1" - implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + implementation group: 'com.google.guava', name: 'guava', version: "${guava_version}" api group: 'org.json', name: 'json', version: '20231013' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:"${versions.log4j}" api project(':common') diff --git a/protocol/build.gradle b/protocol/build.gradle index 5bbff68e51..b5d7929041 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -30,7 +30,7 @@ plugins { } dependencies { - implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + implementation group: 'com.google.guava', name: 'guava', version: "${guava_version}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" diff --git a/sql/build.gradle b/sql/build.gradle index 81872e6035..10bb4b24bb 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -46,7 +46,7 @@ dependencies { antlr "org.antlr:antlr4:4.7.1" implementation "org.antlr:antlr4-runtime:4.7.1" - implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + implementation group: 'com.google.guava', name: 'guava', version: "${guava_version}" implementation group: 'org.json', name: 'json', version:'20231013' implementation project(':common') implementation project(':core') From 82ef68e2b25c7c10740e74968bbe960c000c1cee Mon Sep 17 00:00:00 2001 From: panguixin <panguixin@bytedance.com> Date: Thu, 1 Aug 2024 23:10:13 +0800 Subject: [PATCH 20/96] Support common format geo point (#2801) --------- Signed-off-by: panguixin <panguixin@bytedance.com> --- .../sql/legacy/SQLIntegTestCase.java | 8 +- .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../opensearch/sql/sql/GeopointFormatsIT.java | 60 +++++++++++++ integ-test/src/test/resources/geopoints.json | 12 +++ .../geopoint_index_mapping.json | 9 ++ .../data/utils/OpenSearchJsonContent.java | 50 ++++------- .../value/OpenSearchExprValueFactory.java | 59 ++++++++++-- .../data/utils/OpenSearchJsonContentTest.java | 31 +++++++ .../value/OpenSearchExprValueFactoryTest.java | 89 +++++++++++++------ 10 files changed, 256 insertions(+), 68 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java create mode 100644 integ-test/src/test/resources/geopoints.json create mode 100644 integ-test/src/test/resources/indexDefinitions/geopoint_index_mapping.json create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContentTest.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 63c44bf831..c6d15a305d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.legacy.TestUtils.getDogs3IndexMapping; import static org.opensearch.sql.legacy.TestUtils.getEmployeeNestedTypeIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getGeopointIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getMappingFile; @@ -724,7 +725,12 @@ public enum Index { TestsConstants.TEST_INDEX_NESTED_WITH_NULLS, "multi_nested", getNestedTypeIndexMapping(), - "src/test/resources/nested_with_nulls.json"); + "src/test/resources/nested_with_nulls.json"), + GEOPOINTS( + TestsConstants.TEST_INDEX_GEOPOINT, + "dates", + getGeopointIndexMapping(), + "src/test/resources/geopoints.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 65cacf16d2..195dda0cbd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -245,6 +245,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getGeopointIndexMapping() { + String mappingFile = "geopoint_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 29bc9813fa..73838feb4f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -57,6 +57,7 @@ public class TestsConstants { public static final String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard"; public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; + public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint"; public static final String DATASOURCES = ".ql-datasources"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java new file mode 100644 index 0000000000..f25eeec241 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class GeopointFormatsIT extends SQLIntegTestCase { + + @Override + public void init() throws Exception { + loadIndex(Index.GEOPOINTS); + } + + @Test + public void testReadingGeopoints() throws IOException { + String query = String.format("SELECT point FROM %s LIMIT 5", Index.GEOPOINTS.getName()); + JSONObject result = executeJdbcRequest(query); + verifySchema(result, schema("point", null, "geo_point")); + verifyDataRows( + result, + rows(Map.of("lon", 74, "lat", 40.71)), + rows(Map.of("lon", 74, "lat", 40.71)), + rows(Map.of("lon", 74, "lat", 40.71)), + rows(Map.of("lon", 74, "lat", 40.71)), + rows(Map.of("lon", 74, "lat", 40.71))); + } + + private static final double TOLERANCE = 1E-5; + + public void testReadingGeoHash() throws IOException { + String query = String.format("SELECT point FROM %s WHERE _id='6'", Index.GEOPOINTS.getName()); + JSONObject result = executeJdbcRequest(query); + verifySchema(result, schema("point", null, "geo_point")); + Pair<Double, Double> point = getGeoValue(result); + assertEquals(40.71, point.getLeft(), TOLERANCE); + assertEquals(74, point.getRight(), TOLERANCE); + } + + private Pair<Double, Double> getGeoValue(JSONObject result) { + JSONObject geoRaw = + (JSONObject) ((JSONArray) ((JSONArray) result.get("datarows")).get(0)).get(0); + double lat = geoRaw.getDouble("lat"); + double lon = geoRaw.getDouble("lon"); + return Pair.of(lat, lon); + } +} diff --git a/integ-test/src/test/resources/geopoints.json b/integ-test/src/test/resources/geopoints.json new file mode 100644 index 0000000000..95900fe811 --- /dev/null +++ b/integ-test/src/test/resources/geopoints.json @@ -0,0 +1,12 @@ +{"index": {"_id": "1"}} +{"point": {"lat": 40.71, "lon": 74.00}} +{"index": {"_id": "2"}} +{"point": "40.71,74.00"} +{"index": {"_id": "3"}} +{"point": [74.00, 40.71]} +{"index": {"_id": "4"}} +{"point": "POINT (74.00 40.71)"} +{"index": {"_id": "5"}} +{"point": {"type": "Point", "coordinates": [74.00, 40.71]}} +{"index": {"_id": "6"}} +{"point": "txhxegj0uyp3"} diff --git a/integ-test/src/test/resources/indexDefinitions/geopoint_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/geopoint_index_mapping.json new file mode 100644 index 0000000000..61340530d8 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/geopoint_index_mapping.json @@ -0,0 +1,9 @@ +{ + "mappings": { + "properties": { + "point": { + "type": "geo_point" + } + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java index bdb15428e1..4446c1f979 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java @@ -7,11 +7,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Iterators; +import java.io.IOException; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.OpenSearchParseException; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.geo.GeoUtils; +import org.opensearch.common.xcontent.json.JsonXContentParser; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; /** The Implementation of Content to represent {@link JsonNode}. */ @RequiredArgsConstructor @@ -122,25 +130,17 @@ public Object objectValue() { @Override public Pair<Double, Double> geoValue() { final JsonNode value = value(); - if (value.has("lat") && value.has("lon")) { - Double lat = 0d; - Double lon = 0d; - try { - lat = extractDoubleValue(value.get("lat")); - } catch (Exception exception) { - throw new IllegalStateException( - "latitude must be number value, but got value: " + value.get("lat")); - } - try { - lon = extractDoubleValue(value.get("lon")); - } catch (Exception exception) { - throw new IllegalStateException( - "longitude must be number value, but got value: " + value.get("lon")); - } - return Pair.of(lat, lon); - } else { - throw new IllegalStateException( - "geo point must in format of {\"lat\": number, \"lon\": number}"); + try (XContentParser parser = + new JsonXContentParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + value.traverse())) { + parser.nextToken(); + GeoPoint point = new GeoPoint(); + GeoUtils.parseGeoPoint(parser, point, true); + return Pair.of(point.getLat(), point.getLon()); + } catch (IOException ex) { + throw new OpenSearchParseException("error parsing geo point", ex); } } @@ -148,16 +148,4 @@ public Pair<Double, Double> geoValue() { private JsonNode value() { return value; } - - /** Get doubleValue from JsonNode if possible. */ - private Double extractDoubleValue(JsonNode node) { - if (node.isTextual()) { - return Double.valueOf(node.textValue()); - } - if (node.isNumber()) { - return node.doubleValue(); - } else { - throw new IllegalStateException("node must be a number"); - } - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3cb182de5b..417aaddaee 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -39,6 +39,7 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.opensearch.OpenSearchParseException; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; @@ -62,7 +63,6 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; -import org.opensearch.sql.opensearch.data.type.OpenSearchGeoPointType; import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; @@ -134,10 +134,6 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) { .put( OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) - .put( - OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), - (c, dt) -> - new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), c.geoValue().getRight())) .put( OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary), (c, dt) -> new OpenSearchExprBinaryValue(c.stringValue())) @@ -193,8 +189,11 @@ private ExprValue parse( return ExprNullValue.of(); } - ExprType type = fieldType.get(); - if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + final ExprType type = fieldType.get(); + + if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) { + return parseGeoPoint(content, supportArrays); + } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) || content.isArray()) { return parseArray(content, field, type, supportArrays); } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) @@ -362,6 +361,49 @@ private ExprValue parseArray( return new ExprCollectionValue(result); } + /** + * Parse geo point content. + * + * @param content Content to parse. + * @param supportArrays Parsing the whole array or not + * @return Geo point value parsed from content. + */ + private ExprValue parseGeoPoint(Content content, boolean supportArrays) { + // there is only one point in doc. + if (content.isArray() == false) { + final var pair = content.geoValue(); + return new OpenSearchExprGeoPointValue(pair.getLeft(), pair.getRight()); + } + + var elements = content.array(); + var first = elements.next(); + // an array in the [longitude, latitude] format. + if (first.isNumber()) { + double lon = first.doubleValue(); + var second = elements.next(); + if (second.isNumber() == false) { + throw new OpenSearchParseException("lat must be a number, got " + second.objectValue()); + } + return new OpenSearchExprGeoPointValue(second.doubleValue(), lon); + } + + // there are multi points in doc + var pair = first.geoValue(); + var firstPoint = new OpenSearchExprGeoPointValue(pair.getLeft(), pair.getRight()); + if (supportArrays) { + List<ExprValue> result = new ArrayList<>(); + result.add(firstPoint); + elements.forEachRemaining( + e -> { + var p = e.geoValue(); + result.add(new OpenSearchExprGeoPointValue(p.getLeft(), p.getRight())); + }); + return new ExprCollectionValue(result); + } else { + return firstPoint; + } + } + /** * Parse inner array value. Can be object type and recurse continues. * @@ -375,8 +417,7 @@ private ExprValue parseInnerArrayValue( Content content, String prefix, ExprType type, boolean supportArrays) { if (type instanceof OpenSearchIpType || type instanceof OpenSearchBinaryType - || type instanceof OpenSearchDateType - || type instanceof OpenSearchGeoPointType) { + || type instanceof OpenSearchDateType) { return parse(content, prefix, Optional.of(type), supportArrays); } else if (content.isString()) { return parse(content, prefix, Optional.of(OpenSearchDataType.of(STRING)), supportArrays); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContentTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContentTest.java new file mode 100644 index 0000000000..c2cf0328bd --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContentTest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.utils; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.junit.jupiter.api.Test; +import org.opensearch.OpenSearchParseException; + +public class OpenSearchJsonContentTest { + @Test + public void testGetValueWithIOException() throws IOException { + JsonNode jsonNode = mock(JsonNode.class); + JsonParser jsonParser = mock(JsonParser.class); + when(jsonNode.traverse()).thenReturn(jsonParser); + when(jsonParser.nextToken()).thenThrow(new IOException()); + OpenSearchJsonContent content = new OpenSearchJsonContent(jsonNode); + OpenSearchParseException exception = + assertThrows(OpenSearchParseException.class, content::geoValue); + assertTrue(exception.getMessage().contains("error parsing geo point")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 83e26f85e4..6b4d825ab1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -47,6 +47,8 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.junit.jupiter.api.Test; +import org.opensearch.OpenSearchParseException; +import org.opensearch.geometry.utils.Geohash; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprTimeValue; @@ -597,6 +599,18 @@ public void constructArrayOfGeoPoints() { .get("geoV")); } + @Test + public void constructArrayOfGeoPointsReturnsFirstIndex() { + assertEquals( + new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + tupleValue( + "{\"geoV\":[" + + "{\"lat\":42.60355556,\"lon\":-97.25263889}," + + "{\"lat\":-33.6123556,\"lon\":66.287449}" + + "]}") + .get("geoV")); + } + @Test public void constructArrayOfIPsReturnsFirstIndex() { assertEquals( @@ -671,14 +685,50 @@ public void constructIP() { tupleValue("{\"ipV\":\"192.168.0.1\"}").get("ipV")); } + private static final double TOLERANCE = 1E-5; + @Test public void constructGeoPoint() { + final double lat = 42.60355556; + final double lon = -97.25263889; + final var expectedGeoPointValue = new OpenSearchExprGeoPointValue(lat, lon); + // An object with a latitude and longitude. assertEquals( - new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), - tupleValue("{\"geoV\":{\"lat\":42.60355556,\"lon\":-97.25263889}}").get("geoV")); + expectedGeoPointValue, + tupleValue(String.format("{\"geoV\":{\"lat\":%.8f,\"lon\":%.8f}}", lat, lon)).get("geoV")); + + // A string in the “latitude,longitude” format. assertEquals( - new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), - tupleValue("{\"geoV\":{\"lat\":\"42.60355556\",\"lon\":\"-97.25263889\"}}").get("geoV")); + expectedGeoPointValue, + tupleValue(String.format("{\"geoV\":\"%.8f,%.8f\"}", lat, lon)).get("geoV")); + + // A geohash. + var point = + (OpenSearchExprGeoPointValue.GeoPoint) + tupleValue(String.format("{\"geoV\":\"%s\"}", Geohash.stringEncode(lon, lat))) + .get("geoV") + .value(); + assertEquals(lat, point.getLat(), TOLERANCE); + assertEquals(lon, point.getLon(), TOLERANCE); + + // An array in the [longitude, latitude] format. + assertEquals( + expectedGeoPointValue, + tupleValue(String.format("{\"geoV\":[%.8f, %.8f]}", lon, lat)).get("geoV")); + + // A Well-Known Text POINT in the “POINT(longitude latitude)” format. + assertEquals( + expectedGeoPointValue, + tupleValue(String.format("{\"geoV\":\"POINT (%.8f %.8f)\"}", lon, lat)).get("geoV")); + + // GeoJSON format, where the coordinates are in the [longitude, latitude] format + assertEquals( + expectedGeoPointValue, + tupleValue( + String.format( + "{\"geoV\":{\"type\":\"Point\",\"coordinates\":[%.8f,%.8f]}}", lon, lat)) + .get("geoV")); + assertEquals( new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), constructFromObject("geoV", "42.60355556,-97.25263889")); @@ -686,38 +736,23 @@ public void constructGeoPoint() { @Test public void constructGeoPointFromUnsupportedFormatShouldThrowException() { - IllegalStateException exception = + OpenSearchParseException exception = assertThrows( - IllegalStateException.class, - () -> tupleValue("{\"geoV\":[42.60355556,-97.25263889]}").get("geoV")); - assertEquals( - "geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage()); + OpenSearchParseException.class, + () -> tupleValue("{\"geoV\": [42.60355556, false]}").get("geoV")); + assertEquals("lat must be a number, got false", exception.getMessage()); exception = assertThrows( - IllegalStateException.class, + OpenSearchParseException.class, () -> tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV")); - assertEquals( - "geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage()); - - exception = - assertThrows( - IllegalStateException.class, - () -> tupleValue("{\"geoV\":{\"lat\":-97.25263889}}").get("geoV")); - assertEquals( - "geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage()); + assertEquals("field [lat] missing", exception.getMessage()); exception = assertThrows( - IllegalStateException.class, + OpenSearchParseException.class, () -> tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV")); - assertEquals("latitude must be number value, but got value: true", exception.getMessage()); - - exception = - assertThrows( - IllegalStateException.class, - () -> tupleValue("{\"geoV\":{\"lat\":42.60355556,\"lon\":false}}").get("geoV")); - assertEquals("longitude must be number value, but got value: false", exception.getMessage()); + assertEquals("lat must be a number", exception.getMessage()); } @Test From 14a80a95fb5fa36781b46f28ba52d406927e21c0 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 1 Aug 2024 09:48:33 -0700 Subject: [PATCH 21/96] Add AsyncQueryRequestContext to FlintIndexMetadataService/FlintIndexStateModelService (#2879) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../asyncquery/AsyncQueryExecutorService.java | 2 +- .../AsyncQueryExecutorServiceImpl.java | 4 +- .../spark/dispatcher/AsyncQueryHandler.java | 5 +- .../spark/dispatcher/BatchQueryHandler.java | 5 +- .../sql/spark/dispatcher/IndexDMLHandler.java | 16 +++- .../dispatcher/InteractiveQueryHandler.java | 5 +- .../spark/dispatcher/RefreshQueryHandler.java | 10 +- .../dispatcher/SparkQueryDispatcher.java | 6 +- .../dispatcher/StreamingQueryHandler.java | 5 +- .../flint/FlintIndexMetadataService.java | 11 ++- .../flint/FlintIndexStateModelService.java | 46 ++++++++- .../spark/flint/operation/FlintIndexOp.java | 47 ++++++---- .../flint/operation/FlintIndexOpAlter.java | 8 +- .../flint/operation/FlintIndexOpCancel.java | 6 +- .../flint/operation/FlintIndexOpDrop.java | 6 +- .../flint/operation/FlintIndexOpVacuum.java | 6 +- .../asyncquery/AsyncQueryCoreIntegTest.java | 19 ++-- .../AsyncQueryExecutorServiceImplTest.java | 8 +- .../spark/dispatcher/IndexDMLHandlerTest.java | 11 ++- .../dispatcher/SparkQueryDispatcherTest.java | 15 ++- .../flint/operation/FlintIndexOpTest.java | 37 ++++++-- .../operation/FlintIndexOpVacuumTest.java | 94 ++++++++++++++----- .../FlintStreamingJobHouseKeeperTask.java | 13 ++- .../flint/FlintIndexMetadataServiceImpl.java | 9 +- ...OpenSearchFlintIndexStateModelService.java | 13 ++- ...ransportCancelAsyncQueryRequestAction.java | 5 +- ...AsyncQueryExecutorServiceImplSpecTest.java | 6 +- .../spark/asyncquery/IndexQuerySpecTest.java | 23 +++-- .../asyncquery/model/MockFlintSparkJob.java | 11 ++- .../FlintStreamingJobHouseKeeperTaskTest.java | 8 +- .../FlintIndexMetadataServiceImplTest.java | 27 +++++- ...SearchFlintIndexStateModelServiceTest.java | 13 ++- ...portCancelAsyncQueryRequestActionTest.java | 16 +++- 33 files changed, 384 insertions(+), 132 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java index d38c8554ae..b0c339e93d 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java @@ -39,5 +39,5 @@ CreateAsyncQueryResponse createAsyncQuery( * @param queryId queryId. * @return {@link String} cancelledQueryId. */ - String cancelQuery(String queryId); + String cancelQuery(String queryId, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java index 6d3d5b6765..d304766465 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java @@ -106,11 +106,11 @@ public AsyncQueryExecutionResponse getAsyncQueryResults(String queryId) { } @Override - public String cancelQuery(String queryId) { + public String cancelQuery(String queryId, AsyncQueryRequestContext asyncQueryRequestContext) { Optional<AsyncQueryJobMetadata> asyncQueryJobMetadata = asyncQueryJobMetadataStorageService.getJobMetadata(queryId); if (asyncQueryJobMetadata.isPresent()) { - return sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get()); + return sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get(), asyncQueryRequestContext); } throw new AsyncQueryNotFoundException(String.format("QueryId: %s not found", queryId)); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java index d61ac17aa3..2bafd88b85 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java @@ -12,6 +12,7 @@ import com.amazonaws.services.emrserverless.model.JobRunState; import org.json.JSONObject; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryResponse; @@ -54,7 +55,9 @@ protected abstract JSONObject getResponseFromResultIndex( protected abstract JSONObject getResponseFromExecutor( AsyncQueryJobMetadata asyncQueryJobMetadata); - public abstract String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata); + public abstract String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext); public abstract DispatchQueryResponse submit( DispatchQueryRequest request, DispatchQueryContext context); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 2654f83aad..661ebe27fc 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -16,6 +16,7 @@ import org.json.JSONObject; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.client.StartJobRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; @@ -61,7 +62,9 @@ protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJob } @Override - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { emrServerlessClient.cancelJobRun( asyncQueryJobMetadata.getApplicationId(), asyncQueryJobMetadata.getJobId(), false); return asyncQueryJobMetadata.getQueryId(); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index e8413f469c..f8217142c3 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -62,9 +62,11 @@ public DispatchQueryResponse submit( long startTime = System.currentTimeMillis(); try { IndexQueryDetails indexDetails = context.getIndexQueryDetails(); - FlintIndexMetadata indexMetadata = getFlintIndexMetadata(indexDetails); + FlintIndexMetadata indexMetadata = + getFlintIndexMetadata(indexDetails, context.getAsyncQueryRequestContext()); - getIndexOp(dispatchQueryRequest, indexDetails).apply(indexMetadata); + getIndexOp(dispatchQueryRequest, indexDetails) + .apply(indexMetadata, context.getAsyncQueryRequestContext()); String asyncQueryId = storeIndexDMLResult( @@ -146,9 +148,11 @@ private FlintIndexOp getIndexOp( } } - private FlintIndexMetadata getFlintIndexMetadata(IndexQueryDetails indexDetails) { + private FlintIndexMetadata getFlintIndexMetadata( + IndexQueryDetails indexDetails, AsyncQueryRequestContext asyncQueryRequestContext) { Map<String, FlintIndexMetadata> indexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(indexDetails.openSearchIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + indexDetails.openSearchIndexName(), asyncQueryRequestContext); if (!indexMetadataMap.containsKey(indexDetails.openSearchIndexName())) { throw new IllegalStateException( String.format( @@ -174,7 +178,9 @@ protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJob } @Override - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { throw new IllegalArgumentException("can't cancel index DML query"); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java index ec43bccf11..9a9baedde2 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java @@ -16,6 +16,7 @@ import org.json.JSONObject; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryResponse; @@ -71,7 +72,9 @@ protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJob } @Override - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); getStatementByQueryId( asyncQueryJobMetadata.getSessionId(), diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java index 99984ecc46..38145a143e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java @@ -8,6 +8,7 @@ import java.util.Map; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @@ -51,10 +52,13 @@ public RefreshQueryHandler( } @Override - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { String datasourceName = asyncQueryJobMetadata.getDatasourceName(); Map<String, FlintIndexMetadata> indexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(asyncQueryJobMetadata.getIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + asyncQueryJobMetadata.getIndexName(), asyncQueryRequestContext); if (!indexMetadataMap.containsKey(asyncQueryJobMetadata.getIndexName())) { throw new IllegalStateException( String.format( @@ -62,7 +66,7 @@ public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { } FlintIndexMetadata indexMetadata = indexMetadataMap.get(asyncQueryJobMetadata.getIndexName()); FlintIndexOp jobCancelOp = flintIndexOpFactory.getCancel(datasourceName); - jobCancelOp.apply(indexMetadata); + jobCancelOp.apply(indexMetadata, asyncQueryRequestContext); return asyncQueryJobMetadata.getQueryId(); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index a424db4c34..a6fdd3f102 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -162,9 +162,11 @@ public JSONObject getQueryResponse(AsyncQueryJobMetadata asyncQueryJobMetadata) .getQueryResponse(asyncQueryJobMetadata); } - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { return getAsyncQueryHandlerForExistingQuery(asyncQueryJobMetadata) - .cancelJob(asyncQueryJobMetadata); + .cancelJob(asyncQueryJobMetadata, asyncQueryRequestContext); } private AsyncQueryHandler getAsyncQueryHandlerForExistingQuery( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java index 2fbf2466da..80d4be27cf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java @@ -12,6 +12,7 @@ import java.util.Map; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.client.StartJobRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; @@ -46,7 +47,9 @@ public StreamingQueryHandler( } @Override - public String cancelJob(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public String cancelJob( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { throw new IllegalArgumentException( "can't cancel index DML query, using ALTER auto_refresh=off statement to stop job, using" + " VACUUM statement to stop job and delete data"); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java index ad274e429e..ece14c2a7b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java @@ -6,6 +6,7 @@ package org.opensearch.sql.spark.flint; import java.util.Map; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; /** Interface for FlintIndexMetadataReader */ @@ -15,16 +16,22 @@ public interface FlintIndexMetadataService { * Retrieves a map of {@link FlintIndexMetadata} instances matching the specified index pattern. * * @param indexPattern indexPattern. + * @param asyncQueryRequestContext request context passed to AsyncQueryExecutorService * @return A map of {@link FlintIndexMetadata} instances against indexName, each providing * metadata access for a matched index. Returns an empty list if no indices match the pattern. */ - Map<String, FlintIndexMetadata> getFlintIndexMetadata(String indexPattern); + Map<String, FlintIndexMetadata> getFlintIndexMetadata( + String indexPattern, AsyncQueryRequestContext asyncQueryRequestContext); /** * Performs validation and updates flint index to manual refresh. * * @param indexName indexName. * @param flintIndexOptions flintIndexOptions. + * @param asyncQueryRequestContext request context passed to AsyncQueryExecutorService */ - void updateIndexToManualRefresh(String indexName, FlintIndexOptions flintIndexOptions); + void updateIndexToManualRefresh( + String indexName, + FlintIndexOptions flintIndexOptions, + AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java index 94647f4e07..3872f2d5a0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java @@ -6,20 +6,58 @@ package org.opensearch.sql.spark.flint; import java.util.Optional; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; /** * Abstraction over flint index state storage. Flint index state will maintain the status of each * flint index. */ public interface FlintIndexStateModelService { - FlintIndexStateModel createFlintIndexStateModel(FlintIndexStateModel flintIndexStateModel); - Optional<FlintIndexStateModel> getFlintIndexStateModel(String id, String datasourceName); + /** + * Create Flint index state record + * + * @param flintIndexStateModel the model to be saved + * @param asyncQueryRequestContext the request context passed to AsyncQueryExecutorService + * @return saved model + */ + FlintIndexStateModel createFlintIndexStateModel( + FlintIndexStateModel flintIndexStateModel, AsyncQueryRequestContext asyncQueryRequestContext); + /** + * Get Flint index state record + * + * @param id ID(latestId) of the Flint index state record + * @param datasourceName datasource name + * @param asyncQueryRequestContext the request context passed to AsyncQueryExecutorService + * @return retrieved model + */ + Optional<FlintIndexStateModel> getFlintIndexStateModel( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext); + + /** + * Update Flint index state record + * + * @param flintIndexStateModel the model to be updated + * @param flintIndexState new state + * @param datasourceName Datasource name + * @param asyncQueryRequestContext the request context passed to AsyncQueryExecutorService + * @return Updated model + */ FlintIndexStateModel updateFlintIndexState( FlintIndexStateModel flintIndexStateModel, FlintIndexState flintIndexState, - String datasourceName); + String datasourceName, + AsyncQueryRequestContext asyncQueryRequestContext); - boolean deleteFlintIndexStateModel(String id, String datasourceName); + /** + * Delete Flint index state record + * + * @param id ID(latestId) of the Flint index state record + * @param datasourceName datasource name + * @param asyncQueryRequestContext the request context passed to AsyncQueryExecutorService + * @return true if deleted, otherwise false + */ + boolean deleteFlintIndexStateModel( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java index 244f4aee11..78d217b8dc 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.flint.FlintIndexMetadata; @@ -33,30 +34,33 @@ public abstract class FlintIndexOp { private final EMRServerlessClientFactory emrServerlessClientFactory; /** Apply operation on {@link FlintIndexMetadata} */ - public void apply(FlintIndexMetadata metadata) { + public void apply( + FlintIndexMetadata metadata, AsyncQueryRequestContext asyncQueryRequestContext) { // todo, remove this logic after IndexState feature is enabled in Flint. Optional<String> latestId = metadata.getLatestId(); if (latestId.isEmpty()) { - takeActionWithoutOCC(metadata); + takeActionWithoutOCC(metadata, asyncQueryRequestContext); } else { - FlintIndexStateModel initialFlintIndexStateModel = getFlintIndexStateModel(latestId.get()); + FlintIndexStateModel initialFlintIndexStateModel = + getFlintIndexStateModel(latestId.get(), asyncQueryRequestContext); // 1.validate state. validFlintIndexInitialState(initialFlintIndexStateModel); // 2.begin, move to transitioning state FlintIndexStateModel transitionedFlintIndexStateModel = - moveToTransitioningState(initialFlintIndexStateModel); + moveToTransitioningState(initialFlintIndexStateModel, asyncQueryRequestContext); // 3.runOp try { - runOp(metadata, transitionedFlintIndexStateModel); - commit(transitionedFlintIndexStateModel); + runOp(metadata, transitionedFlintIndexStateModel, asyncQueryRequestContext); + commit(transitionedFlintIndexStateModel, asyncQueryRequestContext); } catch (Throwable e) { LOG.error("Rolling back transient log due to transaction operation failure", e); try { flintIndexStateModelService.updateFlintIndexState( transitionedFlintIndexStateModel, initialFlintIndexStateModel.getIndexState(), - datasourceName); + datasourceName, + asyncQueryRequestContext); } catch (Exception ex) { LOG.error("Failed to rollback transient log", ex); } @@ -66,9 +70,11 @@ public void apply(FlintIndexMetadata metadata) { } @NotNull - private FlintIndexStateModel getFlintIndexStateModel(String latestId) { + private FlintIndexStateModel getFlintIndexStateModel( + String latestId, AsyncQueryRequestContext asyncQueryRequestContext) { Optional<FlintIndexStateModel> flintIndexOptional = - flintIndexStateModelService.getFlintIndexStateModel(latestId, datasourceName); + flintIndexStateModelService.getFlintIndexStateModel( + latestId, datasourceName, asyncQueryRequestContext); if (flintIndexOptional.isEmpty()) { String errorMsg = String.format(Locale.ROOT, "no state found. docId: %s", latestId); LOG.error(errorMsg); @@ -77,7 +83,8 @@ private FlintIndexStateModel getFlintIndexStateModel(String latestId) { return flintIndexOptional.get(); } - private void takeActionWithoutOCC(FlintIndexMetadata metadata) { + private void takeActionWithoutOCC( + FlintIndexMetadata metadata, AsyncQueryRequestContext asyncQueryRequestContext) { // take action without occ. FlintIndexStateModel fakeModel = FlintIndexStateModel.builder() @@ -89,7 +96,7 @@ private void takeActionWithoutOCC(FlintIndexMetadata metadata) { .lastUpdateTime(System.currentTimeMillis()) .error("") .build(); - runOp(metadata, fakeModel); + runOp(metadata, fakeModel, asyncQueryRequestContext); } private void validFlintIndexInitialState(FlintIndexStateModel flintIndex) { @@ -103,13 +110,14 @@ private void validFlintIndexInitialState(FlintIndexStateModel flintIndex) { } } - private FlintIndexStateModel moveToTransitioningState(FlintIndexStateModel flintIndex) { + private FlintIndexStateModel moveToTransitioningState( + FlintIndexStateModel flintIndex, AsyncQueryRequestContext asyncQueryRequestContext) { LOG.debug("Moving to transitioning state before committing."); FlintIndexState transitioningState = transitioningState(); try { flintIndex = flintIndexStateModelService.updateFlintIndexState( - flintIndex, transitioningState(), datasourceName); + flintIndex, transitioningState(), datasourceName, asyncQueryRequestContext); } catch (Exception e) { String errorMsg = String.format(Locale.ROOT, "Moving to transition state:%s failed.", transitioningState); @@ -119,16 +127,18 @@ private FlintIndexStateModel moveToTransitioningState(FlintIndexStateModel flint return flintIndex; } - private void commit(FlintIndexStateModel flintIndex) { + private void commit( + FlintIndexStateModel flintIndex, AsyncQueryRequestContext asyncQueryRequestContext) { LOG.debug("Committing the transaction and moving to stable state."); FlintIndexState stableState = stableState(); try { if (stableState == FlintIndexState.NONE) { LOG.info("Deleting index state with docId: " + flintIndex.getLatestId()); flintIndexStateModelService.deleteFlintIndexStateModel( - flintIndex.getLatestId(), datasourceName); + flintIndex.getLatestId(), datasourceName, asyncQueryRequestContext); } else { - flintIndexStateModelService.updateFlintIndexState(flintIndex, stableState, datasourceName); + flintIndexStateModelService.updateFlintIndexState( + flintIndex, stableState, datasourceName, asyncQueryRequestContext); } } catch (Exception e) { String errorMsg = @@ -192,7 +202,10 @@ public void cancelStreamingJob(FlintIndexStateModel flintIndexStateModel) /** get transitioningState */ abstract FlintIndexState transitioningState(); - abstract void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndex); + abstract void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndex, + AsyncQueryRequestContext asyncQueryRequestContext); /** get stableState */ abstract FlintIndexState stableState(); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java index 9955320253..4a00195ebf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java @@ -8,6 +8,7 @@ import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; import org.opensearch.sql.spark.flint.FlintIndexMetadata; @@ -48,11 +49,14 @@ FlintIndexState transitioningState() { @SneakyThrows @Override - void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndexStateModel) { + void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndexStateModel, + AsyncQueryRequestContext asyncQueryRequestContext) { LOG.debug( "Running alter index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); this.flintIndexMetadataService.updateIndexToManualRefresh( - flintIndexMetadata.getOpensearchIndexName(), flintIndexOptions); + flintIndexMetadata.getOpensearchIndexName(), flintIndexOptions, asyncQueryRequestContext); cancelStreamingJob(flintIndexStateModel); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java index 02c8e39c66..504a8f93c9 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java @@ -8,6 +8,7 @@ import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.flint.FlintIndexMetadata; import org.opensearch.sql.spark.flint.FlintIndexState; @@ -38,7 +39,10 @@ FlintIndexState transitioningState() { /** cancel EMR-S job, wait cancelled state upto 15s. */ @SneakyThrows @Override - void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndexStateModel) { + void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndexStateModel, + AsyncQueryRequestContext asyncQueryRequestContext) { LOG.debug( "Performing drop index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java index 6613c29870..fc9b644fc7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java @@ -8,6 +8,7 @@ import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.flint.FlintIndexMetadata; import org.opensearch.sql.spark.flint.FlintIndexState; @@ -40,7 +41,10 @@ FlintIndexState transitioningState() { /** cancel EMR-S job, wait cancelled state upto 15s. */ @SneakyThrows @Override - void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndexStateModel) { + void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndexStateModel, + AsyncQueryRequestContext asyncQueryRequestContext) { LOG.debug( "Performing drop index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java index a0ef955adf..06aaf8ef9f 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.flint.FlintIndexClient; import org.opensearch.sql.spark.flint.FlintIndexMetadata; @@ -42,7 +43,10 @@ FlintIndexState transitioningState() { } @Override - public void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndex) { + public void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndex, + AsyncQueryRequestContext asyncQueryRequestContext) { LOG.info("Vacuuming Flint index {}", flintIndexMetadata.getOpensearchIndexName()); flintIndexClient.deleteIndex(flintIndexMetadata.getOpensearchIndexName()); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index d82d3bdab7..ff92762a7c 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -249,7 +249,8 @@ public void createAlterIndexQuery() { assertNull(response.getSessionId()); verifyGetQueryIdCalled(); verify(flintIndexMetadataService) - .updateIndexToManualRefresh(eq(indexName), flintIndexOptionsArgumentCaptor.capture()); + .updateIndexToManualRefresh( + eq(indexName), flintIndexOptionsArgumentCaptor.capture(), eq(asyncQueryRequestContext)); FlintIndexOptions flintIndexOptions = flintIndexOptionsArgumentCaptor.getValue(); assertFalse(flintIndexOptions.autoRefresh()); verifyCancelJobRunCalled(); @@ -430,7 +431,7 @@ public void cancelInteractiveQuery() { when(statementStorageService.updateStatementState(statementModel, StatementState.CANCELLED)) .thenReturn(canceledStatementModel); - String result = asyncQueryExecutorService.cancelQuery(QUERY_ID); + String result = asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext); assertEquals(QUERY_ID, result); verify(statementStorageService).updateStatementState(statementModel, StatementState.CANCELLED); @@ -441,14 +442,15 @@ public void cancelIndexDMLQuery() { givenJobMetadataExists(getBaseAsyncQueryJobMetadataBuilder().jobId(DROP_INDEX_JOB_ID)); assertThrows( - IllegalArgumentException.class, () -> asyncQueryExecutorService.cancelQuery(QUERY_ID)); + IllegalArgumentException.class, + () -> asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext)); } @Test public void cancelRefreshQuery() { givenJobMetadataExists( getBaseAsyncQueryJobMetadataBuilder().jobType(JobType.BATCH).indexName(INDEX_NAME)); - when(flintIndexMetadataService.getFlintIndexMetadata(INDEX_NAME)) + when(flintIndexMetadataService.getFlintIndexMetadata(INDEX_NAME, asyncQueryRequestContext)) .thenReturn( ImmutableMap.of( INDEX_NAME, @@ -463,7 +465,7 @@ public void cancelRefreshQuery() { new GetJobRunResult() .withJobRun(new JobRun().withJobRunId(JOB_ID).withState("Cancelled"))); - String result = asyncQueryExecutorService.cancelQuery(QUERY_ID); + String result = asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext); assertEquals(QUERY_ID, result); verifyCancelJobRunCalled(); @@ -475,7 +477,8 @@ public void cancelStreamingQuery() { givenJobMetadataExists(getBaseAsyncQueryJobMetadataBuilder().jobType(JobType.STREAMING)); assertThrows( - IllegalArgumentException.class, () -> asyncQueryExecutorService.cancelQuery(QUERY_ID)); + IllegalArgumentException.class, + () -> asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext)); } @Test @@ -483,7 +486,7 @@ public void cancelBatchQuery() { givenJobMetadataExists(getBaseAsyncQueryJobMetadataBuilder().jobId(JOB_ID)); givenCancelJobRunSucceed(); - String result = asyncQueryExecutorService.cancelQuery(QUERY_ID); + String result = asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext); assertEquals(QUERY_ID, result); verifyCancelJobRunCalled(); @@ -500,7 +503,7 @@ private void givenSparkExecutionEngineConfigIsSupplied() { } private void givenFlintIndexMetadataExists(String indexName) { - when(flintIndexMetadataService.getFlintIndexMetadata(indexName)) + when(flintIndexMetadataService.getFlintIndexMetadata(indexName, asyncQueryRequestContext)) .thenReturn( ImmutableMap.of( indexName, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index dbc51bb0ad..5d8d9a3b63 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -206,7 +206,8 @@ void testCancelJobWithJobNotFound() { AsyncQueryNotFoundException asyncQueryNotFoundException = Assertions.assertThrows( - AsyncQueryNotFoundException.class, () -> jobExecutorService.cancelQuery(EMR_JOB_ID)); + AsyncQueryNotFoundException.class, + () -> jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext)); Assertions.assertEquals( "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); @@ -218,9 +219,10 @@ void testCancelJobWithJobNotFound() { void testCancelJob() { when(asyncQueryJobMetadataStorageService.getJobMetadata(EMR_JOB_ID)) .thenReturn(Optional.of(getAsyncQueryJobMetadata())); - when(sparkQueryDispatcher.cancelJob(getAsyncQueryJobMetadata())).thenReturn(EMR_JOB_ID); + when(sparkQueryDispatcher.cancelJob(getAsyncQueryJobMetadata(), asyncQueryRequestContext)) + .thenReturn(EMR_JOB_ID); - String jobId = jobExecutorService.cancelQuery(EMR_JOB_ID); + String jobId = jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext); Assertions.assertEquals(EMR_JOB_ID, jobId); verifyNoInteractions(sparkExecutionEngineConfigSupplier); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java index 877d6ec32b..9a3c4e663e 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.sql.datasource.model.DataSourceStatus.ACTIVE; @@ -27,6 +28,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.config.SparkSubmitParameterModifier; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @@ -50,6 +52,7 @@ class IndexDMLHandlerTest { @Mock private IndexDMLResultStorageService indexDMLResultStorageService; @Mock private FlintIndexOpFactory flintIndexOpFactory; @Mock private SparkSubmitParameterModifier sparkSubmitParameterModifier; + @Mock private AsyncQueryRequestContext asyncQueryRequestContext; @InjectMocks IndexDMLHandler indexDMLHandler; @@ -82,8 +85,10 @@ public void testWhenIndexDetailsAreNotFound() { .queryId(QUERY_ID) .dataSourceMetadata(metadata) .indexQueryDetails(indexQueryDetails) + .asyncQueryRequestContext(asyncQueryRequestContext) .build(); - Mockito.when(flintIndexMetadataService.getFlintIndexMetadata(any())) + Mockito.when( + flintIndexMetadataService.getFlintIndexMetadata(any(), eq(asyncQueryRequestContext))) .thenReturn(new HashMap<>()); DispatchQueryResponse dispatchQueryResponse = @@ -107,10 +112,12 @@ public void testWhenIndexDetailsWithInvalidQueryActionType() { .queryId(QUERY_ID) .dataSourceMetadata(metadata) .indexQueryDetails(indexQueryDetails) + .asyncQueryRequestContext(asyncQueryRequestContext) .build(); HashMap<String, FlintIndexMetadata> flintMetadataMap = new HashMap<>(); flintMetadataMap.put(indexQueryDetails.openSearchIndexName(), flintIndexMetadata); - when(flintIndexMetadataService.getFlintIndexMetadata(indexQueryDetails.openSearchIndexName())) + when(flintIndexMetadataService.getFlintIndexMetadata( + indexQueryDetails.openSearchIndexName(), asyncQueryRequestContext)) .thenReturn(flintMetadataMap); indexDMLHandler.submit(dispatchQueryRequest, dispatchQueryContext); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index a7a79c758e..592309cb75 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -871,7 +871,8 @@ void testCancelJob() { .withJobRunId(EMR_JOB_ID) .withApplicationId(EMRS_APPLICATION_ID)); - String queryId = sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata()); + String queryId = + sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); Assertions.assertEquals(QUERY_ID, queryId); } @@ -884,7 +885,8 @@ void testCancelQueryWithSession() { String queryId = sparkQueryDispatcher.cancelJob( - asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID)); + asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID), + asyncQueryRequestContext); verifyNoInteractions(emrServerlessClient); verify(statement, times(1)).cancel(); @@ -900,7 +902,8 @@ void testCancelQueryWithInvalidSession() { IllegalArgumentException.class, () -> sparkQueryDispatcher.cancelJob( - asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, "invalid"))); + asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, "invalid"), + asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); verifyNoInteractions(session); @@ -916,7 +919,8 @@ void testCancelQueryWithInvalidStatementId() { IllegalArgumentException.class, () -> sparkQueryDispatcher.cancelJob( - asyncQueryJobMetadataWithSessionId("invalid", MOCK_SESSION_ID))); + asyncQueryJobMetadataWithSessionId("invalid", MOCK_SESSION_ID), + asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); verifyNoInteractions(statement); @@ -933,7 +937,8 @@ void testCancelQueryWithNoSessionId() { .withJobRunId(EMR_JOB_ID) .withApplicationId(EMRS_APPLICATION_ID)); - String queryId = sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata()); + String queryId = + sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); Assertions.assertEquals(QUERY_ID, queryId); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java index 0c82733ae6..8105629822 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java @@ -16,6 +16,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.execution.xcontent.XContentSerializerUtil; import org.opensearch.sql.spark.flint.FlintIndexMetadata; @@ -28,21 +29,26 @@ public class FlintIndexOpTest { @Mock private FlintIndexStateModelService flintIndexStateModelService; @Mock private EMRServerlessClientFactory mockEmrServerlessClientFactory; + @Mock private AsyncQueryRequestContext asyncQueryRequestContext; @Test public void testApplyWithTransitioningStateFailure() { FlintIndexMetadata metadata = mock(FlintIndexMetadata.class); when(metadata.getLatestId()).thenReturn(Optional.of("latestId")); FlintIndexStateModel fakeModel = getFlintIndexStateModel(metadata); - when(flintIndexStateModelService.getFlintIndexStateModel(eq("latestId"), any())) + when(flintIndexStateModelService.getFlintIndexStateModel( + eq("latestId"), any(), eq(asyncQueryRequestContext))) .thenReturn(Optional.of(fakeModel)); - when(flintIndexStateModelService.updateFlintIndexState(any(), any(), any())) + when(flintIndexStateModelService.updateFlintIndexState( + any(), any(), any(), eq(asyncQueryRequestContext))) .thenThrow(new RuntimeException("Transitioning state failed")); FlintIndexOp flintIndexOp = new TestFlintIndexOp(flintIndexStateModelService, "myS3", mockEmrServerlessClientFactory); IllegalStateException illegalStateException = - Assertions.assertThrows(IllegalStateException.class, () -> flintIndexOp.apply(metadata)); + Assertions.assertThrows( + IllegalStateException.class, + () -> flintIndexOp.apply(metadata, asyncQueryRequestContext)); Assertions.assertEquals( "Moving to transition state:DELETING failed.", illegalStateException.getMessage()); @@ -53,9 +59,11 @@ public void testApplyWithCommitFailure() { FlintIndexMetadata metadata = mock(FlintIndexMetadata.class); when(metadata.getLatestId()).thenReturn(Optional.of("latestId")); FlintIndexStateModel fakeModel = getFlintIndexStateModel(metadata); - when(flintIndexStateModelService.getFlintIndexStateModel(eq("latestId"), any())) + when(flintIndexStateModelService.getFlintIndexStateModel( + eq("latestId"), any(), eq(asyncQueryRequestContext))) .thenReturn(Optional.of(fakeModel)); - when(flintIndexStateModelService.updateFlintIndexState(any(), any(), any())) + when(flintIndexStateModelService.updateFlintIndexState( + any(), any(), any(), eq(asyncQueryRequestContext))) .thenReturn( FlintIndexStateModel.copy(fakeModel, XContentSerializerUtil.buildMetadata(1, 2))) .thenThrow(new RuntimeException("Commit state failed")) @@ -65,7 +73,9 @@ public void testApplyWithCommitFailure() { new TestFlintIndexOp(flintIndexStateModelService, "myS3", mockEmrServerlessClientFactory); IllegalStateException illegalStateException = - Assertions.assertThrows(IllegalStateException.class, () -> flintIndexOp.apply(metadata)); + Assertions.assertThrows( + IllegalStateException.class, + () -> flintIndexOp.apply(metadata, asyncQueryRequestContext)); Assertions.assertEquals( "commit failed. target stable state: [DELETED]", illegalStateException.getMessage()); @@ -76,9 +86,11 @@ public void testApplyWithRollBackFailure() { FlintIndexMetadata metadata = mock(FlintIndexMetadata.class); when(metadata.getLatestId()).thenReturn(Optional.of("latestId")); FlintIndexStateModel fakeModel = getFlintIndexStateModel(metadata); - when(flintIndexStateModelService.getFlintIndexStateModel(eq("latestId"), any())) + when(flintIndexStateModelService.getFlintIndexStateModel( + eq("latestId"), any(), eq(asyncQueryRequestContext))) .thenReturn(Optional.of(fakeModel)); - when(flintIndexStateModelService.updateFlintIndexState(any(), any(), any())) + when(flintIndexStateModelService.updateFlintIndexState( + any(), any(), any(), eq(asyncQueryRequestContext))) .thenReturn( FlintIndexStateModel.copy(fakeModel, XContentSerializerUtil.buildMetadata(1, 2))) .thenThrow(new RuntimeException("Commit state failed")) @@ -87,7 +99,9 @@ public void testApplyWithRollBackFailure() { new TestFlintIndexOp(flintIndexStateModelService, "myS3", mockEmrServerlessClientFactory); IllegalStateException illegalStateException = - Assertions.assertThrows(IllegalStateException.class, () -> flintIndexOp.apply(metadata)); + Assertions.assertThrows( + IllegalStateException.class, + () -> flintIndexOp.apply(metadata, asyncQueryRequestContext)); Assertions.assertEquals( "commit failed. target stable state: [DELETED]", illegalStateException.getMessage()); @@ -125,7 +139,10 @@ FlintIndexState transitioningState() { } @Override - void runOp(FlintIndexMetadata flintIndexMetadata, FlintIndexStateModel flintIndex) {} + void runOp( + FlintIndexMetadata flintIndexMetadata, + FlintIndexStateModel flintIndex, + AsyncQueryRequestContext asyncQueryRequestContext) {} @Override FlintIndexState stableState() { diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java index 60fa13dc93..26858c18fe 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java @@ -16,6 +16,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.flint.FlintIndexClient; import org.opensearch.sql.spark.flint.FlintIndexMetadata; @@ -38,6 +39,7 @@ class FlintIndexOpVacuumTest { @Mock EMRServerlessClientFactory emrServerlessClientFactory; @Mock FlintIndexStateModel flintIndexStateModel; @Mock FlintIndexStateModel transitionedFlintIndexStateModel; + @Mock AsyncQueryRequestContext asyncQueryRequestContext; RuntimeException testException = new RuntimeException("Test Exception"); @@ -55,110 +57,154 @@ public void setUp() { @Test public void testApplyWithEmptyLatestId() { - flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITHOUT_LATEST_ID); + flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITHOUT_LATEST_ID, asyncQueryRequestContext); verify(flintIndexClient).deleteIndex(INDEX_NAME); } @Test public void testApplyWithFlintIndexStateNotFound() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.empty()); assertThrows( IllegalStateException.class, - () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); } @Test public void testApplyWithNotDeletedState() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.ACTIVE); assertThrows( IllegalStateException.class, - () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); } @Test public void testApplyWithUpdateFlintIndexStateThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, FlintIndexState.VACUUMING, DATASOURCE_NAME)) + flintIndexStateModel, + FlintIndexState.VACUUMING, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenThrow(testException); assertThrows( IllegalStateException.class, - () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); } @Test public void testApplyWithRunOpThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, FlintIndexState.VACUUMING, DATASOURCE_NAME)) + flintIndexStateModel, + FlintIndexState.VACUUMING, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenReturn(transitionedFlintIndexStateModel); doThrow(testException).when(flintIndexClient).deleteIndex(INDEX_NAME); assertThrows( - Exception.class, () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + Exception.class, + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); verify(flintIndexStateModelService) .updateFlintIndexState( - transitionedFlintIndexStateModel, FlintIndexState.DELETED, DATASOURCE_NAME); + transitionedFlintIndexStateModel, + FlintIndexState.DELETED, + DATASOURCE_NAME, + asyncQueryRequestContext); } @Test public void testApplyWithRunOpThrowAndRollbackThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, FlintIndexState.VACUUMING, DATASOURCE_NAME)) + flintIndexStateModel, + FlintIndexState.VACUUMING, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenReturn(transitionedFlintIndexStateModel); doThrow(testException).when(flintIndexClient).deleteIndex(INDEX_NAME); when(flintIndexStateModelService.updateFlintIndexState( - transitionedFlintIndexStateModel, FlintIndexState.DELETED, DATASOURCE_NAME)) + transitionedFlintIndexStateModel, + FlintIndexState.DELETED, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenThrow(testException); assertThrows( - Exception.class, () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + Exception.class, + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); } @Test public void testApplyWithDeleteFlintIndexStateModelThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, FlintIndexState.VACUUMING, DATASOURCE_NAME)) + flintIndexStateModel, + FlintIndexState.VACUUMING, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenReturn(transitionedFlintIndexStateModel); - when(flintIndexStateModelService.deleteFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.deleteFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenThrow(testException); assertThrows( IllegalStateException.class, - () -> flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID)); + () -> + flintIndexOpVacuum.apply( + FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); } @Test public void testApplyHappyPath() { - when(flintIndexStateModelService.getFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME)) + when(flintIndexStateModelService.getFlintIndexStateModel( + LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(flintIndexStateModel)); when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, FlintIndexState.VACUUMING, DATASOURCE_NAME)) + flintIndexStateModel, + FlintIndexState.VACUUMING, + DATASOURCE_NAME, + asyncQueryRequestContext)) .thenReturn(transitionedFlintIndexStateModel); when(transitionedFlintIndexStateModel.getLatestId()).thenReturn(LATEST_ID); - flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID); + flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext); - verify(flintIndexStateModelService).deleteFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME); + verify(flintIndexStateModelService) + .deleteFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext); verify(flintIndexClient).deleteIndex(INDEX_NAME); } } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java index 31b1ecb49c..2dd0a4a7cf 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java @@ -17,6 +17,7 @@ import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; import org.opensearch.sql.spark.flint.FlintIndexMetadata; import org.opensearch.sql.spark.flint.FlintIndexMetadataService; @@ -29,6 +30,8 @@ public class FlintStreamingJobHouseKeeperTask implements Runnable { private final DataSourceService dataSourceService; private final FlintIndexMetadataService flintIndexMetadataService; private final FlintIndexOpFactory flintIndexOpFactory; + private final NullAsyncQueryRequestContext nullAsyncQueryRequestContext = + new NullAsyncQueryRequestContext(); private static final Logger LOGGER = LogManager.getLogger(FlintStreamingJobHouseKeeperTask.class); protected static final AtomicBoolean isRunning = new AtomicBoolean(false); @@ -91,7 +94,9 @@ private void dropAutoRefreshIndex( String autoRefreshIndex, FlintIndexMetadata flintIndexMetadata, String datasourceName) { // When the datasource is deleted. Possibly Replace with VACUUM Operation. LOGGER.info("Attempting to drop auto refresh index: {}", autoRefreshIndex); - flintIndexOpFactory.getDrop(datasourceName).apply(flintIndexMetadata); + flintIndexOpFactory + .getDrop(datasourceName) + .apply(flintIndexMetadata, nullAsyncQueryRequestContext); LOGGER.info("Successfully dropped index: {}", autoRefreshIndex); } @@ -100,7 +105,9 @@ private void alterAutoRefreshIndex( LOGGER.info("Attempting to alter index: {}", autoRefreshIndex); FlintIndexOptions flintIndexOptions = new FlintIndexOptions(); flintIndexOptions.setOption(FlintIndexOptions.AUTO_REFRESH, "false"); - flintIndexOpFactory.getAlter(flintIndexOptions, datasourceName).apply(flintIndexMetadata); + flintIndexOpFactory + .getAlter(flintIndexOptions, datasourceName) + .apply(flintIndexMetadata, nullAsyncQueryRequestContext); LOGGER.info("Successfully altered index: {}", autoRefreshIndex); } @@ -119,7 +126,7 @@ private String getDataSourceName(FlintIndexMetadata flintIndexMetadata) { private Map<String, FlintIndexMetadata> getAllAutoRefreshIndices() { Map<String, FlintIndexMetadata> flintIndexMetadataHashMap = - flintIndexMetadataService.getFlintIndexMetadata("flint_*"); + flintIndexMetadataService.getFlintIndexMetadata("flint_*", nullAsyncQueryRequestContext); return flintIndexMetadataHashMap.entrySet().stream() .filter(entry -> entry.getValue().getFlintIndexOptions().autoRefresh()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java index 893b33b39d..b8352d15b2 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java @@ -33,6 +33,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.client.Client; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; /** Implementation of {@link FlintIndexMetadataService} */ @@ -49,7 +50,8 @@ public class FlintIndexMetadataServiceImpl implements FlintIndexMetadataService Arrays.asList(AUTO_REFRESH, INCREMENTAL_REFRESH, WATERMARK_DELAY, CHECKPOINT_LOCATION)); @Override - public Map<String, FlintIndexMetadata> getFlintIndexMetadata(String indexPattern) { + public Map<String, FlintIndexMetadata> getFlintIndexMetadata( + String indexPattern, AsyncQueryRequestContext asyncQueryRequestContext) { GetMappingsResponse mappingsResponse = client.admin().indices().prepareGetMappings().setIndices(indexPattern).get(); Map<String, FlintIndexMetadata> indexMetadataMap = new HashMap<>(); @@ -73,7 +75,10 @@ public Map<String, FlintIndexMetadata> getFlintIndexMetadata(String indexPattern } @Override - public void updateIndexToManualRefresh(String indexName, FlintIndexOptions flintIndexOptions) { + public void updateIndexToManualRefresh( + String indexName, + FlintIndexOptions flintIndexOptions, + AsyncQueryRequestContext asyncQueryRequestContext) { GetMappingsResponse mappingsResponse = client.admin().indices().prepareGetMappings().setIndices(indexName).get(); Map<String, Object> flintMetadataMap = diff --git a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java index 5781c3e44b..eba338e912 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java @@ -7,6 +7,7 @@ import java.util.Optional; import lombok.RequiredArgsConstructor; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.execution.statestore.OpenSearchStateStoreUtil; import org.opensearch.sql.spark.execution.statestore.StateStore; import org.opensearch.sql.spark.execution.xcontent.FlintIndexStateModelXContentSerializer; @@ -20,7 +21,8 @@ public class OpenSearchFlintIndexStateModelService implements FlintIndexStateMod public FlintIndexStateModel updateFlintIndexState( FlintIndexStateModel flintIndexStateModel, FlintIndexState flintIndexState, - String datasourceName) { + String datasourceName, + AsyncQueryRequestContext asyncQueryRequestContext) { return stateStore.updateState( flintIndexStateModel, flintIndexState, @@ -29,14 +31,16 @@ public FlintIndexStateModel updateFlintIndexState( } @Override - public Optional<FlintIndexStateModel> getFlintIndexStateModel(String id, String datasourceName) { + public Optional<FlintIndexStateModel> getFlintIndexStateModel( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext) { return stateStore.get( id, serializer::fromXContent, OpenSearchStateStoreUtil.getIndexName(datasourceName)); } @Override public FlintIndexStateModel createFlintIndexStateModel( - FlintIndexStateModel flintIndexStateModel) { + FlintIndexStateModel flintIndexStateModel, + AsyncQueryRequestContext asyncQueryRequestContext) { return stateStore.create( flintIndexStateModel.getId(), flintIndexStateModel, @@ -45,7 +49,8 @@ public FlintIndexStateModel createFlintIndexStateModel( } @Override - public boolean deleteFlintIndexStateModel(String id, String datasourceName) { + public boolean deleteFlintIndexStateModel( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext) { return stateStore.delete(id, OpenSearchStateStoreUtil.getIndexName(datasourceName)); } } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java index 232a280db5..ce80351f70 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java @@ -13,6 +13,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionRequest; import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.tasks.Task; @@ -41,7 +42,9 @@ protected void doExecute( CancelAsyncQueryActionRequest request, ActionListener<CancelAsyncQueryActionResponse> listener) { try { - String jobId = asyncQueryExecutorService.cancelQuery(request.getQueryId()); + String jobId = + asyncQueryExecutorService.cancelQuery( + request.getQueryId(), new NullAsyncQueryRequestContext()); listener.onResponse( new CancelAsyncQueryActionResponse( String.format("Deleted async query with id: %s", jobId))); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java index 3ff806bf50..ede8a348b4 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java @@ -71,7 +71,8 @@ public void withoutSessionCreateAsyncQueryThenGetResultThenCancel() { emrsClient.getJobRunResultCalled(1); // 3. cancel async query. - String cancelQueryId = asyncQueryExecutorService.cancelQuery(response.getQueryId()); + String cancelQueryId = + asyncQueryExecutorService.cancelQuery(response.getQueryId(), asyncQueryRequestContext); assertEquals(response.getQueryId(), cancelQueryId); emrsClient.cancelJobRunCalled(1); } @@ -163,7 +164,8 @@ public void withSessionCreateAsyncQueryThenGetResultThenCancel() { assertEquals(StatementState.WAITING.getState(), asyncQueryResults.getStatus()); // 3. cancel async query. - String cancelQueryId = asyncQueryExecutorService.cancelQuery(response.getQueryId()); + String cancelQueryId = + asyncQueryExecutorService.cancelQuery(response.getQueryId(), asyncQueryRequestContext); assertEquals(response.getQueryId(), cancelQueryId); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java index 2eed7b13a0..29c42446b3 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java @@ -152,7 +152,9 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> asyncQueryExecutorService.cancelQuery(response.getQueryId())); + () -> + asyncQueryExecutorService.cancelQuery( + response.getQueryId(), asyncQueryRequestContext)); assertEquals("can't cancel index DML query", exception.getMessage()); }); } @@ -326,7 +328,9 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> asyncQueryExecutorService.cancelQuery(response.getQueryId())); + () -> + asyncQueryExecutorService.cancelQuery( + response.getQueryId(), asyncQueryRequestContext)); assertEquals("can't cancel index DML query", exception.getMessage()); }); } @@ -901,7 +905,9 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> asyncQueryExecutorService.cancelQuery(response.getQueryId())); + () -> + asyncQueryExecutorService.cancelQuery( + response.getQueryId(), asyncQueryRequestContext)); assertEquals( "can't cancel index DML query, using ALTER auto_refresh=off statement to stop" + " job, using VACUUM statement to stop job and delete data", @@ -944,7 +950,9 @@ public GetJobRunResult getJobRunResult( flintIndexJob.refreshing(); // 2. Cancel query - String cancelResponse = asyncQueryExecutorService.cancelQuery(response.getQueryId()); + String cancelResponse = + asyncQueryExecutorService.cancelQuery( + response.getQueryId(), asyncQueryRequestContext); assertNotNull(cancelResponse); assertTrue(clusterService.state().routingTable().hasIndex(mockDS.indexName)); @@ -992,7 +1000,9 @@ public GetJobRunResult getJobRunResult( IllegalStateException illegalStateException = Assertions.assertThrows( IllegalStateException.class, - () -> asyncQueryExecutorService.cancelQuery(response.getQueryId())); + () -> + asyncQueryExecutorService.cancelQuery( + response.getQueryId(), asyncQueryRequestContext)); Assertions.assertEquals( "Transaction failed as flint index is not in a valid state.", illegalStateException.getMessage()); @@ -1038,6 +1048,7 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. Cancel query Assertions.assertThrows( IllegalStateException.class, - () -> asyncQueryExecutorService.cancelQuery(response.getQueryId())); + () -> + asyncQueryExecutorService.cancelQuery(response.getQueryId(), asyncQueryRequestContext)); } } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java index 6c82188ee6..0dc8f02820 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java @@ -18,6 +18,7 @@ public class MockFlintSparkJob { private FlintIndexStateModel stateModel; private FlintIndexStateModelService flintIndexStateModelService; private String datasource; + private AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); public MockFlintSparkJob( FlintIndexStateModelService flintIndexStateModelService, String latestId, String datasource) { @@ -34,12 +35,15 @@ public MockFlintSparkJob( .lastUpdateTime(System.currentTimeMillis()) .error("") .build(); - stateModel = flintIndexStateModelService.createFlintIndexStateModel(stateModel); + stateModel = + flintIndexStateModelService.createFlintIndexStateModel( + stateModel, asyncQueryRequestContext); } public void transition(FlintIndexState newState) { stateModel = - flintIndexStateModelService.updateFlintIndexState(stateModel, newState, datasource); + flintIndexStateModelService.updateFlintIndexState( + stateModel, newState, datasource, asyncQueryRequestContext); } public void refreshing() { @@ -68,7 +72,8 @@ public void deleted() { public void assertState(FlintIndexState expected) { Optional<FlintIndexStateModel> stateModelOpt = - flintIndexStateModelService.getFlintIndexStateModel(stateModel.getId(), datasource); + flintIndexStateModelService.getFlintIndexStateModel( + stateModel.getId(), datasource, asyncQueryRequestContext); assertTrue(stateModelOpt.isPresent()); assertEquals(expected, stateModelOpt.get().getIndexState()); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java b/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java index c5964a61e3..0a3a180932 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java @@ -20,6 +20,7 @@ import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceSpec; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.asyncquery.model.MockFlintIndex; import org.opensearch.sql.spark.asyncquery.model.MockFlintSparkJob; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; @@ -393,13 +394,16 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { FlintIndexMetadataService flintIndexMetadataService = new FlintIndexMetadataService() { @Override - public Map<String, FlintIndexMetadata> getFlintIndexMetadata(String indexPattern) { + public Map<String, FlintIndexMetadata> getFlintIndexMetadata( + String indexPattern, AsyncQueryRequestContext asyncQueryRequestContext) { throw new RuntimeException("Couldn't fetch details from ElasticSearch"); } @Override public void updateIndexToManualRefresh( - String indexName, FlintIndexOptions flintIndexOptions) {} + String indexName, + FlintIndexOptions flintIndexOptions, + AsyncQueryRequestContext asyncQueryRequestContext) {} }; FlintStreamingJobHouseKeeperTask flintStreamingJobHouseKeeperTask = new FlintStreamingJobHouseKeeperTask( diff --git a/async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java b/async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java index f6baa82dd2..b1321cc132 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java @@ -29,6 +29,7 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName; import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType; @@ -39,6 +40,8 @@ public class FlintIndexMetadataServiceImplTest { @Mock(answer = RETURNS_DEEP_STUBS) private Client client; + @Mock private AsyncQueryRequestContext asyncQueryRequestContext; + @SneakyThrows @Test void testGetJobIdFromFlintSkippingIndexMetadata() { @@ -56,8 +59,11 @@ void testGetJobIdFromFlintSkippingIndexMetadata() { .indexQueryActionType(IndexQueryActionType.DROP) .indexType(FlintIndexType.SKIPPING) .build(); + Map<String, FlintIndexMetadata> indexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(indexQueryDetails.openSearchIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + indexQueryDetails.openSearchIndexName(), asyncQueryRequestContext); + Assertions.assertEquals( "00fhelvq7peuao0", indexMetadataMap.get(indexQueryDetails.openSearchIndexName()).getJobId()); @@ -80,8 +86,11 @@ void testGetJobIdFromFlintSkippingIndexMetadataWithIndexState() { .indexQueryActionType(IndexQueryActionType.DROP) .indexType(FlintIndexType.SKIPPING) .build(); + Map<String, FlintIndexMetadata> indexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(indexQueryDetails.openSearchIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + indexQueryDetails.openSearchIndexName(), asyncQueryRequestContext); + FlintIndexMetadata metadata = indexMetadataMap.get(indexQueryDetails.openSearchIndexName()); Assertions.assertEquals("00fhelvq7peuao0", metadata.getJobId()); } @@ -103,8 +112,11 @@ void testGetJobIdFromFlintCoveringIndexMetadata() { .indexType(FlintIndexType.COVERING) .build(); FlintIndexMetadataService flintIndexMetadataService = new FlintIndexMetadataServiceImpl(client); + Map<String, FlintIndexMetadata> indexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(indexQueryDetails.openSearchIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + indexQueryDetails.openSearchIndexName(), asyncQueryRequestContext); + Assertions.assertEquals( "00fdmvv9hp8u0o0q", indexMetadataMap.get(indexQueryDetails.openSearchIndexName()).getJobId()); @@ -126,8 +138,11 @@ void testGetJobIDWithNPEException() { .indexQueryActionType(IndexQueryActionType.DROP) .indexType(FlintIndexType.COVERING) .build(); + Map<String, FlintIndexMetadata> flintIndexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata(indexQueryDetails.openSearchIndexName()); + flintIndexMetadataService.getFlintIndexMetadata( + indexQueryDetails.openSearchIndexName(), asyncQueryRequestContext); + Assertions.assertFalse( flintIndexMetadataMap.containsKey("flint_mys3_default_http_logs_cv1_index")); } @@ -148,8 +163,10 @@ void testGetJobIDWithNPEExceptionForMultipleIndices() { indexMappingsMap.put(indexName, mappings); mockNodeClientIndicesMappings("flint_mys3*", indexMappingsMap); FlintIndexMetadataService flintIndexMetadataService = new FlintIndexMetadataServiceImpl(client); + Map<String, FlintIndexMetadata> flintIndexMetadataMap = - flintIndexMetadataService.getFlintIndexMetadata("flint_mys3*"); + flintIndexMetadataService.getFlintIndexMetadata("flint_mys3*", asyncQueryRequestContext); + Assertions.assertFalse( flintIndexMetadataMap.containsKey("flint_mys3_default_http_logs_cv1_index")); Assertions.assertTrue( diff --git a/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java b/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java index 977f77b397..4faff41fe6 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java @@ -16,6 +16,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.execution.statestore.StateStore; import org.opensearch.sql.spark.execution.xcontent.FlintIndexStateModelXContentSerializer; @@ -30,6 +31,7 @@ public class OpenSearchFlintIndexStateModelServiceTest { @Mock FlintIndexState flintIndexState; @Mock FlintIndexStateModel responseFlintIndexStateModel; @Mock FlintIndexStateModelXContentSerializer flintIndexStateModelXContentSerializer; + @Mock AsyncQueryRequestContext asyncQueryRequestContext; @InjectMocks OpenSearchFlintIndexStateModelService openSearchFlintIndexStateModelService; @@ -40,7 +42,7 @@ void updateFlintIndexState() { FlintIndexStateModel result = openSearchFlintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, flintIndexState, DATASOURCE); + flintIndexStateModel, flintIndexState, DATASOURCE, asyncQueryRequestContext); assertEquals(responseFlintIndexStateModel, result); } @@ -51,7 +53,8 @@ void getFlintIndexStateModel() { .thenReturn(Optional.of(responseFlintIndexStateModel)); Optional<FlintIndexStateModel> result = - openSearchFlintIndexStateModelService.getFlintIndexStateModel("ID", DATASOURCE); + openSearchFlintIndexStateModelService.getFlintIndexStateModel( + "ID", DATASOURCE, asyncQueryRequestContext); assertEquals(responseFlintIndexStateModel, result.get()); } @@ -63,7 +66,8 @@ void createFlintIndexStateModel() { when(flintIndexStateModel.getDatasourceName()).thenReturn(DATASOURCE); FlintIndexStateModel result = - openSearchFlintIndexStateModelService.createFlintIndexStateModel(flintIndexStateModel); + openSearchFlintIndexStateModelService.createFlintIndexStateModel( + flintIndexStateModel, asyncQueryRequestContext); assertEquals(responseFlintIndexStateModel, result); } @@ -73,7 +77,8 @@ void deleteFlintIndexStateModel() { when(mockStateStore.delete(any(), any())).thenReturn(true); boolean result = - openSearchFlintIndexStateModelService.deleteFlintIndexStateModel(ID, DATASOURCE); + openSearchFlintIndexStateModelService.deleteFlintIndexStateModel( + ID, DATASOURCE, asyncQueryRequestContext); assertTrue(result); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java index 2ff76b9b57..a2581fdea2 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java @@ -7,6 +7,8 @@ package org.opensearch.sql.spark.transport; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.constants.TestConstants.EMR_JOB_ID; @@ -24,6 +26,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionRequest; import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.tasks.Task; @@ -36,7 +39,6 @@ public class TransportCancelAsyncQueryRequestActionTest { @Mock private TransportCancelAsyncQueryRequestAction action; @Mock private Task task; @Mock private ActionListener<CancelAsyncQueryActionResponse> actionListener; - @Mock private AsyncQueryExecutorServiceImpl asyncQueryExecutorService; @Captor @@ -54,8 +56,12 @@ public void setUp() { @Test public void testDoExecute() { CancelAsyncQueryActionRequest request = new CancelAsyncQueryActionRequest(EMR_JOB_ID); - when(asyncQueryExecutorService.cancelQuery(EMR_JOB_ID)).thenReturn(EMR_JOB_ID); + when(asyncQueryExecutorService.cancelQuery( + eq(EMR_JOB_ID), any(NullAsyncQueryRequestContext.class))) + .thenReturn(EMR_JOB_ID); + action.doExecute(task, request, actionListener); + Mockito.verify(actionListener).onResponse(deleteJobActionResponseArgumentCaptor.capture()); CancelAsyncQueryActionResponse cancelAsyncQueryActionResponse = deleteJobActionResponseArgumentCaptor.getValue(); @@ -66,8 +72,12 @@ public void testDoExecute() { @Test public void testDoExecuteWithException() { CancelAsyncQueryActionRequest request = new CancelAsyncQueryActionRequest(EMR_JOB_ID); - doThrow(new RuntimeException("Error")).when(asyncQueryExecutorService).cancelQuery(EMR_JOB_ID); + doThrow(new RuntimeException("Error")) + .when(asyncQueryExecutorService) + .cancelQuery(eq(EMR_JOB_ID), any(NullAsyncQueryRequestContext.class)); + action.doExecute(task, request, actionListener); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); From 0e70a502bbc64779c428dd0dcc2d9d65f6cfb591 Mon Sep 17 00:00:00 2001 From: qianheng <qh2995096578@outlook.com> Date: Mon, 5 Aug 2024 23:25:47 +0800 Subject: [PATCH 22/96] add TakeOrderedOperator (#2863) --------- Signed-off-by: Heng Qian <qianheng@amazon.com> --- .../org/opensearch/sql/executor/Explain.java | 14 + .../sql/planner/DefaultImplementor.java | 9 +- .../sql/planner/physical/PhysicalPlanDSL.java | 5 + .../physical/PhysicalPlanNodeVisitor.java | 4 + .../sql/planner/physical/SortHelper.java | 70 ++ .../sql/planner/physical/SortOperator.java | 43 +- .../planner/physical/TakeOrderedOperator.java | 88 +++ .../opensearch/sql/executor/ExplainTest.java | 21 + .../sql/planner/DefaultImplementorTest.java | 26 + .../physical/PhysicalPlanNodeVisitorTest.java | 4 + .../physical/TakeOrderedOperatorTest.java | 607 ++++++++++++++++++ docs/user/optimization/optimization.rst | 29 +- .../OpenSearchExecutionProtector.java | 12 + .../OpenSearchExecutionProtectorTest.java | 11 + 14 files changed, 883 insertions(+), 60 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/TakeOrderedOperator.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/TakeOrderedOperatorTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/Explain.java b/core/src/main/java/org/opensearch/sql/executor/Explain.java index 0f05b99383..fffbe6f693 100644 --- a/core/src/main/java/org/opensearch/sql/executor/Explain.java +++ b/core/src/main/java/org/opensearch/sql/executor/Explain.java @@ -30,6 +30,7 @@ import org.opensearch.sql.planner.physical.RemoveOperator; import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; +import org.opensearch.sql.planner.physical.TakeOrderedOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.TableScanOperator; @@ -73,6 +74,19 @@ public ExplainResponseNode visitSort(SortOperator node, Object context) { ImmutableMap.of("sortList", describeSortList(node.getSortList())))); } + @Override + public ExplainResponseNode visitTakeOrdered(TakeOrderedOperator node, Object context) { + return explain( + node, + context, + explainNode -> + explainNode.setDescription( + ImmutableMap.of( + "limit", node.getLimit(), + "offset", node.getOffset(), + "sortList", describeSortList(node.getSortList())))); + } + @Override public ExplainResponseNode visitTableScan(TableScanOperator node, Object context) { return explain( diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index b53d17b38f..f962c3e4bf 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -38,6 +38,7 @@ import org.opensearch.sql.planner.physical.RemoveOperator; import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; +import org.opensearch.sql.planner.physical.TakeOrderedOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.read.TableScanBuilder; @@ -129,7 +130,13 @@ public PhysicalPlan visitValues(LogicalValues node, C context) { @Override public PhysicalPlan visitLimit(LogicalLimit node, C context) { - return new LimitOperator(visitChild(node, context), node.getLimit(), node.getOffset()); + PhysicalPlan child = visitChild(node, context); + // Optimize sort + limit to take ordered operator + if (child instanceof SortOperator sortChild) { + return new TakeOrderedOperator( + sortChild.getInput(), node.getLimit(), node.getOffset(), sortChild.getSortList()); + } + return new LimitOperator(child, node.getLimit(), node.getOffset()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java index 147f0e08dc..0c2764112d 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java @@ -64,6 +64,11 @@ public static SortOperator sort(PhysicalPlan input, Pair<SortOption, Expression> return new SortOperator(input, Arrays.asList(sorts)); } + public static TakeOrderedOperator takeOrdered( + PhysicalPlan input, Integer limit, Integer offset, Pair<SortOption, Expression>... sorts) { + return new TakeOrderedOperator(input, limit, offset, Arrays.asList(sorts)); + } + public static DedupeOperator dedupe(PhysicalPlan input, Expression... expressions) { return new DedupeOperator(input, Arrays.asList(expressions)); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 99b5cc8020..67d7a05135 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -72,6 +72,10 @@ public R visitSort(SortOperator node, C context) { return visitNode(node, context); } + public R visitTakeOrdered(TakeOrderedOperator node, C context) { + return visitNode(node, context); + } + public R visitRareTopN(RareTopNOperator node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java b/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java new file mode 100644 index 0000000000..ea117ee6df --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java @@ -0,0 +1,70 @@ +package org.opensearch.sql.planner.physical; + +import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; +import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; + +import com.google.common.collect.Ordering; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.utils.ExprValueOrdering; +import org.opensearch.sql.expression.Expression; + +public interface SortHelper { + + /** + * Construct an expr comparator for sorting on ExprValue. + * + * @param sortList list of sort fields and their related sort options. + * @return A comparator for ExprValue + */ + static Comparator<ExprValue> constructExprComparator( + List<Pair<SortOption, Expression>> sortList) { + return (o1, o2) -> compareWithExpressions(o1, o2, constructComparator(sortList)); + } + + /** + * Construct an expr ordering for efficiently taking the top-k elements on ExprValue. + * + * @param sortList list of sort fields and their related sort options. + * @return An guava ordering for ExprValue + */ + static Ordering<ExprValue> constructExprOrdering(List<Pair<SortOption, Expression>> sortList) { + return Ordering.from(constructExprComparator(sortList)); + } + + private static List<Pair<Expression, Comparator<ExprValue>>> constructComparator( + List<Pair<SortOption, Expression>> sortList) { + List<Pair<Expression, Comparator<ExprValue>>> comparators = new ArrayList<>(); + for (Pair<SortOption, Expression> pair : sortList) { + SortOption option = pair.getLeft(); + ExprValueOrdering ordering = + ASC.equals(option.getSortOrder()) + ? ExprValueOrdering.natural() + : ExprValueOrdering.natural().reverse(); + ordering = + NULL_FIRST.equals(option.getNullOrder()) ? ordering.nullsFirst() : ordering.nullsLast(); + comparators.add(Pair.of(pair.getRight(), ordering)); + } + return comparators; + } + + private static int compareWithExpressions( + ExprValue o1, ExprValue o2, List<Pair<Expression, Comparator<ExprValue>>> comparators) { + for (Pair<Expression, Comparator<ExprValue>> comparator : comparators) { + Expression expression = comparator.getKey(); + int result = + comparator + .getValue() + .compare( + expression.valueOf(o1.bindingTuples()), expression.valueOf(o2.bindingTuples())); + if (result != 0) { + return result; + } + } + return 0; + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/SortOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/SortOperator.java index e3116baedf..b635f01d18 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/SortOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/SortOperator.java @@ -5,25 +5,18 @@ package org.opensearch.sql.planner.physical; -import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; -import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; - import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.PriorityQueue; -import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.Singular; import lombok.ToString; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.utils.ExprValueOrdering; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.planner.physical.SortOperator.Sorter.SorterBuilder; /** * Sort Operator.The input data is sorted by the sort fields in the {@link SortOperator#sortList}. @@ -36,7 +29,7 @@ public class SortOperator extends PhysicalPlan { @Getter private final PhysicalPlan input; @Getter private final List<Pair<SortOption, Expression>> sortList; - @EqualsAndHashCode.Exclude private final Sorter sorter; + @EqualsAndHashCode.Exclude private final Comparator<ExprValue> sorter; @EqualsAndHashCode.Exclude private Iterator<ExprValue> iterator; /** @@ -49,18 +42,7 @@ public class SortOperator extends PhysicalPlan { public SortOperator(PhysicalPlan input, List<Pair<SortOption, Expression>> sortList) { this.input = input; this.sortList = sortList; - SorterBuilder sorterBuilder = Sorter.builder(); - for (Pair<SortOption, Expression> pair : sortList) { - SortOption option = pair.getLeft(); - ExprValueOrdering ordering = - ASC.equals(option.getSortOrder()) - ? ExprValueOrdering.natural() - : ExprValueOrdering.natural().reverse(); - ordering = - NULL_FIRST.equals(option.getNullOrder()) ? ordering.nullsFirst() : ordering.nullsLast(); - sorterBuilder.comparator(Pair.of(pair.getRight(), ordering)); - } - this.sorter = sorterBuilder.build(); + this.sorter = SortHelper.constructExprComparator(sortList); } @Override @@ -94,27 +76,6 @@ public ExprValue next() { return iterator.next(); } - @Builder - public static class Sorter implements Comparator<ExprValue> { - @Singular private final List<Pair<Expression, Comparator<ExprValue>>> comparators; - - @Override - public int compare(ExprValue o1, ExprValue o2) { - for (Pair<Expression, Comparator<ExprValue>> comparator : comparators) { - Expression expression = comparator.getKey(); - int result = - comparator - .getValue() - .compare( - expression.valueOf(o1.bindingTuples()), expression.valueOf(o2.bindingTuples())); - if (result != 0) { - return result; - } - } - return 0; - } - } - private Iterator<ExprValue> iterator(PriorityQueue<ExprValue> result) { return new Iterator<ExprValue>() { @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/TakeOrderedOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/TakeOrderedOperator.java new file mode 100644 index 0000000000..a6e0f968e6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/TakeOrderedOperator.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import com.google.common.collect.Ordering; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.Expression; + +/** + * TakeOrdered Operator. This operator will sort input data as the order of {@link this#sortList} + * specifies and return {@link this#limit} rows from the {@link this#offset} index. + * + * <p>Functionally, this operator is a combination of {@link SortOperator} and {@link + * LimitOperator}. But it can reduce the time complexity from O(nlogn) to O(n), and memory from O(n) + * to O(k) due to use guava {@link com.google.common.collect.Ordering}. + * + * <p>Overall, it's an optimization to replace `Limit(Sort)` in physical plan level since it's all + * about execution. Because most execution engine may not support this operator, it doesn't have a + * related logical operator. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class TakeOrderedOperator extends PhysicalPlan { + @Getter private final PhysicalPlan input; + + @Getter private final List<Pair<SortOption, Expression>> sortList; + @Getter private final Integer limit; + @Getter private final Integer offset; + @EqualsAndHashCode.Exclude private final Ordering<ExprValue> ordering; + @EqualsAndHashCode.Exclude private Iterator<ExprValue> iterator; + + /** + * TakeOrdered Operator Constructor. + * + * @param input input {@link PhysicalPlan} + * @param limit the limit value from LimitOperator + * @param offset the offset value from LimitOperator + * @param sortList list of sort field from SortOperator + */ + public TakeOrderedOperator( + PhysicalPlan input, + Integer limit, + Integer offset, + List<Pair<SortOption, Expression>> sortList) { + this.input = input; + this.sortList = sortList; + this.limit = limit; + this.offset = offset; + this.ordering = SortHelper.constructExprOrdering(sortList); + } + + @Override + public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { + return visitor.visitTakeOrdered(this, context); + } + + @Override + public void open() { + super.open(); + iterator = ordering.leastOf(input, offset + limit).stream().skip(offset).iterator(); + } + + @Override + public List<PhysicalPlan> getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java b/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java index 897347f22d..eaeae07242 100644 --- a/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java @@ -27,6 +27,7 @@ import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.remove; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.rename; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.sort; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.takeOrdered; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.values; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.window; @@ -220,6 +221,26 @@ void can_explain_limit() { explain.apply(plan)); } + @Test + void can_explain_takeOrdered() { + Pair<Sort.SortOption, Expression> sort = + ImmutablePair.of(Sort.SortOption.DEFAULT_ASC, ref("a", INTEGER)); + PhysicalPlan plan = takeOrdered(tableScan, 10, 5, sort); + assertEquals( + new ExplainResponse( + new ExplainResponseNode( + "TakeOrderedOperator", + Map.of( + "limit", + 10, + "offset", + 5, + "sortList", + Map.of("a", Map.of("sortOrder", "ASC", "nullOrder", "NULL_FIRST"))), + singletonList(tableScan.explainNode()))), + explain.apply(plan)); + } + @Test void can_explain_nested() { Set<String> nestedOperatorArgs = Set.of("message.info", "message"); diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java index 45d8f6c03c..8e71fc2bec 100644 --- a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java @@ -278,4 +278,30 @@ public void visitPaginate_should_remove_it_from_tree() { new ProjectOperator(new ValuesOperator(List.of(List.of())), List.of(), List.of()); assertEquals(physicalPlanTree, logicalPlanTree.accept(implementor, null)); } + + @Test + public void visitLimit_support_return_takeOrdered() { + // replace SortOperator + LimitOperator with TakeOrderedOperator + Pair<Sort.SortOption, Expression> sort = + ImmutablePair.of(Sort.SortOption.DEFAULT_ASC, ref("a", INTEGER)); + var logicalValues = values(emptyList()); + var logicalSort = sort(logicalValues, sort); + var logicalLimit = limit(logicalSort, 10, 5); + PhysicalPlan physicalPlanTree = + PhysicalPlanDSL.takeOrdered(PhysicalPlanDSL.values(emptyList()), 10, 5, sort); + assertEquals(physicalPlanTree, logicalLimit.accept(implementor, null)); + + // don't replace if LimitOperator's child is not SortOperator + Pair<ReferenceExpression, Expression> newEvalField = + ImmutablePair.of(ref("name1", STRING), ref("name", STRING)); + var logicalEval = eval(logicalSort, newEvalField); + logicalLimit = limit(logicalEval, 10, 5); + physicalPlanTree = + PhysicalPlanDSL.limit( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.sort(PhysicalPlanDSL.values(emptyList()), sort), newEvalField), + 10, + 5); + assertEquals(physicalPlanTree, logicalLimit.accept(implementor, null)); + } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index c91ae8787c..17fb128ace 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.remove; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.rename; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.sort; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.takeOrdered; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.values; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.window; @@ -117,6 +118,8 @@ public static Stream<Arguments> getPhysicalPlanForTest() { PhysicalPlan sort = sort(plan, Pair.of(SortOption.DEFAULT_ASC, ref)); + PhysicalPlan takeOrdered = takeOrdered(plan, 1, 1, Pair.of(SortOption.DEFAULT_ASC, ref)); + PhysicalPlan dedupe = dedupe(plan, ref); PhysicalPlan values = values(emptyList()); @@ -140,6 +143,7 @@ public static Stream<Arguments> getPhysicalPlanForTest() { Arguments.of(remove, "remove"), Arguments.of(eval, "eval"), Arguments.of(sort, "sort"), + Arguments.of(takeOrdered, "takeOrdered"), Arguments.of(dedupe, "dedupe"), Arguments.of(values, "values"), Arguments.of(rareTopN, "rareTopN"), diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/TakeOrderedOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/TakeOrderedOperatorTest.java new file mode 100644 index 0000000000..f2fcb84910 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/TakeOrderedOperatorTest.java @@ -0,0 +1,607 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.expression.DSL.ref; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.limit; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.sort; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.takeOrdered; + +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.Expression; + +/** + * To make sure {@link TakeOrderedOperator} can replace {@link SortOperator} + {@link + * LimitOperator}, this UT will replica all tests in {@link SortOperatorTest} and add more test + * cases on different limit and offset. + */ +@ExtendWith(MockitoExtension.class) +class TakeOrderedOperatorTest extends PhysicalPlanTestBase { + private static PhysicalPlan inputPlan; + + @Getter + @Setter + private static class Wrapper { + Iterator<ExprValue> iterator = Collections.emptyIterator(); + } + + private static final Wrapper wrapper = new Wrapper(); + + @BeforeAll + public static void setUp() { + inputPlan = Mockito.mock(PhysicalPlan.class); + when(inputPlan.hasNext()) + .thenAnswer((InvocationOnMock invocation) -> wrapper.iterator.hasNext()); + when(inputPlan.next()).thenAnswer((InvocationOnMock invocation) -> wrapper.iterator.next()); + } + + /** + * construct the map which contain null value, because {@link ImmutableMap} doesn't support null + * value. + */ + private static final Map<String, Object> NULL_MAP = + new HashMap<>() { + { + put("size", 399); + put("response", null); + } + }; + + @Test + public void sort_one_field_asc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_with_duplication() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_asc_with_null_value() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_asc_with_missing_value() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_desc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_DESC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 2, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_desc_with_null_value() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_DESC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP)); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP)); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_with_duplicate_value() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + List<Pair<SortOption, Expression>> sortList = + List.of(Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 3, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_two_fields_both_asc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + List<Pair<SortOption, Expression>> sortList = + List.of( + Pair.of(SortOption.DEFAULT_ASC, ref("size", INTEGER)), + Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 5, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 1, + sortList, + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_two_fields_both_desc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + List<Pair<SortOption, Expression>> sortList = + List.of( + Pair.of(SortOption.DEFAULT_DESC, ref("size", INTEGER)), + Pair.of(SortOption.DEFAULT_DESC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 5, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP)); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_two_fields_asc_and_desc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + List<Pair<SortOption, Expression>> sortList = + List.of( + Pair.of(SortOption.DEFAULT_ASC, ref("size", INTEGER)), + Pair.of(SortOption.DEFAULT_DESC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 5, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP)); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 1, + sortList, + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 499, "response", 404))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_two_fields_desc_and_asc() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + List<Pair<SortOption, Expression>> sortList = + List.of( + Pair.of(SortOption.DEFAULT_DESC, ref("size", INTEGER)), + Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + + test_takeOrdered_with_sort_limit( + inputList, + 5, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 0, + sortList, + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + test_takeOrdered_with_sort_limit( + inputList, + 4, + 1, + sortList, + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 320, "response", 200))); + + test_takeOrdered_with_sort_limit(inputList, 0, 1, sortList); + } + + @Test + public void sort_one_field_without_input() { + wrapper.setIterator(Collections.emptyIterator()); + assertEquals( + 0, + execute( + takeOrdered( + inputPlan, 1, 0, Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER)))) + .size()); + } + + @Test + public void offset_exceeds_row_number() { + List<ExprValue> inputList = + Arrays.asList( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(NULL_MAP)); + + wrapper.setIterator(inputList.iterator()); + PhysicalPlan plan = + takeOrdered(inputPlan, 1, 6, Pair.of(SortOption.DEFAULT_ASC, ref("response", INTEGER))); + List<ExprValue> result = execute(plan); + assertEquals(0, result.size()); + } + + private void test_takeOrdered_with_sort_limit( + List<ExprValue> inputList, + int limit, + int offset, + List<Pair<SortOption, Expression>> sortList, + ExprValue... expected) { + wrapper.setIterator(inputList.iterator()); + List<ExprValue> compareResult = + execute(limit(sort(inputPlan, sortList.toArray(Pair[]::new)), limit, offset)); + wrapper.setIterator(inputList.iterator()); + List<ExprValue> testResult = + execute(takeOrdered(inputPlan, limit, offset, sortList.toArray(Pair[]::new))); + assertEquals(compareResult, testResult); + if (expected.length == 0) { + assertEquals(0, testResult.size()); + } else { + assertThat(testResult, contains(expected)); + } + } +} diff --git a/docs/user/optimization/optimization.rst b/docs/user/optimization/optimization.rst index 835fe96eba..454c9ec066 100644 --- a/docs/user/optimization/optimization.rst +++ b/docs/user/optimization/optimization.rst @@ -237,31 +237,24 @@ If sort that includes expression, which cannot be merged into query DSL, also ex }, "children": [ { - "name": "LimitOperator", + "name": "TakeOrderedOperator", "description": { "limit": 10, - "offset": 0 + "offset": 0, + "sortList": { + "abs(age)": { + "sortOrder": "ASC", + "nullOrder": "NULL_FIRST" + } + } }, "children": [ { - "name": "SortOperator", + "name": "OpenSearchIndexScan", "description": { - "sortList": { - "abs(age)": { - "sortOrder": "ASC", - "nullOrder": "NULL_FIRST" - } - } + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)" }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)" - }, - "children": [] - } - ] + "children": [] } ] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index 0905c2f4b4..28827b0a54 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -23,6 +23,7 @@ import org.opensearch.sql.planner.physical.RemoveOperator; import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; +import org.opensearch.sql.planner.physical.TakeOrderedOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.TableScanOperator; @@ -130,6 +131,17 @@ public PhysicalPlan visitSort(SortOperator node, Object context) { return doProtect(new SortOperator(visitInput(node.getInput(), context), node.getSortList())); } + /** Decorate with {@link ResourceMonitorPlan}. */ + @Override + public PhysicalPlan visitTakeOrdered(TakeOrderedOperator node, Object context) { + return doProtect( + new TakeOrderedOperator( + visitInput(node.getInput(), context), + node.getLimit(), + node.getOffset(), + node.getSortList())); + } + /** * Values are a sequence of rows of literal value in memory which doesn't need memory protection. */ diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index b2dc042110..5cd11c6cd4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -68,6 +68,7 @@ import org.opensearch.sql.planner.physical.NestedOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanDSL; +import org.opensearch.sql.planner.physical.TakeOrderedOperator; @ExtendWith(MockitoExtension.class) @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -306,6 +307,16 @@ void do_nothing_with_CursorCloseOperator_and_children() { verify(child, never()).accept(executionProtector, null); } + @Test + public void test_visitTakeOrdered() { + Pair<Sort.SortOption, Expression> sort = + ImmutablePair.of(Sort.SortOption.DEFAULT_ASC, ref("a", INTEGER)); + TakeOrderedOperator takeOrdered = + PhysicalPlanDSL.takeOrdered(PhysicalPlanDSL.values(emptyList()), 10, 5, sort); + assertEquals( + resourceMonitor(takeOrdered), executionProtector.visitTakeOrdered(takeOrdered, null)); + } + PhysicalPlan resourceMonitor(PhysicalPlan input) { return new ResourceMonitorPlan(input, resourceMonitor); } From 7e73f124e2dc63a69750c8034558d6870c46ccf9 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Mon, 5 Aug 2024 11:42:42 -0700 Subject: [PATCH 23/96] Test utils update to fix IT tests for serverless (#2869) Signed-off-by: Manasvini B S <manasvis@amazon.com> --- .../org/opensearch/sql/sql/AggregationIT.java | 9 ++- .../org/opensearch/sql/sql/ScoreQueryIT.java | 8 +-- .../org/opensearch/sql/util/MatcherUtils.java | 31 ++++++++++ .../org/opensearch/sql/util/TestUtils.java | 62 +++++++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java index 29358bd1c3..901c2a41e4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java @@ -9,14 +9,17 @@ import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verify; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySome; import static org.opensearch.sql.util.TestUtils.getResponseBody; +import static org.opensearch.sql.util.TestUtils.roundOfResponse; import java.io.IOException; import java.util.List; import java.util.Locale; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -396,8 +399,9 @@ public void testMaxDoublePushedDown() throws IOException { @Test public void testAvgDoublePushedDown() throws IOException { var response = executeQuery(String.format("SELECT avg(num3)" + " from %s", TEST_INDEX_CALCS)); + JSONArray responseJSON = roundOfResponse(response.getJSONArray("datarows")); verifySchema(response, schema("avg(num3)", null, "double")); - verifyDataRows(response, rows(-6.12D)); + verify(responseJSON, rows(-6.12D)); } @Test @@ -456,8 +460,9 @@ public void testAvgDoubleInMemory() throws IOException { executeQuery( String.format( "SELECT avg(num3)" + " OVER(PARTITION BY datetime1) from %s", TEST_INDEX_CALCS)); + JSONArray roundOfResponse = roundOfResponse(response.getJSONArray("datarows")); verifySchema(response, schema("avg(num3) OVER(PARTITION BY datetime1)", null, "double")); - verifySome(response.getJSONArray("datarows"), rows(-6.12D)); + verifySome(roundOfResponse, rows(-6.12D)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java index 6616746d99..a1f71dcf6c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java @@ -8,7 +8,7 @@ import static org.hamcrest.Matchers.containsString; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataAddressRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -123,8 +123,7 @@ public void scoreQueryTest() throws IOException { TestsConstants.TEST_INDEX_ACCOUNT), "jdbc")); verifySchema(result, schema("address", null, "text"), schema("_score", null, "float")); - verifyDataRows( - result, rows("154 Douglass Street", 650.1515), rows("565 Hall Street", 3.2507575)); + verifyDataAddressRows(result, rows("154 Douglass Street"), rows("565 Hall Street")); } @Test @@ -154,7 +153,8 @@ public void scoreQueryDefaultBoostQueryTest() throws IOException { + "where score(matchQuery(address, 'Powell')) order by _score desc limit 2", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc")); + verifySchema(result, schema("address", null, "text"), schema("_score", null, "float")); - verifyDataRows(result, rows("305 Powell Street", 6.501515)); + verifyDataAddressRows(result, rows("305 Powell Street")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index 26a60cb4e5..d4db502407 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -159,6 +159,11 @@ public static void verifyDataRows(JSONObject response, Matcher<JSONArray>... mat verify(response.getJSONArray("datarows"), matchers); } + @SafeVarargs + public static void verifyDataAddressRows(JSONObject response, Matcher<JSONArray>... matchers) { + verifyAddressRow(response.getJSONArray("datarows"), matchers); + } + @SafeVarargs public static void verifyColumn(JSONObject response, Matcher<JSONObject>... matchers) { verify(response.getJSONArray("schema"), matchers); @@ -183,6 +188,32 @@ public static <T> void verify(JSONArray array, Matcher<T>... matchers) { assertThat(objects, containsInAnyOrder(matchers)); } + // TODO: this is temporary fix for fixing serverless tests to pass as it creates multiple shards + // leading to score differences. + public static <T> void verifyAddressRow(JSONArray array, Matcher<T>... matchers) { + // List to store the processed elements from the JSONArray + List<T> objects = new ArrayList<>(); + + // Iterate through each element in the JSONArray + array + .iterator() + .forEachRemaining( + o -> { + // Check if o is a JSONArray with exactly 2 elements + if (o instanceof JSONArray && ((JSONArray) o).length() == 2) { + // Check if the second element is a BigDecimal/_score value + if (((JSONArray) o).get(1) instanceof BigDecimal) { + // Remove the _score element from response data rows to skip the assertion as it + // will be different when compared against multiple shards + ((JSONArray) o).remove(1); + } + } + objects.add((T) o); + }); + assertEquals(matchers.length, objects.size()); + assertThat(objects, containsInAnyOrder(matchers)); + } + @SafeVarargs @SuppressWarnings("unchecked") public static <T> void verifyInOrder(JSONArray array, Matcher<T>... matchers) { diff --git a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java index 589fb1f9ae..bce83e7ccb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java @@ -17,6 +17,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -27,6 +29,7 @@ import java.util.List; import java.util.Locale; import java.util.stream.Collectors; +import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -34,6 +37,7 @@ import org.opensearch.client.Client; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.client.RestClient; import org.opensearch.common.xcontent.XContentType; import org.opensearch.sql.legacy.cursor.CursorType; @@ -123,10 +127,45 @@ public static Response performRequest(RestClient client, Request request) { } return response; } catch (IOException e) { + if (isRefreshPolicyError(e)) { + try { + return retryWithoutRefreshPolicy(request, client); + } catch (IOException ex) { + throw new IllegalStateException("Failed to perform request without refresh policy.", ex); + } + } throw new IllegalStateException("Failed to perform request", e); } } + /** + * Checks if the IOException is due to an unsupported refresh policy. + * + * @param e The IOException to check. + * @return true if the exception is due to a refresh policy error, false otherwise. + */ + private static boolean isRefreshPolicyError(IOException e) { + return e instanceof ResponseException + && ((ResponseException) e).getResponse().getStatusLine().getStatusCode() == 400 + && e.getMessage().contains("true refresh policy is not supported."); + } + + /** + * Attempts to perform the request without the refresh policy. + * + * @param request The original request. + * @param client client connection + * @return The response after retrying the request. + * @throws IOException If the request fails. + */ + private static Response retryWithoutRefreshPolicy(Request request, RestClient client) + throws IOException { + Request req = + new Request(request.getMethod(), request.getEndpoint().replaceAll("refresh=true", "")); + req.setEntity(request.getEntity()); + return client.performRequest(req); + } + public static String getAccountIndexMapping() { return "{ \"mappings\": {" + " \"properties\": {\n" @@ -772,6 +811,29 @@ public static String getResponseBody(Response response, boolean retainNewLines) return sb.toString(); } + // TODO: this is temporary fix for fixing serverless tests to pass with 2 digit precision value + public static JSONArray roundOfResponse(JSONArray array) { + JSONArray responseJSON = new JSONArray(); + array + .iterator() + .forEachRemaining( + o -> { + JSONArray jsonArray = new JSONArray(); + ((JSONArray) o) + .iterator() + .forEachRemaining( + i -> { + if (i instanceof BigDecimal) { + jsonArray.put(((BigDecimal) i).setScale(2, RoundingMode.HALF_UP)); + } else { + jsonArray.put(i); + } + }); + responseJSON.put(jsonArray); + }); + return responseJSON; + } + public static String fileToString( final String filePathFromProjectRoot, final boolean removeNewLines) throws IOException { From 7022a0946015c41cbd02888e3c712c6b67123982 Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Wed, 7 Aug 2024 10:50:00 +0800 Subject: [PATCH 24/96] Correct regular expression range (#2836) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- .../java/org/opensearch/sql/common/grok/GrokCompiler.java | 2 +- .../main/java/org/opensearch/sql/common/grok/GrokUtils.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java index aba96ad4cb..05fdbd57ed 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java @@ -29,7 +29,7 @@ public class GrokCompiler implements Serializable { // We don't want \n and commented line - private static final Pattern patternLinePattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$"); + private static final Pattern patternLinePattern = Pattern.compile("^([a-zA-Z0-9_]+)\\s+(.*)$"); /** {@code Grok} patterns definitions. */ private final Map<String, String> grokPatternDefinitions = new HashMap<>(); diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java index 4b145bbbe8..2a309bba8f 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java @@ -24,8 +24,8 @@ public class GrokUtils { Pattern.compile( "%\\{" + "(?<name>" - + "(?<pattern>[A-z0-9]+)" - + "(?::(?<subname>[A-z0-9_:;,\\-\\/\\s\\.']+))?" + + "(?<pattern>[a-zA-Z0-9_]+)" + + "(?::(?<subname>[a-zA-Z0-9_:;,\\-\\/\\s\\.']+))?" + ")" + "(?:=(?<definition>" + "(?:" From 4a735ea9bca6313a06616a99a944b8e512baeb66 Mon Sep 17 00:00:00 2001 From: qianheng <qianheng@amazon.com> Date: Fri, 9 Aug 2024 12:08:48 +0800 Subject: [PATCH 25/96] Push down limit through eval (#2876) --- .../optimizer/LogicalPlanOptimizer.java | 2 + .../planner/optimizer/pattern/Patterns.java | 5 ++ .../planner/optimizer/rule/EvalPushDown.java | 82 +++++++++++++++++++ .../optimizer/LogicalPlanOptimizerTest.java | 23 ++++++ .../org/opensearch/sql/ppl/ExplainIT.java | 13 +++ .../ppl/explain_limit_push.json | 27 ++++++ 6 files changed, 152 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java index 5c115f0db8..e805b0dea5 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.stream.Collectors; import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.rule.EvalPushDown; import org.opensearch.sql.planner.optimizer.rule.MergeFilterAndFilter; import org.opensearch.sql.planner.optimizer.rule.PushFilterUnderSort; import org.opensearch.sql.planner.optimizer.rule.read.CreateTableScanBuilder; @@ -46,6 +47,7 @@ public static LogicalPlanOptimizer create() { */ new MergeFilterAndFilter(), new PushFilterUnderSort(), + EvalPushDown.PUSH_DOWN_LIMIT, /* * Phase 2: Transformations that rely on data source push down capability */ diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java index ee4e9a20cc..ef2607e018 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java @@ -12,6 +12,7 @@ import java.util.Optional; import lombok.experimental.UtilityClass; import org.opensearch.sql.planner.logical.LogicalAggregation; +import org.opensearch.sql.planner.logical.LogicalEval; import org.opensearch.sql.planner.logical.LogicalFilter; import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalLimit; @@ -63,6 +64,10 @@ public static <T extends LogicalPlan> Pattern<LogicalProject> project(Pattern<T> return Pattern.typeOf(LogicalProject.class).with(source(pattern)); } + public static Pattern<LogicalEval> evalCapture() { + return Pattern.typeOf(LogicalEval.class).capturedAs(Capture.newCapture()); + } + /** Pattern for {@link TableScanBuilder} and capture it meanwhile. */ public static Pattern<TableScanBuilder> scanBuilder() { return Pattern.typeOf(TableScanBuilder.class).capturedAs(Capture.newCapture()); diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java new file mode 100644 index 0000000000..17eaed0e8c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/EvalPushDown.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.optimizer.rule; + +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.evalCapture; +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.limit; +import static org.opensearch.sql.planner.optimizer.rule.EvalPushDown.EvalPushDownBuilder.match; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.matching.pattern.CapturePattern; +import com.facebook.presto.matching.pattern.WithPattern; +import java.util.List; +import java.util.function.BiFunction; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalEval; +import org.opensearch.sql.planner.logical.LogicalLimit; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.Rule; + +/** + * Rule template for all rules related to push down logical plans under eval, so these plans can + * avoid blocking by eval and may have chances to be pushed down into table scan by rules in {@link + * org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown}. + */ +public class EvalPushDown<T extends LogicalPlan> implements Rule<T> { + + // TODO: Add more rules to push down sort and project + /** Push down optimize rule for limit operator. Transform `limit -> eval` to `eval -> limit` */ + public static final Rule<LogicalLimit> PUSH_DOWN_LIMIT = + match(limit(evalCapture())) + .apply( + (limit, logicalEval) -> { + List<LogicalPlan> child = logicalEval.getChild(); + limit.replaceChildPlans(child); + logicalEval.replaceChildPlans(List.of(limit)); + return logicalEval; + }); + + private final Capture<LogicalEval> capture; + + @Accessors(fluent = true) + @Getter + private final Pattern<T> pattern; + + private final BiFunction<T, LogicalEval, LogicalPlan> pushDownFunction; + + @SuppressWarnings("unchecked") + public EvalPushDown( + WithPattern<T> pattern, BiFunction<T, LogicalEval, LogicalPlan> pushDownFunction) { + this.pattern = pattern; + this.capture = ((CapturePattern<LogicalEval>) pattern.getPattern()).capture(); + this.pushDownFunction = pushDownFunction; + } + + @Override + public LogicalPlan apply(T plan, Captures captures) { + LogicalEval logicalEval = captures.get(capture); + return pushDownFunction.apply(plan, logicalEval); + } + + static class EvalPushDownBuilder<T extends LogicalPlan> { + + private WithPattern<T> pattern; + + public static <T extends LogicalPlan> EvalPushDown.EvalPushDownBuilder<T> match( + Pattern<T> pattern) { + EvalPushDown.EvalPushDownBuilder<T> builder = new EvalPushDown.EvalPushDownBuilder<>(); + builder.pattern = (WithPattern<T>) pattern; + return builder; + } + + public EvalPushDown<T> apply(BiFunction<T, LogicalEval, LogicalPlan> pushDownFunction) { + return new EvalPushDown<>(pattern, pushDownFunction); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java index c25e415cfa..20996503b4 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.longValue; import static org.opensearch.sql.data.type.ExprCoreType.*; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.eval; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.filter; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.highlight; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.limit; @@ -43,6 +44,7 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.planner.logical.LogicalPaginate; @@ -345,6 +347,27 @@ void table_scan_builder_support_offset_push_down_can_apply_its_rule() { assertEquals(project(tableScanBuilder), optimized); } + /** Limit - Eval --> Eval - Limit. */ + @Test + void push_limit_under_eval() { + Pair<ReferenceExpression, Expression> evalExpr = + Pair.of(DSL.ref("name1", STRING), DSL.ref("name", STRING)); + assertEquals( + eval(limit(tableScanBuilder, 10, 5), evalExpr), + optimize(limit(eval(relation("schema", table), evalExpr), 10, 5))); + } + + /** Limit - Eval - Scan --> Eval - Scan. */ + @Test + void push_limit_through_eval_into_scan() { + when(tableScanBuilder.pushDownLimit(any())).thenReturn(true); + Pair<ReferenceExpression, Expression> evalExpr = + Pair.of(DSL.ref("name1", STRING), DSL.ref("name", STRING)); + assertEquals( + eval(tableScanBuilder, evalExpr), + optimize(limit(eval(relation("schema", table), evalExpr), 10, 5))); + } + private LogicalPlan optimize(LogicalPlan plan) { final LogicalPlanOptimizer optimizer = LogicalPlanOptimizer.create(); return optimizer.optimize(plan); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index fce975ef92..c6b21e1605 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -76,6 +76,19 @@ public void testSortPushDownExplain() throws Exception { + "| fields age")); } + @Test + public void testLimitPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_limit_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + "| eval ageMinus = age - 30 " + + "| head 5 " + + "| fields ageMinus")); + } + String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json new file mode 100644 index 0000000000..51a627ea4d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json @@ -0,0 +1,27 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[ageMinus]" + }, + "children": [ + { + "name": "EvalOperator", + "description": { + "expressions": { + "ageMinus": "-(age, 30)" + } + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, searchDone=false)" + }, + "children": [] + } + ] + } + ] + } +} From 6972487e855599745df909292d699d37f82282c7 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Tue, 13 Aug 2024 09:58:40 -0700 Subject: [PATCH 26/96] [Tests] IT tests and test utils update to fix failing tests for serverless (#2902) Signed-off-by: Manasvini B S <manasvis@amazon.com> --- .../sql/legacy/CsvFormatResponseIT.java | 29 ++- .../org/opensearch/sql/ppl/CsvFormatIT.java | 5 +- .../opensearch/sql/ppl/DedupCommandIT.java | 61 ++++++- .../opensearch/sql/ppl/ParseCommandIT.java | 114 +++++++++--- .../org/opensearch/sql/ppl/SortCommandIT.java | 74 +++++++- .../org/opensearch/sql/sql/ConditionalIT.java | 66 ++++++- .../org/opensearch/sql/sql/CsvFormatIT.java | 5 +- .../sql/sql/MathematicalFunctionIT.java | 22 ++- .../java/org/opensearch/sql/sql/NestedIT.java | 166 ++++++++++++++---- .../org/opensearch/sql/sql/RawFormatIT.java | 4 +- .../org/opensearch/sql/util/TestUtils.java | 36 +++- 11 files changed, 491 insertions(+), 91 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index 9a416c9683..b75da57c57 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -689,13 +690,18 @@ public void sanitizeTest() throws IOException { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), false); - List<String> lines = csvResult.getLines(); - assertEquals(5, lines.size()); - assertEquals(lines.get(0), "'+Amber JOHnny,Duke Willmington+"); - assertEquals(lines.get(1), "'-Hattie,Bond-"); - assertEquals(lines.get(2), "'=Nanette,Bates="); - assertEquals(lines.get(3), "'@Dale,Adams@"); - assertEquals(lines.get(4), "\",Elinor\",\"Ratliff,,,\""); + List<String> actualLines = csvResult.getLines(); + assertEquals(5, actualLines.size()); + + List<String> expectedLines = + Arrays.asList( + "'+Amber JOHnny,Duke Willmington+", + "'-Hattie,Bond-", + "'=Nanette,Bates=", + "'@Dale,Adams@", + "\",Elinor\",\"Ratliff,,,\""); + + assertContainsSameItems(expectedLines, actualLines); } @Test @@ -719,6 +725,15 @@ private void verifyFieldOrder(final String[] expectedFields) throws IOException verifyFieldOrder(expectedFields, query); } + private void assertContainsSameItems(List<String> expectedLines, List<String> actualLines) { + Collections.sort(expectedLines); + Collections.sort(actualLines); + assertEquals(expectedLines.size(), actualLines.size()); + for (int i = 0; i < expectedLines.size(); i++) { + assertEquals(expectedLines.get(i), actualLines.get(i)); + } + } + private void verifyFieldOrder(final String[] expectedFields, final String query) throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java index a9eb18c2a1..21240bf416 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -27,7 +28,7 @@ public void sanitizeTest() throws IOException { Locale.ROOT, "source=%s | fields firstname, lastname", TEST_INDEX_BANK_CSV_SANITIZE)); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "'+Amber JOHnny,Duke Willmington+%n" @@ -47,7 +48,7 @@ public void escapeSanitizeTest() throws IOException { "source=%s | fields firstname, lastname", TEST_INDEX_BANK_CSV_SANITIZE), false); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "+Amber JOHnny,Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java index 7a6cf16bb4..b69fce6785 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java @@ -11,6 +11,9 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -35,7 +38,16 @@ public void testConsecutiveDedup() throws IOException { executeQuery( String.format( "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK)); - verifyDataRows(result, rows(true), rows(false), rows(true), rows(false)); + List<Object[]> actualRows = extractActualRows(result); + List<Object[]> expectedRows = getExpectedDedupRows(actualRows); + assertTrue("Deduplication was not consecutive", expectedRows != null); + assertEquals( + "Row count after deduplication does not match", expectedRows.size(), actualRows.size()); + + // Verify the expected and actual rows match + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } @Test @@ -62,4 +74,51 @@ public void testKeepEmptyDedup() throws IOException { rows("Virginia", null), rows("Dillard", 48086)); } + + private List<Object[]> extractActualRows(JSONObject result) { + JSONArray dataRows = result.getJSONArray("datarows"); + List<Object[]> actualRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + actualRows.add(new Object[] {row.get(0)}); + } + return actualRows; + } + + // Create the expected deduplicated rows + private List<Object[]> getExpectedDedupRows(List<Object[]> actualRows) { + if (verifyConsecutiveDeduplication(actualRows)) { + return createExpectedRows(actualRows); + } + return null; + } + + // Verify consecutive deduplication + private boolean verifyConsecutiveDeduplication(List<Object[]> actualRows) { + Object previousValue = null; + + for (Object[] currentRow : actualRows) { + Object currentValue = currentRow[0]; + if (previousValue != null && currentValue.equals(previousValue)) { + return false; // If consecutive values are the same, deduplication fails + } + previousValue = currentValue; + } + return true; + } + + // Create the expected rows after deduplication + private List<Object[]> createExpectedRows(List<Object[]> actualRows) { + List<Object[]> expectedRows = new ArrayList<>(); + Object previousValue = null; + + for (Object[] currentRow : actualRows) { + Object currentValue = currentRow[0]; + if (previousValue == null || !currentValue.equals(previousValue)) { + expectedRows.add(currentRow); + } + previousValue = currentValue; + } + return expectedRows; + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java index 7f25f6f160..5e672812c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java @@ -6,11 +6,13 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.verifyOrder; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Assert; import org.junit.Test; public class ParseCommandIT extends PPLIntegTestCase { @@ -26,15 +28,23 @@ public void testParseCommand() throws IOException { executeQuery( String.format( "source=%s | parse email '.+@(?<host>.+)' | fields email, host", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("amberduke@pyrami.com", "pyrami.com"), - rows("hattiebond@netagy.com", "netagy.com"), - rows("nanettebates@quility.com", "quility.com"), - rows("daleadams@boink.com", "boink.com"), - rows("elinorratliff@scentric.com", "scentric.com"), - rows("virginiaayala@filodyne.com", "filodyne.com"), - rows("dillardmcpherson@quailcom.com", "quailcom.com")); + + // Create the expected rows + List<Object[]> expectedRows = + new ArrayList<>( + List.of( + new Object[] {"amberduke@pyrami.com", "pyrami.com"}, + new Object[] {"hattiebond@netagy.com", "netagy.com"}, + new Object[] {"nanettebates@quility.com", "quility.com"}, + new Object[] {"daleadams@boink.com", "boink.com"}, + new Object[] {"elinorratliff@scentric.com", "scentric.com"}, + new Object[] {"virginiaayala@filodyne.com", "filodyne.com"}, + new Object[] {"dillardmcpherson@quailcom.com", "quailcom.com"})); + + List<Object[]> actualRows = convertJsonToRows(result, 2); + sortRowsByFirstColumn(expectedRows); + sortRowsByFirstColumn(actualRows); + compareRows(expectedRows, actualRows); } @Test @@ -43,15 +53,23 @@ public void testParseCommandReplaceOriginalField() throws IOException { executeQuery( String.format( "source=%s | parse email '.+@(?<email>.+)' | fields email", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("pyrami.com"), - rows("netagy.com"), - rows("quility.com"), - rows("boink.com"), - rows("scentric.com"), - rows("filodyne.com"), - rows("quailcom.com")); + + // Create the expected rows + List<Object[]> expectedRows = + new ArrayList<>( + List.of( + new Object[] {"pyrami.com"}, + new Object[] {"netagy.com"}, + new Object[] {"quility.com"}, + new Object[] {"boink.com"}, + new Object[] {"scentric.com"}, + new Object[] {"filodyne.com"}, + new Object[] {"quailcom.com"})); + + List<Object[]> actualRows = convertJsonToRows(result, 1); + sortRowsByFirstColumn(expectedRows); + sortRowsByFirstColumn(actualRows); + compareRows(expectedRows, actualRows); } @Test @@ -62,14 +80,52 @@ public void testParseCommandWithOtherRunTimeFields() throws IOException { "source=%s | parse email '.+@(?<host>.+)' | " + "eval eval_result=1 | fields host, eval_result", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("pyrami.com", 1), - rows("netagy.com", 1), - rows("quility.com", 1), - rows("boink.com", 1), - rows("scentric.com", 1), - rows("filodyne.com", 1), - rows("quailcom.com", 1)); + + // Create the expected rows as List<Object[]> + List<Object[]> expectedRows = + new ArrayList<>( + List.of( + new Object[] {"pyrami.com", 1}, + new Object[] {"netagy.com", 1}, + new Object[] {"quility.com", 1}, + new Object[] {"boink.com", 1}, + new Object[] {"scentric.com", 1}, + new Object[] {"filodyne.com", 1}, + new Object[] {"quailcom.com", 1})); + + List<Object[]> actualRows = convertJsonToRows(result, 2); + sortRowsByFirstColumn(expectedRows); + sortRowsByFirstColumn(actualRows); + compareRows(expectedRows, actualRows); + } + + // Convert JSON response to List<Object[]> + private List<Object[]> convertJsonToRows(JSONObject result, int columnCount) { + JSONArray dataRows = result.getJSONArray("datarows"); + List<Object[]> rows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + Object[] rowData = new Object[columnCount]; + for (int j = 0; j < columnCount; j++) { + rowData[j] = row.get(j); + } + rows.add(rowData); + } + return rows; + } + + // Sort rows by the first column + private void sortRowsByFirstColumn(List<Object[]> rows) { + rows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + } + + private void compareRows(List<Object[]> expectedRows, List<Object[]> actualRows) { + if (expectedRows.size() != actualRows.size()) { + Assert.fail( + "Row count is different. expectedRows:" + expectedRows + ", actualRows: " + actualRows); + } + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index c90a506252..1061f0bd9d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -12,6 +12,12 @@ import static org.opensearch.sql.util.MatcherUtils.verifyOrder; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; @@ -38,17 +44,77 @@ public void testSortWithNullValue() throws IOException { String.format( "source=%s | sort balance | fields firstname, balance", TEST_INDEX_BANK_WITH_NULL_VALUES)); + + JSONArray dataRows = result.getJSONArray("datarows"); + + // Filter null balance rows + List<Object[]> nullRows = filterRows(dataRows, 1, true); + + // Verify the set values for null balances as rows with null balance can return in any order + List<Object[]> expectedNullRows = + Arrays.asList( + new Object[] {"Hattie", null}, + new Object[] {"Elinor", null}, + new Object[] {"Virginia", null}); + assertSetEquals(expectedNullRows, nullRows); + + // Filter non-null balance rows and create filtered result + List<Object[]> nonNullRows = filterRows(dataRows, 1, false); + JSONObject filteredResult = createFilteredResult(result, nonNullRows); + verifyOrder( - result, - rows("Hattie", null), - rows("Elinor", null), - rows("Virginia", null), + filteredResult, rows("Dale", 4180), rows("Nanette", 32838), rows("Amber JOHnny", 39225), rows("Dillard", 48086)); } + private void assertSetEquals(List<Object[]> expected, List<Object[]> actual) { + Set<List<Object>> expectedSet = new HashSet<>(); + for (Object[] arr : expected) { + expectedSet.add(Arrays.asList(arr)); + } + + Set<List<Object>> actualSet = new HashSet<>(); + for (Object[] arr : actual) { + actualSet.add(Arrays.asList(arr)); + } + + assertEquals(expectedSet, actualSet); + } + + // Filter rows by null or non-null values based on the specified column index + private List<Object[]> filterRows(JSONArray dataRows, int columnIndex, boolean isNull) { + List<Object[]> filteredRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + if ((isNull && row.isNull(columnIndex)) || (!isNull && !row.isNull(columnIndex))) { + Object[] rowData = new Object[row.length()]; + for (int j = 0; j < row.length(); j++) { + rowData[j] = row.isNull(j) ? null : row.get(j); + } + filteredRows.add(rowData); + } + } + return filteredRows; + } + + // Create a new JSONObject with filtered rows and updated metadata + private JSONObject createFilteredResult(JSONObject originalResult, List<Object[]> filteredRows) { + JSONArray jsonArray = new JSONArray(); + for (Object[] row : filteredRows) { + jsonArray.put(new JSONArray(row)); + } + + JSONObject filteredResult = new JSONObject(); + filteredResult.put("schema", originalResult.getJSONArray("schema")); + filteredResult.put("total", jsonArray.length()); + filteredResult.put("datarows", jsonArray); + filteredResult.put("size", jsonArray.length()); + return filteredResult; + } + @Test public void testSortStringField() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java index deb41653e2..9cf4fa2e8a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.core.JsonFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; import org.opensearch.action.search.SearchResponse; @@ -69,10 +72,17 @@ public void ifnullWithNullInputTest() { schema("IFNULL(null, firstname)", "IFNULL1", "keyword"), schema("IFNULL(firstname, null)", "IFNULL2", "keyword"), schema("IFNULL(null, null)", "IFNULL3", "byte")); - verifyDataRows( - response, - rows("Hattie", "Hattie", LITERAL_NULL.value()), - rows("Elinor", "Elinor", LITERAL_NULL.value())); + // Retrieve the actual data rows + JSONArray dataRows = response.getJSONArray("datarows"); + + // Create expected rows dynamically based on the actual data received + // IFNULL1 will be firstname + // IFNULL2 will be firstname + List<Object[]> expectedRows = + createExpectedRows(dataRows, new int[] {0, 0}, LITERAL_NULL.value()); + + // Verify the actual data rows against the expected rows + verifyRows(dataRows, expectedRows); } @Test @@ -216,10 +226,50 @@ public void ifWithTrueAndFalseCondition() throws IOException { schema("IF(2 > 0, firstname, lastname)", "IF1", "keyword"), schema("firstname", "IF2", "text"), schema("lastname", "IF3", "keyword")); - verifyDataRows( - response, - rows("Duke Willmington", "Amber JOHnny", "Amber JOHnny", "Duke Willmington"), - rows("Bond", "Hattie", "Hattie", "Bond")); + + // Retrieve the actual data rows + JSONArray dataRows = response.getJSONArray("datarows"); + + // Create expected rows based on the actual data received as data can be different for the + // different data sources + // IF0 will be lastname as 2 < 0 is false + // IF1 will be firstname as 2 > 0 is true + List<Object[]> expectedRows = createExpectedRows(dataRows, new int[] {0, 1, 1, 0}); + + // Verify the actual data rows against the expected rows + verifyRows(dataRows, expectedRows); + } + + // Convert a JSONArray to a List<Object[]> with dynamic row construction + private List<Object[]> createExpectedRows( + JSONArray dataRows, int[] columnIndices, Object... staticValues) { + List<Object[]> expectedRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + Object[] rowData = new Object[columnIndices.length + staticValues.length]; + int k = 0; + for (int j = 0; j < columnIndices.length; j++) { + rowData[k++] = row.get(columnIndices[j]); + } + for (Object staticValue : staticValues) { + rowData[k++] = staticValue; + } + expectedRows.add(rowData); + } + return expectedRows; + } + + // Verify the actual data rows against the expected rows + private void verifyRows(JSONArray dataRows, List<Object[]> expectedRows) { + for (int i = 0; i < dataRows.length(); i++) { + JSONArray actualRow = dataRows.getJSONArray(i); + Object[] expectedRow = expectedRows.get(i); + Object[] actualRowData = new Object[expectedRow.length]; + for (int j = 0; j < actualRowData.length; j++) { + actualRowData[j] = actualRow.isNull(j) ? LITERAL_NULL.value() : actualRow.get(j); + } + assertArrayEquals(expectedRow, actualRowData); + } } private SearchHits query(String query) throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java index d481e0ad49..d400ad646f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; import static org.opensearch.sql.protocol.response.format.CsvResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -30,7 +31,7 @@ public void sanitizeTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), "csv"); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "'+Amber JOHnny,Duke Willmington+%n" @@ -48,7 +49,7 @@ public void escapeSanitizeTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), "csv&sanitize=false"); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "+Amber JOHnny,Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index 60b7632ad0..b7f2ced5fb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.Locale; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -96,9 +97,24 @@ public void testE() throws IOException { @Test public void testExpm1() throws IOException { JSONObject result = - executeQuery("select expm1(account_number) FROM " + TEST_INDEX_BANK + " LIMIT 2"); - verifySchema(result, schema("expm1(account_number)", null, "double")); - verifyDataRows(result, rows(Math.expm1(1)), rows(Math.expm1(6))); + executeQuery( + "select account_number, expm1(account_number) FROM " + TEST_INDEX_BANK + " LIMIT 2"); + verifySchema( + result, + schema("account_number", null, "long"), + schema("expm1(account_number)", null, "double")); + JSONArray dataRows = result.getJSONArray("datarows"); + + // Extract and calculate expected values dynamically + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + long accountNumber = row.getLong(0); // Extract the account_number + double actualExpm1Value = row.getDouble(1); // Extract the expm1 value + double expectedExpm1Value = Math.expm1(accountNumber); // Calculate the expected expm1 value + + assertEquals( + expectedExpm1Value, actualExpm1Value, 0.000001); // Delta for floating-point comparison + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 96bbae94e5..4ae683c229 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -16,6 +16,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -448,14 +452,19 @@ public void nested_function_all_subfields() { schema("nested(message.author)", null, "keyword"), schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword")); - verifyDataRows( - result, - rows("e", 1, "a"), - rows("f", 2, "b"), - rows("g", 1, "c"), - rows("h", 4, "c"), - rows("i", 5, "a"), - rows("zz", 6, "zz")); + + // Define expected rows as a list (author, dayOfWeek, info) + List<List<Object>> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a"), + Arrays.asList("f", 2, "b"), + Arrays.asList("g", 1, "c"), + Arrays.asList("h", 4, "c"), + Arrays.asList("i", 5, "a"), + Arrays.asList("zz", 6, "zz")); + + List<List<Object>> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -470,14 +479,19 @@ public void nested_function_all_subfields_and_specified_subfield() { schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword"), schema("nested(comment.data)", null, "keyword")); - verifyDataRows( - result, - rows("e", 1, "a", "ab"), - rows("f", 2, "b", "aa"), - rows("g", 1, "c", "aa"), - rows("h", 4, "c", "ab"), - rows("i", 5, "a", "ab"), - rows("zz", 6, "zz", new JSONArray(List.of("aa", "bb")))); + + // Convert the expected rows to a List<List<Object>> for comparison + List<List<Object>> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", "ab"), + Arrays.asList("f", 2, "b", "aa"), + Arrays.asList("g", 1, "c", "aa"), + Arrays.asList("h", 4, "c", "ab"), + Arrays.asList("i", 5, "a", "ab"), + Arrays.asList("zz", 6, "zz", Arrays.asList("aa", "bb"))); + + List<List<Object>> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -513,14 +527,19 @@ public void nested_function_all_subfields_for_two_nested_fields() { schema("nested(message.info)", null, "keyword"), schema("nested(comment.data)", null, "keyword"), schema("nested(comment.likes)", null, "long")); - verifyDataRows( - result, - rows("e", 1, "a", "ab", 3), - rows("f", 2, "b", "aa", 2), - rows("g", 1, "c", "aa", 3), - rows("h", 4, "c", "ab", 1), - rows("i", 5, "a", "ab", 1), - rows("zz", 6, "zz", new JSONArray(List.of("aa", "bb")), 10)); + + // Define expected rows + List<List<Object>> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", "ab", 3), + Arrays.asList("f", 2, "b", "aa", 2), + Arrays.asList("g", 1, "c", "aa", 3), + Arrays.asList("h", 4, "c", "ab", 1), + Arrays.asList("i", 5, "a", "ab", 1), + Arrays.asList("zz", 6, "zz", Arrays.asList("aa", "bb"), 10)); + + List<List<Object>> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -535,14 +554,18 @@ public void nested_function_all_subfields_and_non_nested_field() { schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword"), schema("myNum", null, "long")); - verifyDataRows( - result, - rows("e", 1, "a", 1), - rows("f", 2, "b", 2), - rows("g", 1, "c", 3), - rows("h", 4, "c", 4), - rows("i", 5, "a", 4), - rows("zz", 6, "zz", new JSONArray(List.of(3, 4)))); + + List<List<Object>> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", 1), + Arrays.asList("f", 2, "b", 2), + Arrays.asList("g", 1, "c", 3), + Arrays.asList("h", 4, "c", 4), + Arrays.asList("i", 5, "a", 4), + Arrays.asList("zz", 6, "zz", Arrays.asList(3, 4))); + + List<List<Object>> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -591,4 +614,83 @@ public void nested_function_all_subfields_in_wrong_clause() { + " \"status\": 500\n" + "}")); } + + // Extract rows based on schema + private List<List<Object>> extractActualRowsBasedOnSchemaOrder(JSONObject result) { + JSONArray dataRows = result.getJSONArray("datarows"); + JSONArray schema = result.getJSONArray("schema"); + + Map<String, Integer> schemaIndexMap = createSchemaIndexMap(schema); + return extractRows(dataRows, schema, schemaIndexMap); + } + + // Create a map of schema names to their indices + private Map<String, Integer> createSchemaIndexMap(JSONArray schema) { + Map<String, Integer> schemaIndexMap = new HashMap<>(); + for (int i = 0; i < schema.length(); i++) { + schemaIndexMap.put(schema.getJSONObject(i).getString("name"), i); + } + return schemaIndexMap; + } + + // Extract rows based on the schema order and expected order + private List<List<Object>> extractRows( + JSONArray dataRows, JSONArray schema, Map<String, Integer> schemaIndexMap) { + // Define the expected order for the first three fields + List<String> expectedOrder = + Arrays.asList( + "nested(message.author)", "nested(message.dayOfWeek)", "nested(message.info)"); + List<List<Object>> actualList = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + List<Object> extractedRow = new ArrayList<>(); + + // Extract fields in the expected order + extractExpectedFields(extractedRow, row, expectedOrder, schemaIndexMap); + + // Add remaining fields in the schema order + addRemainingFields(extractedRow, row, schema, expectedOrder); + + actualList.add(extractedRow); + } + return actualList; + } + + // Extract fields in the expected order + private void extractExpectedFields( + List<Object> extractedRow, + JSONArray row, + List<String> expectedOrder, + Map<String, Integer> schemaIndexMap) { + for (String fieldName : expectedOrder) { + int fieldIndex = schemaIndexMap.get(fieldName); + Object fieldValue = row.get(fieldIndex); + extractedRow.add(fieldValue); + } + } + + // Add remaining fields in the schema order, skipping those in the expected order + private void addRemainingFields( + List<Object> extractedRow, JSONArray row, JSONArray schema, List<String> expectedOrder) { + for (int j = 0; j < schema.length(); j++) { + String fieldName = schema.getJSONObject(j).getString("name"); + if (!expectedOrder.contains(fieldName)) { + Object fieldValue = row.get(j); + // Convert JSONArrays to lists if necessary + if (fieldValue instanceof JSONArray) { + extractedRow.add(((JSONArray) fieldValue).toList()); + } else { + extractedRow.add(fieldValue); + } + } + } + } + + // Sort lists and assert equality + private void sortAndAssertEquals(List<List<Object>> expectedList, List<List<Object>> actualList) { + Comparator<List<Object>> comparator = Comparator.comparing(Object::toString); + expectedList.sort(comparator); + actualList.sort(comparator); + assertEquals(expectedList, actualList); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java index d0a5a37db3..0f085a1cde 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_RAW_SANITIZE; import static org.opensearch.sql.protocol.response.format.RawResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -31,7 +32,8 @@ public void rawFormatWithPipeFieldTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_RAW_SANITIZE), "raw"); - assertEquals( + + assertRowsEqual( StringUtils.format( "firstname|lastname%n" + "+Amber JOHnny|Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java index bce83e7ccb..d8bf9153f3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java @@ -6,8 +6,7 @@ package org.opensearch.sql.util; import static com.google.common.base.Strings.isNullOrEmpty; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.opensearch.sql.executor.pagination.PlanSerializer.CURSOR_PREFIX; import java.io.BufferedReader; @@ -25,12 +24,15 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Assert; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; @@ -166,6 +168,36 @@ private static Response retryWithoutRefreshPolicy(Request request, RestClient cl return client.performRequest(req); } + /** + * Compares two multiline strings representing rows of addresses to ensure they are equivalent. + * This method checks if the entire content of the expected and actual strings are the same. If + * they differ, it breaks down the strings into lines and performs a step-by-step comparison: + * + * @param expected The expected string representing rows of data. + * @param actual The actual string to compare against the expected. + */ + public static void assertRowsEqual(String expected, String actual) { + if (expected.equals(actual)) { + return; + } + + List<String> expectedLines = List.of(expected.split("\n")); + List<String> actualLines = List.of(actual.split("\n")); + + if (expectedLines.size() != actualLines.size()) { + Assert.fail("Line count is different. expected=" + expected + ", actual=" + actual); + } + + if (!expectedLines.get(0).equals(actualLines.get(0))) { + Assert.fail("Header is different. expected=" + expected + ", actual=" + actual); + } + + Set<String> expectedItems = new HashSet<>(expectedLines.subList(1, expectedLines.size())); + Set<String> actualItems = new HashSet<>(actualLines.subList(1, actualLines.size())); + + assertEquals(expectedItems, actualItems); + } + public static String getAccountIndexMapping() { return "{ \"mappings\": {" + " \"properties\": {\n" From 05c961ecaf7936895b40b796babb29b27bcbbdad Mon Sep 17 00:00:00 2001 From: Adi Suresh <adsuresh@amazon.com> Date: Tue, 13 Aug 2024 14:37:43 -0500 Subject: [PATCH 27/96] Add flags for Iceberg and Lake Formation and Security Lake as a data source type. (#2858) Previously, Iceberg catalog was set as the default catalog. This poses problems as the behavior to fall back to default Spark catalog is only correct in some versions of Iceberg. Rather than always opt into Iceberg, Iceberg should be an option. Additionally, the Lake Formation flag enabled Lake Formation for the EMR job. This did not work as expected because EMR system space does not work with Flint. Instead Lake Formation can be enabled using the Iceberg catalog implementation. This changes adds Security Lake as a data source type. Security Lake as a data source is simply specific options set on top of the base S3Glue data source. --------- Signed-off-by: Adi Suresh <adsuresh@amazon.com> --- .../spark/data/constants/SparkConstants.java | 29 ++- .../parameter/SparkSubmitParameters.java | 4 + .../SparkSubmitParametersBuilder.java | 14 +- .../dispatcher/SparkQueryDispatcherTest.java | 59 +---- .../SparkSubmitParametersBuilderTest.java | 2 + ...3GlueDataSourceSparkParameterComposer.java | 84 ++++++- .../config/AsyncExecutorServiceModule.java | 6 +- .../AsyncQueryExecutorServiceSpec.java | 67 ++++- ...eDataSourceSparkParameterComposerTest.java | 230 +++++++++++++++++- .../sql/datasource/model/DataSourceType.java | 3 +- .../glue/GlueDataSourceFactory.java | 17 ++ .../glue/SecurityLakeDataSourceFactory.java | 57 +++++ .../glue/GlueDataSourceFactoryTest.java | 63 +++++ .../glue/SecurityLakeSourceFactoryTest.java | 141 +++++++++++ .../ppl/admin/connectors/s3glue_connector.rst | 7 +- .../connectors/security_lake_connector.rst | 78 ++++++ .../org/opensearch/sql/plugin/SQLPlugin.java | 2 + 17 files changed, 760 insertions(+), 103 deletions(-) create mode 100644 datasources/src/main/java/org/opensearch/sql/datasources/glue/SecurityLakeDataSourceFactory.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/glue/SecurityLakeSourceFactoryTest.java create mode 100644 docs/user/ppl/admin/connectors/security_lake_connector.rst diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index 5b25bc175a..e87dbba03e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -92,18 +92,35 @@ public class SparkConstants { public static final String FLINT_SESSION_CLASS_NAME = "org.apache.spark.sql.FlintREPL"; public static final String SPARK_CATALOG = "spark.sql.catalog.spark_catalog"; + public static final String SPARK_CATALOG_CATALOG_IMPL = SPARK_CATALOG + ".catalog-impl"; + public static final String SPARK_CATALOG_CLIENT_REGION = SPARK_CATALOG + ".client.region"; + public static final String SPARK_CATALOG_CLIENT_FACTORY = SPARK_CATALOG + ".client.factory"; + public static final String SPARK_CATALOG_CLIENT_ASSUME_ROLE_ARN = + SPARK_CATALOG + ".client.assume-role.arn"; + public static final String SPARK_CATALOG_CLIENT_ASSUME_ROLE_REGION = + SPARK_CATALOG + ".client.assume-role.region"; + public static final String SPARK_CATALOG_LF_SESSION_TAG_KEY = + SPARK_CATALOG + ".client.assume-role.tags.LakeFormationAuthorizedCaller"; + public static final String SPARK_CATALOG_GLUE_ACCOUNT_ID = SPARK_CATALOG + ".glue.account-id"; + public static final String SPARK_CATALOG_GLUE_LF_ENABLED = + SPARK_CATALOG + ".glue.lakeformation-enabled"; + public static final String ICEBERG_SESSION_CATALOG = "org.apache.iceberg.spark.SparkSessionCatalog"; public static final String ICEBERG_SPARK_EXTENSION = "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"; - public static final String ICEBERG_SPARK_RUNTIME_PACKAGE = - "/usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar"; - public static final String SPARK_CATALOG_CATALOG_IMPL = - "spark.sql.catalog.spark_catalog.catalog-impl"; + public static final String ICEBERG_SPARK_JARS = + "org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.5.0,software.amazon.awssdk:bundle:2.26.30"; public static final String ICEBERG_GLUE_CATALOG = "org.apache.iceberg.aws.glue.GlueCatalog"; + public static final String ICEBERG_ASSUME_ROLE_CLIENT_FACTORY = + "org.apache.iceberg.aws.AssumeRoleAwsClientFactory"; + public static final String ICEBERG_LF_CLIENT_FACTORY = + "org.apache.iceberg.aws.lakeformation.LakeFormationAwsClientFactory"; + // The following option is needed in Iceberg 1.5 when reading timestamp types that do not + // contain timezone in parquet files. The timezone is assumed to be GMT. + public static final String ICEBERG_TS_WO_TZ = + "spark.sql.iceberg.handle-timestamp-without-timezone"; - public static final String EMR_LAKEFORMATION_OPTION = - "spark.emr-serverless.lakeformation.enabled"; public static final String FLINT_ACCELERATE_USING_COVERING_INDEX = "spark.flint.optimizer.covering.enabled"; } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java index 2e142ed117..84fd49b712 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java @@ -30,6 +30,10 @@ public void deleteConfigItem(String key) { config.remove(key); } + public String getConfigItem(String key) { + return config.get(key); + } + @Override public String toString() { StringBuilder stringBuilder = new StringBuilder(); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java index 3fe7d99373..d9d5859f64 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java @@ -27,20 +27,13 @@ import static org.opensearch.sql.spark.data.constants.SparkConstants.GLUE_HIVE_CATALOG_FACTORY_CLASS; import static org.opensearch.sql.spark.data.constants.SparkConstants.HADOOP_CATALOG_CREDENTIALS_PROVIDER_FACTORY_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.HIVE_METASTORE_CLASS_KEY; -import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_GLUE_CATALOG; -import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SESSION_CATALOG; -import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SPARK_EXTENSION; -import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SPARK_RUNTIME_PACKAGE; import static org.opensearch.sql.spark.data.constants.SparkConstants.JAVA_HOME_LOCATION; import static org.opensearch.sql.spark.data.constants.SparkConstants.PPL_STANDALONE_PACKAGE; import static org.opensearch.sql.spark.data.constants.SparkConstants.S3_AWS_CREDENTIALS_PROVIDER_KEY; -import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG; -import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CATALOG_IMPL; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_DRIVER_ENV_FLINT_CLUSTER_NAME_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_DRIVER_ENV_JAVA_HOME_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_EXECUTOR_ENV_FLINT_CLUSTER_NAME_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_EXECUTOR_ENV_JAVA_HOME_KEY; -import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_JARS_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_JAR_PACKAGES_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_JAR_REPOSITORIES_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_LAUNCHER_PACKAGE; @@ -71,7 +64,6 @@ private void setDefaultConfigs() { setConfigItem( HADOOP_CATALOG_CREDENTIALS_PROVIDER_FACTORY_KEY, DEFAULT_GLUE_CATALOG_CREDENTIALS_PROVIDER_FACTORY_KEY); - setConfigItem(SPARK_JARS_KEY, ICEBERG_SPARK_RUNTIME_PACKAGE); setConfigItem( SPARK_JAR_PACKAGES_KEY, SPARK_STANDALONE_PACKAGE + "," + SPARK_LAUNCHER_PACKAGE + "," + PPL_STANDALONE_PACKAGE); @@ -85,12 +77,8 @@ private void setDefaultConfigs() { setConfigItem(FLINT_INDEX_STORE_SCHEME_KEY, FLINT_DEFAULT_SCHEME); setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, FLINT_DEFAULT_AUTH); setConfigItem(FLINT_CREDENTIALS_PROVIDER_KEY, EMR_ASSUME_ROLE_CREDENTIALS_PROVIDER); - setConfigItem( - SPARK_SQL_EXTENSIONS_KEY, - ICEBERG_SPARK_EXTENSION + "," + FLINT_SQL_EXTENSION + "," + FLINT_PPL_EXTENSION); + setConfigItem(SPARK_SQL_EXTENSIONS_KEY, FLINT_SQL_EXTENSION + "," + FLINT_PPL_EXTENSION); setConfigItem(HIVE_METASTORE_CLASS_KEY, GLUE_HIVE_CATALOG_FACTORY_CLASS); - setConfigItem(SPARK_CATALOG, ICEBERG_SESSION_CATALOG); - setConfigItem(SPARK_CATALOG_CATALOG_IMPL, ICEBERG_GLUE_CATALOG); } private void setConfigItem(String key, String value) { diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 592309cb75..ee840e8b4c 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -203,39 +203,6 @@ void testDispatchSelectQuery() { verifyNoInteractions(flintIndexMetadataService); } - @Test - void testDispatchSelectQueryWithLakeFormation() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "select * from my_glue.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadataWithLakeFormation(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); - } - @Test void testDispatchSelectQueryWithBasicAuthIndexStoreDatasource() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); @@ -1085,7 +1052,6 @@ private String constructExpectedSparkSubmitParameterString(String query, String + getConfParam( "spark.hadoop.fs.s3.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider", "spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory", - "spark.jars=/usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar", "spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT", "spark.jars.repositories=https://aws.oss.sonatype.org/content/repositories/snapshots", "spark.emr-serverless.driverEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/", @@ -1097,10 +1063,8 @@ private String constructExpectedSparkSubmitParameterString(String query, String "spark.datasource.flint.scheme=SCHEMA", "spark.datasource.flint.auth=basic", "spark.datasource.flint.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider", - "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.opensearch.flint.spark.FlintSparkExtensions,org.opensearch.flint.spark.FlintPPLSparkExtensions", - "spark.hadoop.hive.metastore.client.factory.class=com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory", - "spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog", - "spark.sql.catalog.spark_catalog.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog") + "spark.sql.extensions=org.opensearch.flint.spark.FlintSparkExtensions,org.opensearch.flint.spark.FlintPPLSparkExtensions", + "spark.hadoop.hive.metastore.client.factory.class=com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory") + getConfParam("spark.flint.job.query=" + query) + (jobType != null ? getConfParam("spark.flint.job.type=" + jobType) : "") + getConfParam( @@ -1149,25 +1113,6 @@ private DataSourceMetadata constructMyGlueDataSourceMetadataWithBasicAuth() { .build(); } - private DataSourceMetadata constructMyGlueDataSourceMetadataWithLakeFormation() { - - Map<String, String> properties = new HashMap<>(); - properties.put("glue.auth.type", "iam_role"); - properties.put( - "glue.auth.role_arn", "arn:aws:iam::924196221507:role/FlintOpensearchServiceRole"); - properties.put( - "glue.indexstore.opensearch.uri", - "https://search-flint-dp-benchmark-cf5crj5mj2kfzvgwdeynkxnefy.eu-west-1.es.amazonaws.com"); - properties.put("glue.indexstore.opensearch.auth", "awssigv4"); - properties.put("glue.indexstore.opensearch.region", "eu-west-1"); - properties.put("glue.lakeformation.enabled", "true"); - return new DataSourceMetadata.Builder() - .setName(MY_GLUE) - .setConnector(DataSourceType.S3GLUE) - .setProperties(properties) - .build(); - } - private DataSourceMetadata constructPrometheusDataSourceType() { return new DataSourceMetadata.Builder() .setName("my_prometheus") diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java index 8947cb61f7..8fb975d187 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.parameter; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -93,6 +94,7 @@ public void testOverrideConfigItem() { params.setConfigItem(SPARK_JARS_KEY, "Overridden"); String result = params.toString(); + assertEquals("Overridden", params.getConfigItem(SPARK_JARS_KEY)); assertTrue(result.contains(String.format("%s=Overridden", SPARK_JARS_KEY))); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposer.java b/async-query/src/main/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposer.java index 26dbf3529a..189e140416 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposer.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposer.java @@ -5,15 +5,16 @@ package org.opensearch.sql.spark.parameter; +import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_ICEBERG_ENABLED; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_REGION; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_URI; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED; +import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_LAKEFORMATION_SESSION_TAG; import static org.opensearch.sql.datasources.glue.GlueDataSourceFactory.GLUE_ROLE_ARN; import static org.opensearch.sql.spark.data.constants.SparkConstants.DRIVER_ENV_ASSUME_ROLE_ARN_KEY; -import static org.opensearch.sql.spark.data.constants.SparkConstants.EMR_LAKEFORMATION_OPTION; import static org.opensearch.sql.spark.data.constants.SparkConstants.EXECUTOR_ENV_ASSUME_ROLE_ARN_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_ACCELERATE_USING_COVERING_INDEX; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DATA_SOURCE_KEY; @@ -25,19 +26,50 @@ import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_HOST_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_PORT_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_SCHEME_KEY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_PPL_EXTENSION; +import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_SQL_EXTENSION; import static org.opensearch.sql.spark.data.constants.SparkConstants.HIVE_METASTORE_GLUE_ARN_KEY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_ASSUME_ROLE_CLIENT_FACTORY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_GLUE_CATALOG; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_LF_CLIENT_FACTORY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SESSION_CATALOG; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SPARK_EXTENSION; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_SPARK_JARS; +import static org.opensearch.sql.spark.data.constants.SparkConstants.ICEBERG_TS_WO_TZ; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CATALOG_IMPL; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CLIENT_ASSUME_ROLE_ARN; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CLIENT_ASSUME_ROLE_REGION; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CLIENT_FACTORY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_CLIENT_REGION; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_GLUE_ACCOUNT_ID; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_GLUE_LF_ENABLED; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_CATALOG_LF_SESSION_TAG_KEY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_JAR_PACKAGES_KEY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_SQL_EXTENSIONS_KEY; +import com.amazonaws.arn.Arn; import java.net.URI; import java.net.URISyntaxException; +import java.util.Optional; import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasources.auth.AuthenticationType; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.config.SparkExecutionEngineConfigClusterSetting; +import org.opensearch.sql.spark.config.SparkExecutionEngineConfigClusterSettingLoader; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; +@RequiredArgsConstructor public class S3GlueDataSourceSparkParameterComposer implements DataSourceSparkParameterComposer { public static final String FLINT_BASIC_AUTH = "basic"; + public static final String FALSE = "false"; + public static final String TRUE = "true"; + + private final SparkExecutionEngineConfigClusterSettingLoader settingLoader; @Override public void compose( @@ -45,7 +77,16 @@ public void compose( SparkSubmitParameters params, DispatchQueryRequest dispatchQueryRequest, AsyncQueryRequestContext context) { - String roleArn = metadata.getProperties().get(GLUE_ROLE_ARN); + final Optional<SparkExecutionEngineConfigClusterSetting> maybeClusterSettings = + settingLoader.load(); + if (!maybeClusterSettings.isPresent()) { + throw new RuntimeException("No cluster settings present"); + } + final SparkExecutionEngineConfigClusterSetting clusterSetting = maybeClusterSettings.get(); + final String region = clusterSetting.getRegion(); + + final String roleArn = metadata.getProperties().get(GLUE_ROLE_ARN); + final String accountId = Arn.fromString(roleArn).getAccountId(); params.setConfigItem(DRIVER_ENV_ASSUME_ROLE_ARN_KEY, roleArn); params.setConfigItem(EXECUTOR_ENV_ASSUME_ROLE_ARN_KEY, roleArn); @@ -53,11 +94,40 @@ public void compose( params.setConfigItem("spark.sql.catalog." + metadata.getName(), FLINT_DELEGATE_CATALOG); params.setConfigItem(FLINT_DATA_SOURCE_KEY, metadata.getName()); - final boolean lakeFormationEnabled = - BooleanUtils.toBoolean(metadata.getProperties().get(GLUE_LAKEFORMATION_ENABLED)); - params.setConfigItem(EMR_LAKEFORMATION_OPTION, Boolean.toString(lakeFormationEnabled)); - params.setConfigItem( - FLINT_ACCELERATE_USING_COVERING_INDEX, Boolean.toString(!lakeFormationEnabled)); + final boolean icebergEnabled = + BooleanUtils.toBoolean(metadata.getProperties().get(GLUE_ICEBERG_ENABLED)); + if (icebergEnabled) { + params.setConfigItem( + SPARK_JAR_PACKAGES_KEY, + params.getConfigItem(SPARK_JAR_PACKAGES_KEY) + "," + ICEBERG_SPARK_JARS); + params.setConfigItem(SPARK_CATALOG, ICEBERG_SESSION_CATALOG); + params.setConfigItem(SPARK_CATALOG_CATALOG_IMPL, ICEBERG_GLUE_CATALOG); + params.setConfigItem( + SPARK_SQL_EXTENSIONS_KEY, + ICEBERG_SPARK_EXTENSION + "," + FLINT_SQL_EXTENSION + "," + FLINT_PPL_EXTENSION); + + params.setConfigItem(SPARK_CATALOG_CLIENT_REGION, region); + params.setConfigItem(SPARK_CATALOG_GLUE_ACCOUNT_ID, accountId); + params.setConfigItem(SPARK_CATALOG_CLIENT_ASSUME_ROLE_ARN, roleArn); + params.setConfigItem(SPARK_CATALOG_CLIENT_ASSUME_ROLE_REGION, region); + params.setConfigItem(ICEBERG_TS_WO_TZ, TRUE); + + final boolean lakeFormationEnabled = + BooleanUtils.toBoolean(metadata.getProperties().get(GLUE_LAKEFORMATION_ENABLED)); + if (lakeFormationEnabled) { + final String sessionTag = metadata.getProperties().get(GLUE_LAKEFORMATION_SESSION_TAG); + if (StringUtils.isBlank(sessionTag)) { + throw new IllegalArgumentException(GLUE_LAKEFORMATION_SESSION_TAG + " is required"); + } + + params.setConfigItem(FLINT_ACCELERATE_USING_COVERING_INDEX, FALSE); + params.setConfigItem(SPARK_CATALOG_GLUE_LF_ENABLED, TRUE); + params.setConfigItem(SPARK_CATALOG_CLIENT_FACTORY, ICEBERG_LF_CLIENT_FACTORY); + params.setConfigItem(SPARK_CATALOG_LF_SESSION_TAG_KEY, sessionTag); + } else { + params.setConfigItem(SPARK_CATALOG_CLIENT_FACTORY, ICEBERG_ASSUME_ROLE_CLIENT_FACTORY); + } + } setFlintIndexStoreHost( params, diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index 05f7d1095c..9cc69b2fb7 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -159,7 +159,11 @@ public FlintIndexStateModelService flintIndexStateModelService( public SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider( Settings settings, SparkExecutionEngineConfigClusterSettingLoader clusterSettingLoader) { SparkParameterComposerCollection collection = new SparkParameterComposerCollection(); - collection.register(DataSourceType.S3GLUE, new S3GlueDataSourceSparkParameterComposer()); + collection.register( + DataSourceType.S3GLUE, new S3GlueDataSourceSparkParameterComposer(clusterSettingLoader)); + collection.register( + DataSourceType.SECURITY_LAKE, + new S3GlueDataSourceSparkParameterComposer(clusterSettingLoader)); collection.register(new OpenSearchExtraParameterComposer(clusterSettingLoader)); return new SparkSubmitParametersBuilderProvider(collection); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index d8e3b80175..641b083d53 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -18,12 +18,17 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.Resources; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -42,6 +47,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.plugins.Plugin; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.datasources.encryptor.EncryptorImpl; @@ -58,6 +64,7 @@ import org.opensearch.sql.spark.client.StartJobRequest; import org.opensearch.sql.spark.config.OpenSearchSparkSubmitParameterModifier; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; +import org.opensearch.sql.spark.config.SparkExecutionEngineConfigClusterSettingLoader; import org.opensearch.sql.spark.dispatcher.DatasourceEmbeddedQueryIdProvider; import org.opensearch.sql.spark.dispatcher.QueryHandlerFactory; import org.opensearch.sql.spark.dispatcher.SparkQueryDispatcher; @@ -100,6 +107,10 @@ public class AsyncQueryExecutorServiceSpec extends OpenSearchIntegTestCase { public static final String MYS3_DATASOURCE = "mys3"; public static final String MYGLUE_DATASOURCE = "my_glue"; + public static final String ACCOUNT_ID = "accountId"; + public static final String APPLICATION_ID = "appId"; + public static final String REGION = "us-west-2"; + public static final String ROLE_ARN = "roleArn"; protected ClusterService clusterService; protected org.opensearch.sql.common.setting.Settings pluginSettings; @@ -262,7 +273,13 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( SparkParameterComposerCollection sparkParameterComposerCollection = new SparkParameterComposerCollection(); sparkParameterComposerCollection.register( - DataSourceType.S3GLUE, new S3GlueDataSourceSparkParameterComposer()); + DataSourceType.S3GLUE, + new S3GlueDataSourceSparkParameterComposer( + getSparkExecutionEngineConfigClusterSettingLoader())); + sparkParameterComposerCollection.register( + DataSourceType.SECURITY_LAKE, + new S3GlueDataSourceSparkParameterComposer( + getSparkExecutionEngineConfigClusterSettingLoader())); SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider = new SparkSubmitParametersBuilderProvider(sparkParameterComposerCollection); QueryHandlerFactory queryHandlerFactory = @@ -372,14 +389,56 @@ public EMRServerlessClient getClient(String accountId) { public SparkExecutionEngineConfig sparkExecutionEngineConfig( AsyncQueryRequestContext asyncQueryRequestContext) { return SparkExecutionEngineConfig.builder() - .applicationId("appId") - .region("us-west-2") - .executionRoleARN("roleArn") + .applicationId(APPLICATION_ID) + .region(REGION) + .executionRoleARN(ROLE_ARN) .sparkSubmitParameterModifier(new OpenSearchSparkSubmitParameterModifier("")) .clusterName("myCluster") .build(); } + public static class TestSettings extends org.opensearch.sql.common.setting.Settings { + final Map<Key, Object> values; + + public TestSettings() { + values = new HashMap<>(); + } + + /** Get Setting Value. */ + @Override + public <T> T getSettingValue(Key key) { + return (T) values.get(key); + } + + @Override + public List<String> getSettings() { + return values.keySet().stream().map(Key::getKeyValue).collect(Collectors.toList()); + } + + public <T> void putSettingValue(Key key, T value) { + values.put(key, value); + } + } + + public SparkExecutionEngineConfigClusterSettingLoader + getSparkExecutionEngineConfigClusterSettingLoader() { + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("accountId", ACCOUNT_ID); + jsonObject.addProperty("applicationId", APPLICATION_ID); + jsonObject.addProperty("region", REGION); + jsonObject.addProperty("executionRoleARN", ROLE_ARN); + jsonObject.addProperty("sparkSubmitParameters", ""); + + // Convert JsonObject to JSON string + final String jsonString = gson.toJson(jsonObject); + + final TestSettings settings = new TestSettings(); + settings.putSettingValue(Key.SPARK_EXECUTION_ENGINE_CONFIG, jsonString); + + return new SparkExecutionEngineConfigClusterSettingLoader(settings); + } + public void enableSession(boolean enabled) { // doNothing } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposerTest.java index 55e62d52f0..3e12aa78d0 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposerTest.java @@ -7,21 +7,28 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_JAR_PACKAGES_KEY; import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceStatus; import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.datasources.auth.AuthenticationType; import org.opensearch.sql.datasources.glue.GlueDataSourceFactory; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; +import org.opensearch.sql.spark.config.SparkExecutionEngineConfigClusterSettingLoader; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @ExtendWith(MockitoExtension.class) @@ -33,18 +40,20 @@ class S3GlueDataSourceSparkParameterComposerTest { public static final String PASSWORD = "PASSWORD"; public static final String REGION = "REGION"; public static final String TRUE = "true"; - public static final String ROLE_ARN = "ROLE_ARN"; + public static final String ROLE_ARN = "arn:aws:iam::123456789012:role/ROLE_NAME"; + public static final String APP_ID = "APP_ID"; + public static final String CLUSTER_NAME = "CLUSTER_NAME"; + public static final String ACCOUNT_ID = "123456789012"; + public static final String SESSION_TAG = "SESSION_TAG"; private static final String COMMON_EXPECTED_PARAMS = " --class org.apache.spark.sql.FlintJob " + getConfList( - "spark.emr-serverless.driverEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=ROLE_ARN", - "spark.executorEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=ROLE_ARN", - "spark.hive.metastore.glue.role.arn=ROLE_ARN", + "spark.emr-serverless.driverEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.executorEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.hive.metastore.glue.role.arn=arn:aws:iam::123456789012:role/ROLE_NAME", "spark.sql.catalog.DATASOURCE_NAME=org.opensearch.sql.FlintDelegatingSessionCatalog", "spark.flint.datasource.name=DATASOURCE_NAME", - "spark.emr-serverless.lakeformation.enabled=true", - "spark.flint.optimizer.covering.enabled=false", "spark.datasource.flint.host=test.host.com", "spark.datasource.flint.port=9200", "spark.datasource.flint.scheme=https"); @@ -57,7 +66,7 @@ public void testBasicAuth() { getDataSourceMetadata(AuthenticationType.BASICAUTH, VALID_URI); SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); - new S3GlueDataSourceSparkParameterComposer() + new S3GlueDataSourceSparkParameterComposer(getSparkExecutionEngineConfigClusterSettingLoader()) .compose( dataSourceMetadata, sparkSubmitParameters, @@ -79,7 +88,7 @@ public void testComposeWithSigV4Auth() { getDataSourceMetadata(AuthenticationType.AWSSIGV4AUTH, VALID_URI); SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); - new S3GlueDataSourceSparkParameterComposer() + new S3GlueDataSourceSparkParameterComposer(getSparkExecutionEngineConfigClusterSettingLoader()) .compose( dataSourceMetadata, sparkSubmitParameters, @@ -99,7 +108,7 @@ public void testComposeWithNoAuth() { getDataSourceMetadata(AuthenticationType.NOAUTH, VALID_URI); SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); - new S3GlueDataSourceSparkParameterComposer() + new S3GlueDataSourceSparkParameterComposer(getSparkExecutionEngineConfigClusterSettingLoader()) .compose( dataSourceMetadata, sparkSubmitParameters, @@ -120,7 +129,8 @@ public void testComposeWithBadUri() { assertThrows( IllegalArgumentException.class, () -> - new S3GlueDataSourceSparkParameterComposer() + new S3GlueDataSourceSparkParameterComposer( + getSparkExecutionEngineConfigClusterSettingLoader()) .compose( dataSourceMetadata, sparkSubmitParameters, @@ -128,6 +138,174 @@ public void testComposeWithBadUri() { new NullAsyncQueryRequestContext())); } + @Test + public void testIcebergEnabled() { + final Map<String, String> properties = + ImmutableMap.<String, String>builder() + .put(GlueDataSourceFactory.GLUE_ROLE_ARN, ROLE_ARN) + .put(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED, TRUE) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_URI, VALID_URI) + .put( + GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH, + AuthenticationType.BASICAUTH.getName()) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME, USERNAME) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD, PASSWORD) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_REGION, REGION) + .build(); + + final String expectedParams = + " --class org.apache.spark.sql.FlintJob " + + getConfList( + "spark.jars.packages=package,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.5.0,software.amazon.awssdk:bundle:2.26.30", + "spark.emr-serverless.driverEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.executorEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.hive.metastore.glue.role.arn=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.sql.catalog.DATASOURCE_NAME=org.opensearch.sql.FlintDelegatingSessionCatalog", + "spark.flint.datasource.name=DATASOURCE_NAME", + "spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog", + "spark.sql.catalog.spark_catalog.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog", + "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.opensearch.flint.spark.FlintSparkExtensions,org.opensearch.flint.spark.FlintPPLSparkExtensions", + "spark.sql.catalog.spark_catalog.client.region=REGION", + "spark.sql.catalog.spark_catalog.glue.account-id=123456789012", + "spark.sql.catalog.spark_catalog.client.assume-role.arn=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.sql.catalog.spark_catalog.client.assume-role.region=REGION", + "spark.sql.iceberg.handle-timestamp-without-timezone=true", + "spark.sql.catalog.spark_catalog.client.factory=org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + "spark.datasource.flint.host=test.host.com", + "spark.datasource.flint.port=9200", + "spark.datasource.flint.scheme=https", + "spark.datasource.flint.auth=basic", + "spark.datasource.flint.auth.username=USERNAME", + "spark.datasource.flint.auth.password=PASSWORD"); + + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(properties); + SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); + sparkSubmitParameters.setConfigItem(SPARK_JAR_PACKAGES_KEY, "package"); + + new S3GlueDataSourceSparkParameterComposer(getSparkExecutionEngineConfigClusterSettingLoader()) + .compose( + dataSourceMetadata, + sparkSubmitParameters, + dispatchQueryRequest, + new NullAsyncQueryRequestContext()); + + assertEquals(expectedParams, sparkSubmitParameters.toString()); + } + + @Test + public void testIcebergWithLakeFormationEnabled() { + final Map<String, String> properties = + ImmutableMap.<String, String>builder() + .put(GlueDataSourceFactory.GLUE_ROLE_ARN, ROLE_ARN) + .put(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED, TRUE) + .put(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED, TRUE) + .put(GlueDataSourceFactory.GLUE_LAKEFORMATION_SESSION_TAG, SESSION_TAG) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_URI, VALID_URI) + .put( + GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH, + AuthenticationType.BASICAUTH.getName()) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME, USERNAME) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD, PASSWORD) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_REGION, REGION) + .build(); + + final String expectedParams = + " --class org.apache.spark.sql.FlintJob " + + getConfList( + "spark.jars.packages=package,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.5.0,software.amazon.awssdk:bundle:2.26.30", + "spark.emr-serverless.driverEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.executorEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.hive.metastore.glue.role.arn=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.sql.catalog.DATASOURCE_NAME=org.opensearch.sql.FlintDelegatingSessionCatalog", + "spark.flint.datasource.name=DATASOURCE_NAME", + "spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog", + "spark.sql.catalog.spark_catalog.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog", + "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.opensearch.flint.spark.FlintSparkExtensions,org.opensearch.flint.spark.FlintPPLSparkExtensions", + "spark.sql.catalog.spark_catalog.client.region=REGION", + "spark.sql.catalog.spark_catalog.glue.account-id=123456789012", + "spark.sql.catalog.spark_catalog.client.assume-role.arn=arn:aws:iam::123456789012:role/ROLE_NAME", + "spark.sql.catalog.spark_catalog.client.assume-role.region=REGION", + "spark.sql.iceberg.handle-timestamp-without-timezone=true", + "spark.flint.optimizer.covering.enabled=false", + "spark.sql.catalog.spark_catalog.glue.lakeformation-enabled=true", + "spark.sql.catalog.spark_catalog.client.factory=org.apache.iceberg.aws.lakeformation.LakeFormationAwsClientFactory", + "spark.sql.catalog.spark_catalog.client.assume-role.tags.LakeFormationAuthorizedCaller=SESSION_TAG", + "spark.datasource.flint.host=test.host.com", + "spark.datasource.flint.port=9200", + "spark.datasource.flint.scheme=https", + "spark.datasource.flint.auth=basic", + "spark.datasource.flint.auth.username=USERNAME", + "spark.datasource.flint.auth.password=PASSWORD"); + + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(properties); + SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); + sparkSubmitParameters.setConfigItem(SPARK_JAR_PACKAGES_KEY, "package"); + + new S3GlueDataSourceSparkParameterComposer(getSparkExecutionEngineConfigClusterSettingLoader()) + .compose( + dataSourceMetadata, + sparkSubmitParameters, + dispatchQueryRequest, + new NullAsyncQueryRequestContext()); + + assertEquals(expectedParams, sparkSubmitParameters.toString()); + } + + @Test + public void testIcebergWithLakeFormationEnabledNoSessionTag() { + final Map<String, String> properties = + ImmutableMap.<String, String>builder() + .put(GlueDataSourceFactory.GLUE_ROLE_ARN, ROLE_ARN) + .put(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED, TRUE) + .put(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED, TRUE) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_URI, VALID_URI) + .put( + GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH, + AuthenticationType.BASICAUTH.getName()) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME, USERNAME) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD, PASSWORD) + .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_REGION, REGION) + .build(); + + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(properties); + SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); + + final S3GlueDataSourceSparkParameterComposer composer = + new S3GlueDataSourceSparkParameterComposer( + getSparkExecutionEngineConfigClusterSettingLoader()); + assertThrows( + IllegalArgumentException.class, + () -> + composer.compose( + dataSourceMetadata, + sparkSubmitParameters, + dispatchQueryRequest, + new NullAsyncQueryRequestContext())); + } + + @Test + public void testNoClusterConfigAvailable() { + DataSourceMetadata dataSourceMetadata = + getDataSourceMetadata(AuthenticationType.BASICAUTH, VALID_URI); + SparkSubmitParameters sparkSubmitParameters = new SparkSubmitParameters(); + + final OpenSearchSettings settings = Mockito.mock(OpenSearchSettings.class); + Mockito.when(settings.getSettingValue(Key.SPARK_EXECUTION_ENGINE_CONFIG)).thenReturn(null); + + final S3GlueDataSourceSparkParameterComposer composer = + new S3GlueDataSourceSparkParameterComposer( + new SparkExecutionEngineConfigClusterSettingLoader(settings)); + + assertThrows( + RuntimeException.class, + () -> + composer.compose( + dataSourceMetadata, + sparkSubmitParameters, + dispatchQueryRequest, + new NullAsyncQueryRequestContext())); + } + private DataSourceMetadata getDataSourceMetadata( AuthenticationType authenticationType, String uri) { return new DataSourceMetadata.Builder() @@ -140,10 +318,20 @@ private DataSourceMetadata getDataSourceMetadata( .build(); } + private DataSourceMetadata getDataSourceMetadata(Map<String, String> properties) { + return new DataSourceMetadata.Builder() + .setConnector(DataSourceType.S3GLUE) + .setName("DATASOURCE_NAME") + .setDescription("DESCRIPTION") + .setResultIndex("RESULT_INDEX") + .setDataSourceStatus(DataSourceStatus.ACTIVE) + .setProperties(properties) + .build(); + } + private Map<String, String> getProperties(AuthenticationType authType, String uri) { return ImmutableMap.<String, String>builder() .put(GlueDataSourceFactory.GLUE_ROLE_ARN, ROLE_ARN) - .put(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED, TRUE) .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_URI, uri) .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH, authType.getName()) .put(GlueDataSourceFactory.GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME, USERNAME) @@ -152,6 +340,26 @@ private Map<String, String> getProperties(AuthenticationType authType, String ur .build(); } + private SparkExecutionEngineConfigClusterSettingLoader + getSparkExecutionEngineConfigClusterSettingLoader() { + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("accountId", ACCOUNT_ID); + jsonObject.addProperty("applicationId", APP_ID); + jsonObject.addProperty("region", REGION); + jsonObject.addProperty("executionRoleARN", ROLE_ARN); + jsonObject.addProperty("sparkSubmitParameters", ""); + + // Convert JsonObject to JSON string + final String jsonString = gson.toJson(jsonObject); + + final OpenSearchSettings settings = Mockito.mock(OpenSearchSettings.class); + Mockito.when(settings.getSettingValue(Key.SPARK_EXECUTION_ENGINE_CONFIG)) + .thenReturn(jsonString); + + return new SparkExecutionEngineConfigClusterSettingLoader(settings); + } + private static String getConfList(String... params) { return Arrays.stream(params) .map(param -> String.format(" --conf %s ", param)) diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java index c727c3c531..c74964fc00 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java @@ -15,12 +15,13 @@ public class DataSourceType { public static DataSourceType OPENSEARCH = new DataSourceType("OPENSEARCH"); public static DataSourceType SPARK = new DataSourceType("SPARK"); public static DataSourceType S3GLUE = new DataSourceType("S3GLUE"); + public static DataSourceType SECURITY_LAKE = new DataSourceType("SECURITY_LAKE"); // Map from uppercase DataSourceType name to DataSourceType object private static Map<String, DataSourceType> knownValues = new HashMap<>(); static { - register(PROMETHEUS, OPENSEARCH, SPARK, S3GLUE); + register(PROMETHEUS, OPENSEARCH, SPARK, S3GLUE, SECURITY_LAKE); } private final String name; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java b/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java index e0c13ff005..11a33a2969 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java @@ -5,6 +5,8 @@ import java.util.Map; import java.util.Set; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; @@ -29,7 +31,9 @@ public class GlueDataSourceFactory implements DataSourceFactory { "glue.indexstore.opensearch.auth.password"; public static final String GLUE_INDEX_STORE_OPENSEARCH_REGION = "glue.indexstore.opensearch.region"; + public static final String GLUE_ICEBERG_ENABLED = "glue.iceberg.enabled"; public static final String GLUE_LAKEFORMATION_ENABLED = "glue.lakeformation.enabled"; + public static final String GLUE_LAKEFORMATION_SESSION_TAG = "glue.lakeformation.session_tag"; @Override public DataSourceType getDataSourceType() { @@ -76,5 +80,18 @@ private void validateGlueDataSourceConfiguration(Map<String, String> dataSourceM DatasourceValidationUtils.validateHost( dataSourceMetadataConfig.get(GLUE_INDEX_STORE_OPENSEARCH_URI), pluginSettings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST)); + + // validate Lake Formation config + if (BooleanUtils.toBoolean(dataSourceMetadataConfig.get(GLUE_LAKEFORMATION_ENABLED))) { + if (!BooleanUtils.toBoolean(dataSourceMetadataConfig.get(GLUE_ICEBERG_ENABLED))) { + throw new IllegalArgumentException( + "Lake Formation can only be enabled when Iceberg is enabled."); + } + + if (StringUtils.isBlank(dataSourceMetadataConfig.get(GLUE_LAKEFORMATION_SESSION_TAG))) { + throw new IllegalArgumentException( + "Lake Formation session tag must be specified when enabling Lake Formation"); + } + } } } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/glue/SecurityLakeDataSourceFactory.java b/datasources/src/main/java/org/opensearch/sql/datasources/glue/SecurityLakeDataSourceFactory.java new file mode 100644 index 0000000000..0f336a08d1 --- /dev/null +++ b/datasources/src/main/java/org/opensearch/sql/datasources/glue/SecurityLakeDataSourceFactory.java @@ -0,0 +1,57 @@ +package org.opensearch.sql.datasources.glue; + +import java.util.Map; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; + +public class SecurityLakeDataSourceFactory extends GlueDataSourceFactory { + + private final Settings pluginSettings; + + public static final String TRUE = "true"; + + public SecurityLakeDataSourceFactory(final Settings pluginSettings) { + super(pluginSettings); + this.pluginSettings = pluginSettings; + } + + @Override + public DataSourceType getDataSourceType() { + return DataSourceType.SECURITY_LAKE; + } + + @Override + public DataSource createDataSource(DataSourceMetadata metadata) { + validateProperties(metadata.getProperties()); + metadata.getProperties().put(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED, TRUE); + metadata.getProperties().put(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED, TRUE); + return super.createDataSource(metadata); + } + + private void validateProperties(Map<String, String> properties) { + // validate Lake Formation config + if (properties.get(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED) != null + && !BooleanUtils.toBoolean(properties.get(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED))) { + throw new IllegalArgumentException( + GlueDataSourceFactory.GLUE_ICEBERG_ENABLED + + " cannot be false when using Security Lake data source."); + } + + if (properties.get(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED) != null + && !BooleanUtils.toBoolean( + properties.get(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED))) { + throw new IllegalArgumentException( + GLUE_LAKEFORMATION_ENABLED + " cannot be false when using Security Lake data source."); + } + + if (StringUtils.isBlank(properties.get(GLUE_LAKEFORMATION_SESSION_TAG))) { + throw new IllegalArgumentException( + GlueDataSourceFactory.GLUE_LAKEFORMATION_SESSION_TAG + + " must be specified when using Security Lake data source"); + } + } +} diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactoryTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactoryTest.java index 52f8ec9cd1..2833717265 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactoryTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactoryTest.java @@ -210,4 +210,67 @@ void testCreateGLueDatSourceWithInvalidFlintHostSyntax() { Assertions.assertEquals( "Invalid flint host in properties.", illegalArgumentException.getMessage()); } + + @Test + @SneakyThrows + void testCreateGlueDataSourceWithLakeFormationNoIceberg() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST)) + .thenReturn(Collections.emptyList()); + GlueDataSourceFactory glueDatasourceFactory = new GlueDataSourceFactory(settings); + + HashMap<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.lakeformation.enabled", "true"); + properties.put("glue.iceberg.enabled", "false"); + properties.put("glue.lakeformation.session_tag", "session_tag"); + + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_glue") + .setConnector(DataSourceType.S3GLUE) + .setProperties(properties) + .build(); + + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows( + IllegalArgumentException.class, () -> glueDatasourceFactory.createDataSource(metadata)); + Assertions.assertEquals( + "Lake Formation can only be enabled when Iceberg is enabled.", + illegalArgumentException.getMessage()); + } + + @Test + @SneakyThrows + void testCreateGlueDataSourceWithLakeFormationNoSessionTags() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST)) + .thenReturn(Collections.emptyList()); + GlueDataSourceFactory glueDatasourceFactory = new GlueDataSourceFactory(settings); + + HashMap<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.lakeformation.enabled", "true"); + properties.put("glue.iceberg.enabled", "true"); + + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_glue") + .setConnector(DataSourceType.S3GLUE) + .setProperties(properties) + .build(); + + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows( + IllegalArgumentException.class, () -> glueDatasourceFactory.createDataSource(metadata)); + Assertions.assertEquals( + "Lake Formation session tag must be specified when enabling Lake Formation", + illegalArgumentException.getMessage()); + } } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/glue/SecurityLakeSourceFactoryTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/glue/SecurityLakeSourceFactoryTest.java new file mode 100644 index 0000000000..561d549826 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/glue/SecurityLakeSourceFactoryTest.java @@ -0,0 +1,141 @@ +package org.opensearch.sql.datasources.glue; + +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; + +@ExtendWith(MockitoExtension.class) +public class SecurityLakeSourceFactoryTest { + + @Mock private Settings settings; + + @Test + void testGetConnectorType() { + SecurityLakeDataSourceFactory securityLakeDataSourceFactory = + new SecurityLakeDataSourceFactory(settings); + Assertions.assertEquals( + DataSourceType.SECURITY_LAKE, securityLakeDataSourceFactory.getDataSourceType()); + } + + @Test + @SneakyThrows + void testCreateSecurityLakeDataSource() { + when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST)) + .thenReturn(Collections.emptyList()); + SecurityLakeDataSourceFactory securityLakeDataSourceFactory = + new SecurityLakeDataSourceFactory(settings); + + Map<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.lakeformation.session_tag", "session_tag"); + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_sl") + .setConnector(DataSourceType.SECURITY_LAKE) + .setProperties(properties) + .build(); + DataSource dataSource = securityLakeDataSourceFactory.createDataSource(metadata); + Assertions.assertEquals(DataSourceType.SECURITY_LAKE, dataSource.getConnectorType()); + + Assertions.assertEquals( + properties.get(GlueDataSourceFactory.GLUE_ICEBERG_ENABLED), + SecurityLakeDataSourceFactory.TRUE); + Assertions.assertEquals( + properties.get(GlueDataSourceFactory.GLUE_LAKEFORMATION_ENABLED), + SecurityLakeDataSourceFactory.TRUE); + } + + @Test + @SneakyThrows + void testCreateSecurityLakeDataSourceIcebergCannotBeDisabled() { + SecurityLakeDataSourceFactory securityLakeDataSourceFactory = + new SecurityLakeDataSourceFactory(settings); + + Map<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.iceberg.enabled", "false"); + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_sl") + .setConnector(DataSourceType.SECURITY_LAKE) + .setProperties(properties) + .build(); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> securityLakeDataSourceFactory.createDataSource(metadata)); + } + + @Test + @SneakyThrows + void testCreateSecurityLakeDataSourceLakeFormationCannotBeDisabled() { + SecurityLakeDataSourceFactory securityLakeDataSourceFactory = + new SecurityLakeDataSourceFactory(settings); + + Map<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.iceberg.enabled", "true"); + properties.put("glue.lakeformation.enabled", "false"); + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_sl") + .setConnector(DataSourceType.SECURITY_LAKE) + .setProperties(properties) + .build(); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> securityLakeDataSourceFactory.createDataSource(metadata)); + } + + @Test + @SneakyThrows + void testCreateGlueDataSourceWithLakeFormationNoSessionTags() { + SecurityLakeDataSourceFactory securityLakeDataSourceFactory = + new SecurityLakeDataSourceFactory(settings); + + HashMap<String, String> properties = new HashMap<>(); + properties.put("glue.auth.type", "iam_role"); + properties.put("glue.auth.role_arn", "role_arn"); + properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200"); + properties.put("glue.indexstore.opensearch.auth", "noauth"); + properties.put("glue.indexstore.opensearch.region", "us-west-2"); + properties.put("glue.iceberg.enabled", "true"); + properties.put("glue.lakeformation.enabled", "true"); + + DataSourceMetadata metadata = + new DataSourceMetadata.Builder() + .setName("my_sl") + .setConnector(DataSourceType.SECURITY_LAKE) + .setProperties(properties) + .build(); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> securityLakeDataSourceFactory.createDataSource(metadata)); + } +} diff --git a/docs/user/ppl/admin/connectors/s3glue_connector.rst b/docs/user/ppl/admin/connectors/s3glue_connector.rst index 5e91df70e5..48f19a9d1e 100644 --- a/docs/user/ppl/admin/connectors/s3glue_connector.rst +++ b/docs/user/ppl/admin/connectors/s3glue_connector.rst @@ -42,7 +42,9 @@ Glue Connector Properties. * Basic Auth required ``glue.indexstore.opensearch.auth.username`` and ``glue.indexstore.opensearch.auth.password`` * AWSSigV4 Auth requires ``glue.indexstore.opensearch.auth.region`` and ``glue.auth.role_arn`` * ``glue.indexstore.opensearch.region`` [Required for awssigv4 auth] -* ``glue.lakeformation.enabled`` determines whether to enable lakeformation for queries. Default value is ``"false"`` if not specified +* ``glue.iceberg.enabled`` determines whether to enable Iceberg for the session. Default value is ``"false"`` if not specified. +* ``glue.lakeformation.enabled`` determines whether to enable Lake Formation for queries when Iceberg is also enabled. If Iceberg is not enabled, then this property has no effect. Default value is ``"false"`` if not specified. +* ``glue.lakeformation.session_tag`` what session tag to use when assuming the data source role. This property is required when both Iceberg and Lake Formation are enabled. Sample Glue dataSource configuration ======================================== @@ -71,8 +73,7 @@ Glue datasource configuration:: "glue.auth.role_arn": "role_arn", "glue.indexstore.opensearch.uri": "http://adsasdf.amazonopensearch.com:9200", "glue.indexstore.opensearch.auth" :"awssigv4", - "glue.indexstore.opensearch.auth.region" :"awssigv4", - "glue.lakeformation.enabled": "true" + "glue.indexstore.opensearch.auth.region" :"us-east-1" }, "resultIndex": "query_execution_result" }] diff --git a/docs/user/ppl/admin/connectors/security_lake_connector.rst b/docs/user/ppl/admin/connectors/security_lake_connector.rst new file mode 100644 index 0000000000..6afddca131 --- /dev/null +++ b/docs/user/ppl/admin/connectors/security_lake_connector.rst @@ -0,0 +1,78 @@ +.. highlight:: sh + +==================== +Security Lake Connector +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + + +Introduction +============ + +Security Lake connector provides a way to query Security Lake tables. + +Required resources for Security Lake Connector +======================================== +* ``EMRServerless Spark Execution Engine Config Setting``: Since we execute s3Glue queries on top of spark execution engine, we require this configuration. + More details: `ExecutionEngine Config <../../../interfaces/asyncqueryinterface.rst#id2>`_ +* ``S3``: This is where the data lies. +* ``Glue``: Metadata store: Glue takes care of table metadata. +* ``Lake Formation``: AWS service that performs authorization on Security Lake tables +* ``Security Lake``: AWS service that orchestrates creation of S3 files, Glue tables, and Lake Formation permissions. +* ``Opensearch IndexStore``: Index for s3 data lies in opensearch and also acts as temporary buffer for query results. + +We currently only support emr-serverless as spark execution engine and Glue as metadata store. we will add more support in future. + +Glue Connector Properties. + +* ``resultIndex`` is a new parameter specific to glue connector. Stores the results of queries executed on the data source. If unavailable, it defaults to .query_execution_result. +* ``glue.auth.type`` [Required] + * This parameters provides the authentication type information required for execution engine to connect to glue. + * S3 Glue connector currently only supports ``iam_role`` authentication and the below parameters is required. + * ``glue.auth.role_arn`` +* ``glue.indexstore.opensearch.*`` [Required] + * This parameters provides the Opensearch domain host information for glue connector. This opensearch instance is used for writing index data back and also + * ``glue.indexstore.opensearch.uri`` [Required] + * ``glue.indexstore.opensearch.auth`` [Required] + * Accepted values include ["noauth", "basicauth", "awssigv4"] + * Basic Auth required ``glue.indexstore.opensearch.auth.username`` and ``glue.indexstore.opensearch.auth.password`` + * AWSSigV4 Auth requires ``glue.indexstore.opensearch.auth.region`` and ``glue.auth.role_arn`` + * ``glue.indexstore.opensearch.region`` [Required for awssigv4 auth] +* ``glue.lakeformation.session_tag`` [Required] + * What session tag to use when assuming the data source role. + +Sample Glue dataSource configuration +======================================== + +Glue datasource configuration:: + + [{ + "name" : "my_sl", + "connector": "security_lake", + "properties" : { + "glue.auth.type": "iam_role", + "glue.auth.role_arn": "role_arn", + "glue.indexstore.opensearch.uri": "http://adsasdf.amazonopensearch.com:9200", + "glue.indexstore.opensearch.auth" :"awssigv4", + "glue.indexstore.opensearch.auth.region" :"us-east-1", + "glue.lakeformation.session_tag": "sesson_tag" + }, + "resultIndex": "query_execution_result" + }] + +Sample Security Lake datasource queries APIS +===================================== + +Sample Queries + +* Select Query : ``select * from mysl.amazon_security_lake_glue_db_eu_west_1.amazon_security_lake_table_eu_west_1_vpc_flow_2_0 limit 1`` +* Create Covering Index Query: ``create index srcip_time on mysl.amazon_security_lake_glue_db_eu_west_1.amazon_security_lake_table_eu_west_1_vpc_flow_2_0 (src_endpoint.ip, time) WITH (auto_refresh=true)`` + +These queries would work only top of async queries. Documentation: `Async Query APIs <../../../interfaces/asyncqueryinterface.rst>`_ + +Documentation for Index Queries: https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index a1b1e32955..971ef5e928 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -60,6 +60,7 @@ import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; import org.opensearch.sql.datasources.encryptor.EncryptorImpl; import org.opensearch.sql.datasources.glue.GlueDataSourceFactory; +import org.opensearch.sql.datasources.glue.SecurityLakeDataSourceFactory; import org.opensearch.sql.datasources.model.transport.*; import org.opensearch.sql.datasources.rest.RestDataSourceQueryAction; import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; @@ -326,6 +327,7 @@ private DataSourceServiceImpl createDataSourceService() { .add(new PrometheusStorageFactory(pluginSettings)) .add(new SparkStorageFactory(this.client, pluginSettings)) .add(new GlueDataSourceFactory(pluginSettings)) + .add(new SecurityLakeDataSourceFactory(pluginSettings)) .build(), dataSourceMetadataStorage, dataSourceUserAuthorizationHelper); From 7815c96597585c6c47ed6f6189cd1c6c8b56f0a8 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Thu, 15 Aug 2024 12:59:28 -0700 Subject: [PATCH 28/96] Merge feature/pit branch to main (#2936) * Add pit for join queries (#2703) * Add search after for join Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Enable search after by default Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix tests Signed-off-by: Rupal Mahajan <maharup@amazon.com> * ignore joinWithGeoIntersectNL Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rerun CI with scroll Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code and retrigger CI with search_after true Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Address comments Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code change Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix scroll condition Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit before query execution Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from join request builder to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused methods Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit in parent class's run() Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add comment for fetching subsequent result in NestedLoopsElasticExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update comment Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add javadoc for pit handler Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit interface Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit handler unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix failed unit test CI Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix spotless error Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rename pit class and add logs Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix pit delete unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit for multi query (#2753) * Add search after for join Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Enable search after by default Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix tests Signed-off-by: Rupal Mahajan <maharup@amazon.com> * ignore joinWithGeoIntersectNL Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rerun CI with scroll Signed-off-by: Rupal Mahajan <maharup@amazon.com> * draft Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code and retrigger CI with search_after true Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Address comments Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code change Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix scroll condition Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit before query execution Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Refactor get response with pit method Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update remaining scroll search calls Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix integ test failures Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from join request builder to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused methods Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from request to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix pit.delete call missed while merge Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move getResponseWithHits method to util class Signed-off-by: Rupal Mahajan <maharup@amazon.com> * add try catch for create delete pit in minus executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * move all common fields to ElasticHitsExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * add javadoc for ElasticHitsExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add missing javadoc Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Forcing an empty commit as last commit is stuck processing updates Signed-off-by: Rupal Mahajan <maharup@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> Signed-off-by: Manasvini B S <manasvis@amazon.com> Co-authored-by: Rupal Mahajan <maharup@amazon.com> --- .../sql/common/setting/Settings.java | 1 + docs/dev/opensearch-pagination.md | 40 ++++ docs/user/admin/settings.rst | 44 +++++ .../org/opensearch/sql/legacy/JoinIT.java | 6 +- .../legacy/executor/ElasticHitsExecutor.java | 91 ++++++++- .../executor/join/ElasticJoinExecutor.java | 76 ++++---- .../join/HashJoinElasticExecutor.java | 39 ++-- .../join/NestedLoopsElasticExecutor.java | 33 ++-- .../join/QueryPlanElasticExecutor.java | 5 +- .../legacy/executor/multi/MinusExecutor.java | 181 ++++++++++-------- .../legacy/executor/multi/UnionExecutor.java | 3 +- .../sql/legacy/pit/PointInTimeHandler.java | 18 ++ .../legacy/pit/PointInTimeHandlerImpl.java | 83 ++++++++ .../pit/PointInTimeHandlerImplTest.java | 81 ++++++++ .../setting/OpenSearchSettings.java | 14 ++ 15 files changed, 557 insertions(+), 158 deletions(-) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java create mode 100644 legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 7346ee6722..b6643f3209 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -23,6 +23,7 @@ public enum Key { SQL_SLOWLOG("plugins.sql.slowlog"), SQL_CURSOR_KEEP_ALIVE("plugins.sql.cursor.keep_alive"), SQL_DELETE_ENABLED("plugins.sql.delete.enabled"), + SQL_PAGINATION_API_SEARCH_AFTER("plugins.sql.pagination.api"), /** PPL Settings. */ PPL_ENABLED("plugins.ppl.enabled"), diff --git a/docs/dev/opensearch-pagination.md b/docs/dev/opensearch-pagination.md index 4982b13d7f..1919af30fe 100644 --- a/docs/dev/opensearch-pagination.md +++ b/docs/dev/opensearch-pagination.md @@ -477,4 +477,44 @@ Response: } +``` + +#### plugins.sql.pagination.api + +This setting controls whether the SQL search queries in OpenSearch use Point-In-Time (PIT) with search_after or the traditional scroll mechanism for fetching paginated results. + +- Default Value: true +- Possible Values: true or false +- When set to true, the search query in the background uses PIT with search_after instead of scroll to retrieve paginated results. The Cursor Id returned to the user will encode relevant pagination query-related information, which will be used to fetch the subsequent pages of results. +- This setting is node-level. +- This setting can be updated dynamically. + +Example: + +``` +>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ + "transient" : { + "plugins.sql.pagination.api" : "true" + } +}' +``` + +Response: + +``` +{ + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "sql" : { + "pagination" : { + "api" : "true" + } + } + } + } +} + + ``` diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 6b24e41f87..236406e2c7 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -196,6 +196,50 @@ Result set:: Note: the legacy settings of ``opendistro.sql.cursor.keep_alive`` is deprecated, it will fallback to the new settings if you request an update with the legacy name. +plugins.sql.pagination.api +================================ + +Description +----------- + +This setting controls whether the SQL search queries in OpenSearch use Point-In-Time (PIT) with search_after or the traditional scroll mechanism for fetching paginated results. + +1. Default Value: true +2. Possible Values: true or false +3. When set to true, the search query in the background uses PIT with search_after instead of scroll to retrieve paginated results. The Cursor Id returned to the user will encode relevant pagination query-related information, which will be used to fetch the subsequent pages of results. +4. This setting is node-level. +5. This setting can be updated dynamically. + + +Example +------- + +You can update the setting with a new value like this. + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.sql.pagination.api" : "true" + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "sql" : { + "pagination" : { + "api" : "true" + } + } + } + } + } + plugins.query.size_limit =========================== diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java index 8019454b77..8c2ea96474 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinIT.java @@ -288,6 +288,8 @@ public void hintMultiSearchCanRunFewTimesNL() throws IOException { Assert.assertThat(hits.length(), equalTo(42)); } + // TODO: Fix joinWithGeoIntersectNL test when SQL_PAGINATION_API_SEARCH_AFTER is true + @Ignore @Test public void joinWithGeoIntersectNL() throws IOException { @@ -455,7 +457,7 @@ public void joinParseCheckSelectedFieldsSplitNLConditionOrderGT() throws IOExcep "SELECT /*! USE_NL*/ a.firstname, a.lastname, a.gender, d.firstname, d.age FROM %s a" + " JOIN %s d on a.age < d.age WHERE (d.firstname = 'Lynn' OR d.firstname =" + " 'Obrien') AND a.firstname = 'Mcgee'", - TEST_INDEX_PEOPLE, + TEST_INDEX_PEOPLE2, TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); @@ -501,7 +503,7 @@ public void joinParseCheckSelectedFieldsSplitNLConditionOrderLT() throws IOExcep "SELECT /*! USE_NL*/ a.firstname, a.lastname, a.gender, d.firstname, d.age FROM %s a" + " JOIN %s d on a.age > d.age WHERE (d.firstname = 'Sandoval' OR d.firstname =" + " 'Hewitt') AND a.firstname = 'Fulton'", - TEST_INDEX_PEOPLE, + TEST_INDEX_PEOPLE2, TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java index 62a6d63ef7..2b80575e1e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java @@ -5,13 +5,96 @@ package org.opensearch.sql.legacy.executor; +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.io.IOException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.sql.legacy.domain.Select; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; + +/** Executor for search requests with pagination. */ +public abstract class ElasticHitsExecutor { + protected static final Logger LOG = LogManager.getLogger(); + protected PointInTimeHandler pit; + protected Client client; + + /** + * Executes search request + * + * @throws IOException If an input or output exception occurred + * @throws SqlParseException If parsing exception occurred + */ + protected abstract void run() throws IOException, SqlParseException; + + /** + * Get search hits after execution + * + * @return Search hits + */ + protected abstract SearchHits getHits(); + + /** + * Get response for search request with pit/scroll + * + * @param request search request + * @param select sql select + * @param size fetch size + * @param previousResponse response for previous request + * @param pit point in time + * @return search response for subsequent request + */ + public SearchResponse getResponseWithHits( + SearchRequestBuilder request, + Select select, + int size, + SearchResponse previousResponse, + PointInTimeHandler pit) { + // Set Size + request.setSize(size); + SearchResponse responseWithHits; -/** Created by Eliran on 21/8/2016. */ -public interface ElasticHitsExecutor { - void run() throws IOException, SqlParseException; + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + // Set sort field for search_after + boolean ordered = select.isOrderdSelect(); + if (!ordered) { + request.addSort(DOC_FIELD_NAME, ASC); + } + // Set PIT + request.setPointInTime(new PointInTimeBuilder(pit.getPitId())); + // from and size is alternate method to paginate result. + // If select has from clause, search after is not required. + if (previousResponse != null && select.getFrom().isEmpty()) { + request.searchAfter(previousResponse.getHits().getSortFields()); + } + responseWithHits = request.get(); + } else { + // Set scroll + TimeValue keepAlive = LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE); + if (previousResponse != null) { + responseWithHits = + client + .prepareSearchScroll(previousResponse.getScrollId()) + .setScroll(keepAlive) + .execute() + .actionGet(); + } else { + request.setScroll(keepAlive); + responseWithHits = request.get(); + } + } - SearchHits getHits(); + return responseWithHits; + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index f0ffafc470..061868c9b5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.executor.join; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -12,15 +14,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.stream.Stream; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; -import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; @@ -28,11 +27,11 @@ import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.search.sort.FieldSortBuilder; -import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.legacy.domain.Field; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.SqlElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.JoinRequestBuilder; @@ -41,16 +40,16 @@ import org.opensearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; /** Created by Eliran on 15/9/2015. */ -public abstract class ElasticJoinExecutor implements ElasticHitsExecutor { - private static final Logger LOG = LogManager.getLogger(); +public abstract class ElasticJoinExecutor extends ElasticHitsExecutor { protected List<SearchHit> results; // Keep list to avoid copy to new array in SearchHits protected MetaSearchResult metaResults; protected final int MAX_RESULTS_ON_ONE_FETCH = 10000; private Set<String> aliasesOnReturn; private boolean allFieldsReturn; + protected String[] indices; - protected ElasticJoinExecutor(JoinRequestBuilder requestBuilder) { + protected ElasticJoinExecutor(Client client, JoinRequestBuilder requestBuilder) { metaResults = new MetaSearchResult(); aliasesOnReturn = new HashSet<>(); List<Field> firstTableReturnedField = requestBuilder.getFirstTable().getReturnedFields(); @@ -58,6 +57,8 @@ protected ElasticJoinExecutor(JoinRequestBuilder requestBuilder) { allFieldsReturn = (firstTableReturnedField == null || firstTableReturnedField.size() == 0) && (secondTableReturnedField == null || secondTableReturnedField.size() == 0); + indices = getIndices(requestBuilder); + this.client = client; } public void sendResponse(RestChannel channel) throws IOException { @@ -85,10 +86,22 @@ public void sendResponse(RestChannel channel) throws IOException { } public void run() throws IOException, SqlParseException { - long timeBefore = System.currentTimeMillis(); - results = innerRun(); - long joinTimeInMilli = System.currentTimeMillis() - timeBefore; - this.metaResults.setTookImMilli(joinTimeInMilli); + try { + long timeBefore = System.currentTimeMillis(); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = new PointInTimeHandlerImpl(client, indices); + pit.create(); + } + results = innerRun(); + long joinTimeInMilli = System.currentTimeMillis() - timeBefore; + this.metaResults.setTookImMilli(joinTimeInMilli); + } catch (Exception e) { + LOG.error("Failed during join query run.", e); + } finally { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit.delete(); + } + } } protected abstract List<SearchHit> innerRun() throws IOException, SqlParseException; @@ -103,7 +116,7 @@ public SearchHits getHits() { public static ElasticJoinExecutor createJoinExecutor( Client client, SqlElasticRequestBuilder requestBuilder) { if (requestBuilder instanceof HashJoinQueryPlanRequestBuilder) { - return new QueryPlanElasticExecutor((HashJoinQueryPlanRequestBuilder) requestBuilder); + return new QueryPlanElasticExecutor(client, (HashJoinQueryPlanRequestBuilder) requestBuilder); } else if (requestBuilder instanceof HashJoinElasticRequestBuilder) { HashJoinElasticRequestBuilder hashJoin = (HashJoinElasticRequestBuilder) requestBuilder; return new HashJoinElasticExecutor(client, hashJoin); @@ -256,23 +269,22 @@ protected void updateMetaSearchResults(SearchResponse searchResponse) { this.metaResults.updateTimeOut(searchResponse.isTimedOut()); } - protected SearchResponse scrollOneTimeWithMax( - Client client, TableInJoinRequestBuilder tableRequest) { - SearchRequestBuilder scrollRequest = - tableRequest - .getRequestBuilder() - .setScroll(new TimeValue(60000)) - .setSize(MAX_RESULTS_ON_ONE_FETCH); - boolean ordered = tableRequest.getOriginalSelect().isOrderdSelect(); - if (!ordered) { - scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); - } - SearchResponse responseWithHits = scrollRequest.get(); - // on ordered select - not using SCAN , elastic returns hits on first scroll - // es5.0 elastic always return docs on scan - // if(!ordered) - // responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()) - // .setScroll(new TimeValue(600000)).get(); - return responseWithHits; + public SearchResponse getResponseWithHits( + TableInJoinRequestBuilder tableRequest, int size, SearchResponse previousResponse) { + + return getResponseWithHits( + tableRequest.getRequestBuilder(), + tableRequest.getOriginalSelect(), + size, + previousResponse, + pit); + } + + public String[] getIndices(JoinRequestBuilder joinRequestBuilder) { + return Stream.concat( + Stream.of(joinRequestBuilder.getFirstTable().getOriginalSelect().getIndexArr()), + Stream.of(joinRequestBuilder.getSecondTable().getOriginalSelect().getIndexArr())) + .distinct() + .toArray(String[]::new); } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java index 06a913205d..0e33ab9eef 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java @@ -20,7 +20,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -36,16 +35,13 @@ /** Created by Eliran on 22/8/2015. */ public class HashJoinElasticExecutor extends ElasticJoinExecutor { private HashJoinElasticRequestBuilder requestBuilder; - - private Client client; private boolean useQueryTermsFilterOptimization = false; private final int MAX_RESULTS_FOR_FIRST_TABLE = 100000; HashJoinComparisonStructure hashJoinComparisonStructure; private Set<String> alreadyMatched; public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requestBuilder) { - super(requestBuilder); - this.client = client; + super(client, requestBuilder); this.requestBuilder = requestBuilder; this.useQueryTermsFilterOptimization = requestBuilder.isUseTermFiltersOptimization(); this.hashJoinComparisonStructure = @@ -54,7 +50,6 @@ public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requ } public List<SearchHit> innerRun() throws IOException, SqlParseException { - Map<String, Map<String, List<Object>>> optimizationTermsFilterStructure = initOptimizationStructure(); @@ -124,16 +119,12 @@ private List<SearchHit> createCombinedResults(TableInJoinRequestBuilder secondTa Integer hintLimit = secondTableRequest.getHintLimit(); SearchResponse searchResponse; boolean finishedScrolling; + if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { - searchResponse = secondTableRequest.getRequestBuilder().setSize(hintLimit).get(); + searchResponse = getResponseWithHits(secondTableRequest, hintLimit, null); finishedScrolling = true; } else { - searchResponse = - secondTableRequest - .getRequestBuilder() - .setScroll(new TimeValue(60000)) - .setSize(MAX_RESULTS_ON_ONE_FETCH) - .get(); + searchResponse = getResponseWithHits(secondTableRequest, MAX_RESULTS_ON_ONE_FETCH, null); // es5.0 no need to scroll again! // searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) // .setScroll(new TimeValue(600000)).get(); @@ -214,11 +205,7 @@ private List<SearchHit> createCombinedResults(TableInJoinRequestBuilder secondTa if (secondTableHits.length > 0 && (hintLimit == null || fetchedSoFarFromSecondTable >= hintLimit)) { searchResponse = - client - .prepareSearchScroll(searchResponse.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits(secondTableRequest, MAX_RESULTS_ON_ONE_FETCH, searchResponse); } else { break; } @@ -292,12 +279,13 @@ private List<SearchHit> fetchAllHits(TableInJoinRequestBuilder tableInJoinReques private List<SearchHit> scrollTillLimit( TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { - SearchResponse scrollResp = scrollOneTimeWithMax(client, tableInJoinRequest); + SearchResponse response = + getResponseWithHits(tableInJoinRequest, MAX_RESULTS_ON_ONE_FETCH, null); - updateMetaSearchResults(scrollResp); + updateMetaSearchResults(response); List<SearchHit> hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; - SearchHit[] hits = scrollResp.getHits().getHits(); + SearchHit[] hits = response.getHits().getHits(); if (hintLimit == null) { hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; @@ -311,13 +299,8 @@ private List<SearchHit> scrollTillLimit( System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } - scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); - hits = scrollResp.getHits().getHits(); + response = getResponseWithHits(tableInJoinRequest, MAX_RESULTS_FOR_FIRST_TABLE, response); + hits = response.getHits().getHits(); } return hitsWithScan; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index 56c5f96af5..9356a0058e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -18,7 +18,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -39,11 +38,9 @@ public class NestedLoopsElasticExecutor extends ElasticJoinExecutor { private static final Logger LOG = LogManager.getLogger(); private final NestedLoopsElasticRequestBuilder nestedLoopsRequest; - private final Client client; public NestedLoopsElasticExecutor(Client client, NestedLoopsElasticRequestBuilder nestedLoops) { - super(nestedLoops); - this.client = client; + super(client, nestedLoops); this.nestedLoopsRequest = nestedLoops; } @@ -111,11 +108,26 @@ protected List<SearchHit> innerRun() throws SqlParseException { if (!BackOffRetryStrategy.isHealthy()) { throw new IllegalStateException("Memory circuit is broken"); } - firstTableResponse = - client - .prepareSearchScroll(firstTableResponse.getScrollId()) - .setScroll(new TimeValue(600000)) - .get(); + /* Fetching next result page. + Using scroll api - only scrollId from previous response is required for scroll request. + Using pit with search_after - we need to recreate search request along with pitId and + sort fields from previous response. + Here we are finding required size for recreating search request with pit and search after. + Conditions for size are similar as firstFetch(). + In case of scroll, this size will be ignored and size from first request will be used. + */ + Integer hintLimit = nestedLoopsRequest.getFirstTable().getHintLimit(); + if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { + firstTableResponse = + getResponseWithHits( + nestedLoopsRequest.getFirstTable(), hintLimit, firstTableResponse); + } else { + firstTableResponse = + getResponseWithHits( + nestedLoopsRequest.getFirstTable(), + MAX_RESULTS_ON_ONE_FETCH, + firstTableResponse); + } } else { finishedWithFirstTable = true; } @@ -287,12 +299,11 @@ private FetchWithScrollResponse firstFetch(TableInJoinRequestBuilder tableReques boolean needScrollForFirstTable = false; SearchResponse responseWithHits; if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { - responseWithHits = tableRequest.getRequestBuilder().setSize(hintLimit).get(); needScrollForFirstTable = false; } else { // scroll request with max. - responseWithHits = scrollOneTimeWithMax(client, tableRequest); + responseWithHits = getResponseWithHits(tableRequest, MAX_RESULTS_ON_ONE_FETCH, null); if (responseWithHits.getHits().getTotalHits() != null && responseWithHits.getHits().getTotalHits().value < MAX_RESULTS_ON_ONE_FETCH) { needScrollForFirstTable = true; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java index f4b2f5421d..d8e9d41376 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.join; import java.util.List; +import org.opensearch.client.Client; import org.opensearch.search.SearchHit; import org.opensearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; import org.opensearch.sql.legacy.query.planner.core.QueryPlanner; @@ -19,8 +20,8 @@ class QueryPlanElasticExecutor extends ElasticJoinExecutor { private final QueryPlanner queryPlanner; - QueryPlanElasticExecutor(HashJoinQueryPlanRequestBuilder request) { - super(request); + QueryPlanElasticExecutor(Client client, HashJoinQueryPlanRequestBuilder request) { + super(client, request); this.queryPlanner = request.plan(); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index 03e16424e7..f58b25e821 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.executor.multi; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -18,7 +20,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.ArrayUtils; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -28,16 +30,16 @@ import org.opensearch.sql.legacy.domain.Where; import org.opensearch.sql.legacy.domain.hints.Hint; import org.opensearch.sql.legacy.domain.hints.HintType; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; -import org.opensearch.sql.legacy.executor.join.ElasticUtils; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.multi.MultiQueryRequestBuilder; import org.opensearch.sql.legacy.utils.Util; /** Created by Eliran on 26/8/2016. */ -public class MinusExecutor implements ElasticHitsExecutor { - private Client client; +public class MinusExecutor extends ElasticHitsExecutor { private MultiQueryRequestBuilder builder; private SearchHits minusHits; private boolean useTermsOptimization; @@ -63,45 +65,63 @@ public MinusExecutor(Client client, MultiQueryRequestBuilder builder) { @Override public void run() throws SqlParseException { - if (this.useTermsOptimization && this.fieldsOrderFirstTable.length != 1) { - throw new SqlParseException( - "Terms optimization failed: terms optimization for minus execution is supported with one" - + " field"); - } - if (this.useTermsOptimization && !this.useScrolling) { - throw new SqlParseException( - "Terms optimization failed: using scrolling is required for terms optimization"); - } - if (!this.useScrolling || !this.useTermsOptimization) { - Set<ComperableHitResult> comperableHitResults; - if (!this.useScrolling) { - // 1. get results from first search , put in set - // 2. get reults from second search - // 2.1 for each result remove from set - comperableHitResults = simpleOneTimeQueryEach(); + try { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = + new PointInTimeHandlerImpl( + client, + ArrayUtils.concat( + builder.getOriginalSelect(true).getIndexArr(), + builder.getOriginalSelect(false).getIndexArr())); + pit.create(); + } + + if (this.useTermsOptimization && this.fieldsOrderFirstTable.length != 1) { + throw new SqlParseException( + "Terms optimization failed: terms optimization for minus execution is supported with" + + " one field"); + } + if (this.useTermsOptimization && !this.useScrolling) { + throw new SqlParseException( + "Terms optimization failed: using scrolling is required for terms optimization"); + } + if (!this.useScrolling || !this.useTermsOptimization) { + Set<ComperableHitResult> comperableHitResults; + if (!this.useScrolling) { + // 1. get results from first search , put in set + // 2. get reults from second search + // 2.1 for each result remove from set + comperableHitResults = simpleOneTimeQueryEach(); + } else { + // if scrolling + // 1. get all results in scrolls (till some limit) . put on set + // 2. scroll on second table + // 3. on each scroll result remove items from set + comperableHitResults = runWithScrollings(); + } + fillMinusHitsFromResults(comperableHitResults); + return; } else { - // if scrolling - // 1. get all results in scrolls (till some limit) . put on set - // 2. scroll on second table - // 3. on each scroll result remove items from set - comperableHitResults = runWithScrollings(); + // if scrolling and optimization + // 0. save the original second table where , init set + // 1. on each scroll on first table , create miniSet + // 1.1 build where from all results (terms filter) , and run query + // 1.1.1 on each result remove from miniSet + // 1.1.2 add all results left from miniset to bigset + Select firstSelect = this.builder.getOriginalSelect(true); + MinusOneFieldAndOptimizationResult optimizationResult = + runWithScrollingAndAddFilter(fieldsOrderFirstTable[0], fieldsOrderSecondTable[0]); + String fieldName = getFieldName(firstSelect.getFields().get(0)); + Set<Object> results = optimizationResult.getFieldValues(); + SearchHit someHit = optimizationResult.getSomeHit(); + fillMinusHitsFromOneField(fieldName, results, someHit); + } + } catch (Exception e) { + LOG.error("Failed during multi query run.", e); + } finally { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit.delete(); } - fillMinusHitsFromResults(comperableHitResults); - return; - } else { - // if scrolling and optimization - // 0. save the original second table where , init set - // 1. on each scroll on first table , create miniSet - // 1.1 build where from all results (terms filter) , and run query - // 1.1.1 on each result remove from miniSet - // 1.1.2 add all results left from miniset to bigset - Select firstSelect = this.builder.getOriginalSelect(true); - MinusOneFieldAndOptimizationResult optimizationResult = - runWithScrollingAndAddFilter(fieldsOrderFirstTable[0], fieldsOrderSecondTable[0]); - String fieldName = getFieldName(firstSelect.getFields().get(0)); - Set<Object> results = optimizationResult.getFieldValues(); - SearchHit someHit = optimizationResult.getSomeHit(); - fillMinusHitsFromOneField(fieldName, results, someHit); } } @@ -187,11 +207,12 @@ private void fillMinusHitsFromResults(Set<ComperableHitResult> comperableHitResu private Set<ComperableHitResult> runWithScrollings() { SearchResponse scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, - this.builder.getFirstSearchRequest(), + getResponseWithHits( + builder.getFirstSearchRequest(), builder.getOriginalSelect(true), - this.maxDocsToFetchOnEachScrollShard); + maxDocsToFetchOnEachScrollShard, + null, + pit); Set<ComperableHitResult> results = new HashSet<>(); SearchHit[] hits = scrollResp.getHits().getHits(); @@ -199,7 +220,6 @@ private Set<ComperableHitResult> runWithScrollings() { return new HashSet<>(); } int totalDocsFetchedFromFirstTable = 0; - // fetch from first table . fill set. while (hits != null && hits.length != 0) { totalDocsFetchedFromFirstTable += hits.length; @@ -208,19 +228,21 @@ private Set<ComperableHitResult> runWithScrollings() { break; } scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getFirstSearchRequest(), + builder.getOriginalSelect(true), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, + getResponseWithHits( this.builder.getSecondSearchRequest(), builder.getOriginalSelect(false), - this.maxDocsToFetchOnEachScrollShard); + this.maxDocsToFetchOnEachScrollShard, + null, + pit); hits = scrollResp.getHits().getHits(); if (hits == null || hits.length == 0) { @@ -234,11 +256,12 @@ private Set<ComperableHitResult> runWithScrollings() { break; } scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getSecondSearchRequest(), + builder.getOriginalSelect(false), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } @@ -303,11 +326,12 @@ private boolean checkIfOnlyOneField(Select firstSelect, Select secondSelect) { private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( String firstFieldName, String secondFieldName) throws SqlParseException { SearchResponse scrollResp = - ElasticUtils.scrollOneTimeWithHits( - this.client, - this.builder.getFirstSearchRequest(), + getResponseWithHits( + builder.getFirstSearchRequest(), builder.getOriginalSelect(true), - this.maxDocsToFetchOnEachScrollShard); + maxDocsToFetchOnEachScrollShard, + null, + pit); Set<Object> results = new HashSet<>(); int currentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); @@ -335,14 +359,16 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( break; } SearchResponse responseForSecondTable = - ElasticUtils.scrollOneTimeWithHits( - this.client, + getResponseWithHits( queryAction.getRequestBuilder(), secondQuerySelect, - this.maxDocsToFetchOnEachScrollShard); + this.maxDocsToFetchOnEachScrollShard, + null, + pit); SearchHits secondQuerySearchHits = responseForSecondTable.getHits(); SearchHit[] secondQueryHits = secondQuerySearchHits.getHits(); + while (secondQueryHits.length > 0) { totalDocsFetchedFromSecondTable += secondQueryHits.length; removeValuesFromSetAccordingToHits(secondFieldName, currentSetFromResults, secondQueryHits); @@ -350,11 +376,12 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( break; } responseForSecondTable = - client - .prepareSearchScroll(responseForSecondTable.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + queryAction.getRequestBuilder(), + secondQuerySelect, + maxDocsToFetchOnEachScrollShard, + responseForSecondTable, + pit); secondQueryHits = responseForSecondTable.getHits().getHits(); } results.addAll(currentSetFromResults); @@ -363,13 +390,13 @@ private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter( "too many results for first table, stoping at:" + totalDocsFetchedFromFirstTable); break; } - scrollResp = - client - .prepareSearchScroll(scrollResp.getScrollId()) - .setScroll(new TimeValue(600000)) - .execute() - .actionGet(); + getResponseWithHits( + builder.getFirstSearchRequest(), + builder.getOriginalSelect(true), + maxDocsToFetchOnEachScrollShard, + scrollResp, + pit); hits = scrollResp.getHits().getHits(); } return new MinusOneFieldAndOptimizationResult(results, someHit); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java index 6b8b64c4e8..375c40a5c1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java @@ -23,11 +23,10 @@ import org.opensearch.sql.legacy.utils.Util; /** Created by Eliran on 21/8/2016. */ -public class UnionExecutor implements ElasticHitsExecutor { +public class UnionExecutor extends ElasticHitsExecutor { private MultiQueryRequestBuilder multiQueryBuilder; private SearchHits results; - private Client client; private int currentId; public UnionExecutor(Client client, MultiQueryRequestBuilder builder) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java new file mode 100644 index 0000000000..66339cc70a --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandler.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +/** Point In Time */ +public interface PointInTimeHandler { + /** Create Point In Time */ + void create(); + + /** Delete Point In Time */ + void delete(); + + /** Get Point In Time Identifier */ + String getPitId(); +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java new file mode 100644 index 0000000000..64535749e8 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; + +import lombok.Getter; +import lombok.Setter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitRequest; +import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; + +/** Handler for Point In Time */ +public class PointInTimeHandlerImpl implements PointInTimeHandler { + private Client client; + private String[] indices; + @Getter @Setter private String pitId; + private static final Logger LOG = LogManager.getLogger(); + + /** + * Constructor for class + * + * @param client OpenSearch client + * @param indices list of indices + */ + public PointInTimeHandlerImpl(Client client, String[] indices) { + this.client = client; + this.indices = indices; + } + + /** Create PIT for given indices */ + @Override + public void create() { + CreatePitRequest createPitRequest = + new CreatePitRequest( + LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE), false, indices); + client.createPit( + createPitRequest, + new ActionListener<>() { + @Override + public void onResponse(CreatePitResponse createPitResponse) { + pitId = createPitResponse.getId(); + LOG.info("Created Point In Time {} successfully.", pitId); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error occurred while creating PIT", e); + } + }); + } + + /** Delete PIT */ + @Override + public void delete() { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + client.deletePits( + deletePitRequest, + new ActionListener<>() { + @Override + public void onResponse(DeletePitResponse deletePitResponse) { + LOG.info( + "Delete Point In Time {} status: {}", + pitId, + deletePitResponse.status().getStatus()); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error occurred while deleting PIT", e); + } + }); + } +} diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java new file mode 100644 index 0000000000..42f1af4563 --- /dev/null +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy.pit; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; + +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; + +public class PointInTimeHandlerImplTest { + + @Mock private Client mockClient; + private String[] indices = {"index1", "index2"}; + private PointInTimeHandlerImpl pointInTimeHandlerImpl; + @Captor private ArgumentCaptor<ActionListener<DeletePitResponse>> listenerCaptor; + private final String PIT_ID = "testId"; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + pointInTimeHandlerImpl = new PointInTimeHandlerImpl(mockClient, indices); + } + + @Test + public void testCreate() { + when(LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE)) + .thenReturn(new TimeValue(10000)); + + CreatePitResponse mockCreatePitResponse = mock(CreatePitResponse.class); + when(mockCreatePitResponse.getId()).thenReturn(PIT_ID); + + CompletableFuture<CreatePitResponse> completableFuture = + CompletableFuture.completedFuture(mockCreatePitResponse); + + doAnswer( + invocation -> { + ActionListener<CreatePitResponse> actionListener = invocation.getArgument(1); + actionListener.onResponse(mockCreatePitResponse); + return completableFuture; + }) + .when(mockClient) + .createPit(any(), any()); + + pointInTimeHandlerImpl.create(); + + assertEquals(PIT_ID, pointInTimeHandlerImpl.getPitId()); + } + + @Test + public void testDelete() { + DeletePitResponse mockedResponse = mock(DeletePitResponse.class); + RestStatus mockRestStatus = mock(RestStatus.class); + when(mockedResponse.status()).thenReturn(mockRestStatus); + when(mockedResponse.status().getStatus()).thenReturn(200); + pointInTimeHandlerImpl.setPitId(PIT_ID); + pointInTimeHandlerImpl.delete(); + verify(mockClient).deletePits(any(), listenerCaptor.capture()); + listenerCaptor.getValue().onResponse(mockedResponse); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 475a584623..494b906b55 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,6 +71,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting<?> SQL_PAGINATION_API_SEARCH_AFTER_SETTING = + Setting.boolSetting( + Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting<?> PPL_ENABLED_SETTING = Setting.boolSetting( Key.PPL_ENABLED.getKeyValue(), @@ -237,6 +244,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.SQL_DELETE_ENABLED, SQL_DELETE_ENABLED_SETTING, new Updater(Key.SQL_DELETE_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.SQL_PAGINATION_API_SEARCH_AFTER, + SQL_PAGINATION_API_SEARCH_AFTER_SETTING, + new Updater(Key.SQL_PAGINATION_API_SEARCH_AFTER)); register( settingBuilder, clusterSettings, @@ -397,6 +410,7 @@ public static List<Setting<?>> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(SQL_DELETE_ENABLED_SETTING) + .add(SQL_PAGINATION_API_SEARCH_AFTER_SETTING) .add(PPL_ENABLED_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) From 69853fe99c4ee940fc739373e0ba04418bda71c6 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Fri, 23 Aug 2024 14:19:06 -0700 Subject: [PATCH 29/96] Add pit for pagination query (#2940) * Add pit for join queries (#2703) * Add search after for join Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Enable search after by default Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix tests Signed-off-by: Rupal Mahajan <maharup@amazon.com> * ignore joinWithGeoIntersectNL Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rerun CI with scroll Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code and retrigger CI with search_after true Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Address comments Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code change Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix scroll condition Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit before query execution Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from join request builder to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused methods Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit in parent class's run() Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add comment for fetching subsequent result in NestedLoopsElasticExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update comment Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add javadoc for pit handler Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit interface Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit handler unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix failed unit test CI Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix spotless error Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rename pit class and add logs Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix pit delete unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit for multi query (#2753) * Add search after for join Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Enable search after by default Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix tests Signed-off-by: Rupal Mahajan <maharup@amazon.com> * ignore joinWithGeoIntersectNL Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rerun CI with scroll Signed-off-by: Rupal Mahajan <maharup@amazon.com> * draft Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code and retrigger CI with search_after true Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Address comments Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused code change Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update pit keep alive time with SQL_CURSOR_KEEP_ALIVE Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix scroll condition Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit before query execution Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Refactor get response with pit method Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Update remaining scroll search calls Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix integ test failures Signed-off-by: Rupal Mahajan <maharup@amazon.com> * nit Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from join request builder to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Remove unused methods Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move pit from request to executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Fix pit.delete call missed while merge Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Move getResponseWithHits method to util class Signed-off-by: Rupal Mahajan <maharup@amazon.com> * add try catch for create delete pit in minus executor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * move all common fields to ElasticHitsExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * add javadoc for ElasticHitsExecutor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add missing javadoc Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Forcing an empty commit as last commit is stuck processing updates Signed-off-by: Rupal Mahajan <maharup@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Add pit to default cursor Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Run CI without pit unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> * Rerun CI without pit unit test Signed-off-by: Rupal Mahajan <maharup@amazon.com> * FIx unit tests for PIT changes Signed-off-by: Manasvini B S <manasvis@amazon.com> * Addressed comments Signed-off-by: Manasvini B S <manasvis@amazon.com> --------- Signed-off-by: Rupal Mahajan <maharup@amazon.com> Signed-off-by: Manasvini B S <manasvis@amazon.com> Co-authored-by: Rupal Mahajan <maharup@amazon.com> --- .../sql/legacy/cursor/DefaultCursor.java | 122 +++++++++++++++++- .../executor/cursor/CursorCloseExecutor.java | 30 ++++- .../executor/cursor/CursorResultExecutor.java | 62 +++++++-- .../format/PrettyFormatRestExecutor.java | 45 ++++++- .../executor/format/SelectResultSet.java | 28 +++- .../executor/join/ElasticJoinExecutor.java | 9 +- .../legacy/executor/multi/MinusExecutor.java | 9 +- .../legacy/pit/PointInTimeHandlerImpl.java | 63 ++++----- .../sql/legacy/query/DefaultQueryAction.java | 16 ++- .../pit/PointInTimeHandlerImplTest.java | 118 ++++++++++++----- .../unittest/cursor/DefaultCursorTest.java | 58 ++++++++- .../query/DefaultQueryActionTest.java | 63 +++++++-- 12 files changed, 513 insertions(+), 110 deletions(-) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java b/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java index c5be0066fc..2b0de9022c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java @@ -5,8 +5,19 @@ package org.opensearch.sql.legacy.cursor; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,6 +29,16 @@ import lombok.Setter; import org.json.JSONArray; import org.json.JSONObject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.executor.format.Schema; /** @@ -40,6 +61,10 @@ public class DefaultCursor implements Cursor { private static final String SCROLL_ID = "s"; private static final String SCHEMA_COLUMNS = "c"; private static final String FIELD_ALIAS_MAP = "a"; + private static final String PIT_ID = "p"; + private static final String SEARCH_REQUEST = "r"; + private static final String SORT_FIELDS = "h"; + private static final ObjectMapper objectMapper = new ObjectMapper(); /** * To get mappings for index to check if type is date needed for @@ -70,11 +95,28 @@ public class DefaultCursor implements Cursor { /** To get next batch of result */ private String scrollId; + /** To get Point In Time */ + private String pitId; + + /** To get next batch of result with search after api */ + private SearchSourceBuilder searchSourceBuilder; + + /** To get last sort values * */ + private Object[] sortFields; + /** To reduce the number of rows left by fetchSize */ @NonNull private Integer fetchSize; private Integer limit; + /** + * {@link NamedXContentRegistry} from {@link SearchModule} used for construct {@link QueryBuilder} + * from DSL query string. + */ + private static final NamedXContentRegistry xContentRegistry = + new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + @Override public CursorType getType() { return type; @@ -82,19 +124,56 @@ public CursorType getType() { @Override public String generateCursorId() { - if (rowsLeft <= 0 || Strings.isNullOrEmpty(scrollId)) { + if (rowsLeft <= 0 || isCursorIdNullOrEmpty()) { return null; } JSONObject json = new JSONObject(); json.put(FETCH_SIZE, fetchSize); json.put(ROWS_LEFT, rowsLeft); json.put(INDEX_PATTERN, indexPattern); - json.put(SCROLL_ID, scrollId); json.put(SCHEMA_COLUMNS, getSchemaAsJson()); json.put(FIELD_ALIAS_MAP, fieldAliasMap); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + json.put(PIT_ID, pitId); + String sortFieldValue = + AccessController.doPrivileged( + (PrivilegedAction<String>) + () -> { + try { + return objectMapper.writeValueAsString(sortFields); + } catch (JsonProcessingException e) { + throw new RuntimeException( + "Failed to parse sort fields from JSON string.", e); + } + }); + json.put(SORT_FIELDS, sortFieldValue); + setSearchRequestString(json, searchSourceBuilder); + } else { + json.put(SCROLL_ID, scrollId); + } return String.format("%s:%s", type.getId(), encodeCursor(json)); } + private void setSearchRequestString(JSONObject cursorJson, SearchSourceBuilder sourceBuilder) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + XContentBuilder builder = XContentFactory.jsonBuilder(outputStream); + sourceBuilder.toXContent(builder, null); + builder.close(); + + String searchRequestBase64 = Base64.getEncoder().encodeToString(outputStream.toByteArray()); + cursorJson.put("searchSourceBuilder", searchRequestBase64); + } catch (IOException ex) { + throw new RuntimeException("Failed to set search request string on cursor json.", ex); + } + } + + private boolean isCursorIdNullOrEmpty() { + return LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER) + ? Strings.isNullOrEmpty(pitId) + : Strings.isNullOrEmpty(scrollId); + } + public static DefaultCursor from(String cursorId) { /** * It is assumed that cursorId here is the second part of the original cursor passed by the @@ -105,13 +184,50 @@ public static DefaultCursor from(String cursorId) { cursor.setFetchSize(json.getInt(FETCH_SIZE)); cursor.setRowsLeft(json.getLong(ROWS_LEFT)); cursor.setIndexPattern(json.getString(INDEX_PATTERN)); - cursor.setScrollId(json.getString(SCROLL_ID)); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + populateCursorForPit(json, cursor); + } else { + cursor.setScrollId(json.getString(SCROLL_ID)); + } cursor.setColumns(getColumnsFromSchema(json.getJSONArray(SCHEMA_COLUMNS))); cursor.setFieldAliasMap(fieldAliasMap(json.getJSONObject(FIELD_ALIAS_MAP))); return cursor; } + private static void populateCursorForPit(JSONObject json, DefaultCursor cursor) { + cursor.setPitId(json.getString(PIT_ID)); + + cursor.setSortFields(getSortFieldsFromJson(json)); + + // Retrieve and set the SearchSourceBuilder from the JSON field + String searchSourceBuilderBase64 = json.getString("searchSourceBuilder"); + byte[] bytes = Base64.getDecoder().decode(searchSourceBuilderBase64); + ByteArrayInputStream streamInput = new ByteArrayInputStream(bytes); + try { + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, IGNORE_DEPRECATIONS, streamInput); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.fromXContent(parser); + cursor.setSearchSourceBuilder(sourceBuilder); + } catch (IOException ex) { + throw new RuntimeException("Failed to get searchSourceBuilder from cursor Id", ex); + } + } + + private static Object[] getSortFieldsFromJson(JSONObject json) { + return AccessController.doPrivileged( + (PrivilegedAction<Object[]>) + () -> { + try { + return objectMapper.readValue(json.getString(SORT_FIELDS), Object[].class); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to parse sort fields from JSON string.", e); + } + }); + } + private JSONArray getSchemaAsJson() { JSONArray schemaJson = new JSONArray(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java index 7282eaed4c..222ca5d9fc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.cursor; import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Map; import org.apache.logging.log4j.LogManager; @@ -18,8 +19,11 @@ import org.opensearch.rest.RestChannel; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; public class CursorCloseExecutor implements CursorRestExecutor { @@ -79,14 +83,26 @@ public String execute(Client client, Map<String, String> params) throws Exceptio } private String handleDefaultCursorCloseRequest(Client client, DefaultCursor cursor) { - String scrollId = cursor.getScrollId(); - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollId).get(); - if (clearScrollResponse.isSucceeded()) { - return SUCCEEDED_TRUE; + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); + try { + pit.delete(); + return SUCCEEDED_TRUE; + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + return SUCCEEDED_FALSE; + } } else { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - return SUCCEEDED_FALSE; + String scrollId = cursor.getScrollId(); + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(scrollId).get(); + if (clearScrollResponse.isSucceeded()) { + return SUCCEEDED_TRUE; + } else { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + return SUCCEEDED_FALSE; + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index 66c69f3430..0af3ca243b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -6,6 +6,8 @@ package org.opensearch.sql.legacy.executor.cursor; import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Arrays; import java.util.Map; @@ -14,6 +16,7 @@ import org.json.JSONException; import org.opensearch.OpenSearchException; import org.opensearch.action.search.ClearScrollResponse; +import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; @@ -21,7 +24,8 @@ import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.sql.common.setting.Settings; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; import org.opensearch.sql.legacy.esdomain.LocalClusterState; @@ -29,6 +33,8 @@ import org.opensearch.sql.legacy.executor.format.Protocol; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; public class CursorResultExecutor implements CursorRestExecutor { @@ -91,14 +97,27 @@ public String execute(Client client, Map<String, String> params) throws Exceptio } private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) { - String previousScrollId = cursor.getScrollId(); LocalClusterState clusterState = LocalClusterState.state(); - TimeValue scrollTimeout = clusterState.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); - SearchResponse scrollResponse = - client.prepareSearchScroll(previousScrollId).setScroll(scrollTimeout).get(); + TimeValue paginationTimeout = clusterState.getSettingValue(SQL_CURSOR_KEEP_ALIVE); + + SearchResponse scrollResponse = null; + if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + SearchSourceBuilder source = cursor.getSearchSourceBuilder(); + source.searchAfter(cursor.getSortFields()); + source.pointInTimeBuilder(new PointInTimeBuilder(pitId)); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(source); + scrollResponse = client.search(searchRequest).actionGet(); + } else { + String previousScrollId = cursor.getScrollId(); + scrollResponse = + client.prepareSearchScroll(previousScrollId).setScroll(paginationTimeout).get(); + } SearchHits searchHits = scrollResponse.getHits(); SearchHit[] searchHitArray = searchHits.getHits(); String newScrollId = scrollResponse.getScrollId(); + String newPitId = scrollResponse.pointInTimeId(); int rowsLeft = (int) cursor.getRowsLeft(); int fetch = cursor.getFetchSize(); @@ -124,16 +143,37 @@ private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) { if (rowsLeft <= 0) { /** Clear the scroll context on last page */ - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(newScrollId).get(); - if (!clearScrollResponse.isSucceeded()) { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.info("Error closing the cursor context {} ", newScrollId); + if (newScrollId != null) { + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(newScrollId).get(); + if (!clearScrollResponse.isSucceeded()) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error closing the cursor context {} ", newScrollId); + } + } + if (newPitId != null) { + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, newPitId); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", newPitId); + } } } cursor.setRowsLeft(rowsLeft); - cursor.setScrollId(newScrollId); + if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + cursor.setPitId(newPitId); + cursor.setSearchSourceBuilder(cursor.getSearchSourceBuilder()); + cursor.setSortFields( + scrollResponse + .getHits() + .getAt(scrollResponse.getHits().getHits().length - 1) + .getSortValues()); + } else { + cursor.setScrollId(newScrollId); + } Protocol protocol = new Protocol(client, searchHits, format.name().toLowerCase(), cursor); return protocol.cursorFormat(); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 00feabf5d8..5f758e7d87 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -5,23 +5,31 @@ package org.opensearch.sql.legacy.executor.format; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.sql.legacy.cursor.Cursor; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.QueryActionElasticExecutor; import org.opensearch.sql.legacy.executor.RestExecutor; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.QueryAction; +import org.opensearch.sql.legacy.query.SqlOpenSearchRequestBuilder; import org.opensearch.sql.legacy.query.join.BackOffRetryStrategy; public class PrettyFormatRestExecutor implements RestExecutor { @@ -90,15 +98,32 @@ public String execute(Client client, Map<String, String> params, QueryAction que private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction queryAction) throws SqlParseException { - SearchResponse response = (SearchResponse) queryAction.explain().get(); - String scrollId = response.getScrollId(); + PointInTimeHandler pit = null; + SearchResponse response; + SqlOpenSearchRequestBuilder sqlOpenSearchRequestBuilder = queryAction.explain(); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + pit = new PointInTimeHandlerImpl(client, queryAction.getSelect().getIndexArr()); + pit.create(); + SearchRequestBuilder searchRequest = queryAction.getRequestBuilder(); + searchRequest.setPointInTime(new PointInTimeBuilder(pit.getPitId())); + response = searchRequest.get(); + } else { + response = (SearchResponse) sqlOpenSearchRequestBuilder.get(); + } Protocol protocol; - if (!Strings.isNullOrEmpty(scrollId)) { + if (isDefaultCursor(response, queryAction)) { DefaultCursor defaultCursor = new DefaultCursor(); - defaultCursor.setScrollId(scrollId); defaultCursor.setLimit(queryAction.getSelect().getRowCount()); defaultCursor.setFetchSize(queryAction.getSqlRequest().fetchSize()); + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + defaultCursor.setPitId(pit.getPitId()); + defaultCursor.setSearchSourceBuilder(queryAction.getRequestBuilder().request().source()); + defaultCursor.setSortFields( + response.getHits().getAt(response.getHits().getHits().length - 1).getSortValues()); + } else { + defaultCursor.setScrollId(response.getScrollId()); + } protocol = new Protocol(client, queryAction, response.getHits(), format, defaultCursor); } else { protocol = new Protocol(client, queryAction, response.getHits(), format, Cursor.NULL_CURSOR); @@ -106,4 +131,16 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction return protocol; } + + private boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + if (searchResponse.getHits().getTotalHits().value < queryAction.getSqlRequest().fetchSize()) { + return false; + } else { + return true; + } + } else { + return !Strings.isNullOrEmpty(searchResponse.getScrollId()); + } + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index c60691cb7c..261816cddc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -40,6 +40,7 @@ import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.aggregations.metrics.Percentiles; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.cursor.Cursor; import org.opensearch.sql.legacy.cursor.DefaultCursor; import org.opensearch.sql.legacy.domain.ColumnTypeProvider; @@ -49,11 +50,14 @@ import org.opensearch.sql.legacy.domain.Query; import org.opensearch.sql.legacy.domain.Select; import org.opensearch.sql.legacy.domain.TableOnJoinSelect; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMapping; import org.opensearch.sql.legacy.exception.SqlFeatureNotImplementedException; import org.opensearch.sql.legacy.executor.Format; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; +import org.opensearch.sql.legacy.pit.PointInTimeHandler; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.utils.SQLFunctions; public class SelectResultSet extends ResultSet { @@ -564,12 +568,24 @@ private void populateDefaultCursor(DefaultCursor cursor) { long rowsLeft = rowsLeft(cursor.getFetchSize(), cursor.getLimit()); if (rowsLeft <= 0) { // close the cursor - String scrollId = cursor.getScrollId(); - ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollId).get(); - if (!clearScrollResponse.isSucceeded()) { - Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.error("Error closing the cursor context {} ", scrollId); + if (LocalClusterState.state().getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { + String pitId = cursor.getPitId(); + PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pitId); + } + } else { + // close the cursor + String scrollId = cursor.getScrollId(); + ClearScrollResponse clearScrollResponse = + client.prepareClearScroll().addScrollId(scrollId).get(); + if (!clearScrollResponse.isSucceeded()) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.error("Error closing the cursor context {} ", scrollId); + } } return; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index 061868c9b5..c589edcf50 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -31,6 +31,8 @@ import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.metrics.MetricName; +import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.SqlElasticRequestBuilder; import org.opensearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; @@ -99,7 +101,12 @@ public void run() throws IOException, SqlParseException { LOG.error("Failed during join query run.", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - pit.delete(); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pit); + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index f58b25e821..06186d0695 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -33,6 +33,8 @@ import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.executor.ElasticHitsExecutor; +import org.opensearch.sql.legacy.metrics.MetricName; +import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.query.DefaultQueryAction; import org.opensearch.sql.legacy.query.multi.MultiQueryRequestBuilder; @@ -120,7 +122,12 @@ public void run() throws SqlParseException { LOG.error("Failed during multi query run.", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - pit.delete(); + try { + pit.delete(); + } catch (RuntimeException e) { + Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); + LOG.info("Error deleting point in time {} ", pit); + } } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java index 64535749e8..8d61c03388 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java @@ -7,16 +7,19 @@ import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; +import java.util.concurrent.ExecutionException; import lombok.Getter; import lombok.Setter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.CreatePitAction; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitAction; import org.opensearch.action.search.DeletePitRequest; import org.opensearch.action.search.DeletePitResponse; import org.opensearch.client.Client; -import org.opensearch.core.action.ActionListener; +import org.opensearch.common.action.ActionFuture; import org.opensearch.sql.legacy.esdomain.LocalClusterState; /** Handler for Point In Time */ @@ -37,47 +40,45 @@ public PointInTimeHandlerImpl(Client client, String[] indices) { this.indices = indices; } + /** + * Constructor for class + * + * @param client OpenSearch client + * @param pitId Point In Time ID + */ + public PointInTimeHandlerImpl(Client client, String pitId) { + this.client = client; + this.pitId = pitId; + } + /** Create PIT for given indices */ @Override public void create() { CreatePitRequest createPitRequest = new CreatePitRequest( LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE), false, indices); - client.createPit( - createPitRequest, - new ActionListener<>() { - @Override - public void onResponse(CreatePitResponse createPitResponse) { - pitId = createPitResponse.getId(); - LOG.info("Created Point In Time {} successfully.", pitId); - } - - @Override - public void onFailure(Exception e) { - LOG.error("Error occurred while creating PIT", e); - } - }); + ActionFuture<CreatePitResponse> execute = + client.execute(CreatePitAction.INSTANCE, createPitRequest); + try { + CreatePitResponse pitResponse = execute.get(); + pitId = pitResponse.getId(); + LOG.info("Created Point In Time {} successfully.", pitId); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while creating PIT.", e); + } } /** Delete PIT */ @Override public void delete() { DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); - client.deletePits( - deletePitRequest, - new ActionListener<>() { - @Override - public void onResponse(DeletePitResponse deletePitResponse) { - LOG.info( - "Delete Point In Time {} status: {}", - pitId, - deletePitResponse.status().getStatus()); - } - - @Override - public void onFailure(Exception e) { - LOG.error("Error occurred while deleting PIT", e); - } - }); + ActionFuture<DeletePitResponse> execute = + client.execute(DeletePitAction.INSTANCE, deletePitRequest); + try { + DeletePitResponse deletePitResponse = execute.get(); + LOG.info("Delete Point In Time {} status: {}", pitId, deletePitResponse.status().getStatus()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while deleting PIT.", e); + } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java index 18c9708df8..9877b17a8f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.query; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; @@ -100,7 +102,19 @@ public void checkAndSetScroll() { .getNumericalMetric(MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL) .increment(); Metrics.getInstance().getNumericalMetric(MetricName.DEFAULT_CURSOR_REQUEST_TOTAL).increment(); - request.setSize(fetchSize).setScroll(timeValue); + request.setSize(fetchSize); + // Set scroll or search after for pagination + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + // search after requires results to be in specific order + // set sort field for search_after + boolean ordered = select.isOrderdSelect(); + if (!ordered) { + request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + } + // Request also requires PointInTime, but we should create pit while execution. + } else { + request.setScroll(timeValue); + } } else { request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); setLimit(select.getOffset(), rowCount != null ? rowCount : Select.DEFAULT_LIMIT); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java index 42f1af4563..eba4ae0346 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -2,80 +2,132 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.legacy.pit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; -import java.util.concurrent.CompletableFuture; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import lombok.SneakyThrows; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.action.search.CreatePitAction; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitAction; +import org.opensearch.action.search.DeletePitRequest; import org.opensearch.action.search.DeletePitResponse; import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; public class PointInTimeHandlerImplTest { @Mock private Client mockClient; private String[] indices = {"index1", "index2"}; private PointInTimeHandlerImpl pointInTimeHandlerImpl; - @Captor private ArgumentCaptor<ActionListener<DeletePitResponse>> listenerCaptor; private final String PIT_ID = "testId"; + private CreatePitResponse mockCreatePitResponse; + private DeletePitResponse mockDeletePitResponse; + private ActionFuture<CreatePitResponse> mockActionFuture; + private ActionFuture<DeletePitResponse> mockActionFutureDelete; + + @Mock private OpenSearchSettings settings; @Before public void setUp() { MockitoAnnotations.initMocks(this); pointInTimeHandlerImpl = new PointInTimeHandlerImpl(mockClient, indices); - } - @Test - public void testCreate() { - when(LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE)) + doReturn(Collections.emptyList()).when(settings).getSettings(); + when(settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)) .thenReturn(new TimeValue(10000)); + LocalClusterState.state().setPluginSettings(settings); - CreatePitResponse mockCreatePitResponse = mock(CreatePitResponse.class); + mockCreatePitResponse = mock(CreatePitResponse.class); + mockDeletePitResponse = mock(DeletePitResponse.class); + mockActionFuture = mock(ActionFuture.class); + mockActionFutureDelete = mock(ActionFuture.class); + when(mockClient.execute(any(CreatePitAction.class), any(CreatePitRequest.class))) + .thenReturn(mockActionFuture); + when(mockClient.execute(any(DeletePitAction.class), any(DeletePitRequest.class))) + .thenReturn(mockActionFutureDelete); + RestStatus mockRestStatus = mock(RestStatus.class); + when(mockDeletePitResponse.status()).thenReturn(mockRestStatus); + when(mockDeletePitResponse.status().getStatus()).thenReturn(200); when(mockCreatePitResponse.getId()).thenReturn(PIT_ID); + } - CompletableFuture<CreatePitResponse> completableFuture = - CompletableFuture.completedFuture(mockCreatePitResponse); + @SneakyThrows + @Test + public void testCreate() { + when(mockActionFuture.get()).thenReturn(mockCreatePitResponse); + try { + pointInTimeHandlerImpl.create(); + } catch (RuntimeException e) { + fail("Expected no exception while creating PIT, but got: " + e.getMessage()); + } + verify(mockClient).execute(any(CreatePitAction.class), any(CreatePitRequest.class)); + verify(mockActionFuture).get(); + verify(mockCreatePitResponse).getId(); + } - doAnswer( - invocation -> { - ActionListener<CreatePitResponse> actionListener = invocation.getArgument(1); - actionListener.onResponse(mockCreatePitResponse); - return completableFuture; - }) - .when(mockClient) - .createPit(any(), any()); + @SneakyThrows + @Test + public void testCreateForFailure() { + ExecutionException executionException = + new ExecutionException("Error occurred while creating PIT.", new Throwable()); + when(mockActionFuture.get()).thenThrow(executionException); - pointInTimeHandlerImpl.create(); + RuntimeException thrownException = + assertThrows(RuntimeException.class, () -> pointInTimeHandlerImpl.create()); - assertEquals(PIT_ID, pointInTimeHandlerImpl.getPitId()); + verify(mockClient).execute(any(CreatePitAction.class), any(CreatePitRequest.class)); + assertNotNull(thrownException.getCause()); + assertEquals("Error occurred while creating PIT.", thrownException.getMessage()); + verify(mockActionFuture).get(); } + @SneakyThrows @Test public void testDelete() { - DeletePitResponse mockedResponse = mock(DeletePitResponse.class); - RestStatus mockRestStatus = mock(RestStatus.class); - when(mockedResponse.status()).thenReturn(mockRestStatus); - when(mockedResponse.status().getStatus()).thenReturn(200); - pointInTimeHandlerImpl.setPitId(PIT_ID); - pointInTimeHandlerImpl.delete(); - verify(mockClient).deletePits(any(), listenerCaptor.capture()); - listenerCaptor.getValue().onResponse(mockedResponse); + when(mockActionFutureDelete.get()).thenReturn(mockDeletePitResponse); + try { + pointInTimeHandlerImpl.delete(); + } catch (RuntimeException e) { + fail("Expected no exception while deleting PIT, but got: " + e.getMessage()); + } + verify(mockClient).execute(any(DeletePitAction.class), any(DeletePitRequest.class)); + verify(mockActionFutureDelete).get(); + } + + @SneakyThrows + @Test + public void testDeleteForFailure() { + ExecutionException executionException = + new ExecutionException("Error occurred while deleting PIT.", new Throwable()); + when(mockActionFutureDelete.get()).thenThrow(executionException); + + RuntimeException thrownException = + assertThrows(RuntimeException.class, () -> pointInTimeHandlerImpl.delete()); + + verify(mockClient).execute(any(DeletePitAction.class), any(DeletePitRequest.class)); + assertNotNull(thrownException.getCause()); + assertEquals("Error occurred while deleting PIT.", thrownException.getMessage()); + verify(mockActionFutureDelete).get(); } } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java index 1b9662035d..deff7132b0 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/cursor/DefaultCursorTest.java @@ -9,14 +9,47 @@ import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Collections; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.cursor.CursorType; import org.opensearch.sql.legacy.cursor.DefaultCursor; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; public class DefaultCursorTest { + @Mock private OpenSearchSettings settings; + + @Mock private SearchSourceBuilder sourceBuilder; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + // Required for Pagination queries using PIT instead of Scroll + doReturn(Collections.emptyList()).when(settings).getSettings(); + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + LocalClusterState.state().setPluginSettings(settings); + + // Mock the toXContent method of SearchSourceBuilder + try { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(new ByteArrayOutputStream()); + when(sourceBuilder.toXContent(any(XContentBuilder.class), any())).thenReturn(xContentBuilder); + } catch (Exception e) { + throw new RuntimeException(e); + } + } @Test public void checkCursorType() { @@ -25,7 +58,26 @@ public void checkCursorType() { } @Test - public void cursorShouldStartWithCursorTypeID() { + public void cursorShouldStartWithCursorTypeIDForPIT() { + DefaultCursor cursor = new DefaultCursor(); + cursor.setRowsLeft(50); + cursor.setPitId("dbdskbcdjksbcjkdsbcjk+//"); + cursor.setIndexPattern("myIndex"); + cursor.setFetchSize(500); + cursor.setFieldAliasMap(Collections.emptyMap()); + cursor.setColumns(new ArrayList<>()); + + // Set the mocked SearchSourceBuilder to the cursor + cursor.setSearchSourceBuilder(sourceBuilder); + + assertThat(cursor.generateCursorId(), startsWith(cursor.getType().getId() + ":")); + } + + @Test + public void cursorShouldStartWithCursorTypeIDForScroll() { + // Disable PIT for pagination and use scroll instead + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + DefaultCursor cursor = new DefaultCursor(); cursor.setRowsLeft(50); cursor.setScrollId("dbdskbcdjksbcjkdsbcjk+//"); @@ -33,6 +85,10 @@ public void cursorShouldStartWithCursorTypeID() { cursor.setFetchSize(500); cursor.setFieldAliasMap(Collections.emptyMap()); cursor.setColumns(new ArrayList<>()); + + // Set the mocked SearchSourceBuilder to the cursor + cursor.setSearchSourceBuilder(sourceBuilder); + assertThat(cursor.generateCursorId(), startsWith(cursor.getType().getId() + ":")); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java index 755d604a65..ec6ab00f97 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java @@ -8,16 +8,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; +import static org.mockito.Mockito.*; + +import java.util.*; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -27,6 +20,8 @@ import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; import org.opensearch.script.Script; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.domain.Field; import org.opensearch.sql.legacy.domain.KVValue; @@ -149,6 +144,12 @@ public void testIfScrollShouldBeOpenWithDifferentFormats() { queryAction.setFormat(Format.JDBC); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(settingFetchSize); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -168,6 +169,12 @@ public void testIfScrollShouldBeOpen() { mockLocalClusterStateAndInitializeMetrics(timeValue); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(settingFetchSize); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -195,6 +202,12 @@ public void testIfScrollShouldBeOpenWithDifferentFetchSize() { doReturn(mockRequestBuilder).when(mockRequestBuilder).setSize(userFetchSize); queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(20); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setScroll(timeValue); } @@ -216,7 +229,9 @@ public void testIfScrollShouldBeOpenWithDifferentValidFetchSizeAndLimit() { queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(userFetchSize); - Mockito.verify(mockRequestBuilder).setScroll(timeValue); + Mockito.verify(mockRequestBuilder).addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + // Skip setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); /** fetchSize > LIMIT - no scroll */ userFetchSize = 5000; @@ -226,6 +241,14 @@ public void testIfScrollShouldBeOpenWithDifferentValidFetchSizeAndLimit() { queryAction.checkAndSetScroll(); Mockito.verify(mockRequestBuilder).setSize(limit); Mockito.verify(mockRequestBuilder, never()).setScroll(timeValue); + + // Verify setScroll when SQL_PAGINATION_API_SEARCH_AFTER is set to false + mockLocalClusterStateAndIntializeMetricsForScroll(timeValue); + /** fetchSize <= LIMIT - open scroll */ + userFetchSize = 1500; + doReturn(userFetchSize).when(mockSqlRequest).fetchSize(); + queryAction.checkAndSetScroll(); + Mockito.verify(mockRequestBuilder).setScroll(timeValue); } private void mockLocalClusterStateAndInitializeMetrics(TimeValue time) { @@ -236,6 +259,24 @@ private void mockLocalClusterStateAndInitializeMetrics(TimeValue time) { .when(mockLocalClusterState) .getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW); doReturn(2L).when(mockLocalClusterState).getSettingValue(Settings.Key.METRICS_ROLLING_INTERVAL); + doReturn(true) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); + + Metrics.getInstance().registerDefaultMetrics(); + } + + private void mockLocalClusterStateAndIntializeMetricsForScroll(TimeValue time) { + LocalClusterState mockLocalClusterState = mock(LocalClusterState.class); + LocalClusterState.state(mockLocalClusterState); + doReturn(time).when(mockLocalClusterState).getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); + doReturn(3600L) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW); + doReturn(2L).when(mockLocalClusterState).getSettingValue(Settings.Key.METRICS_ROLLING_INTERVAL); + doReturn(false) + .when(mockLocalClusterState) + .getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); Metrics.getInstance().registerDefaultMetrics(); } From 83bc3d2cdcea815200ad6e0d2b608cea1a88c2e2 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 29 Aug 2024 12:54:03 -0700 Subject: [PATCH 30/96] Add mvQuery attribute in IndexQueryDetails (#2946) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../dispatcher/model/IndexQueryDetails.java | 6 ++++++ .../sql/spark/utils/SQLQueryUtils.java | 11 +++++++++++ .../sql/spark/utils/SQLQueryUtilsTest.java | 19 +++++++++++++++---- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java index 5596d1b425..2ca997f6b0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java @@ -31,6 +31,7 @@ public class IndexQueryDetails { // materialized view special case where // table name and mv name are combined. private String mvName; + private String mvQuery; private FlintIndexType indexType; private IndexQueryDetails() {} @@ -73,6 +74,11 @@ public IndexQueryDetailsBuilder mvName(String mvName) { return this; } + public IndexQueryDetailsBuilder mvQuery(String mvQuery) { + indexQueryDetails.mvQuery = mvQuery; + return this; + } + public IndexQueryDetailsBuilder indexType(FlintIndexType indexType) { indexQueryDetails.indexType = indexType; return this; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index 0bb9cb4b85..ff08a8f41e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -13,6 +13,7 @@ import lombok.Getter; import lombok.experimental.UtilityClass; import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; @@ -20,6 +21,7 @@ import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsBaseVisitor; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsLexer; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser; +import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser.MaterializedViewQueryContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.IdentifierReferenceContext; @@ -353,6 +355,15 @@ public Void visitAlterMaterializedViewStatement( return super.visitAlterMaterializedViewStatement(ctx); } + @Override + public Void visitMaterializedViewQuery(MaterializedViewQueryContext ctx) { + int a = ctx.start.getStartIndex(); + int b = ctx.stop.getStopIndex(); + String query = ctx.start.getInputStream().getText(new Interval(a, b)); + indexQueryDetailsBuilder.mvQuery(query); + return super.visitMaterializedViewQuery(ctx); + } + private String propertyKey(FlintSparkSqlExtensionsParser.PropertyKeyContext key) { if (key.STRING() != null) { return key.STRING().getText(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index bf6fe9e5db..fe7777606c 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -181,11 +181,22 @@ void testExtractionFromFlintCoveringIndexQueries() { } } + @Test + void testExtractionFromCreateMVQuery() { + String mvQuery = "select * from my_glue.default.logs"; + String query = "CREATE MATERIALIZED VIEW mv_1 AS " + mvQuery + " WITH (auto_refresh = true)"; + + assertTrue(SQLQueryUtils.isFlintExtensionQuery(query)); + IndexQueryDetails indexQueryDetails = SQLQueryUtils.extractIndexDetails(query); + assertNull(indexQueryDetails.getIndexName()); + assertNull(indexQueryDetails.getFullyQualifiedTableName()); + assertEquals(mvQuery, indexQueryDetails.getMvQuery()); + assertEquals("mv_1", indexQueryDetails.getMvName()); + } + @Test void testExtractionFromFlintMVQuery() { String[] mvQueries = { - "CREATE MATERIALIZED VIEW mv_1 AS query=select * from my_glue.default.logs WITH" - + " (auto_refresh = true)", "DROP MATERIALIZED VIEW mv_1", "VACUUM MATERIALIZED VIEW mv_1", "ALTER MATERIALIZED VIEW mv_1 WITH (auto_refresh = false)", @@ -200,6 +211,7 @@ void testExtractionFromFlintMVQuery() { assertNull(indexQueryDetails.getIndexName()); assertNull(fullyQualifiedTableName); + assertNull(indexQueryDetails.getMvQuery()); assertEquals("mv_1", indexQueryDetails.getMvName()); } } @@ -428,8 +440,7 @@ public static IndexQuery index() { } public static IndexQuery mv() { - return new IndexQuery( - "CREATE MATERIALIZED VIEW mv_1 AS query=select * from my_glue.default.logs"); + return new IndexQuery("CREATE MATERIALIZED VIEW mv_1 AS select * from my_glue.default.logs"); } public IndexQuery withProperty(String key, String value) { From e1ee3b130ab19b99bffd2c80c5c9bb40ac9bb145 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 29 Aug 2024 15:07:29 -0700 Subject: [PATCH 31/96] Add AsyncQueryRequestContext to update/get in StatementStorageService (#2943) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../asyncquery/AsyncQueryExecutorService.java | 3 +- .../AsyncQueryExecutorServiceImpl.java | 6 ++- .../spark/dispatcher/AsyncQueryHandler.java | 15 ++++-- .../spark/dispatcher/BatchQueryHandler.java | 8 +++- .../sql/spark/dispatcher/IndexDMLHandler.java | 8 +++- .../dispatcher/InteractiveQueryHandler.java | 22 ++++++--- .../dispatcher/SparkQueryDispatcher.java | 6 ++- .../execution/session/InteractiveSession.java | 6 ++- .../sql/spark/execution/session/Session.java | 2 +- .../spark/execution/statement/Statement.java | 3 +- .../statestore/StatementStorageService.java | 7 ++- .../asyncquery/AsyncQueryCoreIntegTest.java | 17 ++++--- .../AsyncQueryExecutorServiceImplTest.java | 14 ++++-- .../spark/dispatcher/IndexDMLHandlerTest.java | 4 +- .../dispatcher/SparkQueryDispatcherTest.java | 21 +++++---- .../OpenSearchStatementStorageService.java | 12 +++-- .../TransportGetAsyncQueryResultAction.java | 3 +- ...AsyncQueryExecutorServiceImplSpecTest.java | 24 ++++++---- .../AsyncQueryGetResultSpecTest.java | 13 ++++-- .../asyncquery/IndexQuerySpecAlterTest.java | 46 ++++++++++++------- .../spark/asyncquery/IndexQuerySpecTest.java | 39 ++++++++++------ .../asyncquery/IndexQuerySpecVacuumTest.java | 3 +- .../execution/statement/StatementTest.java | 19 +++++--- ...ransportGetAsyncQueryResultActionTest.java | 20 ++++++-- 24 files changed, 217 insertions(+), 104 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java index b0c339e93d..1240545acd 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java @@ -31,7 +31,8 @@ CreateAsyncQueryResponse createAsyncQuery( * @param queryId queryId. * @return {@link AsyncQueryExecutionResponse} */ - AsyncQueryExecutionResponse getAsyncQueryResults(String queryId); + AsyncQueryExecutionResponse getAsyncQueryResults( + String queryId, AsyncQueryRequestContext asyncQueryRequestContext); /** * Cancels running async query and returns the cancelled queryId. diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java index d304766465..5933343ba4 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java @@ -74,12 +74,14 @@ public CreateAsyncQueryResponse createAsyncQuery( } @Override - public AsyncQueryExecutionResponse getAsyncQueryResults(String queryId) { + public AsyncQueryExecutionResponse getAsyncQueryResults( + String queryId, AsyncQueryRequestContext asyncQueryRequestContext) { Optional<AsyncQueryJobMetadata> jobMetadata = asyncQueryJobMetadataStorageService.getJobMetadata(queryId); if (jobMetadata.isPresent()) { String sessionId = jobMetadata.get().getSessionId(); - JSONObject jsonObject = sparkQueryDispatcher.getQueryResponse(jobMetadata.get()); + JSONObject jsonObject = + sparkQueryDispatcher.getQueryResponse(jobMetadata.get(), asyncQueryRequestContext); if (JobRunState.SUCCESS.toString().equals(jsonObject.getString(STATUS_FIELD))) { DefaultSparkSqlFunctionResponseHandle sparkSqlFunctionResponseHandle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java index 2bafd88b85..441846d678 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java @@ -21,8 +21,10 @@ /** Process async query request. */ public abstract class AsyncQueryHandler { - public JSONObject getQueryResponse(AsyncQueryJobMetadata asyncQueryJobMetadata) { - JSONObject result = getResponseFromResultIndex(asyncQueryJobMetadata); + public JSONObject getQueryResponse( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { + JSONObject result = getResponseFromResultIndex(asyncQueryJobMetadata, asyncQueryRequestContext); if (result.has(DATA_FIELD)) { JSONObject items = result.getJSONObject(DATA_FIELD); @@ -35,7 +37,8 @@ public JSONObject getQueryResponse(AsyncQueryJobMetadata asyncQueryJobMetadata) result.put(ERROR_FIELD, error); return result; } else { - JSONObject statement = getResponseFromExecutor(asyncQueryJobMetadata); + JSONObject statement = + getResponseFromExecutor(asyncQueryJobMetadata, asyncQueryRequestContext); // Consider statement still running if state is success but query result unavailable if (isSuccessState(statement)) { @@ -50,10 +53,12 @@ private boolean isSuccessState(JSONObject statement) { } protected abstract JSONObject getResponseFromResultIndex( - AsyncQueryJobMetadata asyncQueryJobMetadata); + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext); protected abstract JSONObject getResponseFromExecutor( - AsyncQueryJobMetadata asyncQueryJobMetadata); + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext); public abstract String cancelJob( AsyncQueryJobMetadata asyncQueryJobMetadata, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 661ebe27fc..bce1918631 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -41,7 +41,9 @@ public class BatchQueryHandler extends AsyncQueryHandler { protected final SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider; @Override - protected JSONObject getResponseFromResultIndex(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { // either empty json when the result is not available or data with status // Fetch from Result Index return jobExecutionResponseReader.getResultWithJobId( @@ -49,7 +51,9 @@ protected JSONObject getResponseFromResultIndex(AsyncQueryJobMetadata asyncQuery } @Override - protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromExecutor( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { JSONObject result = new JSONObject(); // make call to EMR Serverless when related result index documents are not available GetJobRunResult getJobRunResult = diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index f8217142c3..d5885e6f2a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -162,14 +162,18 @@ private FlintIndexMetadata getFlintIndexMetadata( } @Override - protected JSONObject getResponseFromResultIndex(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( queryId, asyncQueryJobMetadata.getResultIndex()); } @Override - protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromExecutor( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { // Consider statement still running if result doc created in submit() is not available yet JSONObject result = new JSONObject(); result.put(STATUS_FIELD, StatementState.RUNNING.getState()); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java index 9a9baedde2..7be6809912 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java @@ -50,21 +50,26 @@ public class InteractiveQueryHandler extends AsyncQueryHandler { protected final SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider; @Override - protected JSONObject getResponseFromResultIndex(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( queryId, asyncQueryJobMetadata.getResultIndex()); } @Override - protected JSONObject getResponseFromExecutor(AsyncQueryJobMetadata asyncQueryJobMetadata) { + protected JSONObject getResponseFromExecutor( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { JSONObject result = new JSONObject(); String queryId = asyncQueryJobMetadata.getQueryId(); Statement statement = getStatementByQueryId( asyncQueryJobMetadata.getSessionId(), queryId, - asyncQueryJobMetadata.getDatasourceName()); + asyncQueryJobMetadata.getDatasourceName(), + asyncQueryRequestContext); StatementState statementState = statement.getStatementState(); result.put(STATUS_FIELD, statementState.getState()); result.put(ERROR_FIELD, Optional.of(statement.getStatementModel().getError()).orElse("")); @@ -79,7 +84,8 @@ public String cancelJob( getStatementByQueryId( asyncQueryJobMetadata.getSessionId(), queryId, - asyncQueryJobMetadata.getDatasourceName()) + asyncQueryJobMetadata.getDatasourceName(), + asyncQueryRequestContext) .cancel(); return queryId; } @@ -148,12 +154,16 @@ public DispatchQueryResponse submit( .build(); } - private Statement getStatementByQueryId(String sessionId, String queryId, String datasourceName) { + private Statement getStatementByQueryId( + String sessionId, + String queryId, + String datasourceName, + AsyncQueryRequestContext asyncQueryRequestContext) { Optional<Session> session = sessionManager.getSession(sessionId, datasourceName); if (session.isPresent()) { // todo, statementId == jobId if statement running in session. StatementId statementId = new StatementId(queryId); - Optional<Statement> statement = session.get().get(statementId); + Optional<Statement> statement = session.get().get(statementId, asyncQueryRequestContext); if (statement.isPresent()) { return statement.get(); } else { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index a6fdd3f102..710f472acb 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -157,9 +157,11 @@ private boolean isEligibleForIndexDMLHandling(IndexQueryDetails indexQueryDetail && !indexQueryDetails.getFlintIndexOptions().autoRefresh())); } - public JSONObject getQueryResponse(AsyncQueryJobMetadata asyncQueryJobMetadata) { + public JSONObject getQueryResponse( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { return getAsyncQueryHandlerForExistingQuery(asyncQueryJobMetadata) - .getQueryResponse(asyncQueryJobMetadata); + .getQueryResponse(asyncQueryJobMetadata, asyncQueryRequestContext); } public String cancelJob( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java index aeedaef4e7..2915e2a3e1 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java @@ -121,9 +121,10 @@ public StatementId submit( } @Override - public Optional<Statement> get(StatementId stID) { + public Optional<Statement> get( + StatementId stID, AsyncQueryRequestContext asyncQueryRequestContext) { return statementStorageService - .getStatement(stID.getId(), sessionModel.getDatasourceName()) + .getStatement(stID.getId(), sessionModel.getDatasourceName(), asyncQueryRequestContext) .map( model -> Statement.builder() @@ -137,6 +138,7 @@ public Optional<Statement> get(StatementId stID) { .queryId(model.getQueryId()) .statementStorageService(statementStorageService) .statementModel(model) + .asyncQueryRequestContext(asyncQueryRequestContext) .build()); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java index fad097ca1b..4c083d79c4 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java @@ -35,7 +35,7 @@ void open( * @param stID {@link StatementId} * @return {@link Statement} */ - Optional<Statement> get(StatementId stID); + Optional<Statement> get(StatementId stID, AsyncQueryRequestContext asyncQueryRequestContext); SessionModel getSessionModel(); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java index 3237a5d372..272f0edf4a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java @@ -70,7 +70,8 @@ public void cancel() { throw new IllegalStateException(errorMsg); } this.statementModel = - statementStorageService.updateStatementState(statementModel, StatementState.CANCELLED); + statementStorageService.updateStatementState( + statementModel, StatementState.CANCELLED, asyncQueryRequestContext); } public StatementState getStatementState() { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java index 39f1ecf704..b9446809fb 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java @@ -20,7 +20,10 @@ StatementModel createStatement( StatementModel statementModel, AsyncQueryRequestContext asyncQueryRequestContext); StatementModel updateStatementState( - StatementModel oldStatementModel, StatementState statementState); + StatementModel oldStatementModel, + StatementState statementState, + AsyncQueryRequestContext asyncQueryRequestContext); - Optional<StatementModel> getStatement(String id, String datasourceName); + Optional<StatementModel> getStatement( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index ff92762a7c..feb8c8c0ac 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -377,7 +377,8 @@ public void getResultOfInteractiveQuery() { when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) .thenReturn(result); - AsyncQueryExecutionResponse response = asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID); + AsyncQueryExecutionResponse response = + asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID, asyncQueryRequestContext); assertEquals("SUCCESS", response.getStatus()); assertEquals(SESSION_ID, response.getSessionId()); @@ -395,7 +396,8 @@ public void getResultOfIndexDMLQuery() { when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) .thenReturn(result); - AsyncQueryExecutionResponse response = asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID); + AsyncQueryExecutionResponse response = + asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID, asyncQueryRequestContext); assertEquals("SUCCESS", response.getStatus()); assertNull(response.getSessionId()); @@ -413,7 +415,8 @@ public void getResultOfRefreshQuery() { JSONObject result = getValidExecutionResponse(); when(jobExecutionResponseReader.getResultWithJobId(JOB_ID, RESULT_INDEX)).thenReturn(result); - AsyncQueryExecutionResponse response = asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID); + AsyncQueryExecutionResponse response = + asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID, asyncQueryRequestContext); assertEquals("SUCCESS", response.getStatus()); assertNull(response.getSessionId()); @@ -428,13 +431,15 @@ public void cancelInteractiveQuery() { final StatementModel statementModel = givenStatementExists(); StatementModel canceledStatementModel = StatementModel.copyWithState(statementModel, StatementState.CANCELLED, ImmutableMap.of()); - when(statementStorageService.updateStatementState(statementModel, StatementState.CANCELLED)) + when(statementStorageService.updateStatementState( + statementModel, StatementState.CANCELLED, asyncQueryRequestContext)) .thenReturn(canceledStatementModel); String result = asyncQueryExecutorService.cancelQuery(QUERY_ID, asyncQueryRequestContext); assertEquals(QUERY_ID, result); - verify(statementStorageService).updateStatementState(statementModel, StatementState.CANCELLED); + verify(statementStorageService) + .updateStatementState(statementModel, StatementState.CANCELLED, asyncQueryRequestContext); } @Test @@ -596,7 +601,7 @@ private StatementModel givenStatementExists() { .statementId(new StatementId(QUERY_ID)) .statementState(StatementState.RUNNING) .build(); - when(statementStorageService.getStatement(QUERY_ID, DATASOURCE_NAME)) + when(statementStorageService.getStatement(QUERY_ID, DATASOURCE_NAME, asyncQueryRequestContext)) .thenReturn(Optional.of(statementModel)); return statementModel; } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index 5d8d9a3b63..1491f0bd61 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -152,7 +152,7 @@ void testGetAsyncQueryResultsWithJobNotFoundException() { AsyncQueryNotFoundException asyncQueryNotFoundException = Assertions.assertThrows( AsyncQueryNotFoundException.class, - () -> jobExecutorService.getAsyncQueryResults(EMR_JOB_ID)); + () -> jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext)); Assertions.assertEquals( "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); @@ -166,10 +166,12 @@ void testGetAsyncQueryResultsWithInProgressJob() { .thenReturn(Optional.of(getAsyncQueryJobMetadata())); JSONObject jobResult = new JSONObject(); jobResult.put("status", JobRunState.PENDING.toString()); - when(sparkQueryDispatcher.getQueryResponse(getAsyncQueryJobMetadata())).thenReturn(jobResult); + when(sparkQueryDispatcher.getQueryResponse( + getAsyncQueryJobMetadata(), asyncQueryRequestContext)) + .thenReturn(jobResult); AsyncQueryExecutionResponse asyncQueryExecutionResponse = - jobExecutorService.getAsyncQueryResults(EMR_JOB_ID); + jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext); Assertions.assertNull(asyncQueryExecutionResponse.getResults()); Assertions.assertNull(asyncQueryExecutionResponse.getSchema()); @@ -183,10 +185,12 @@ void testGetAsyncQueryResultsWithSuccessJob() throws IOException { .thenReturn(Optional.of(getAsyncQueryJobMetadata())); JSONObject jobResult = new JSONObject(getJson("select_query_response.json")); jobResult.put("status", JobRunState.SUCCESS.toString()); - when(sparkQueryDispatcher.getQueryResponse(getAsyncQueryJobMetadata())).thenReturn(jobResult); + when(sparkQueryDispatcher.getQueryResponse( + getAsyncQueryJobMetadata(), asyncQueryRequestContext)) + .thenReturn(jobResult); AsyncQueryExecutionResponse asyncQueryExecutionResponse = - jobExecutorService.getAsyncQueryResults(EMR_JOB_ID); + jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext); Assertions.assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); Assertions.assertEquals(1, asyncQueryExecutionResponse.getSchema().getColumns().size()); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java index 9a3c4e663e..570a7cab7d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java @@ -66,7 +66,9 @@ class IndexDMLHandlerTest { @Test public void getResponseFromExecutor() { - JSONObject result = new IndexDMLHandler(null, null, null, null).getResponseFromExecutor(null); + JSONObject result = + new IndexDMLHandler(null, null, null, null) + .getResponseFromExecutor(null, asyncQueryRequestContext); assertEquals("running", result.getString(STATUS_FIELD)); assertEquals("", result.getString(ERROR_FIELD)); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index ee840e8b4c..b6369292a6 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -847,7 +847,7 @@ void testCancelJob() { @Test void testCancelQueryWithSession() { doReturn(Optional.of(session)).when(sessionManager).getSession(MOCK_SESSION_ID, MY_GLUE); - doReturn(Optional.of(statement)).when(session).get(any()); + doReturn(Optional.of(statement)).when(session).get(any(), eq(asyncQueryRequestContext)); doNothing().when(statement).cancel(); String queryId = @@ -919,7 +919,8 @@ void testGetQueryResponse() { when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)) .thenReturn(new JSONObject()); - JSONObject result = sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata()); + JSONObject result = + sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); Assertions.assertEquals("PENDING", result.get("status")); } @@ -927,7 +928,7 @@ void testGetQueryResponse() { @Test void testGetQueryResponseWithSession() { doReturn(Optional.of(session)).when(sessionManager).getSession(MOCK_SESSION_ID, MY_GLUE); - doReturn(Optional.of(statement)).when(session).get(any()); + doReturn(Optional.of(statement)).when(session).get(any(), eq(asyncQueryRequestContext)); when(statement.getStatementModel().getError()).thenReturn("mock error"); doReturn(StatementState.WAITING).when(statement).getStatementState(); doReturn(new JSONObject()) @@ -936,7 +937,8 @@ void testGetQueryResponseWithSession() { JSONObject result = sparkQueryDispatcher.getQueryResponse( - asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID)); + asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID), + asyncQueryRequestContext); verifyNoInteractions(emrServerlessClient); Assertions.assertEquals("waiting", result.get("status")); @@ -954,7 +956,8 @@ void testGetQueryResponseWithInvalidSession() { IllegalArgumentException.class, () -> sparkQueryDispatcher.getQueryResponse( - asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID))); + asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID), + asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); Assertions.assertEquals("no session found. " + MOCK_SESSION_ID, exception.getMessage()); @@ -963,7 +966,7 @@ void testGetQueryResponseWithInvalidSession() { @Test void testGetQueryResponseWithStatementNotExist() { doReturn(Optional.of(session)).when(sessionManager).getSession(MOCK_SESSION_ID, MY_GLUE); - doReturn(Optional.empty()).when(session).get(any()); + doReturn(Optional.empty()).when(session).get(any(), eq(asyncQueryRequestContext)); doReturn(new JSONObject()) .when(jobExecutionResponseReader) .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); @@ -973,7 +976,8 @@ void testGetQueryResponseWithStatementNotExist() { IllegalArgumentException.class, () -> sparkQueryDispatcher.getQueryResponse( - asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID))); + asyncQueryJobMetadataWithSessionId(MOCK_STATEMENT_ID, MOCK_SESSION_ID), + asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); Assertions.assertEquals( @@ -989,7 +993,8 @@ void testGetQueryResponseWithSuccess() { queryResult.put(DATA_FIELD, resultMap); when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)).thenReturn(queryResult); - JSONObject result = sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata()); + JSONObject result = + sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); verify(jobExecutionResponseReader, times(1)).getResultWithJobId(EMR_JOB_ID, null); Assertions.assertEquals( diff --git a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java index 67d0609ca5..527cd24bc8 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java @@ -40,14 +40,17 @@ public StatementModel createStatement( } @Override - public Optional<StatementModel> getStatement(String id, String datasourceName) { + public Optional<StatementModel> getStatement( + String id, String datasourceName, AsyncQueryRequestContext asyncQueryRequestContext) { return stateStore.get( id, serializer::fromXContent, OpenSearchStateStoreUtil.getIndexName(datasourceName)); } @Override public StatementModel updateStatementState( - StatementModel oldStatementModel, StatementState statementState) { + StatementModel oldStatementModel, + StatementState statementState, + AsyncQueryRequestContext asyncQueryRequestContext) { try { return stateStore.updateState( oldStatementModel, @@ -63,7 +66,10 @@ public StatementModel updateStatementState( throw new IllegalStateException(errorMsg); } catch (VersionConflictEngineException e) { StatementModel statementModel = - getStatement(oldStatementModel.getId(), oldStatementModel.getDatasourceName()) + getStatement( + oldStatementModel.getId(), + oldStatementModel.getDatasourceName(), + asyncQueryRequestContext) .orElse(oldStatementModel); String errorMsg = String.format( diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java index 0e9da0c13c..250837e0cd 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java @@ -16,6 +16,7 @@ import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService; import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; import org.opensearch.sql.spark.transport.format.AsyncQueryResultResponseFormatter; import org.opensearch.sql.spark.transport.model.AsyncQueryResult; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionRequest; @@ -50,7 +51,7 @@ protected void doExecute( try { String jobId = request.getQueryId(); AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(jobId); + asyncQueryExecutorService.getAsyncQueryResults(jobId, new NullAsyncQueryRequestContext()); ResponseFormatter<AsyncQueryResult> formatter = new AsyncQueryResultResponseFormatter(JsonResponseFormatter.Style.PRETTY); String responseContent = diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java index ede8a348b4..db0adfc156 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java @@ -66,7 +66,8 @@ public void withoutSessionCreateAsyncQueryThenGetResultThenCancel() { // 2. fetch async query result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("RUNNING", asyncQueryResults.getStatus()); emrsClient.getJobRunResultCalled(1); @@ -152,13 +153,15 @@ public void withSessionCreateAsyncQueryThenGetResultThenCancel() { asyncQueryRequestContext); assertNotNull(response.getSessionId()); Optional<StatementModel> statementModel = - statementStorageService.getStatement(response.getQueryId(), MYS3_DATASOURCE); + statementStorageService.getStatement( + response.getQueryId(), MYS3_DATASOURCE, asyncQueryRequestContext); assertTrue(statementModel.isPresent()); assertEquals(StatementState.WAITING, statementModel.get().getStatementState()); // 2. fetch async query result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("", asyncQueryResults.getError()); assertTrue(Strings.isEmpty(asyncQueryResults.getError())); assertEquals(StatementState.WAITING.getState(), asyncQueryResults.getStatus()); @@ -211,13 +214,15 @@ public void reuseSessionWhenCreateAsyncQuery() { .must(QueryBuilders.termQuery(SESSION_ID, first.getSessionId())))); Optional<StatementModel> firstModel = - statementStorageService.getStatement(first.getQueryId(), MYS3_DATASOURCE); + statementStorageService.getStatement( + first.getQueryId(), MYS3_DATASOURCE, asyncQueryRequestContext); assertTrue(firstModel.isPresent()); assertEquals(StatementState.WAITING, firstModel.get().getStatementState()); assertEquals(first.getQueryId(), firstModel.get().getStatementId().getId()); assertEquals(first.getQueryId(), firstModel.get().getQueryId()); Optional<StatementModel> secondModel = - statementStorageService.getStatement(second.getQueryId(), MYS3_DATASOURCE); + statementStorageService.getStatement( + second.getQueryId(), MYS3_DATASOURCE, asyncQueryRequestContext); assertEquals(StatementState.WAITING, secondModel.get().getStatementState()); assertEquals(second.getQueryId(), secondModel.get().getStatementId().getId()); assertEquals(second.getQueryId(), secondModel.get().getQueryId()); @@ -311,7 +316,8 @@ public void withSessionCreateAsyncQueryFailed() { asyncQueryRequestContext); assertNotNull(response.getSessionId()); Optional<StatementModel> statementModel = - statementStorageService.getStatement(response.getQueryId(), MYS3_DATASOURCE); + statementStorageService.getStatement( + response.getQueryId(), MYS3_DATASOURCE, asyncQueryRequestContext); assertTrue(statementModel.isPresent()); assertEquals(StatementState.WAITING, statementModel.get().getStatementState()); @@ -334,10 +340,12 @@ public void withSessionCreateAsyncQueryFailed() { .error("mock error") .metadata(submitted.getMetadata()) .build(); - statementStorageService.updateStatementState(mocked, StatementState.FAILED); + statementStorageService.updateStatementState( + mocked, StatementState.FAILED, asyncQueryRequestContext); AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals(StatementState.FAILED.getState(), asyncQueryResults.getStatus()); assertEquals("mock error", asyncQueryResults.getError()); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java index e0f04761c7..7ccbad969d 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java @@ -450,7 +450,8 @@ AssertionHelper withInteraction(Interaction interaction) { AssertionHelper assertQueryResults(String status, List<ExprValue> data) { AsyncQueryExecutionResponse results = - queryService.getAsyncQueryResults(createQueryResponse.getQueryId()); + queryService.getAsyncQueryResults( + createQueryResponse.getQueryId(), asyncQueryRequestContext); assertEquals(status, results.getStatus()); assertEquals(data, results.getResults()); return this; @@ -458,7 +459,8 @@ AssertionHelper assertQueryResults(String status, List<ExprValue> data) { AssertionHelper assertFormattedQueryResults(String expected) { AsyncQueryExecutionResponse results = - queryService.getAsyncQueryResults(createQueryResponse.getQueryId()); + queryService.getAsyncQueryResults( + createQueryResponse.getQueryId(), asyncQueryRequestContext); ResponseFormatter<AsyncQueryResult> formatter = new AsyncQueryResultResponseFormatter(JsonResponseFormatter.Style.COMPACT); @@ -515,8 +517,11 @@ void emrJobWriteResultDoc(Map<String, Object> resultDoc) { /** Simulate EMR-S updates query_execution_request with state */ void emrJobUpdateStatementState(StatementState newState) { - StatementModel stmt = statementStorageService.getStatement(queryId, MYS3_DATASOURCE).get(); - statementStorageService.updateStatementState(stmt, newState); + StatementModel stmt = + statementStorageService + .getStatement(queryId, MYS3_DATASOURCE, asyncQueryRequestContext) + .get(); + statementStorageService.updateStatementState(stmt, newState, asyncQueryRequestContext); } void emrJobUpdateJobState(JobRunState jobState) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java index 70a43e42d5..d69c7d4864 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java @@ -86,7 +86,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.cancelJobRunCalled(1); @@ -155,7 +156,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.cancelJobRunCalled(1); @@ -237,7 +239,8 @@ public CancelJobRunResult cancelJobRun( // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.cancelJobRunCalled(1); @@ -303,7 +306,7 @@ public void testAlterIndexQueryConvertingToAutoRefresh() { assertEquals( "RUNNING", asyncQueryExecutorService - .getAsyncQueryResults(response.getQueryId()) + .getAsyncQueryResults(response.getQueryId(), asyncQueryRequestContext) .getStatus()); flintIndexJob.assertState(FlintIndexState.ACTIVE); @@ -369,7 +372,7 @@ public void testAlterIndexQueryWithOutAnyAutoRefresh() { assertEquals( "RUNNING", asyncQueryExecutorService - .getAsyncQueryResults(response.getQueryId()) + .getAsyncQueryResults(response.getQueryId(), asyncQueryRequestContext) .getStatus()); flintIndexJob.assertState(FlintIndexState.ACTIVE); @@ -442,7 +445,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Altering to full refresh only allows: [auto_refresh, incremental_refresh]" @@ -517,7 +521,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Altering to incremental refresh only allows: [auto_refresh, incremental_refresh," @@ -586,7 +591,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Conversion to incremental refresh index cannot proceed due to missing" @@ -648,7 +654,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Conversion to incremental refresh index cannot proceed due to missing" @@ -712,7 +719,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Conversion to incremental refresh index cannot proceed due to missing" @@ -776,7 +784,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.getJobRunResultCalled(1); @@ -837,7 +846,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.getJobRunResultCalled(1); @@ -896,7 +906,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( "Transaction failed as flint index is not in a valid state.", @@ -963,7 +974,8 @@ public CancelJobRunResult cancelJobRun( // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); emrsClient.startJobRunCalled(0); emrsClient.cancelJobRunCalled(1); @@ -1028,7 +1040,8 @@ public CancelJobRunResult cancelJobRun( // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals("Internal Server Error.", asyncQueryExecutionResponse.getError()); emrsClient.startJobRunCalled(0); @@ -1094,7 +1107,8 @@ public CancelJobRunResult cancelJobRun( // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals("Internal Server Error.", asyncQueryExecutionResponse.getError()); emrsClient.startJobRunCalled(0); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java index 29c42446b3..920981abf1 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java @@ -143,7 +143,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2.fetch result AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryResults.getStatus()); assertNull(asyncQueryResults.getError()); emrsClient.cancelJobRunCalled(1); @@ -193,7 +194,8 @@ public CancelJobRunResult cancelJobRun( // 2.fetch result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryResults.getStatus()); assertNull(asyncQueryResults.getError()); }); @@ -233,7 +235,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryResults.getStatus()); assertEquals("Cancel job operation timed out.", asyncQueryResults.getError()); }); @@ -270,7 +273,8 @@ public CancelJobRunResult cancelJobRun( // 2.fetch result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryResults.getStatus()); assertNull(asyncQueryResults.getError()); } @@ -319,7 +323,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2.fetch result AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryResults.getStatus()); assertNull(asyncQueryResults.getError()); emrsClient.cancelJobRunCalled(1); @@ -375,7 +380,8 @@ public CancelJobRunResult cancelJobRun( // 2.fetch result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryResults.getStatus()); assertNull(asyncQueryResults.getError()); @@ -422,7 +428,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryResults.getStatus()); assertEquals("Cancel job operation timed out.", asyncQueryResults.getError()); flintIndexJob.assertState(FlintIndexState.REFRESHING); @@ -470,7 +477,7 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { assertEquals( "SUCCESS", asyncQueryExecutorService - .getAsyncQueryResults(response.getQueryId()) + .getAsyncQueryResults(response.getQueryId(), asyncQueryRequestContext) .getStatus()); flintIndexJob.assertState(FlintIndexState.DELETED); @@ -519,7 +526,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); flintIndexJob.assertState(FlintIndexState.DELETED); emrsClient.startJobRunCalled(0); @@ -569,7 +577,7 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { assertEquals( "SUCCESS", asyncQueryExecutorService - .getAsyncQueryResults(response.getQueryId()) + .getAsyncQueryResults(response.getQueryId(), asyncQueryRequestContext) .getStatus()); flintIndexJob.assertState(FlintIndexState.DELETED); @@ -616,7 +624,7 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { assertEquals( "SUCCESS", asyncQueryExecutorService - .getAsyncQueryResults(response.getQueryId()) + .getAsyncQueryResults(response.getQueryId(), asyncQueryRequestContext) .getStatus()); flintIndexJob.assertState(FlintIndexState.DELETED); @@ -668,7 +676,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { asyncQueryRequestContext); AsyncQueryExecutionResponse asyncQueryExecutionResponse = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); // 2. fetch result assertEquals("FAILED", asyncQueryExecutionResponse.getStatus()); assertEquals( @@ -714,7 +723,8 @@ public CancelJobRunResult cancelJobRun( // 2.fetch result. AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryResults.getStatus()); assertEquals("Internal Server Error.", asyncQueryResults.getError()); @@ -762,7 +772,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { // 2. fetch result AsyncQueryExecutionResponse asyncQueryResults = - asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); assertEquals("FAILED", asyncQueryResults.getStatus()); assertTrue(asyncQueryResults.getError().contains("no state found")); }); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java index 439b2ed2d6..e62b60bfd2 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java @@ -174,7 +174,8 @@ public GetJobRunResult getJobRunResult(String applicationId, String jobId) { new CreateAsyncQueryRequest(mockDS.query, MYS3_DATASOURCE, LangType.SQL, null), asyncQueryRequestContext); - return asyncQueryExecutorService.getAsyncQueryResults(response.getQueryId()); + return asyncQueryExecutorService.getAsyncQueryResults( + response.getQueryId(), asyncQueryRequestContext); } private boolean flintIndexExists(String flintIndexName) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java index e76776e2fc..fe3d5f3177 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java @@ -144,7 +144,8 @@ public void cancelFailedBecauseOfConflict() { st.open(); StatementModel running = - statementStorageService.updateStatementState(st.getStatementModel(), CANCELLED); + statementStorageService.updateStatementState( + st.getStatementModel(), CANCELLED, asyncQueryRequestContext); assertEquals(StatementState.CANCELLED, running.getStatementState()); IllegalStateException exception = assertThrows(IllegalStateException.class, st::cancel); @@ -265,7 +266,7 @@ public void newStatementFieldAssert() { Session session = sessionManager.createSession(createSessionRequest(), asyncQueryRequestContext); StatementId statementId = session.submit(queryRequest(), asyncQueryRequestContext); - Optional<Statement> statement = session.get(statementId); + Optional<Statement> statement = session.get(statementId, asyncQueryRequestContext); assertTrue(statement.isPresent()); assertEquals(session.getSessionId(), statement.get().getSessionId()); @@ -301,7 +302,7 @@ public void getStatementSuccess() { sessionStorageService.updateSessionState(session.getSessionModel(), SessionState.RUNNING); StatementId statementId = session.submit(queryRequest(), asyncQueryRequestContext); - Optional<Statement> statement = session.get(statementId); + Optional<Statement> statement = session.get(statementId, asyncQueryRequestContext); assertTrue(statement.isPresent()); assertEquals(WAITING, statement.get().getStatementState()); assertEquals(statementId, statement.get().getStatementId()); @@ -314,7 +315,8 @@ public void getStatementNotExist() { // App change state to running sessionStorageService.updateSessionState(session.getSessionModel(), SessionState.RUNNING); - Optional<Statement> statement = session.get(StatementId.newStatementId("not-exist-id")); + Optional<Statement> statement = + session.get(StatementId.newStatementId("not-exist-id"), asyncQueryRequestContext); assertFalse(statement.isPresent()); } @@ -332,7 +334,8 @@ public TestStatement assertSessionState(StatementState expected) { assertEquals(expected, st.getStatementModel().getStatementState()); Optional<StatementModel> model = - statementStorageService.getStatement(st.getStatementId().getId(), TEST_DATASOURCE_NAME); + statementStorageService.getStatement( + st.getStatementId().getId(), TEST_DATASOURCE_NAME, st.getAsyncQueryRequestContext()); assertTrue(model.isPresent()); assertEquals(expected, model.get().getStatementState()); @@ -343,7 +346,8 @@ public TestStatement assertStatementId(StatementId expected) { assertEquals(expected, st.getStatementModel().getStatementId()); Optional<StatementModel> model = - statementStorageService.getStatement(st.getStatementId().getId(), TEST_DATASOURCE_NAME); + statementStorageService.getStatement( + st.getStatementId().getId(), TEST_DATASOURCE_NAME, st.getAsyncQueryRequestContext()); assertTrue(model.isPresent()); assertEquals(expected, model.get().getStatementId()); return this; @@ -361,7 +365,8 @@ public TestStatement cancel() { public TestStatement run() { StatementModel model = - statementStorageService.updateStatementState(st.getStatementModel(), RUNNING); + statementStorageService.updateStatementState( + st.getStatementModel(), RUNNING, st.getAsyncQueryRequestContext()); st.setStatementModel(model); return this; } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java index 34f10b0083..475eceb37e 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java @@ -7,6 +7,8 @@ package org.opensearch.sql.spark.transport; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -33,6 +35,7 @@ import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl; import org.opensearch.sql.spark.asyncquery.exceptions.AsyncQueryNotFoundException; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionRequest; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse; import org.opensearch.tasks.Task; @@ -64,8 +67,11 @@ public void testDoExecute() { GetAsyncQueryResultActionRequest request = new GetAsyncQueryResultActionRequest("jobId"); AsyncQueryExecutionResponse asyncQueryExecutionResponse = new AsyncQueryExecutionResponse("IN_PROGRESS", null, null, null, null); - when(jobExecutorService.getAsyncQueryResults("jobId")).thenReturn(asyncQueryExecutionResponse); + when(jobExecutorService.getAsyncQueryResults(eq("jobId"), any())) + .thenReturn(asyncQueryExecutionResponse); + action.doExecute(task, request, actionListener); + verify(actionListener).onResponse(createJobActionResponseArgumentCaptor.capture()); GetAsyncQueryResultActionResponse getAsyncQueryResultActionResponse = createJobActionResponseArgumentCaptor.getValue(); @@ -91,8 +97,11 @@ public void testDoExecuteWithSuccessResponse() { tupleValue(ImmutableMap.of("name", "Smith", "age", 30))), null, null); - when(jobExecutorService.getAsyncQueryResults("jobId")).thenReturn(asyncQueryExecutionResponse); + when(jobExecutorService.getAsyncQueryResults(eq("jobId"), any())) + .thenReturn(asyncQueryExecutionResponse); + action.doExecute(task, request, actionListener); + verify(actionListener).onResponse(createJobActionResponseArgumentCaptor.capture()); GetAsyncQueryResultActionResponse getAsyncQueryResultActionResponse = createJobActionResponseArgumentCaptor.getValue(); @@ -130,9 +139,12 @@ public void testDoExecuteWithException() { GetAsyncQueryResultActionRequest request = new GetAsyncQueryResultActionRequest("123"); doThrow(new AsyncQueryNotFoundException("JobId 123 not found")) .when(jobExecutorService) - .getAsyncQueryResults("123"); + .getAsyncQueryResults(eq("123"), any()); + action.doExecute(task, request, actionListener); - verify(jobExecutorService, times(1)).getAsyncQueryResults("123"); + + verify(jobExecutorService, times(1)) + .getAsyncQueryResults(eq("123"), any(NullAsyncQueryRequestContext.class)); verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); From d260e0e449051eac0e3bc73db04103badebeb848 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 29 Aug 2024 21:10:34 -0700 Subject: [PATCH 32/96] Extract validation logic from FlintIndexMetadataServiceImpl (#2944) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../flint/FlintIndexMetadataValidator.java | 88 ++++++++++++++++++ .../FlintIndexMetadataValidatorTest.java | 90 +++++++++++++++++++ .../flint/FlintIndexMetadataServiceImpl.java | 69 +------------- 3 files changed, 179 insertions(+), 68 deletions(-) create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidator.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidatorTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidator.java new file mode 100644 index 0000000000..68ba34c476 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidator.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.flint; + +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.AUTO_REFRESH; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.CHECKPOINT_LOCATION; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.INCREMENTAL_REFRESH; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.WATERMARK_DELAY; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FlintIndexMetadataValidator { + private static final Logger LOGGER = LogManager.getLogger(FlintIndexMetadataValidator.class); + + public static final Set<String> ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS = + new LinkedHashSet<>(Arrays.asList(AUTO_REFRESH, INCREMENTAL_REFRESH)); + public static final Set<String> ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS = + new LinkedHashSet<>( + Arrays.asList(AUTO_REFRESH, INCREMENTAL_REFRESH, WATERMARK_DELAY, CHECKPOINT_LOCATION)); + + /** + * Validate if the flint index options contain valid key/value pairs. Throws + * IllegalArgumentException with description about invalid options. + */ + public static void validateFlintIndexOptions( + String kind, Map<String, Object> existingOptions, Map<String, String> newOptions) { + if ((newOptions.containsKey(INCREMENTAL_REFRESH) + && Boolean.parseBoolean(newOptions.get(INCREMENTAL_REFRESH))) + || ((!newOptions.containsKey(INCREMENTAL_REFRESH) + && Boolean.parseBoolean((String) existingOptions.get(INCREMENTAL_REFRESH))))) { + validateConversionToIncrementalRefresh(kind, existingOptions, newOptions); + } else { + validateConversionToFullRefresh(newOptions); + } + } + + private static void validateConversionToFullRefresh(Map<String, String> newOptions) { + if (!ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS.containsAll(newOptions.keySet())) { + throw new IllegalArgumentException( + String.format( + "Altering to full refresh only allows: %s options", + ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS)); + } + } + + private static void validateConversionToIncrementalRefresh( + String kind, Map<String, Object> existingOptions, Map<String, String> newOptions) { + if (!ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS.containsAll(newOptions.keySet())) { + throw new IllegalArgumentException( + String.format( + "Altering to incremental refresh only allows: %s options", + ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS)); + } + HashMap<String, Object> mergedOptions = new HashMap<>(); + mergedOptions.putAll(existingOptions); + mergedOptions.putAll(newOptions); + List<String> missingAttributes = new ArrayList<>(); + if (!mergedOptions.containsKey(CHECKPOINT_LOCATION) + || StringUtils.isEmpty((String) mergedOptions.get(CHECKPOINT_LOCATION))) { + missingAttributes.add(CHECKPOINT_LOCATION); + } + if (kind.equals("mv") + && (!mergedOptions.containsKey(WATERMARK_DELAY) + || StringUtils.isEmpty((String) mergedOptions.get(WATERMARK_DELAY)))) { + missingAttributes.add(WATERMARK_DELAY); + } + if (missingAttributes.size() > 0) { + String errorMessage = + "Conversion to incremental refresh index cannot proceed due to missing attributes: " + + String.join(", ", missingAttributes) + + "."; + LOGGER.error(errorMessage); + throw new IllegalArgumentException(errorMessage); + } + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidatorTest.java new file mode 100644 index 0000000000..7a1e718c05 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataValidatorTest.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.flint; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.AUTO_REFRESH; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.CHECKPOINT_LOCATION; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.INCREMENTAL_REFRESH; +import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.WATERMARK_DELAY; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlintIndexMetadataValidatorTest { + @Test + public void conversionToIncrementalRefreshWithValidOption() { + Map<String, Object> existingOptions = + ImmutableMap.<String, Object>builder().put(INCREMENTAL_REFRESH, "false").build(); + Map<String, String> newOptions = + ImmutableMap.<String, String>builder() + .put(INCREMENTAL_REFRESH, "true") + .put(CHECKPOINT_LOCATION, "checkpoint_location") + .put(WATERMARK_DELAY, "1") + .build(); + + FlintIndexMetadataValidator.validateFlintIndexOptions("mv", existingOptions, newOptions); + } + + @Test + public void conversionToIncrementalRefreshWithMissingOptions() { + Map<String, Object> existingOptions = + ImmutableMap.<String, Object>builder().put(AUTO_REFRESH, "true").build(); + Map<String, String> newOptions = + ImmutableMap.<String, String>builder().put(INCREMENTAL_REFRESH, "true").build(); + + assertThrows( + IllegalArgumentException.class, + () -> + FlintIndexMetadataValidator.validateFlintIndexOptions( + "mv", existingOptions, newOptions)); + } + + @Test + public void conversionToIncrementalRefreshWithInvalidOption() { + Map<String, Object> existingOptions = + ImmutableMap.<String, Object>builder().put(INCREMENTAL_REFRESH, "false").build(); + Map<String, String> newOptions = + ImmutableMap.<String, String>builder() + .put(INCREMENTAL_REFRESH, "true") + .put("INVALID_OPTION", "1") + .build(); + + assertThrows( + IllegalArgumentException.class, + () -> + FlintIndexMetadataValidator.validateFlintIndexOptions( + "mv", existingOptions, newOptions)); + } + + @Test + public void conversionToFullRefreshWithValidOption() { + Map<String, Object> existingOptions = + ImmutableMap.<String, Object>builder().put(AUTO_REFRESH, "false").build(); + Map<String, String> newOptions = + ImmutableMap.<String, String>builder().put(AUTO_REFRESH, "true").build(); + + FlintIndexMetadataValidator.validateFlintIndexOptions("mv", existingOptions, newOptions); + } + + @Test + public void conversionToFullRefreshWithInvalidOption() { + Map<String, Object> existingOptions = + ImmutableMap.<String, Object>builder().put(AUTO_REFRESH, "false").build(); + Map<String, String> newOptions = + ImmutableMap.<String, String>builder() + .put(AUTO_REFRESH, "true") + .put(WATERMARK_DELAY, "1") + .build(); + + assertThrows( + IllegalArgumentException.class, + () -> + FlintIndexMetadataValidator.validateFlintIndexOptions( + "mv", existingOptions, newOptions)); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java index b8352d15b2..38789dd796 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java @@ -5,10 +5,6 @@ package org.opensearch.sql.spark.flint; -import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.AUTO_REFRESH; -import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.CHECKPOINT_LOCATION; -import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.INCREMENTAL_REFRESH; -import static org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions.WATERMARK_DELAY; import static org.opensearch.sql.spark.flint.FlintIndexMetadata.APP_ID; import static org.opensearch.sql.spark.flint.FlintIndexMetadata.ENV_KEY; import static org.opensearch.sql.spark.flint.FlintIndexMetadata.KIND_KEY; @@ -20,15 +16,9 @@ import static org.opensearch.sql.spark.flint.FlintIndexMetadata.SERVERLESS_EMR_JOB_ID; import static org.opensearch.sql.spark.flint.FlintIndexMetadata.SOURCE_KEY; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import lombok.AllArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -43,11 +33,6 @@ public class FlintIndexMetadataServiceImpl implements FlintIndexMetadataService private static final Logger LOGGER = LogManager.getLogger(FlintIndexMetadataServiceImpl.class); private final Client client; - public static final Set<String> ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS = - new LinkedHashSet<>(Arrays.asList(AUTO_REFRESH, INCREMENTAL_REFRESH)); - public static final Set<String> ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS = - new LinkedHashSet<>( - Arrays.asList(AUTO_REFRESH, INCREMENTAL_REFRESH, WATERMARK_DELAY, CHECKPOINT_LOCATION)); @Override public Map<String, FlintIndexMetadata> getFlintIndexMetadata( @@ -87,63 +72,11 @@ public void updateIndexToManualRefresh( String kind = (String) meta.get("kind"); Map<String, Object> options = (Map<String, Object>) meta.get("options"); Map<String, String> newOptions = flintIndexOptions.getProvidedOptions(); - validateFlintIndexOptions(kind, options, newOptions); + FlintIndexMetadataValidator.validateFlintIndexOptions(kind, options, newOptions); options.putAll(newOptions); client.admin().indices().preparePutMapping(indexName).setSource(flintMetadataMap).get(); } - private void validateFlintIndexOptions( - String kind, Map<String, Object> existingOptions, Map<String, String> newOptions) { - if ((newOptions.containsKey(INCREMENTAL_REFRESH) - && Boolean.parseBoolean(newOptions.get(INCREMENTAL_REFRESH))) - || ((!newOptions.containsKey(INCREMENTAL_REFRESH) - && Boolean.parseBoolean((String) existingOptions.get(INCREMENTAL_REFRESH))))) { - validateConversionToIncrementalRefresh(kind, existingOptions, newOptions); - } else { - validateConversionToFullRefresh(newOptions); - } - } - - private void validateConversionToFullRefresh(Map<String, String> newOptions) { - if (!ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS.containsAll(newOptions.keySet())) { - throw new IllegalArgumentException( - String.format( - "Altering to full refresh only allows: %s options", - ALTER_TO_FULL_REFRESH_ALLOWED_OPTIONS)); - } - } - - private void validateConversionToIncrementalRefresh( - String kind, Map<String, Object> existingOptions, Map<String, String> newOptions) { - if (!ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS.containsAll(newOptions.keySet())) { - throw new IllegalArgumentException( - String.format( - "Altering to incremental refresh only allows: %s options", - ALTER_TO_INCREMENTAL_REFRESH_ALLOWED_OPTIONS)); - } - HashMap<String, Object> mergedOptions = new HashMap<>(); - mergedOptions.putAll(existingOptions); - mergedOptions.putAll(newOptions); - List<String> missingAttributes = new ArrayList<>(); - if (!mergedOptions.containsKey(CHECKPOINT_LOCATION) - || StringUtils.isEmpty((String) mergedOptions.get(CHECKPOINT_LOCATION))) { - missingAttributes.add(CHECKPOINT_LOCATION); - } - if (kind.equals("mv") - && (!mergedOptions.containsKey(WATERMARK_DELAY) - || StringUtils.isEmpty((String) mergedOptions.get(WATERMARK_DELAY)))) { - missingAttributes.add(WATERMARK_DELAY); - } - if (missingAttributes.size() > 0) { - String errorMessage = - "Conversion to incremental refresh index cannot proceed due to missing attributes: " - + String.join(", ", missingAttributes) - + "."; - LOGGER.error(errorMessage); - throw new IllegalArgumentException(errorMessage); - } - } - private FlintIndexMetadata fromMetadata(String indexName, Map<String, Object> metaMap) { FlintIndexMetadata.FlintIndexMetadataBuilder flintIndexMetadataBuilder = FlintIndexMetadata.builder(); From c13f7705fca39c737b11fcf6fb0bf5ce9fb540ef Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 3 Sep 2024 22:13:57 -0700 Subject: [PATCH 33/96] Fix jobType for Batch and IndexDML query (#2955) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../sql/spark/dispatcher/BatchQueryHandler.java | 2 +- .../sql/spark/dispatcher/IndexDMLHandler.java | 4 ++-- .../asyncquery/AsyncQueryCoreIntegTest.java | 17 +++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index bce1918631..36e4c227b8 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -109,7 +109,7 @@ public DispatchQueryResponse submit( .jobId(jobId) .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) - .jobType(JobType.INTERACTIVE) + .jobType(JobType.BATCH) .build(); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index d5885e6f2a..4698bfcccc 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -82,7 +82,7 @@ public DispatchQueryResponse submit( .jobId(DML_QUERY_JOB_ID) .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) - .jobType(JobType.INTERACTIVE) + .jobType(JobType.BATCH) .build(); } catch (Exception e) { LOG.error(e.getMessage()); @@ -100,7 +100,7 @@ public DispatchQueryResponse submit( .jobId(DML_QUERY_JOB_ID) .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) - .jobType(JobType.INTERACTIVE) + .jobType(JobType.BATCH) .build(); } } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index feb8c8c0ac..09767d16bd 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -202,7 +202,7 @@ public void createDropIndexQuery() { verifyGetQueryIdCalled(); verifyCancelJobRunCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } @Test @@ -224,7 +224,7 @@ public void createVacuumIndexQuery() { verifyGetQueryIdCalled(); verify(flintIndexClient).deleteIndex(indexName); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } @Test @@ -255,7 +255,7 @@ public void createAlterIndexQuery() { assertFalse(flintIndexOptions.autoRefresh()); verifyCancelJobRunCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } @Test @@ -280,7 +280,7 @@ public void createStreamingQuery() { verifyGetQueryIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID); + verifyStoreJobMetadataCalled(JOB_ID, JobType.STREAMING); } private void verifyStartJobRunCalled() { @@ -315,7 +315,7 @@ public void createCreateIndexQuery() { assertNull(response.getSessionId()); verifyGetQueryIdCalled(); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID); + verifyStoreJobMetadataCalled(JOB_ID, JobType.BATCH); } @Test @@ -337,7 +337,7 @@ public void createRefreshQuery() { verifyGetQueryIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID); + verifyStoreJobMetadataCalled(JOB_ID, JobType.BATCH); } @Test @@ -363,7 +363,7 @@ public void createInteractiveQuery() { verifyGetSessionIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID); + verifyStoreJobMetadataCalled(JOB_ID, JobType.INTERACTIVE); } @Test @@ -560,7 +560,7 @@ private void verifyGetSessionIdCalled() { assertEquals(APPLICATION_ID, createSessionRequest.getApplicationId()); } - private void verifyStoreJobMetadataCalled(String jobId) { + private void verifyStoreJobMetadataCalled(String jobId, JobType jobType) { verify(asyncQueryJobMetadataStorageService) .storeJobMetadata( asyncQueryJobMetadataArgumentCaptor.capture(), eq(asyncQueryRequestContext)); @@ -568,6 +568,7 @@ private void verifyStoreJobMetadataCalled(String jobId) { assertEquals(QUERY_ID, asyncQueryJobMetadata.getQueryId()); assertEquals(jobId, asyncQueryJobMetadata.getJobId()); assertEquals(DATASOURCE_NAME, asyncQueryJobMetadata.getDatasourceName()); + assertEquals(jobType, asyncQueryJobMetadata.getJobType()); } private void verifyCreateIndexDMLResultCalled() { From a83ab20d90009a9fd30580f2356f28e415d55752 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 3 Sep 2024 22:21:07 -0700 Subject: [PATCH 34/96] Add queryId Spark parameter to batch query (#2952) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../spark/data/constants/SparkConstants.java | 1 + .../spark/dispatcher/BatchQueryHandler.java | 1 + .../dispatcher/StreamingQueryHandler.java | 1 + .../SparkSubmitParametersBuilder.java | 6 ++ .../dispatcher/SparkQueryDispatcherTest.java | 58 ++++++++++++++----- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index e87dbba03e..9b82022d8f 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -86,6 +86,7 @@ public class SparkConstants { "com.amazonaws.emr.AssumeRoleAWSCredentialsProvider"; public static final String JAVA_HOME_LOCATION = "/usr/lib/jvm/java-17-amazon-corretto.x86_64/"; public static final String FLINT_JOB_QUERY = "spark.flint.job.query"; + public static final String FLINT_JOB_QUERY_ID = "spark.flint.job.queryId"; public static final String FLINT_JOB_REQUEST_INDEX = "spark.flint.job.requestIndex"; public static final String FLINT_JOB_SESSION_ID = "spark.flint.job.sessionId"; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 36e4c227b8..c693656150 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -91,6 +91,7 @@ public DispatchQueryResponse submit( sparkSubmitParametersBuilderProvider .getSparkSubmitParametersBuilder() .clusterName(clusterName) + .queryId(context.getQueryId()) .query(dispatchQueryRequest.getQuery()) .dataSource( context.getDataSourceMetadata(), diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java index 80d4be27cf..51e245b57c 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java @@ -82,6 +82,7 @@ public DispatchQueryResponse submit( sparkSubmitParametersBuilderProvider .getSparkSubmitParametersBuilder() .clusterName(clusterName) + .queryId(context.getQueryId()) .query(dispatchQueryRequest.getQuery()) .structuredStreaming(true) .dataSource( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java index d9d5859f64..db74d0a5a7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilder.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_PORT_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_SCHEME_KEY; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_QUERY; +import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_QUERY_ID; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_REQUEST_INDEX; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_SESSION_ID; import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_PPL_EXTENSION; @@ -108,6 +109,11 @@ public SparkSubmitParametersBuilder query(String query) { return this; } + public SparkSubmitParametersBuilder queryId(String queryId) { + setConfigItem(FLINT_JOB_QUERY_ID, queryId); + return this; + } + public SparkSubmitParametersBuilder dataSource( DataSourceMetadata metadata, DispatchQueryRequest dispatchQueryRequest, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index b6369292a6..1587ce6638 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -162,12 +162,14 @@ void setUp() { @Test void testDispatchSelectQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "select * from my_glue.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -206,12 +208,14 @@ void testDispatchSelectQuery() { @Test void testDispatchSelectQueryWithBasicAuthIndexStoreDatasource() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "select * from my_glue.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -310,6 +314,7 @@ void testDispatchSelectQueryFailedCreateSession() { @Test void testDispatchCreateAutoRefreshIndexQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(INDEX_TAG_KEY, "flint_my_glue_default_http_logs_elb_and_requesturi_index"); @@ -318,7 +323,8 @@ void testDispatchCreateAutoRefreshIndexQuery() { String query = "CREATE INDEX elb_and_requestUri ON my_glue.default.http_logs(l_orderkey, l_quantity) WITH" + " (auto_refresh = true)"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query, "streaming"); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, "streaming", QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:streaming:flint_my_glue_default_http_logs_elb_and_requesturi_index", @@ -347,6 +353,7 @@ void testDispatchCreateAutoRefreshIndexQuery() { @Test void testDispatchCreateManualRefreshIndexQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, "my_glue"); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); @@ -354,7 +361,8 @@ void testDispatchCreateManualRefreshIndexQuery() { String query = "CREATE INDEX elb_and_requestUri ON my_glue.default.http_logs(l_orderkey, l_quantity) WITH" + " (auto_refresh = false)"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -383,12 +391,14 @@ void testDispatchCreateManualRefreshIndexQuery() { @Test void testDispatchWithPPLQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "source = my_glue.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -448,12 +458,14 @@ void testDispatchWithSparkUDFQuery() { @Test void testInvalidSQLQueryDispatchToSpark() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "myselect 1"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -492,12 +504,14 @@ void testInvalidSQLQueryDispatchToSpark() { @Test void testDispatchQueryWithoutATableAndDataSourceName() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "show tables"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -526,6 +540,7 @@ void testDispatchQueryWithoutATableAndDataSourceName() { @Test void testDispatchIndexQueryWithoutADatasourceName() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(INDEX_TAG_KEY, "flint_my_glue_default_http_logs_elb_and_requesturi_index"); @@ -534,7 +549,8 @@ void testDispatchIndexQueryWithoutADatasourceName() { String query = "CREATE INDEX elb_and_requestUri ON default.http_logs(l_orderkey, l_quantity) WITH" + " (auto_refresh = true)"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query, "streaming"); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, "streaming", QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:streaming:flint_my_glue_default_http_logs_elb_and_requesturi_index", @@ -563,6 +579,7 @@ void testDispatchIndexQueryWithoutADatasourceName() { @Test void testDispatchMaterializedViewQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(INDEX_TAG_KEY, "flint_mv_1"); @@ -570,7 +587,8 @@ void testDispatchMaterializedViewQuery() { tags.put(JOB_TYPE_TAG_KEY, JobType.STREAMING.getText()); String query = "CREATE MATERIALIZED VIEW mv_1 AS select * from logs WITH" + " (auto_refresh = true)"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query, "streaming"); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, "streaming", QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:streaming:flint_mv_1", @@ -599,12 +617,14 @@ void testDispatchMaterializedViewQuery() { @Test void testDispatchShowMVQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "SHOW MATERIALIZED VIEW IN mys3.default"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -633,12 +653,14 @@ void testDispatchShowMVQuery() { @Test void testRefreshIndexQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "REFRESH SKIPPING INDEX ON my_glue.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -667,12 +689,14 @@ void testRefreshIndexQuery() { @Test void testDispatchDescribeIndexQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, MY_GLUE); tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); String query = "DESCRIBE SKIPPING INDEX ON mys3.default.http_logs"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:batch", @@ -701,6 +725,7 @@ void testDispatchDescribeIndexQuery() { @Test void testDispatchAlterToAutoRefreshIndexQuery() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); HashMap<String, String> tags = new HashMap<>(); tags.put(DATASOURCE_TAG_KEY, "my_glue"); tags.put(INDEX_TAG_KEY, "flint_my_glue_default_http_logs_elb_and_requesturi_index"); @@ -709,7 +734,8 @@ void testDispatchAlterToAutoRefreshIndexQuery() { String query = "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = true)"; - String sparkSubmitParameters = constructExpectedSparkSubmitParameterString(query, "streaming"); + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, "streaming", QUERY_ID); StartJobRequest expected = new StartJobRequest( "TEST_CLUSTER:streaming:flint_my_glue_default_http_logs_elb_and_requesturi_index", @@ -1048,10 +1074,11 @@ void testDispatchQueryWithExtraSparkSubmitParameters() { } private String constructExpectedSparkSubmitParameterString(String query) { - return constructExpectedSparkSubmitParameterString(query, null); + return constructExpectedSparkSubmitParameterString(query, null, null); } - private String constructExpectedSparkSubmitParameterString(String query, String jobType) { + private String constructExpectedSparkSubmitParameterString( + String query, String jobType, String queryId) { query = "\"" + query + "\""; return " --class org.apache.spark.sql.FlintJob " + getConfParam( @@ -1070,6 +1097,7 @@ private String constructExpectedSparkSubmitParameterString(String query, String "spark.datasource.flint.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider", "spark.sql.extensions=org.opensearch.flint.spark.FlintSparkExtensions,org.opensearch.flint.spark.FlintPPLSparkExtensions", "spark.hadoop.hive.metastore.client.factory.class=com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory") + + (queryId != null ? getConfParam("spark.flint.job.queryId=" + queryId) : "") + getConfParam("spark.flint.job.query=" + query) + (jobType != null ? getConfParam("spark.flint.job.type=" + jobType) : "") + getConfParam( From b4a6c60daf1a2c35a740d0245d7b3fb32fd8a0e0 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 4 Sep 2024 14:04:11 -0700 Subject: [PATCH 35/96] Fix SparkExecutionEngineConfigClusterSetting deserialize issue (#2966) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../config/SparkExecutionEngineConfigClusterSetting.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java index adaaa57d31..f940680c06 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java @@ -7,8 +7,10 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; /** * This POJO is just for reading stringified json in `plugins.query.executionengine.spark.config` @@ -16,6 +18,8 @@ */ @Data @Builder +@AllArgsConstructor +@NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) public class SparkExecutionEngineConfigClusterSetting { // optional From 729bb13247f12c3b7b91a92276e79430e9477db3 Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Wed, 4 Sep 2024 15:32:22 -0700 Subject: [PATCH 36/96] Flint query scheduler part 2 (#2961) * Flint query scheduler part 2 Signed-off-by: Louis Chu <clingzhi@amazon.com> * spotless apply Signed-off-by: Louis Chu <clingzhi@amazon.com> * Add UT Signed-off-by: Louis Chu <clingzhi@amazon.com> * Resolve comments Signed-off-by: Louis Chu <clingzhi@amazon.com> * Add more UTs Signed-off-by: Louis Chu <clingzhi@amazon.com> * Resolve comments Signed-off-by: Louis Chu <clingzhi@amazon.com> * Use SQL thread pool Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .../src/main/antlr/FlintSparkSqlExtensions.g4 | 5 +- .../src/main/antlr/SparkSqlBase.g4 | 1 + .../src/main/antlr/SqlBaseLexer.g4 | 2 + .../src/main/antlr/SqlBaseParser.g4 | 22 +- .../dispatcher/model/FlintIndexOptions.java | 6 + .../flint/operation/FlintIndexOpAlter.java | 12 +- .../flint/operation/FlintIndexOpDrop.java | 13 +- .../flint/operation/FlintIndexOpFactory.java | 13 +- .../flint/operation/FlintIndexOpVacuum.java | 11 +- .../spark/scheduler/AsyncQueryScheduler.java | 57 +++++ .../model/AsyncQuerySchedulerRequest.java | 31 +++ .../asyncquery/AsyncQueryCoreIntegTest.java | 110 ++++++++- .../dispatcher/SparkQueryDispatcherTest.java | 2 + .../operation/FlintIndexOpFactoryTest.java | 2 + .../operation/FlintIndexOpVacuumTest.java | 57 ++++- async-query/build.gradle | 2 +- .../OpenSearchAsyncQueryScheduler.java | 58 ++--- .../job/OpenSearchRefreshIndexJob.java | 93 -------- .../job/ScheduledAsyncQueryJobRunner.java | 116 ++++++++++ .../OpenSearchRefreshIndexJobRequest.java | 108 --------- .../model/ScheduledAsyncQueryJobRequest.java | 156 +++++++++++++ .../parser/IntervalScheduleParser.java | 100 +++++++++ ...nSearchScheduleQueryJobRequestParser.java} | 40 ++-- .../config/AsyncExecutorServiceModule.java | 16 +- .../async-query-scheduler-index-mapping.yml | 10 +- .../AsyncQueryExecutorServiceSpec.java | 10 +- .../OpenSearchAsyncQuerySchedulerTest.java | 63 +++--- .../job/OpenSearchRefreshIndexJobTest.java | 145 ------------ .../job/ScheduledAsyncQueryJobRunnerTest.java | 210 ++++++++++++++++++ .../OpenSearchRefreshIndexJobRequestTest.java | 81 ------- .../ScheduledAsyncQueryJobRequestTest.java | 210 ++++++++++++++++++ .../parser/IntervalScheduleParserTest.java | 122 ++++++++++ .../org/opensearch/sql/plugin/SQLPlugin.java | 23 +- 33 files changed, 1371 insertions(+), 536 deletions(-) create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java delete mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java delete mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java rename async-query/src/main/java/org/opensearch/sql/spark/scheduler/{OpenSearchRefreshIndexJobRequestParser.java => parser/OpenSearchScheduleQueryJobRequestParser.java} (57%) delete mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java delete mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java diff --git a/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 b/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 index 2e8d634dad..46e814e9f5 100644 --- a/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 +++ b/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 @@ -156,7 +156,10 @@ indexManagementStatement ; showFlintIndexStatement - : SHOW FLINT (INDEX | INDEXES) IN catalogDb=multipartIdentifier + : SHOW FLINT (INDEX | INDEXES) + IN catalogDb=multipartIdentifier #showFlintIndex + | SHOW FLINT (INDEX | INDEXES) EXTENDED + IN catalogDb=multipartIdentifier #showFlintIndexExtended ; indexJobManagementStatement diff --git a/async-query-core/src/main/antlr/SparkSqlBase.g4 b/async-query-core/src/main/antlr/SparkSqlBase.g4 index 283981e471..c53c61adfd 100644 --- a/async-query-core/src/main/antlr/SparkSqlBase.g4 +++ b/async-query-core/src/main/antlr/SparkSqlBase.g4 @@ -163,6 +163,7 @@ DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; DROP: 'DROP'; EXISTS: 'EXISTS'; +EXTENDED: 'EXTENDED'; FALSE: 'FALSE'; FLINT: 'FLINT'; IF: 'IF'; diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index bde298c23e..acfc0011f5 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -212,6 +212,7 @@ DIRECTORY: 'DIRECTORY'; DISTINCT: 'DISTINCT'; DISTRIBUTE: 'DISTRIBUTE'; DIV: 'DIV'; +DO: 'DO'; DOUBLE: 'DOUBLE'; DROP: 'DROP'; ELSE: 'ELSE'; @@ -467,6 +468,7 @@ WEEK: 'WEEK'; WEEKS: 'WEEKS'; WHEN: 'WHEN'; WHERE: 'WHERE'; +WHILE: 'WHILE'; WINDOW: 'WINDOW'; WITH: 'WITH'; WITHIN: 'WITHIN'; diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index c7aa56cf92..5b8805821b 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -63,6 +63,8 @@ compoundStatement : statement | setStatementWithOptionalVarKeyword | beginEndCompoundBlock + | ifElseStatement + | whileStatement ; setStatementWithOptionalVarKeyword @@ -71,6 +73,16 @@ setStatementWithOptionalVarKeyword LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword ; +whileStatement + : beginLabel? WHILE booleanExpression DO compoundBody END WHILE endLabel? + ; + +ifElseStatement + : IF booleanExpression THEN conditionalBodies+=compoundBody + (ELSE IF booleanExpression THEN conditionalBodies+=compoundBody)* + (ELSE elseBody=compoundBody)? END IF + ; + singleStatement : (statement|setResetStatement) SEMICOLON* EOF ; @@ -406,9 +418,9 @@ query ; insertInto - : INSERT OVERWRITE TABLE? identifierReference (partitionSpec (IF errorCapturingNot EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable - | INSERT INTO TABLE? identifierReference partitionSpec? (IF errorCapturingNot EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable - | INSERT INTO TABLE? identifierReference REPLACE whereClause #insertIntoReplaceWhere + : INSERT OVERWRITE TABLE? identifierReference optionsClause? (partitionSpec (IF errorCapturingNot EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable + | INSERT INTO TABLE? identifierReference optionsClause? partitionSpec? (IF errorCapturingNot EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable + | INSERT INTO TABLE? identifierReference optionsClause? REPLACE whereClause #insertIntoReplaceWhere | INSERT OVERWRITE LOCAL? DIRECTORY path=stringLit rowFormat? createFileFormat? #insertOverwriteHiveDir | INSERT OVERWRITE LOCAL? DIRECTORY (path=stringLit)? tableProvider (OPTIONS options=propertyList)? #insertOverwriteDir ; @@ -1522,6 +1534,7 @@ ansiNonReserved | DIRECTORY | DISTRIBUTE | DIV + | DO | DOUBLE | DROP | ESCAPED @@ -1723,6 +1736,7 @@ ansiNonReserved | VOID | WEEK | WEEKS + | WHILE | WINDOW | YEAR | YEARS @@ -1853,6 +1867,7 @@ nonReserved | DISTINCT | DISTRIBUTE | DIV + | DO | DOUBLE | DROP | ELSE @@ -2092,6 +2107,7 @@ nonReserved | VOID | WEEK | WEEKS + | WHILE | WHEN | WHERE | WINDOW diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java index 79af1c91ab..6c7cc7c5fb 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java @@ -19,6 +19,7 @@ public class FlintIndexOptions { public static final String INCREMENTAL_REFRESH = "incremental_refresh"; public static final String CHECKPOINT_LOCATION = "checkpoint_location"; public static final String WATERMARK_DELAY = "watermark_delay"; + public static final String SCHEDULER_MODE = "scheduler_mode"; private final Map<String, String> options = new HashMap<>(); public void setOption(String key, String value) { @@ -33,6 +34,11 @@ public boolean autoRefresh() { return Boolean.parseBoolean(getOption(AUTO_REFRESH).orElse("false")); } + public boolean isExternalScheduler() { + // Default is false, which means using internal scheduler to refresh the index. + return getOption(SCHEDULER_MODE).map(mode -> "external".equals(mode)).orElse(false); + } + public Map<String, String> getProvidedOptions() { return new HashMap<>(options); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java index 4a00195ebf..de34803823 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java @@ -16,6 +16,7 @@ import org.opensearch.sql.spark.flint.FlintIndexState; import org.opensearch.sql.spark.flint.FlintIndexStateModel; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; /** * Index Operation for Altering the flint index. Only handles alter operation when @@ -25,16 +26,19 @@ public class FlintIndexOpAlter extends FlintIndexOp { private static final Logger LOG = LogManager.getLogger(FlintIndexOpAlter.class); private final FlintIndexMetadataService flintIndexMetadataService; private final FlintIndexOptions flintIndexOptions; + private final AsyncQueryScheduler asyncQueryScheduler; public FlintIndexOpAlter( FlintIndexOptions flintIndexOptions, FlintIndexStateModelService flintIndexStateModelService, String datasourceName, EMRServerlessClientFactory emrServerlessClientFactory, - FlintIndexMetadataService flintIndexMetadataService) { + FlintIndexMetadataService flintIndexMetadataService, + AsyncQueryScheduler asyncQueryScheduler) { super(flintIndexStateModelService, datasourceName, emrServerlessClientFactory); this.flintIndexMetadataService = flintIndexMetadataService; this.flintIndexOptions = flintIndexOptions; + this.asyncQueryScheduler = asyncQueryScheduler; } @Override @@ -57,7 +61,11 @@ void runOp( "Running alter index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); this.flintIndexMetadataService.updateIndexToManualRefresh( flintIndexMetadata.getOpensearchIndexName(), flintIndexOptions, asyncQueryRequestContext); - cancelStreamingJob(flintIndexStateModel); + if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { + asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + } else { + cancelStreamingJob(flintIndexStateModel); + } } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java index fc9b644fc7..3fa5423c10 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java @@ -14,16 +14,21 @@ import org.opensearch.sql.spark.flint.FlintIndexState; import org.opensearch.sql.spark.flint.FlintIndexStateModel; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; /** Operation to drop Flint index */ public class FlintIndexOpDrop extends FlintIndexOp { private static final Logger LOG = LogManager.getLogger(); + private final AsyncQueryScheduler asyncQueryScheduler; + public FlintIndexOpDrop( FlintIndexStateModelService flintIndexStateModelService, String datasourceName, - EMRServerlessClientFactory emrServerlessClientFactory) { + EMRServerlessClientFactory emrServerlessClientFactory, + AsyncQueryScheduler asyncQueryScheduler) { super(flintIndexStateModelService, datasourceName, emrServerlessClientFactory); + this.asyncQueryScheduler = asyncQueryScheduler; } public boolean validate(FlintIndexState state) { @@ -48,7 +53,11 @@ void runOp( LOG.debug( "Performing drop index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); - cancelStreamingJob(flintIndexStateModel); + if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { + asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + } else { + cancelStreamingJob(flintIndexStateModel); + } } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java index 14cf9fa7c9..9f925e0bcf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java @@ -11,6 +11,7 @@ import org.opensearch.sql.spark.flint.FlintIndexClient; import org.opensearch.sql.spark.flint.FlintIndexMetadataService; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; @RequiredArgsConstructor public class FlintIndexOpFactory { @@ -18,10 +19,11 @@ public class FlintIndexOpFactory { private final FlintIndexClient flintIndexClient; private final FlintIndexMetadataService flintIndexMetadataService; private final EMRServerlessClientFactory emrServerlessClientFactory; + private final AsyncQueryScheduler asyncQueryScheduler; public FlintIndexOpDrop getDrop(String datasource) { return new FlintIndexOpDrop( - flintIndexStateModelService, datasource, emrServerlessClientFactory); + flintIndexStateModelService, datasource, emrServerlessClientFactory, asyncQueryScheduler); } public FlintIndexOpAlter getAlter(FlintIndexOptions flintIndexOptions, String datasource) { @@ -30,12 +32,17 @@ public FlintIndexOpAlter getAlter(FlintIndexOptions flintIndexOptions, String da flintIndexStateModelService, datasource, emrServerlessClientFactory, - flintIndexMetadataService); + flintIndexMetadataService, + asyncQueryScheduler); } public FlintIndexOpVacuum getVacuum(String datasource) { return new FlintIndexOpVacuum( - flintIndexStateModelService, datasource, flintIndexClient, emrServerlessClientFactory); + flintIndexStateModelService, + datasource, + flintIndexClient, + emrServerlessClientFactory, + asyncQueryScheduler); } public FlintIndexOpCancel getCancel(String datasource) { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java index 06aaf8ef9f..324ddb5720 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java @@ -14,12 +14,14 @@ import org.opensearch.sql.spark.flint.FlintIndexState; import org.opensearch.sql.spark.flint.FlintIndexStateModel; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; /** Flint index vacuum operation. */ public class FlintIndexOpVacuum extends FlintIndexOp { - private static final Logger LOG = LogManager.getLogger(); + private final AsyncQueryScheduler asyncQueryScheduler; + /** OpenSearch client. */ private final FlintIndexClient flintIndexClient; @@ -27,9 +29,11 @@ public FlintIndexOpVacuum( FlintIndexStateModelService flintIndexStateModelService, String datasourceName, FlintIndexClient flintIndexClient, - EMRServerlessClientFactory emrServerlessClientFactory) { + EMRServerlessClientFactory emrServerlessClientFactory, + AsyncQueryScheduler asyncQueryScheduler) { super(flintIndexStateModelService, datasourceName, emrServerlessClientFactory); this.flintIndexClient = flintIndexClient; + this.asyncQueryScheduler = asyncQueryScheduler; } @Override @@ -48,6 +52,9 @@ public void runOp( FlintIndexStateModel flintIndex, AsyncQueryRequestContext asyncQueryRequestContext) { LOG.info("Vacuuming Flint index {}", flintIndexMetadata.getOpensearchIndexName()); + if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { + asyncQueryScheduler.removeJob(flintIndexMetadata.getOpensearchIndexName()); + } flintIndexClient.deleteIndex(flintIndexMetadata.getOpensearchIndexName()); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java new file mode 100644 index 0000000000..8ac499081e --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java @@ -0,0 +1,57 @@ +package org.opensearch.sql.spark.scheduler; + +import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; + +/** Scheduler interface for scheduling asynchronous query jobs. */ +public interface AsyncQueryScheduler { + + /** + * Schedules a new job in the system. This method creates a new job entry based on the provided + * request parameters. + * + * <p>Use cases: - Creating a new periodic query execution - Setting up a scheduled data refresh + * task + * + * @param asyncQuerySchedulerRequest The request containing job configuration details + * @throws IllegalArgumentException if a job with the same name already exists + * @throws RuntimeException if there's an error during job creation + */ + void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + + /** + * Updates an existing job with new parameters. This method modifies the configuration of an + * already scheduled job. + * + * <p>Use cases: - Changing the schedule of an existing job - Modifying query parameters of a + * scheduled job - Updating resource allocations for a job + * + * @param asyncQuerySchedulerRequest The request containing updated job configuration + * @throws IllegalArgumentException if the job to be updated doesn't exist + * @throws RuntimeException if there's an error during the update process + */ + void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + + /** + * Unschedules a job by marking it as disabled and updating its last update time. This method is + * used when you want to temporarily stop a job from running but keep its configuration and + * history in the system. + * + * <p>Use cases: - Pausing a job that's causing issues without losing its configuration - + * Temporarily disabling a job during maintenance or high-load periods - Allowing for easy + * re-enabling of the job in the future + * + * @param jobId The unique identifier of the job to unschedule + */ + void unscheduleJob(String jobId); + + /** + * Removes a job completely from the scheduler. This method permanently deletes the job and all + * its associated data from the system. + * + * <p>Use cases: - Cleaning up jobs that are no longer needed - Removing obsolete or erroneously + * created jobs - Freeing up resources by deleting unused job configurations + * + * @param jobId The unique identifier of the job to remove + */ + void removeJob(String jobId); +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java new file mode 100644 index 0000000000..b54e5b30ce --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.model; + +import java.time.Instant; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.opensearch.sql.spark.rest.model.LangType; + +/** Represents a job request for a scheduled task. */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AsyncQuerySchedulerRequest { + protected String accountId; + // Scheduler jobid is the opensearch index name until we support multiple jobs per index + protected String jobId; + protected String dataSource; + protected String scheduledQuery; + protected LangType queryLang; + protected Object schedule; + protected boolean enabled; + protected Instant lastUpdateTime; + protected Instant enabledTime; + protected Long lockDurationSeconds; + protected Double jitter; +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 09767d16bd..226e0ff5eb 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -83,6 +83,7 @@ import org.opensearch.sql.spark.rest.model.CreateAsyncQueryRequest; import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse; import org.opensearch.sql.spark.rest.model.LangType; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; /** * This tests async-query-core library end-to-end using mocked implementation of extension points. @@ -112,6 +113,7 @@ public class AsyncQueryCoreIntegTest { @Mock FlintIndexClient flintIndexClient; @Mock AsyncQueryRequestContext asyncQueryRequestContext; @Mock MetricsService metricsService; + @Mock AsyncQueryScheduler asyncQueryScheduler; @Mock SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider; // storage services @@ -159,7 +161,8 @@ public void setUp() { flintIndexStateModelService, flintIndexClient, flintIndexMetadataService, - emrServerlessClientFactory); + emrServerlessClientFactory, + asyncQueryScheduler); QueryHandlerFactory queryHandlerFactory = new QueryHandlerFactory( jobExecutionResponseReader, @@ -205,6 +208,30 @@ public void createDropIndexQuery() { verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } + @Test + public void createDropIndexQueryWithScheduler() { + givenSparkExecutionEngineConfigIsSupplied(); + givenValidDataSourceMetadataExist(); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); + + String indexName = "flint_datasource_name_table_name_index_name_index"; + givenFlintIndexMetadataExistsWithExternalScheduler(indexName); + + CreateAsyncQueryResponse response = + asyncQueryExecutorService.createAsyncQuery( + new CreateAsyncQueryRequest( + "DROP INDEX index_name ON table_name", DATASOURCE_NAME, LangType.SQL), + asyncQueryRequestContext); + + assertEquals(QUERY_ID, response.getQueryId()); + assertNull(response.getSessionId()); + verifyGetQueryIdCalled(); + verifyCreateIndexDMLResultCalled(); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + + verify(asyncQueryScheduler).unscheduleJob(indexName); + } + @Test public void createVacuumIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); @@ -227,6 +254,32 @@ public void createVacuumIndexQuery() { verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } + @Test + public void createVacuumIndexQueryWithScheduler() { + givenSparkExecutionEngineConfigIsSupplied(); + givenValidDataSourceMetadataExist(); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); + + String indexName = "flint_datasource_name_table_name_index_name_index"; + givenFlintIndexMetadataExistsWithExternalScheduler(indexName); + + CreateAsyncQueryResponse response = + asyncQueryExecutorService.createAsyncQuery( + new CreateAsyncQueryRequest( + "VACUUM INDEX index_name ON table_name", DATASOURCE_NAME, LangType.SQL), + asyncQueryRequestContext); + + assertEquals(QUERY_ID, response.getQueryId()); + assertNull(response.getSessionId()); + verifyGetQueryIdCalled(); + + verify(flintIndexClient).deleteIndex(indexName); + verifyCreateIndexDMLResultCalled(); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + + verify(asyncQueryScheduler).removeJob(indexName); + } + @Test public void createAlterIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); @@ -258,6 +311,40 @@ public void createAlterIndexQuery() { verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } + @Test + public void createAlterIndexQueryWithScheduler() { + givenSparkExecutionEngineConfigIsSupplied(); + givenValidDataSourceMetadataExist(); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); + + String indexName = "flint_datasource_name_table_name_index_name_index"; + givenFlintIndexMetadataExistsWithExternalScheduler(indexName); + + CreateAsyncQueryResponse response = + asyncQueryExecutorService.createAsyncQuery( + new CreateAsyncQueryRequest( + "ALTER INDEX index_name ON table_name WITH (auto_refresh = false)", + DATASOURCE_NAME, + LangType.SQL), + asyncQueryRequestContext); + + assertEquals(QUERY_ID, response.getQueryId()); + assertNull(response.getSessionId()); + verifyGetQueryIdCalled(); + + verify(flintIndexMetadataService) + .updateIndexToManualRefresh( + eq(indexName), flintIndexOptionsArgumentCaptor.capture(), eq(asyncQueryRequestContext)); + + FlintIndexOptions flintIndexOptions = flintIndexOptionsArgumentCaptor.getValue(); + assertFalse(flintIndexOptions.autoRefresh()); + + verify(asyncQueryScheduler).unscheduleJob(indexName); + + verifyCreateIndexDMLResultCalled(); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + } + @Test public void createStreamingQuery() { givenSparkExecutionEngineConfigIsSupplied(); @@ -507,7 +594,8 @@ private void givenSparkExecutionEngineConfigIsSupplied() { .build()); } - private void givenFlintIndexMetadataExists(String indexName) { + private void givenFlintIndexMetadataExists( + String indexName, FlintIndexOptions flintIndexOptions) { when(flintIndexMetadataService.getFlintIndexMetadata(indexName, asyncQueryRequestContext)) .thenReturn( ImmutableMap.of( @@ -516,9 +604,27 @@ private void givenFlintIndexMetadataExists(String indexName) { .appId(APPLICATION_ID) .jobId(JOB_ID) .opensearchIndexName(indexName) + .flintIndexOptions(flintIndexOptions) .build())); } + // Overload method for default FlintIndexOptions usage + private void givenFlintIndexMetadataExists(String indexName) { + givenFlintIndexMetadataExists(indexName, new FlintIndexOptions()); + } + + // Method to set up FlintIndexMetadata with external scheduler + private void givenFlintIndexMetadataExistsWithExternalScheduler(String indexName) { + givenFlintIndexMetadataExists(indexName, createExternalSchedulerFlintIndexOptions()); + } + + // Helper method for creating FlintIndexOptions with external scheduler + private FlintIndexOptions createExternalSchedulerFlintIndexOptions() { + FlintIndexOptions options = new FlintIndexOptions(); + options.setOption(FlintIndexOptions.SCHEDULER_MODE, "external"); + return options; + } + private void givenValidDataSourceMetadataExist() { when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( DATASOURCE_NAME, asyncQueryRequestContext)) diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 1587ce6638..d040db24b2 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -86,6 +86,7 @@ import org.opensearch.sql.spark.parameter.SparkSubmitParametersBuilderProvider; import org.opensearch.sql.spark.response.JobExecutionResponseReader; import org.opensearch.sql.spark.rest.model.LangType; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; @ExtendWith(MockitoExtension.class) public class SparkQueryDispatcherTest { @@ -108,6 +109,7 @@ public class SparkQueryDispatcherTest { @Mock private QueryIdProvider queryIdProvider; @Mock private AsyncQueryRequestContext asyncQueryRequestContext; @Mock private MetricsService metricsService; + @Mock private AsyncQueryScheduler asyncQueryScheduler; private DataSourceSparkParameterComposer dataSourceSparkParameterComposer = (datasourceMetadata, sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, "basic"); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java index 3bf438aeb9..62ac98f1a2 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java @@ -17,6 +17,7 @@ import org.opensearch.sql.spark.flint.FlintIndexClient; import org.opensearch.sql.spark.flint.FlintIndexMetadataService; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; @ExtendWith(MockitoExtension.class) class FlintIndexOpFactoryTest { @@ -26,6 +27,7 @@ class FlintIndexOpFactoryTest { @Mock private FlintIndexClient flintIndexClient; @Mock private FlintIndexMetadataService flintIndexMetadataService; @Mock private EMRServerlessClientFactory emrServerlessClientFactory; + @Mock private AsyncQueryScheduler asyncQueryScheduler; @InjectMocks FlintIndexOpFactory flintIndexOpFactory; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java index 26858c18fe..08f8efd488 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -18,11 +19,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; +import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; import org.opensearch.sql.spark.flint.FlintIndexClient; import org.opensearch.sql.spark.flint.FlintIndexMetadata; import org.opensearch.sql.spark.flint.FlintIndexState; import org.opensearch.sql.spark.flint.FlintIndexStateModel; import org.opensearch.sql.spark.flint.FlintIndexStateModelService; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; @ExtendWith(MockitoExtension.class) class FlintIndexOpVacuumTest { @@ -30,16 +33,20 @@ class FlintIndexOpVacuumTest { public static final String DATASOURCE_NAME = "DATASOURCE_NAME"; public static final String LATEST_ID = "LATEST_ID"; public static final String INDEX_NAME = "INDEX_NAME"; + public static final FlintIndexMetadata FLINT_INDEX_METADATA_WITH_LATEST_ID = - FlintIndexMetadata.builder().latestId(LATEST_ID).opensearchIndexName(INDEX_NAME).build(); + createFlintIndexMetadataWithLatestId(); + public static final FlintIndexMetadata FLINT_INDEX_METADATA_WITHOUT_LATEST_ID = - FlintIndexMetadata.builder().opensearchIndexName(INDEX_NAME).build(); + createFlintIndexMetadataWithoutLatestId(); + @Mock FlintIndexClient flintIndexClient; @Mock FlintIndexStateModelService flintIndexStateModelService; @Mock EMRServerlessClientFactory emrServerlessClientFactory; @Mock FlintIndexStateModel flintIndexStateModel; @Mock FlintIndexStateModel transitionedFlintIndexStateModel; @Mock AsyncQueryRequestContext asyncQueryRequestContext; + @Mock AsyncQueryScheduler asyncQueryScheduler; RuntimeException testException = new RuntimeException("Test Exception"); @@ -52,7 +59,33 @@ public void setUp() { flintIndexStateModelService, DATASOURCE_NAME, flintIndexClient, - emrServerlessClientFactory); + emrServerlessClientFactory, + asyncQueryScheduler); + } + + private static FlintIndexMetadata createFlintIndexMetadataWithLatestId() { + return FlintIndexMetadata.builder() + .latestId(LATEST_ID) + .opensearchIndexName(INDEX_NAME) + .flintIndexOptions(new FlintIndexOptions()) + .build(); + } + + private static FlintIndexMetadata createFlintIndexMetadataWithoutLatestId() { + return FlintIndexMetadata.builder() + .opensearchIndexName(INDEX_NAME) + .flintIndexOptions(new FlintIndexOptions()) + .build(); + } + + private FlintIndexMetadata createFlintIndexMetadataWithExternalScheduler() { + FlintIndexOptions flintIndexOptions = new FlintIndexOptions(); + flintIndexOptions.setOption(FlintIndexOptions.SCHEDULER_MODE, "external"); + + return FlintIndexMetadata.builder() + .opensearchIndexName(INDEX_NAME) + .flintIndexOptions(flintIndexOptions) + .build(); } @Test @@ -207,4 +240,22 @@ public void testApplyHappyPath() { .deleteFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext); verify(flintIndexClient).deleteIndex(INDEX_NAME); } + + @Test + public void testRunOpWithExternalScheduler() { + FlintIndexMetadata flintIndexMetadata = createFlintIndexMetadataWithExternalScheduler(); + flintIndexOpVacuum.runOp(flintIndexMetadata, flintIndexStateModel, asyncQueryRequestContext); + + verify(asyncQueryScheduler).removeJob(INDEX_NAME); + verify(flintIndexClient).deleteIndex(INDEX_NAME); + } + + @Test + public void testRunOpWithoutExternalScheduler() { + FlintIndexMetadata flintIndexMetadata = FLINT_INDEX_METADATA_WITHOUT_LATEST_ID; + flintIndexOpVacuum.runOp(flintIndexMetadata, flintIndexStateModel, asyncQueryRequestContext); + + verify(asyncQueryScheduler, never()).removeJob(INDEX_NAME); + verify(flintIndexClient).deleteIndex(INDEX_NAME); + } } diff --git a/async-query/build.gradle b/async-query/build.gradle index abda6161d3..53fdcbe292 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -99,7 +99,7 @@ jacocoTestCoverageVerification { // ignore because XContext IOException 'org.opensearch.sql.spark.execution.statestore.StateStore', 'org.opensearch.sql.spark.rest.*', - 'org.opensearch.sql.spark.scheduler.OpenSearchRefreshIndexJobRequestParser', + 'org.opensearch.sql.spark.scheduler.parser.OpenSearchScheduleQueryJobRequestParser', 'org.opensearch.sql.spark.transport.model.*' ] limit { diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java index c7a66fc6be..9ebde4fe83 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java @@ -8,10 +8,11 @@ import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -34,12 +35,13 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.sql.spark.scheduler.job.OpenSearchRefreshIndexJob; -import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.sql.spark.scheduler.job.ScheduledAsyncQueryJobRunner; +import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; +import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; /** Scheduler class for managing asynchronous query jobs. */ -public class OpenSearchAsyncQueryScheduler { +@RequiredArgsConstructor +public class OpenSearchAsyncQueryScheduler implements AsyncQueryScheduler { public static final String SCHEDULER_INDEX_NAME = ".async-query-scheduler"; public static final String SCHEDULER_PLUGIN_JOB_TYPE = "async-query-scheduler"; private static final String SCHEDULER_INDEX_MAPPING_FILE_NAME = @@ -48,22 +50,14 @@ public class OpenSearchAsyncQueryScheduler { "async-query-scheduler-index-settings.yml"; private static final Logger LOG = LogManager.getLogger(); - private Client client; - private ClusterService clusterService; - - /** Loads job resources, setting up required services and job runner instance. */ - public void loadJobResource(Client client, ClusterService clusterService, ThreadPool threadPool) { - this.client = client; - this.clusterService = clusterService; - OpenSearchRefreshIndexJob openSearchRefreshIndexJob = - OpenSearchRefreshIndexJob.getJobRunnerInstance(); - openSearchRefreshIndexJob.setClusterService(clusterService); - openSearchRefreshIndexJob.setThreadPool(threadPool); - openSearchRefreshIndexJob.setClient(client); - } + private final Client client; + private final ClusterService clusterService; + @Override /** Schedules a new job by indexing it into the job index. */ - public void scheduleJob(OpenSearchRefreshIndexJobRequest request) { + public void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); if (!this.clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) { createAsyncQuerySchedulerIndex(); } @@ -92,19 +86,28 @@ public void scheduleJob(OpenSearchRefreshIndexJobRequest request) { } /** Unschedules a job by marking it as disabled and updating its last update time. */ - public void unscheduleJob(String jobId) throws IOException { - assertIndexExists(); - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(jobId) + @Override + public void unscheduleJob(String jobId) { + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(jobId) .enabled(false) .lastUpdateTime(Instant.now()) .build(); - updateJob(request); + try { + updateJob(request); + LOG.info("Unscheduled job for jobId: {}", jobId); + } catch (IllegalStateException | DocumentMissingException e) { + LOG.error("Failed to unschedule job: {}", jobId, e); + } } /** Updates an existing job with new parameters. */ - public void updateJob(OpenSearchRefreshIndexJobRequest request) throws IOException { + @Override + @SneakyThrows + public void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); assertIndexExists(); UpdateRequest updateRequest = new UpdateRequest(SCHEDULER_INDEX_NAME, request.getName()); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -130,6 +133,7 @@ public void updateJob(OpenSearchRefreshIndexJobRequest request) throws IOExcepti } /** Removes a job by deleting its document from the index. */ + @Override public void removeJob(String jobId) { assertIndexExists(); DeleteRequest deleteRequest = new DeleteRequest(SCHEDULER_INDEX_NAME, jobId); @@ -192,6 +196,6 @@ private void assertIndexExists() { /** Returns the job runner instance for the scheduler. */ public static ScheduledJobRunner getJobRunner() { - return OpenSearchRefreshIndexJob.getJobRunnerInstance(); + return ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); } } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java deleted file mode 100644 index e465a8790f..0000000000 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJob.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.scheduler.job; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.plugins.Plugin; -import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; -import org.opensearch.threadpool.ThreadPool; - -/** - * The job runner class for scheduling refresh index query. - * - * <p>The job runner should be a singleton class if it uses OpenSearch client or other objects - * passed from OpenSearch. Because when registering the job runner to JobScheduler plugin, - * OpenSearch has not invoked plugins' createComponents() method. That is saying the plugin is not - * completely initialized, and the OpenSearch {@link org.opensearch.client.Client}, {@link - * ClusterService} and other objects are not available to plugin and this job runner. - * - * <p>So we have to move this job runner initialization to {@link Plugin} createComponents() method, - * and using singleton job runner to ensure we register a usable job runner instance to JobScheduler - * plugin. - */ -public class OpenSearchRefreshIndexJob implements ScheduledJobRunner { - - private static final Logger log = LogManager.getLogger(OpenSearchRefreshIndexJob.class); - - public static OpenSearchRefreshIndexJob INSTANCE = new OpenSearchRefreshIndexJob(); - - public static OpenSearchRefreshIndexJob getJobRunnerInstance() { - return INSTANCE; - } - - private ClusterService clusterService; - private ThreadPool threadPool; - private Client client; - - private OpenSearchRefreshIndexJob() { - // Singleton class, use getJobRunnerInstance method instead of constructor - } - - public void setClusterService(ClusterService clusterService) { - this.clusterService = clusterService; - } - - public void setThreadPool(ThreadPool threadPool) { - this.threadPool = threadPool; - } - - public void setClient(Client client) { - this.client = client; - } - - @Override - public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { - if (!(jobParameter instanceof OpenSearchRefreshIndexJobRequest)) { - throw new IllegalStateException( - "Job parameter is not instance of OpenSearchRefreshIndexJobRequest, type: " - + jobParameter.getClass().getCanonicalName()); - } - - if (this.clusterService == null) { - throw new IllegalStateException("ClusterService is not initialized."); - } - - if (this.threadPool == null) { - throw new IllegalStateException("ThreadPool is not initialized."); - } - - if (this.client == null) { - throw new IllegalStateException("Client is not initialized."); - } - - Runnable runnable = - () -> { - doRefresh(jobParameter.getName()); - }; - threadPool.generic().submit(runnable); - } - - void doRefresh(String refreshIndex) { - // TODO: add logic to refresh index - log.info("Scheduled refresh index job on : " + refreshIndex); - } -} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java new file mode 100644 index 0000000000..3652acf295 --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.job; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.plugins.Plugin; +import org.opensearch.sql.legacy.executor.AsyncRestExecutor; +import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; +import org.opensearch.sql.spark.rest.model.CreateAsyncQueryRequest; +import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse; +import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; +import org.opensearch.threadpool.ThreadPool; + +/** + * The job runner class for scheduling async query. + * + * <p>The job runner should be a singleton class if it uses OpenSearch client or other objects + * passed from OpenSearch. Because when registering the job runner to JobScheduler plugin, + * OpenSearch has not invoked plugins' createComponents() method. That is saying the plugin is not + * completely initialized, and the OpenSearch {@link org.opensearch.client.Client}, {@link + * ClusterService} and other objects are not available to plugin and this job runner. + * + * <p>So we have to move this job runner initialization to {@link Plugin} createComponents() method, + * and using singleton job runner to ensure we register a usable job runner instance to JobScheduler + * plugin. + */ +public class ScheduledAsyncQueryJobRunner implements ScheduledJobRunner { + // Share SQL plugin thread pool + private static final String ASYNC_QUERY_THREAD_POOL_NAME = + AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME; + private static final Logger LOGGER = LogManager.getLogger(ScheduledAsyncQueryJobRunner.class); + + private static ScheduledAsyncQueryJobRunner INSTANCE = new ScheduledAsyncQueryJobRunner(); + + public static ScheduledAsyncQueryJobRunner getJobRunnerInstance() { + return INSTANCE; + } + + private ClusterService clusterService; + private ThreadPool threadPool; + private Client client; + private AsyncQueryExecutorService asyncQueryExecutorService; + + private ScheduledAsyncQueryJobRunner() { + // Singleton class, use getJobRunnerInstance method instead of constructor + } + + /** Loads job resources, setting up required services and job runner instance. */ + public void loadJobResource( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + AsyncQueryExecutorService asyncQueryExecutorService) { + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.asyncQueryExecutorService = asyncQueryExecutorService; + } + + @Override + public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { + // Parser will convert jobParameter to ScheduledAsyncQueryJobRequest + if (!(jobParameter instanceof ScheduledAsyncQueryJobRequest)) { + throw new IllegalStateException( + "Job parameter is not instance of ScheduledAsyncQueryJobRequest, type: " + + jobParameter.getClass().getCanonicalName()); + } + + if (this.clusterService == null) { + throw new IllegalStateException("ClusterService is not initialized."); + } + + if (this.threadPool == null) { + throw new IllegalStateException("ThreadPool is not initialized."); + } + + if (this.client == null) { + throw new IllegalStateException("Client is not initialized."); + } + + if (this.asyncQueryExecutorService == null) { + throw new IllegalStateException("AsyncQueryExecutorService is not initialized."); + } + + Runnable runnable = + () -> { + try { + doRefresh((ScheduledAsyncQueryJobRequest) jobParameter); + } catch (Throwable throwable) { + LOGGER.error(throwable); + } + }; + threadPool.executor(ASYNC_QUERY_THREAD_POOL_NAME).submit(runnable); + } + + void doRefresh(ScheduledAsyncQueryJobRequest request) { + LOGGER.info("Scheduled refresh index job on job: " + request.getName()); + CreateAsyncQueryRequest createAsyncQueryRequest = + new CreateAsyncQueryRequest( + request.getScheduledQuery(), request.getDataSource(), request.getQueryLang()); + CreateAsyncQueryResponse createAsyncQueryResponse = + asyncQueryExecutorService.createAsyncQuery( + createAsyncQueryRequest, new NullAsyncQueryRequestContext()); + LOGGER.info("Created async query with queryId: " + createAsyncQueryResponse.getQueryId()); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java deleted file mode 100644 index 7eaa4e2d29..0000000000 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.scheduler.model; - -import java.io.IOException; -import java.time.Instant; -import lombok.Builder; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.jobscheduler.spi.schedule.Schedule; - -/** Represents a job request to refresh index. */ -@Builder -public class OpenSearchRefreshIndexJobRequest implements ScheduledJobParameter { - // Constant fields for JSON serialization - public static final String JOB_NAME_FIELD = "jobName"; - public static final String JOB_TYPE_FIELD = "jobType"; - public static final String LAST_UPDATE_TIME_FIELD = "lastUpdateTime"; - public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; - public static final String SCHEDULE_FIELD = "schedule"; - public static final String ENABLED_TIME_FIELD = "enabledTime"; - public static final String ENABLED_TIME_FIELD_READABLE = "enabled_time_field"; - public static final String LOCK_DURATION_SECONDS = "lockDurationSeconds"; - public static final String JITTER = "jitter"; - public static final String ENABLED_FIELD = "enabled"; - - // name is doc id - private final String jobName; - private final String jobType; - private final Schedule schedule; - private final boolean enabled; - private final Instant lastUpdateTime; - private final Instant enabledTime; - private final Long lockDurationSeconds; - private final Double jitter; - - @Override - public String getName() { - return jobName; - } - - public String getJobType() { - return jobType; - } - - @Override - public Schedule getSchedule() { - return schedule; - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public Instant getLastUpdateTime() { - return lastUpdateTime; - } - - @Override - public Instant getEnabledTime() { - return enabledTime; - } - - @Override - public Long getLockDurationSeconds() { - return lockDurationSeconds; - } - - @Override - public Double getJitter() { - return jitter; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) - throws IOException { - builder.startObject(); - builder.field(JOB_NAME_FIELD, getName()).field(ENABLED_FIELD, isEnabled()); - if (getSchedule() != null) { - builder.field(SCHEDULE_FIELD, getSchedule()); - } - if (getJobType() != null) { - builder.field(JOB_TYPE_FIELD, getJobType()); - } - if (getEnabledTime() != null) { - builder.timeField( - ENABLED_TIME_FIELD, ENABLED_TIME_FIELD_READABLE, getEnabledTime().toEpochMilli()); - } - builder.timeField( - LAST_UPDATE_TIME_FIELD, - LAST_UPDATE_TIME_FIELD_READABLE, - getLastUpdateTime().toEpochMilli()); - if (this.lockDurationSeconds != null) { - builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); - } - if (this.jitter != null) { - builder.field(JITTER, this.jitter); - } - builder.endObject(); - return builder; - } -} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java new file mode 100644 index 0000000000..9b85a11888 --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.model; + +import java.io.IOException; +import java.time.Instant; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.sql.spark.rest.model.LangType; +import org.opensearch.sql.spark.scheduler.parser.IntervalScheduleParser; + +/** Represents a job request to refresh index. */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class ScheduledAsyncQueryJobRequest extends AsyncQuerySchedulerRequest + implements ScheduledJobParameter { + // Constant fields for JSON serialization + public static final String ACCOUNT_ID_FIELD = "accountId"; + public static final String JOB_ID_FIELD = "jobId"; + public static final String DATA_SOURCE_NAME_FIELD = "dataSource"; + public static final String SCHEDULED_QUERY_FIELD = "scheduledQuery"; + public static final String QUERY_LANG_FIELD = "queryLang"; + public static final String LAST_UPDATE_TIME_FIELD = "lastUpdateTime"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FIELD = "enabledTime"; + public static final String LOCK_DURATION_SECONDS = "lockDurationSeconds"; + public static final String JITTER = "jitter"; + public static final String ENABLED_FIELD = "enabled"; + private final Schedule schedule; + + @Builder + public ScheduledAsyncQueryJobRequest( + String accountId, + String jobId, + String dataSource, + String scheduledQuery, + LangType queryLang, + Schedule schedule, // Use the OpenSearch Schedule type + boolean enabled, + Instant lastUpdateTime, + Instant enabledTime, + Long lockDurationSeconds, + Double jitter) { + super( + accountId, + jobId, + dataSource, + scheduledQuery, + queryLang, + schedule, + enabled, + lastUpdateTime, + enabledTime, + lockDurationSeconds, + jitter); + this.schedule = schedule; + } + + @Override + public String getName() { + return getJobId(); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public Instant getLastUpdateTime() { + return lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return enabledTime; + } + + @Override + public Schedule getSchedule() { + return schedule; + } + + @Override + public Long getLockDurationSeconds() { + return lockDurationSeconds; + } + + @Override + public Double getJitter() { + return jitter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) + throws IOException { + builder.startObject(); + if (getAccountId() != null) { + builder.field(ACCOUNT_ID_FIELD, getAccountId()); + } + builder.field(JOB_ID_FIELD, getJobId()).field(ENABLED_FIELD, isEnabled()); + if (getDataSource() != null) { + builder.field(DATA_SOURCE_NAME_FIELD, getDataSource()); + } + if (getScheduledQuery() != null) { + builder.field(SCHEDULED_QUERY_FIELD, getScheduledQuery()); + } + if (getQueryLang() != null) { + builder.field(QUERY_LANG_FIELD, getQueryLang()); + } + if (getSchedule() != null) { + builder.field(SCHEDULE_FIELD, getSchedule()); + } + if (getEnabledTime() != null) { + builder.field(ENABLED_TIME_FIELD, getEnabledTime().toEpochMilli()); + } + builder.field(LAST_UPDATE_TIME_FIELD, getLastUpdateTime().toEpochMilli()); + if (this.lockDurationSeconds != null) { + builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); + } + if (this.jitter != null) { + builder.field(JITTER, this.jitter); + } + builder.endObject(); + return builder; + } + + public static ScheduledAsyncQueryJobRequest fromAsyncQuerySchedulerRequest( + AsyncQuerySchedulerRequest request) { + Instant updateTime = + request.getLastUpdateTime() != null ? request.getLastUpdateTime() : Instant.now(); + return ScheduledAsyncQueryJobRequest.builder() + .accountId(request.getAccountId()) + .jobId(request.getJobId()) + .dataSource(request.getDataSource()) + .scheduledQuery(request.getScheduledQuery()) + .queryLang(request.getQueryLang()) + .enabled(request.isEnabled()) + .lastUpdateTime(updateTime) + .enabledTime(request.getEnabledTime()) + .lockDurationSeconds(request.getLockDurationSeconds()) + .jitter(request.getJitter()) + .schedule(IntervalScheduleParser.parse(request.getSchedule(), updateTime)) + .build(); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java new file mode 100644 index 0000000000..2d5a1b332f --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.parser; + +import com.google.common.annotations.VisibleForTesting; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +/** Parse string raw schedule into job scheduler IntervalSchedule */ +public class IntervalScheduleParser { + + private static final Pattern DURATION_PATTERN = + Pattern.compile( + "^(\\d+)\\s*(years?|months?|weeks?|days?|hours?|minutes?|minute|mins?|seconds?|secs?|milliseconds?|millis?|microseconds?|microsecond|micros?|micros|nanoseconds?|nanos?)$", + Pattern.CASE_INSENSITIVE); + + public static Schedule parse(Object schedule, Instant startTime) { + if (schedule == null) { + return null; + } + + if (schedule instanceof Schedule) { + return (Schedule) schedule; + } + + if (!(schedule instanceof String)) { + throw new IllegalArgumentException("Schedule must be a String object for parsing."); + } + + String intervalStr = ((String) schedule).trim().toLowerCase(); + + Matcher matcher = DURATION_PATTERN.matcher(intervalStr); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid interval format: " + intervalStr); + } + + long value = Long.parseLong(matcher.group(1)); + String unitStr = matcher.group(2).toLowerCase(); + + // Convert to a supported unit or directly return an IntervalSchedule + long intervalInMinutes = convertToSupportedUnit(value, unitStr); + + return new IntervalSchedule(startTime, (int) intervalInMinutes, ChronoUnit.MINUTES); + } + + @VisibleForTesting + protected static long convertToSupportedUnit(long value, String unitStr) { + switch (unitStr) { + case "years": + case "year": + throw new IllegalArgumentException("Years cannot be converted to minutes accurately."); + case "months": + case "month": + throw new IllegalArgumentException("Months cannot be converted to minutes accurately."); + case "weeks": + case "week": + return value * 7 * 24 * 60; // Convert weeks to minutes + case "days": + case "day": + return value * 24 * 60; // Convert days to minutes + case "hours": + case "hour": + return value * 60; // Convert hours to minutes + case "minutes": + case "minute": + case "mins": + case "min": + return value; // Already in minutes + case "seconds": + case "second": + case "secs": + case "sec": + return value / 60; // Convert seconds to minutes + case "milliseconds": + case "millisecond": + case "millis": + case "milli": + return value / (60 * 1000); // Convert milliseconds to minutes + case "microseconds": + case "microsecond": + case "micros": + case "micro": + return value / (60 * 1000 * 1000); // Convert microseconds to minutes + case "nanoseconds": + case "nanosecond": + case "nanos": + case "nano": + return value / (60 * 1000 * 1000 * 1000L); // Convert nanoseconds to minutes + default: + throw new IllegalArgumentException("Unsupported time unit: " + unitStr); + } + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java similarity index 57% rename from async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java rename to async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java index 0422e7c015..9e33ef0248 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchRefreshIndexJobRequestParser.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.spark.scheduler; +package org.opensearch.sql.spark.scheduler.parser; import java.io.IOException; import java.time.Instant; @@ -11,9 +11,10 @@ import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.jobscheduler.spi.ScheduledJobParser; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; -import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; +import org.opensearch.sql.spark.rest.model.LangType; +import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; -public class OpenSearchRefreshIndexJobRequestParser { +public class OpenSearchScheduleQueryJobRequestParser { private static Instant parseInstantValue(XContentParser parser) throws IOException { if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { @@ -28,8 +29,8 @@ private static Instant parseInstantValue(XContentParser parser) throws IOExcepti public static ScheduledJobParser getJobParser() { return (parser, id, jobDocVersion) -> { - OpenSearchRefreshIndexJobRequest.OpenSearchRefreshIndexJobRequestBuilder builder = - OpenSearchRefreshIndexJobRequest.builder(); + ScheduledAsyncQueryJobRequest.ScheduledAsyncQueryJobRequestBuilder builder = + ScheduledAsyncQueryJobRequest.builder(); XContentParserUtils.ensureExpectedToken( XContentParser.Token.START_OBJECT, parser.nextToken(), parser); @@ -37,28 +38,37 @@ public static ScheduledJobParser getJobParser() { String fieldName = parser.currentName(); parser.nextToken(); switch (fieldName) { - case OpenSearchRefreshIndexJobRequest.JOB_NAME_FIELD: - builder.jobName(parser.text()); + case ScheduledAsyncQueryJobRequest.ACCOUNT_ID_FIELD: + builder.accountId(parser.text()); break; - case OpenSearchRefreshIndexJobRequest.JOB_TYPE_FIELD: - builder.jobType(parser.text()); + case ScheduledAsyncQueryJobRequest.JOB_ID_FIELD: + builder.jobId(parser.text()); break; - case OpenSearchRefreshIndexJobRequest.ENABLED_FIELD: + case ScheduledAsyncQueryJobRequest.DATA_SOURCE_NAME_FIELD: + builder.dataSource(parser.text()); + break; + case ScheduledAsyncQueryJobRequest.SCHEDULED_QUERY_FIELD: + builder.scheduledQuery(parser.text()); + break; + case ScheduledAsyncQueryJobRequest.QUERY_LANG_FIELD: + builder.queryLang(LangType.fromString(parser.text())); + break; + case ScheduledAsyncQueryJobRequest.ENABLED_FIELD: builder.enabled(parser.booleanValue()); break; - case OpenSearchRefreshIndexJobRequest.ENABLED_TIME_FIELD: + case ScheduledAsyncQueryJobRequest.ENABLED_TIME_FIELD: builder.enabledTime(parseInstantValue(parser)); break; - case OpenSearchRefreshIndexJobRequest.LAST_UPDATE_TIME_FIELD: + case ScheduledAsyncQueryJobRequest.LAST_UPDATE_TIME_FIELD: builder.lastUpdateTime(parseInstantValue(parser)); break; - case OpenSearchRefreshIndexJobRequest.SCHEDULE_FIELD: + case ScheduledAsyncQueryJobRequest.SCHEDULE_FIELD: builder.schedule(ScheduleParser.parse(parser)); break; - case OpenSearchRefreshIndexJobRequest.LOCK_DURATION_SECONDS: + case ScheduledAsyncQueryJobRequest.LOCK_DURATION_SECONDS: builder.lockDurationSeconds(parser.longValue()); break; - case OpenSearchRefreshIndexJobRequest.JITTER: + case ScheduledAsyncQueryJobRequest.JITTER: builder.jitter(parser.doubleValue()); break; default: diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index 9cc69b2fb7..52ffda483c 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -61,6 +61,8 @@ import org.opensearch.sql.spark.parameter.SparkSubmitParametersBuilderProvider; import org.opensearch.sql.spark.response.JobExecutionResponseReader; import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; @RequiredArgsConstructor public class AsyncExecutorServiceModule extends AbstractModule { @@ -136,12 +138,14 @@ public FlintIndexOpFactory flintIndexOpFactory( FlintIndexStateModelService flintIndexStateModelService, FlintIndexClient flintIndexClient, FlintIndexMetadataServiceImpl flintIndexMetadataService, - EMRServerlessClientFactory emrServerlessClientFactory) { + EMRServerlessClientFactory emrServerlessClientFactory, + AsyncQueryScheduler asyncQueryScheduler) { return new FlintIndexOpFactory( flintIndexStateModelService, flintIndexClient, flintIndexMetadataService, - emrServerlessClientFactory); + emrServerlessClientFactory, + asyncQueryScheduler); } @Provides @@ -245,6 +249,14 @@ public SessionConfigSupplier sessionConfigSupplier(Settings settings) { return new OpenSearchSessionConfigSupplier(settings); } + @Provides + @Singleton + public AsyncQueryScheduler asyncQueryScheduler(NodeClient client, ClusterService clusterService) { + OpenSearchAsyncQueryScheduler scheduler = + new OpenSearchAsyncQueryScheduler(client, clusterService); + return scheduler; + } + private void registerStateStoreMetrics(StateStore stateStore) { GaugeMetric<Long> activeSessionMetric = new GaugeMetric<>( diff --git a/async-query/src/main/resources/async-query-scheduler-index-mapping.yml b/async-query/src/main/resources/async-query-scheduler-index-mapping.yml index 36bd1b873e..1aa90e8ed8 100644 --- a/async-query/src/main/resources/async-query-scheduler-index-mapping.yml +++ b/async-query/src/main/resources/async-query-scheduler-index-mapping.yml @@ -8,9 +8,15 @@ # Also "dynamic" is set to "false" so that other fields cannot be added. dynamic: false properties: - name: + accountId: type: keyword - jobType: + jobId: + type: keyword + dataSource: + type: keyword + scheduledQuery: + type: text + queryLang: type: keyword lastUpdateTime: type: date diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index 641b083d53..9b897d36b4 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -100,6 +100,8 @@ import org.opensearch.sql.spark.parameter.SparkSubmitParametersBuilderProvider; import org.opensearch.sql.spark.response.JobExecutionResponseReader; import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; +import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.test.OpenSearchIntegTestCase; @@ -124,6 +126,7 @@ public class AsyncQueryExecutorServiceSpec extends OpenSearchIntegTestCase { protected StateStore stateStore; protected SessionStorageService sessionStorageService; protected StatementStorageService statementStorageService; + protected AsyncQueryScheduler asyncQueryScheduler; protected AsyncQueryRequestContext asyncQueryRequestContext; protected SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); @@ -204,6 +207,7 @@ public void setup() { new OpenSearchSessionStorageService(stateStore, new SessionModelXContentSerializer()); statementStorageService = new OpenSearchStatementStorageService(stateStore, new StatementModelXContentSerializer()); + asyncQueryScheduler = new OpenSearchAsyncQueryScheduler(client, clusterService); } protected FlintIndexOpFactory getFlintIndexOpFactory( @@ -212,7 +216,8 @@ protected FlintIndexOpFactory getFlintIndexOpFactory( flintIndexStateModelService, flintIndexClient, flintIndexMetadataService, - emrServerlessClientFactory); + emrServerlessClientFactory, + asyncQueryScheduler); } @After @@ -298,7 +303,8 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( flintIndexStateModelService, flintIndexClient, new FlintIndexMetadataServiceImpl(client), - emrServerlessClientFactory), + emrServerlessClientFactory, + asyncQueryScheduler), emrServerlessClientFactory, new OpenSearchMetricsService(), sparkSubmitParametersBuilderProvider); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java index de86f111f3..a4a6eb6471 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -42,8 +43,7 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; public class OpenSearchAsyncQuerySchedulerTest { @@ -57,9 +57,6 @@ public class OpenSearchAsyncQuerySchedulerTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) private ClusterService clusterService; - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private ThreadPool threadPool; - @Mock private ActionFuture<IndexResponse> indexResponseActionFuture; @Mock private ActionFuture<UpdateResponse> updateResponseActionFuture; @@ -77,8 +74,7 @@ public class OpenSearchAsyncQuerySchedulerTest { @BeforeEach public void setup() { MockitoAnnotations.openMocks(this); - scheduler = new OpenSearchAsyncQueryScheduler(); - scheduler.loadJobResource(client, clusterService, threadPool); + scheduler = new OpenSearchAsyncQueryScheduler(client, clusterService); } @Test @@ -95,9 +91,9 @@ public void testScheduleJob() { when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -119,9 +115,9 @@ public void testScheduleJobWithExistingJob() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) .thenReturn(Boolean.TRUE); - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -148,9 +144,9 @@ public void testScheduleJobWithExceptions() { .thenReturn(new CreateIndexResponse(true, true, TEST_SCHEDULER_INDEX_NAME)); when(client.index(any(IndexRequest.class))).thenThrow(new RuntimeException("Test exception")); - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -199,14 +195,17 @@ public void testUnscheduleJob() throws IOException { public void testUnscheduleJobWithIndexNotFound() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - assertThrows(IllegalStateException.class, () -> scheduler.unscheduleJob(TEST_JOB_ID)); + scheduler.unscheduleJob(TEST_JOB_ID); + + // Verify that no update operation was performed + verify(client, never()).update(any(UpdateRequest.class)); } @Test public void testUpdateJob() throws IOException { - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -229,9 +228,9 @@ public void testUpdateJob() throws IOException { @Test public void testUpdateJobWithIndexNotFound() { - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -242,9 +241,9 @@ public void testUpdateJobWithIndexNotFound() { @Test public void testUpdateJobWithExceptions() { - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -351,9 +350,9 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { Mockito.when(createIndexResponseActionFuture.actionGet()) .thenReturn(new CreateIndexResponse(false, false, SCHEDULER_INDEX_NAME)); - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -367,9 +366,9 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { @Test public void testUpdateJobNotFound() { - OpenSearchRefreshIndexJobRequest request = - OpenSearchRefreshIndexJobRequest.builder() - .jobName(TEST_JOB_ID) + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java deleted file mode 100644 index cbf137997e..0000000000 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/OpenSearchRefreshIndexJobTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.scheduler.job; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - -import java.time.Instant; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Answers; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.sql.spark.scheduler.model.OpenSearchRefreshIndexJobRequest; -import org.opensearch.threadpool.ThreadPool; - -public class OpenSearchRefreshIndexJobTest { - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private ClusterService clusterService; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private ThreadPool threadPool; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Client client; - - @Mock private JobExecutionContext context; - - private OpenSearchRefreshIndexJob jobRunner; - - private OpenSearchRefreshIndexJob spyJobRunner; - - @BeforeEach - public void setup() { - MockitoAnnotations.openMocks(this); - jobRunner = OpenSearchRefreshIndexJob.getJobRunnerInstance(); - jobRunner.setClient(null); - jobRunner.setClusterService(null); - jobRunner.setThreadPool(null); - } - - @Test - public void testRunJobWithCorrectParameter() { - spyJobRunner = spy(jobRunner); - spyJobRunner.setClusterService(clusterService); - spyJobRunner.setThreadPool(threadPool); - spyJobRunner.setClient(client); - - OpenSearchRefreshIndexJobRequest jobParameter = - OpenSearchRefreshIndexJobRequest.builder() - .jobName("testJob") - .lastUpdateTime(Instant.now()) - .lockDurationSeconds(10L) - .build(); - - spyJobRunner.runJob(jobParameter, context); - - ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class); - verify(threadPool.generic()).submit(captor.capture()); - - Runnable runnable = captor.getValue(); - runnable.run(); - - verify(spyJobRunner).doRefresh(eq(jobParameter.getName())); - } - - @Test - public void testRunJobWithIncorrectParameter() { - jobRunner = OpenSearchRefreshIndexJob.getJobRunnerInstance(); - jobRunner.setClusterService(clusterService); - jobRunner.setThreadPool(threadPool); - jobRunner.setClient(client); - - ScheduledJobParameter wrongParameter = mock(ScheduledJobParameter.class); - - IllegalStateException exception = - assertThrows( - IllegalStateException.class, - () -> jobRunner.runJob(wrongParameter, context), - "Expected IllegalStateException but no exception was thrown"); - - assertEquals( - "Job parameter is not instance of OpenSearchRefreshIndexJobRequest, type: " - + wrongParameter.getClass().getCanonicalName(), - exception.getMessage()); - } - - @Test - public void testRunJobWithUninitializedServices() { - OpenSearchRefreshIndexJobRequest jobParameter = - OpenSearchRefreshIndexJobRequest.builder() - .jobName("testJob") - .lastUpdateTime(Instant.now()) - .build(); - - IllegalStateException exception = - assertThrows( - IllegalStateException.class, - () -> jobRunner.runJob(jobParameter, context), - "Expected IllegalStateException but no exception was thrown"); - assertEquals("ClusterService is not initialized.", exception.getMessage()); - - jobRunner.setClusterService(clusterService); - - exception = - assertThrows( - IllegalStateException.class, - () -> jobRunner.runJob(jobParameter, context), - "Expected IllegalStateException but no exception was thrown"); - assertEquals("ThreadPool is not initialized.", exception.getMessage()); - - jobRunner.setThreadPool(threadPool); - - exception = - assertThrows( - IllegalStateException.class, - () -> jobRunner.runJob(jobParameter, context), - "Expected IllegalStateException but no exception was thrown"); - assertEquals("Client is not initialized.", exception.getMessage()); - } - - @Test - public void testGetJobRunnerInstanceMultipleCalls() { - OpenSearchRefreshIndexJob instance1 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); - OpenSearchRefreshIndexJob instance2 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); - OpenSearchRefreshIndexJob instance3 = OpenSearchRefreshIndexJob.getJobRunnerInstance(); - - assertSame(instance1, instance2); - assertSame(instance2, instance3); - } -} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java new file mode 100644 index 0000000000..cba8d43a2a --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java @@ -0,0 +1,210 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.job; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LogEvent; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.sql.legacy.executor.AsyncRestExecutor; +import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService; +import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; +import org.opensearch.sql.spark.rest.model.CreateAsyncQueryRequest; +import org.opensearch.sql.spark.rest.model.LangType; +import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; +import org.opensearch.threadpool.ThreadPool; + +public class ScheduledAsyncQueryJobRunnerTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ClusterService clusterService; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ThreadPool threadPool; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Client client; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private AsyncQueryExecutorService asyncQueryExecutorService; + + @Mock private JobExecutionContext context; + + private ScheduledAsyncQueryJobRunner jobRunner; + + private ScheduledAsyncQueryJobRunner spyJobRunner; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + jobRunner = ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); + jobRunner.loadJobResource(null, null, null, null); + } + + @Test + public void testRunJobWithCorrectParameter() { + spyJobRunner = spy(jobRunner); + spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); + + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId("testJob") + .lastUpdateTime(Instant.now()) + .lockDurationSeconds(10L) + .scheduledQuery("REFRESH INDEX testIndex") + .dataSource("testDataSource") + .queryLang(LangType.SQL) + .build(); + + CreateAsyncQueryRequest createAsyncQueryRequest = + new CreateAsyncQueryRequest( + request.getScheduledQuery(), request.getDataSource(), request.getQueryLang()); + spyJobRunner.runJob(request, context); + + ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class); + verify(threadPool.executor(AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME)) + .submit(captor.capture()); + + Runnable runnable = captor.getValue(); + runnable.run(); + + verify(spyJobRunner).doRefresh(eq(request)); + verify(asyncQueryExecutorService) + .createAsyncQuery(eq(createAsyncQueryRequest), any(NullAsyncQueryRequestContext.class)); + } + + @Test + public void testRunJobWithIncorrectParameter() { + jobRunner = ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); + jobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); + + ScheduledJobParameter wrongParameter = mock(ScheduledJobParameter.class); + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(wrongParameter, context), + "Expected IllegalStateException but no exception was thrown"); + + assertEquals( + "Job parameter is not instance of ScheduledAsyncQueryJobRequest, type: " + + wrongParameter.getClass().getCanonicalName(), + exception.getMessage()); + } + + @Test + public void testDoRefreshThrowsException() { + spyJobRunner = spy(jobRunner); + spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); + + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId("testJob") + .lastUpdateTime(Instant.now()) + .lockDurationSeconds(10L) + .scheduledQuery("REFRESH INDEX testIndex") + .dataSource("testDataSource") + .queryLang(LangType.SQL) + .build(); + + doThrow(new RuntimeException("Test exception")).when(spyJobRunner).doRefresh(request); + + Logger logger = LogManager.getLogger(ScheduledAsyncQueryJobRunner.class); + Appender mockAppender = mock(Appender.class); + when(mockAppender.getName()).thenReturn("MockAppender"); + when(mockAppender.isStarted()).thenReturn(true); + when(mockAppender.isStopped()).thenReturn(false); + ((org.apache.logging.log4j.core.Logger) logger) + .addAppender((org.apache.logging.log4j.core.Appender) mockAppender); + + spyJobRunner.runJob(request, context); + + ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class); + verify(threadPool.executor(AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME)) + .submit(captor.capture()); + + Runnable runnable = captor.getValue(); + runnable.run(); + + verify(spyJobRunner).doRefresh(eq(request)); + verify(mockAppender).append(any(LogEvent.class)); + } + + @Test + public void testRunJobWithUninitializedServices() { + ScheduledAsyncQueryJobRequest jobParameter = + ScheduledAsyncQueryJobRequest.builder() + .jobId("testJob") + .lastUpdateTime(Instant.now()) + .build(); + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("ClusterService is not initialized.", exception.getMessage()); + + jobRunner.loadJobResource(null, clusterService, null, null); + + exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("ThreadPool is not initialized.", exception.getMessage()); + + jobRunner.loadJobResource(null, clusterService, threadPool, null); + + exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("Client is not initialized.", exception.getMessage()); + + jobRunner.loadJobResource(client, clusterService, threadPool, null); + + exception = + assertThrows( + IllegalStateException.class, + () -> jobRunner.runJob(jobParameter, context), + "Expected IllegalStateException but no exception was thrown"); + assertEquals("AsyncQueryExecutorService is not initialized.", exception.getMessage()); + } + + @Test + public void testGetJobRunnerInstanceMultipleCalls() { + ScheduledAsyncQueryJobRunner instance1 = ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); + ScheduledAsyncQueryJobRunner instance2 = ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); + ScheduledAsyncQueryJobRunner instance3 = ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); + + assertSame(instance1, instance2); + assertSame(instance2, instance3); + } +} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java deleted file mode 100644 index 108f1acfd5..0000000000 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/OpenSearchRefreshIndexJobRequestTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.scheduler.model; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; - -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import org.junit.jupiter.api.Test; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; - -public class OpenSearchRefreshIndexJobRequestTest { - - @Test - public void testBuilderAndGetterMethods() { - Instant now = Instant.now(); - IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); - - OpenSearchRefreshIndexJobRequest jobRequest = - OpenSearchRefreshIndexJobRequest.builder() - .jobName("testJob") - .jobType("testType") - .schedule(schedule) - .enabled(true) - .lastUpdateTime(now) - .enabledTime(now) - .lockDurationSeconds(60L) - .jitter(0.1) - .build(); - - assertEquals("testJob", jobRequest.getName()); - assertEquals("testType", jobRequest.getJobType()); - assertEquals(schedule, jobRequest.getSchedule()); - assertTrue(jobRequest.isEnabled()); - assertEquals(now, jobRequest.getLastUpdateTime()); - assertEquals(now, jobRequest.getEnabledTime()); - assertEquals(60L, jobRequest.getLockDurationSeconds()); - assertEquals(0.1, jobRequest.getJitter()); - } - - @Test - public void testToXContent() throws IOException { - Instant now = Instant.now(); - IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); - - OpenSearchRefreshIndexJobRequest jobRequest = - OpenSearchRefreshIndexJobRequest.builder() - .jobName("testJob") - .jobType("testType") - .schedule(schedule) - .enabled(true) - .lastUpdateTime(now) - .enabledTime(now) - .lockDurationSeconds(60L) - .jitter(0.1) - .build(); - - XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); - jobRequest.toXContent(builder, EMPTY_PARAMS); - String jsonString = builder.toString(); - - assertTrue(jsonString.contains("\"jobName\" : \"testJob\"")); - assertTrue(jsonString.contains("\"jobType\" : \"testType\"")); - assertTrue(jsonString.contains("\"start_time\" : " + now.toEpochMilli())); - assertTrue(jsonString.contains("\"period\" : 1")); - assertTrue(jsonString.contains("\"unit\" : \"Minutes\"")); - assertTrue(jsonString.contains("\"enabled\" : true")); - assertTrue(jsonString.contains("\"lastUpdateTime\" : " + now.toEpochMilli())); - assertTrue(jsonString.contains("\"enabledTime\" : " + now.toEpochMilli())); - assertTrue(jsonString.contains("\"lockDurationSeconds\" : 60")); - assertTrue(jsonString.contains("\"jitter\" : 0.1")); - } -} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java new file mode 100644 index 0000000000..85d1948dc3 --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java @@ -0,0 +1,210 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.model; + +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Test; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.sql.spark.rest.model.LangType; + +public class ScheduledAsyncQueryJobRequestTest { + + @Test + public void testBuilderAndGetterMethods() { + Instant now = Instant.now(); + IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); + + ScheduledAsyncQueryJobRequest jobRequest = + ScheduledAsyncQueryJobRequest.builder() + .accountId("testAccount") + .jobId("testJob") + .dataSource("testDataSource") + .scheduledQuery("SELECT * FROM test") + .queryLang(LangType.SQL) + .schedule(schedule) + .enabled(true) + .lastUpdateTime(now) + .enabledTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + assertEquals("testAccount", jobRequest.getAccountId()); + assertEquals("testJob", jobRequest.getJobId()); + assertEquals("testJob", jobRequest.getName()); + assertEquals("testDataSource", jobRequest.getDataSource()); + assertEquals("SELECT * FROM test", jobRequest.getScheduledQuery()); + assertEquals(LangType.SQL, jobRequest.getQueryLang()); + assertEquals(schedule, jobRequest.getSchedule()); + assertTrue(jobRequest.isEnabled()); + assertEquals(now, jobRequest.getLastUpdateTime()); + assertEquals(now, jobRequest.getEnabledTime()); + assertEquals(60L, jobRequest.getLockDurationSeconds()); + assertEquals(0.1, jobRequest.getJitter()); + } + + @Test + public void testToXContent() throws IOException { + Instant now = Instant.now(); + IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); + + ScheduledAsyncQueryJobRequest request = + ScheduledAsyncQueryJobRequest.builder() + .accountId("testAccount") + .jobId("testJob") + .dataSource("testDataSource") + .scheduledQuery("SELECT * FROM test") + .queryLang(LangType.SQL) + .schedule(schedule) + .enabled(true) + .enabledTime(now) + .lastUpdateTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + request.toXContent(builder, EMPTY_PARAMS); + String jsonString = builder.toString(); + + assertTrue(jsonString.contains("\"accountId\" : \"testAccount\"")); + assertTrue(jsonString.contains("\"jobId\" : \"testJob\"")); + assertTrue(jsonString.contains("\"dataSource\" : \"testDataSource\"")); + assertTrue(jsonString.contains("\"scheduledQuery\" : \"SELECT * FROM test\"")); + assertTrue(jsonString.contains("\"queryLang\" : \"SQL\"")); + assertTrue(jsonString.contains("\"start_time\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"period\" : 1")); + assertTrue(jsonString.contains("\"unit\" : \"Minutes\"")); + assertTrue(jsonString.contains("\"enabled\" : true")); + assertTrue(jsonString.contains("\"lastUpdateTime\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"enabledTime\" : " + now.toEpochMilli())); + assertTrue(jsonString.contains("\"lockDurationSeconds\" : 60")); + assertTrue(jsonString.contains("\"jitter\" : 0.1")); + } + + @Test + public void testFromAsyncQuerySchedulerRequest() { + Instant now = Instant.now(); + AsyncQuerySchedulerRequest request = new AsyncQuerySchedulerRequest(); + request.setJobId("testJob"); + request.setAccountId("testAccount"); + request.setDataSource("testDataSource"); + request.setScheduledQuery("SELECT * FROM test"); + request.setQueryLang(LangType.SQL); + request.setSchedule("1 minutes"); + request.setEnabled(true); + request.setLastUpdateTime(now); + request.setLockDurationSeconds(60L); + request.setJitter(0.1); + + ScheduledAsyncQueryJobRequest jobRequest = + ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(request); + + assertEquals("testJob", jobRequest.getJobId()); + assertEquals("testAccount", jobRequest.getAccountId()); + assertEquals("testDataSource", jobRequest.getDataSource()); + assertEquals("SELECT * FROM test", jobRequest.getScheduledQuery()); + assertEquals(LangType.SQL, jobRequest.getQueryLang()); + assertEquals(new IntervalSchedule(now, 1, ChronoUnit.MINUTES), jobRequest.getSchedule()); + assertTrue(jobRequest.isEnabled()); + assertEquals(60L, jobRequest.getLockDurationSeconds()); + assertEquals(0.1, jobRequest.getJitter()); + } + + @Test + public void testFromAsyncQuerySchedulerRequestWithInvalidSchedule() { + AsyncQuerySchedulerRequest request = new AsyncQuerySchedulerRequest(); + request.setJobId("testJob"); + request.setSchedule(new Object()); // Set schedule to a non-String object + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(request); + }); + + assertEquals("Schedule must be a String object for parsing.", exception.getMessage()); + } + + @Test + public void testEqualsAndHashCode() { + Instant now = Instant.now(); + IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); + + ScheduledAsyncQueryJobRequest request1 = + ScheduledAsyncQueryJobRequest.builder() + .accountId("testAccount") + .jobId("testJob") + .dataSource("testDataSource") + .scheduledQuery("SELECT * FROM test") + .queryLang(LangType.SQL) + .schedule(schedule) + .enabled(true) + .enabledTime(now) + .lastUpdateTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + // Test toString + String toString = request1.toString(); + assertTrue(toString.contains("accountId=testAccount")); + assertTrue(toString.contains("jobId=testJob")); + assertTrue(toString.contains("dataSource=testDataSource")); + assertTrue(toString.contains("scheduledQuery=SELECT * FROM test")); + assertTrue(toString.contains("queryLang=SQL")); + assertTrue(toString.contains("enabled=true")); + assertTrue(toString.contains("lockDurationSeconds=60")); + assertTrue(toString.contains("jitter=0.1")); + + ScheduledAsyncQueryJobRequest request2 = + ScheduledAsyncQueryJobRequest.builder() + .accountId("testAccount") + .jobId("testJob") + .dataSource("testDataSource") + .scheduledQuery("SELECT * FROM test") + .queryLang(LangType.SQL) + .schedule(schedule) + .enabled(true) + .enabledTime(now) + .lastUpdateTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + assertEquals(request1, request2); + assertEquals(request1.hashCode(), request2.hashCode()); + + ScheduledAsyncQueryJobRequest request3 = + ScheduledAsyncQueryJobRequest.builder() + .accountId("differentAccount") + .jobId("testJob") + .dataSource("testDataSource") + .scheduledQuery("SELECT * FROM test") + .queryLang(LangType.SQL) + .schedule(schedule) + .enabled(true) + .enabledTime(now) + .lastUpdateTime(now) + .lockDurationSeconds(60L) + .jitter(0.1) + .build(); + + assertNotEquals(request1, request3); + assertNotEquals(request1.hashCode(), request3.hashCode()); + } +} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java new file mode 100644 index 0000000000..b119c345b9 --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java @@ -0,0 +1,122 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.scheduler.parser; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +public class IntervalScheduleParserTest { + + private Instant startTime; + + @BeforeEach + public void setup() { + startTime = Instant.now(); + } + + @Test + public void testParseValidScheduleString() { + verifyParseSchedule(5, "5 minutes"); + } + + @Test + public void testParseValidScheduleStringWithDifferentUnits() { + verifyParseSchedule(120, "2 hours"); + verifyParseSchedule(1440, "1 day"); + verifyParseSchedule(30240, "3 weeks"); + } + + @Test + public void testParseNullSchedule() { + Schedule schedule = IntervalScheduleParser.parse(null, startTime); + assertNull(schedule); + } + + @Test + public void testParseScheduleObject() { + IntervalSchedule expectedSchedule = new IntervalSchedule(startTime, 10, ChronoUnit.MINUTES); + Schedule schedule = IntervalScheduleParser.parse(expectedSchedule, startTime); + assertEquals(expectedSchedule, schedule); + } + + @Test + public void testParseInvalidScheduleString() { + String scheduleStr = "invalid schedule"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> IntervalScheduleParser.parse(scheduleStr, startTime), + "Expected IllegalArgumentException but no exception was thrown"); + + assertEquals("Invalid interval format: " + scheduleStr.toLowerCase(), exception.getMessage()); + } + + @Test + public void testParseUnsupportedUnits() { + assertThrows( + IllegalArgumentException.class, + () -> IntervalScheduleParser.parse("1 year", startTime), + "Expected IllegalArgumentException but no exception was thrown"); + + assertThrows( + IllegalArgumentException.class, + () -> IntervalScheduleParser.parse("1 month", startTime), + "Expected IllegalArgumentException but no exception was thrown"); + } + + @Test + public void testParseNonStringSchedule() { + Object nonStringSchedule = 12345; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> IntervalScheduleParser.parse(nonStringSchedule, startTime), + "Expected IllegalArgumentException but no exception was thrown"); + + assertEquals("Schedule must be a String object for parsing.", exception.getMessage()); + } + + @Test + public void testParseScheduleWithNanoseconds() { + verifyParseSchedule(1, "60000000000 nanoseconds"); + } + + @Test + public void testParseScheduleWithMilliseconds() { + verifyParseSchedule(1, "60000 milliseconds"); + } + + @Test + public void testParseScheduleWithMicroseconds() { + verifyParseSchedule(1, "60000000 microseconds"); + } + + @Test + public void testUnsupportedTimeUnit() { + assertThrows( + IllegalArgumentException.class, + () -> IntervalScheduleParser.convertToSupportedUnit(10, "unsupportedunit"), + "Expected IllegalArgumentException but no exception was thrown"); + } + + @Test + public void testParseScheduleWithSeconds() { + verifyParseSchedule(2, "120 seconds"); + } + + private void verifyParseSchedule(int expectedMinutes, String scheduleStr) { + Schedule schedule = IntervalScheduleParser.parse(scheduleStr, startTime); + assertEquals(new IntervalSchedule(startTime, expectedMinutes, ChronoUnit.MINUTES), schedule); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 971ef5e928..560c5edadd 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -96,8 +96,8 @@ import org.opensearch.sql.spark.flint.operation.FlintIndexOpFactory; import org.opensearch.sql.spark.rest.RestAsyncQueryManagementAction; import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; -import org.opensearch.sql.spark.scheduler.OpenSearchRefreshIndexJobRequestParser; -import org.opensearch.sql.spark.scheduler.job.OpenSearchRefreshIndexJob; +import org.opensearch.sql.spark.scheduler.job.ScheduledAsyncQueryJobRunner; +import org.opensearch.sql.spark.scheduler.parser.OpenSearchScheduleQueryJobRequestParser; import org.opensearch.sql.spark.storage.SparkStorageFactory; import org.opensearch.sql.spark.transport.TransportCancelAsyncQueryRequestAction; import org.opensearch.sql.spark.transport.TransportCreateAsyncQueryRequestAction; @@ -217,8 +217,6 @@ public Collection<Object> createComponents( this.client = (NodeClient) client; this.dataSourceService = createDataSourceService(); dataSourceService.createDataSource(defaultOpenSearchDataSourceMetadata()); - this.asyncQueryScheduler = new OpenSearchAsyncQueryScheduler(); - this.asyncQueryScheduler.loadJobResource(client, clusterService, threadPool); LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); LocalClusterState.state().setClient(client); @@ -247,11 +245,13 @@ public Collection<Object> createComponents( dataSourceService, injector.getInstance(FlintIndexMetadataServiceImpl.class), injector.getInstance(FlintIndexOpFactory.class)); + AsyncQueryExecutorService asyncQueryExecutorService = + injector.getInstance(AsyncQueryExecutorService.class); + ScheduledAsyncQueryJobRunner.getJobRunnerInstance() + .loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); + return ImmutableList.of( - dataSourceService, - injector.getInstance(AsyncQueryExecutorService.class), - clusterManagerEventListener, - pluginSettings); + dataSourceService, asyncQueryExecutorService, clusterManagerEventListener, pluginSettings); } @Override @@ -266,12 +266,12 @@ public String getJobIndex() { @Override public ScheduledJobRunner getJobRunner() { - return OpenSearchRefreshIndexJob.getJobRunnerInstance(); + return ScheduledAsyncQueryJobRunner.getJobRunnerInstance(); } @Override public ScheduledJobParser getJobParser() { - return OpenSearchRefreshIndexJobRequestParser.getJobParser(); + return OpenSearchScheduleQueryJobRequestParser.getJobParser(); } @Override @@ -342,6 +342,9 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett systemIndexDescriptors.add( new SystemIndexDescriptor( SPARK_REQUEST_BUFFER_INDEX_NAME + "*", "SQL Spark Request Buffer index pattern")); + systemIndexDescriptors.add( + new SystemIndexDescriptor( + OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME, "SQL Scheduler job index")); return systemIndexDescriptors; } } From 6c5c68597c77a9402bc680a45f95f19f5da995fe Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala <sashank.nistala@gmail.com> Date: Wed, 4 Sep 2024 15:39:19 -0700 Subject: [PATCH 37/96] Adds validation to allow only flint queries and sql SELECT queries to security lake type datasource (#2959) * allows only flint queries and select sql queries to security lake datasource Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add sql validator for security lake and refactor validateSparkSqlQuery class Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * spotless fixes Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * address review comments. Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * address comment to extract validate logic into a separate method in tests Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add more tests to get more code coverage Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --- .../dispatcher/SparkQueryDispatcher.java | 4 +- .../sql/spark/utils/SQLQueryUtils.java | 67 +++++++++++-- .../sql/spark/utils/SQLQueryUtilsTest.java | 93 ++++++++++++++++++- .../sql/datasource/model/DataSourceType.java | 2 + 4 files changed, 152 insertions(+), 14 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 710f472acb..c4b5c89540 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -54,7 +54,9 @@ public DispatchQueryResponse dispatch( dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } - List<String> validationErrors = SQLQueryUtils.validateSparkSqlQuery(query); + List<String> validationErrors = + SQLQueryUtils.validateSparkSqlQuery( + dataSourceService.getDataSource(dispatchQueryRequest.getDatasource()), query); if (!validationErrors.isEmpty()) { throw new IllegalArgumentException( "Query is not allowed: " + String.join(", ", validationErrors)); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index ff08a8f41e..ce3bcab06b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -15,9 +15,13 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsBaseVisitor; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsLexer; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser; @@ -25,6 +29,7 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.IdentifierReferenceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.StatementContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName; @@ -38,13 +43,14 @@ */ @UtilityClass public class SQLQueryUtils { + private static final Logger logger = LogManager.getLogger(SQLQueryUtils.class); public static List<FullyQualifiedTableName> extractFullyQualifiedTableNames(String sqlQuery) { SqlBaseParser sqlBaseParser = new SqlBaseParser( new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery)))); sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); - SqlBaseParser.StatementContext statement = sqlBaseParser.statement(); + StatementContext statement = sqlBaseParser.statement(); SparkSqlTableNameVisitor sparkSqlTableNameVisitor = new SparkSqlTableNameVisitor(); statement.accept(sparkSqlTableNameVisitor); return sparkSqlTableNameVisitor.getFullyQualifiedTableNames(); @@ -77,32 +83,73 @@ public static boolean isFlintExtensionQuery(String sqlQuery) { } } - public static List<String> validateSparkSqlQuery(String sqlQuery) { - SparkSqlValidatorVisitor sparkSqlValidatorVisitor = new SparkSqlValidatorVisitor(); + public static List<String> validateSparkSqlQuery(DataSource datasource, String sqlQuery) { SqlBaseParser sqlBaseParser = new SqlBaseParser( new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery)))); sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); try { - SqlBaseParser.StatementContext statement = sqlBaseParser.statement(); - sparkSqlValidatorVisitor.visit(statement); - return sparkSqlValidatorVisitor.getValidationErrors(); - } catch (SyntaxCheckException syntaxCheckException) { + SqlBaseValidatorVisitor sqlParserBaseVisitor = getSparkSqlValidatorVisitor(datasource); + StatementContext statement = sqlBaseParser.statement(); + sqlParserBaseVisitor.visit(statement); + return sqlParserBaseVisitor.getValidationErrors(); + } catch (SyntaxCheckException e) { + logger.error( + String.format( + "Failed to parse sql statement context while validating sql query %s", sqlQuery), + e); return Collections.emptyList(); } } - private static class SparkSqlValidatorVisitor extends SqlBaseParserBaseVisitor<Void> { + private SqlBaseValidatorVisitor getSparkSqlValidatorVisitor(DataSource datasource) { + if (datasource != null + && datasource.getConnectorType() != null + && datasource.getConnectorType().equals(DataSourceType.SECURITY_LAKE)) { + return new SparkSqlSecurityLakeValidatorVisitor(); + } else { + return new SparkSqlValidatorVisitor(); + } + } - @Getter private final List<String> validationErrors = new ArrayList<>(); + /** + * A base class extending SqlBaseParserBaseVisitor for validating Spark Sql Queries. The class + * supports accumulating validation errors on visiting sql statement + */ + @Getter + private static class SqlBaseValidatorVisitor<T> extends SqlBaseParserBaseVisitor<T> { + private final List<String> validationErrors = new ArrayList<>(); + } + /** A generic validator impl for Spark Sql Queries */ + private static class SparkSqlValidatorVisitor extends SqlBaseValidatorVisitor<Void> { @Override public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { - validationErrors.add("Creating user-defined functions is not allowed"); + getValidationErrors().add("Creating user-defined functions is not allowed"); return super.visitCreateFunction(ctx); } } + /** A validator impl specific to Security Lake for Spark Sql Queries */ + private static class SparkSqlSecurityLakeValidatorVisitor extends SqlBaseValidatorVisitor<Void> { + + public SparkSqlSecurityLakeValidatorVisitor() { + // only select statement allowed. hence we add the validation error to all types of statements + // by default + // and remove the validation error only for select statement. + getValidationErrors() + .add( + "Unsupported sql statement for security lake data source. Only select queries are" + + " allowed"); + } + + @Override + public Void visitStatementDefault(SqlBaseParser.StatementDefaultContext ctx) { + getValidationErrors().clear(); + return super.visitStatementDefault(ctx); + } + } + public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor<Void> { @Getter private List<FullyQualifiedTableName> fullyQualifiedTableNames = new LinkedList<>(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index fe7777606c..235fe84c70 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.index; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.mv; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.skippingIndex; @@ -18,7 +19,10 @@ import lombok.Getter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName; import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType; import org.opensearch.sql.spark.dispatcher.model.IndexQueryDetails; @@ -27,6 +31,8 @@ @ExtendWith(MockitoExtension.class) public class SQLQueryUtilsTest { + @Mock private DataSource dataSource; + @Test void testExtractionOfTableNameFromSQLQueries() { String sqlQuery = "select * from my_glue.default.http_logs"; @@ -404,15 +410,96 @@ void testAutoRefresh() { @Test void testValidateSparkSqlQuery_ValidQuery() { - String validQuery = "SELECT * FROM users WHERE age > 18"; - List<String> errors = SQLQueryUtils.validateSparkSqlQuery(validQuery); + List<String> errors = + validateSparkSqlQueryForDataSourceType( + "DELETE FROM Customers WHERE CustomerName='Alfreds Futterkiste'", + DataSourceType.PROMETHEUS); + assertTrue(errors.isEmpty(), "Valid query should not produce any errors"); } + @Test + void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake() { + List<String> errors = + validateSparkSqlQueryForDataSourceType( + "SELECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); + + assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); + } + + @Test + void testValidateSparkSqlQuery_SelectQuery_DataSourceTypeNull() { + List<String> errors = + validateSparkSqlQueryForDataSourceType("SELECT * FROM users WHERE age > 18", null); + + assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); + } + + @Test + void testValidateSparkSqlQuery_InvalidQuery_SyntaxCheckFailureSkippedWithoutValidationError() { + List<String> errors = + validateSparkSqlQueryForDataSourceType( + "SEECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); + + assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); + } + + @Test + void testValidateSparkSqlQuery_nullDatasource() { + List<String> errors = + SQLQueryUtils.validateSparkSqlQuery(null, "SELECT * FROM users WHERE age > 18"); + assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); + } + + private List<String> validateSparkSqlQueryForDataSourceType( + String query, DataSourceType dataSourceType) { + when(this.dataSource.getConnectorType()).thenReturn(dataSourceType); + + return SQLQueryUtils.validateSparkSqlQuery(this.dataSource, query); + } + + @Test + void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake_ValidationFails() { + List<String> errors = + validateSparkSqlQueryForDataSourceType( + "REFRESH INDEX cv1 ON mys3.default.http_logs", DataSourceType.SECURITY_LAKE); + + assertFalse( + errors.isEmpty(), + "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" + + " queries. Given query was REFRESH sql query"); + assertEquals( + errors.get(0), + "Unsupported sql statement for security lake data source. Only select queries are allowed"); + } + + @Test + void + testValidateSparkSqlQuery_NonSelectStatementContainingSelectClause_DataSourceSecurityLake_ValidationFails() { + String query = + "CREATE TABLE AccountSummaryOrWhatever AS " + + "select taxid, address1, count(address1) from dbo.t " + + "group by taxid, address1;"; + + List<String> errors = + validateSparkSqlQueryForDataSourceType(query, DataSourceType.SECURITY_LAKE); + + assertFalse( + errors.isEmpty(), + "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" + + " queries. Given query was REFRESH sql query"); + assertEquals( + errors.get(0), + "Unsupported sql statement for security lake data source. Only select queries are allowed"); + } + @Test void testValidateSparkSqlQuery_InvalidQuery() { + when(dataSource.getConnectorType()).thenReturn(DataSourceType.PROMETHEUS); String invalidQuery = "CREATE FUNCTION myUDF AS 'com.example.UDF'"; - List<String> errors = SQLQueryUtils.validateSparkSqlQuery(invalidQuery); + + List<String> errors = SQLQueryUtils.validateSparkSqlQuery(dataSource, invalidQuery); + assertFalse(errors.isEmpty(), "Invalid query should produce errors"); assertEquals(1, errors.size(), "Should have one error"); assertEquals( diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java index c74964fc00..ac8ae1a5e1 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java @@ -7,9 +7,11 @@ import java.util.HashMap; import java.util.Map; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor +@EqualsAndHashCode public class DataSourceType { public static DataSourceType PROMETHEUS = new DataSourceType("PROMETHEUS"); public static DataSourceType OPENSEARCH = new DataSourceType("OPENSEARCH"); From b14a8cb7eeb5bd74eb1aebfe7cd8d56bfe2c88d8 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 4 Sep 2024 16:09:28 -0700 Subject: [PATCH 38/96] Fix handler for existing query (#2968) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../spark/dispatcher/RefreshQueryHandler.java | 4 +- .../dispatcher/SparkQueryDispatcher.java | 2 +- .../sql/spark/dispatcher/model/JobType.java | 1 + .../asyncquery/AsyncQueryCoreIntegTest.java | 4 +- .../dispatcher/SparkQueryDispatcherTest.java | 39 +++++++++++++------ 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java index 38145a143e..cf5a0c6c59 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java @@ -73,7 +73,7 @@ public String cancelJob( @Override public DispatchQueryResponse submit( DispatchQueryRequest dispatchQueryRequest, DispatchQueryContext context) { - leaseManager.borrow(new LeaseRequest(JobType.BATCH, dispatchQueryRequest.getDatasource())); + leaseManager.borrow(new LeaseRequest(JobType.REFRESH, dispatchQueryRequest.getDatasource())); DispatchQueryResponse resp = super.submit(dispatchQueryRequest, context); DataSourceMetadata dataSourceMetadata = context.getDataSourceMetadata(); @@ -83,7 +83,7 @@ public DispatchQueryResponse submit( .resultIndex(resp.getResultIndex()) .sessionId(resp.getSessionId()) .datasourceName(dataSourceMetadata.getName()) - .jobType(JobType.BATCH) + .jobType(JobType.REFRESH) .indexName(context.getIndexQueryDetails().openSearchIndexName()) .build(); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index c4b5c89540..4df2b5450d 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -179,7 +179,7 @@ private AsyncQueryHandler getAsyncQueryHandlerForExistingQuery( return queryHandlerFactory.getInteractiveQueryHandler(); } else if (IndexDMLHandler.isIndexDMLQuery(asyncQueryJobMetadata.getJobId())) { return queryHandlerFactory.getIndexDMLHandler(); - } else if (asyncQueryJobMetadata.getJobType() == JobType.BATCH) { + } else if (asyncQueryJobMetadata.getJobType() == JobType.REFRESH) { return queryHandlerFactory.getRefreshQueryHandler(asyncQueryJobMetadata.getAccountId()); } else if (asyncQueryJobMetadata.getJobType() == JobType.STREAMING) { return queryHandlerFactory.getStreamingQueryHandler(asyncQueryJobMetadata.getAccountId()); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java index 01f5f422e9..af1f69d74b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java @@ -8,6 +8,7 @@ public enum JobType { INTERACTIVE("interactive"), STREAMING("streaming"), + REFRESH("refresh"), BATCH("batch"); private String text; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 226e0ff5eb..e1c9bb6f39 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -424,7 +424,7 @@ public void createRefreshQuery() { verifyGetQueryIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(JOB_ID, JobType.REFRESH); } @Test @@ -541,7 +541,7 @@ public void cancelIndexDMLQuery() { @Test public void cancelRefreshQuery() { givenJobMetadataExists( - getBaseAsyncQueryJobMetadataBuilder().jobType(JobType.BATCH).indexName(INDEX_NAME)); + getBaseAsyncQueryJobMetadataBuilder().jobType(JobType.REFRESH).indexName(INDEX_NAME)); when(flintIndexMetadataService.getFlintIndexMetadata(INDEX_NAME, asyncQueryRequestContext)) .thenReturn( ImmutableMap.of( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index d040db24b2..5154b71574 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -387,6 +387,7 @@ void testDispatchCreateManualRefreshIndexQuery() { verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + Assertions.assertEquals(JobType.BATCH, dispatchQueryResponse.getJobType()); verifyNoInteractions(flintIndexMetadataService); } @@ -685,6 +686,7 @@ void testRefreshIndexQuery() { verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + Assertions.assertEquals(JobType.REFRESH, dispatchQueryResponse.getJobType()); verifyNoInteractions(flintIndexMetadataService); } @@ -859,12 +861,7 @@ void testDispatchWithUnSupportedDataSourceType() { @Test void testCancelJob() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(emrServerlessClient.cancelJobRun(EMRS_APPLICATION_ID, EMR_JOB_ID, false)) - .thenReturn( - new CancelJobRunResult() - .withJobRunId(EMR_JOB_ID) - .withApplicationId(EMRS_APPLICATION_ID)); + givenCancelJobRunSucceed(); String queryId = sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); @@ -925,17 +922,32 @@ void testCancelQueryWithInvalidStatementId() { @Test void testCancelQueryWithNoSessionId() { + givenCancelJobRunSucceed(); + + String queryId = + sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); + + Assertions.assertEquals(QUERY_ID, queryId); + } + + @Test + void testCancelBatchJob() { + givenCancelJobRunSucceed(); + + String queryId = + sparkQueryDispatcher.cancelJob( + asyncQueryJobMetadata(JobType.BATCH), asyncQueryRequestContext); + + Assertions.assertEquals(QUERY_ID, queryId); + } + + private void givenCancelJobRunSucceed() { when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); when(emrServerlessClient.cancelJobRun(EMRS_APPLICATION_ID, EMR_JOB_ID, false)) .thenReturn( new CancelJobRunResult() .withJobRunId(EMR_JOB_ID) .withApplicationId(EMRS_APPLICATION_ID)); - - String queryId = - sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); - - Assertions.assertEquals(QUERY_ID, queryId); } @Test @@ -1184,11 +1196,16 @@ private DispatchQueryRequest dispatchQueryRequestWithSessionId(String query, Str } private AsyncQueryJobMetadata asyncQueryJobMetadata() { + return asyncQueryJobMetadata(JobType.INTERACTIVE); + } + + private AsyncQueryJobMetadata asyncQueryJobMetadata(JobType jobType) { return AsyncQueryJobMetadata.builder() .queryId(QUERY_ID) .applicationId(EMRS_APPLICATION_ID) .jobId(EMR_JOB_ID) .datasourceName(MY_GLUE) + .jobType(jobType) .build(); } From da622ebd6206b5215f0eceffbbe10218853d6d6d Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Wed, 4 Sep 2024 19:10:59 -0700 Subject: [PATCH 39/96] Add feature flag for async query scheduler (#2973) * Add feature flag for async query scheduler Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix Jacoco verification Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .../spark/data/constants/SparkConstants.java | 4 ++ .../asyncquery/AsyncQueryCoreIntegTest.java | 6 +- ...archAsyncQuerySchedulerConfigComposer.java | 36 ++++++++++ .../parser/IntervalScheduleParser.java | 1 - .../config/AsyncExecutorServiceModule.java | 2 + ...AsyncQuerySchedulerConfigComposerTest.java | 68 ++++++++++++++++++ .../parser/IntervalScheduleParserTest.java | 8 +++ .../sql/common/setting/Settings.java | 4 ++ docs/user/admin/settings.rst | 69 +++++++++++++++++++ .../setting/OpenSearchSettings.java | 27 ++++++++ .../setting/OpenSearchSettingsTest.java | 20 ++++++ 11 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index 9b82022d8f..43815a9904 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -87,6 +87,10 @@ public class SparkConstants { public static final String JAVA_HOME_LOCATION = "/usr/lib/jvm/java-17-amazon-corretto.x86_64/"; public static final String FLINT_JOB_QUERY = "spark.flint.job.query"; public static final String FLINT_JOB_QUERY_ID = "spark.flint.job.queryId"; + public static final String FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED = + "spark.flint.job.externalScheduler.enabled"; + public static final String FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL = + "spark.flint.job.externalScheduler.interval"; public static final String FLINT_JOB_REQUEST_INDEX = "spark.flint.job.requestIndex"; public static final String FLINT_JOB_SESSION_ID = "spark.flint.job.sessionId"; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index e1c9bb6f39..ca4a8736d2 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -227,7 +227,7 @@ public void createDropIndexQueryWithScheduler() { assertNull(response.getSessionId()); verifyGetQueryIdCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); verify(asyncQueryScheduler).unscheduleJob(indexName); } @@ -275,7 +275,7 @@ public void createVacuumIndexQueryWithScheduler() { verify(flintIndexClient).deleteIndex(indexName); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); verify(asyncQueryScheduler).removeJob(indexName); } @@ -342,7 +342,7 @@ public void createAlterIndexQueryWithScheduler() { verify(asyncQueryScheduler).unscheduleJob(indexName); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); } @Test diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java new file mode 100644 index 0000000000..6dce09a406 --- /dev/null +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.config; + +import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED; +import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL; + +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; +import org.opensearch.sql.spark.parameter.GeneralSparkParameterComposer; +import org.opensearch.sql.spark.parameter.SparkSubmitParameters; + +@RequiredArgsConstructor +public class OpenSearchAsyncQuerySchedulerConfigComposer implements GeneralSparkParameterComposer { + private final Settings settings; + + @Override + public void compose( + SparkSubmitParameters sparkSubmitParameters, + DispatchQueryRequest dispatchQueryRequest, + AsyncQueryRequestContext context) { + String externalSchedulerEnabled = + settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED); + String externalSchedulerInterval = + settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL); + sparkSubmitParameters.setConfigItem( + FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, externalSchedulerEnabled); + sparkSubmitParameters.setConfigItem( + FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java index 2d5a1b332f..47e652c570 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParser.java @@ -15,7 +15,6 @@ /** Parse string raw schedule into job scheduler IntervalSchedule */ public class IntervalScheduleParser { - private static final Pattern DURATION_PATTERN = Pattern.compile( "^(\\d+)\\s*(years?|months?|weeks?|days?|hours?|minutes?|minute|mins?|seconds?|secs?|milliseconds?|millis?|microseconds?|microsecond|micros?|micros|nanoseconds?|nanos?)$", diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index 52ffda483c..c6f6ffcd81 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -24,6 +24,7 @@ import org.opensearch.sql.spark.asyncquery.OpenSearchAsyncQueryJobMetadataStorageService; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.client.EMRServerlessClientFactoryImpl; +import org.opensearch.sql.spark.config.OpenSearchAsyncQuerySchedulerConfigComposer; import org.opensearch.sql.spark.config.OpenSearchExtraParameterComposer; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigClusterSettingLoader; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; @@ -168,6 +169,7 @@ public SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider collection.register( DataSourceType.SECURITY_LAKE, new S3GlueDataSourceSparkParameterComposer(clusterSettingLoader)); + collection.register(new OpenSearchAsyncQuerySchedulerConfigComposer(settings)); collection.register(new OpenSearchExtraParameterComposer(clusterSettingLoader)); return new SparkSubmitParametersBuilderProvider(collection); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java new file mode 100644 index 0000000000..7836c63b7a --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java @@ -0,0 +1,68 @@ +package org.opensearch.sql.spark.config; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; +import org.opensearch.sql.spark.parameter.SparkSubmitParameters; + +@ExtendWith(MockitoExtension.class) +public class OpenSearchAsyncQuerySchedulerConfigComposerTest { + + @Mock private Settings settings; + @Mock private SparkSubmitParameters sparkSubmitParameters; + @Mock private DispatchQueryRequest dispatchQueryRequest; + @Mock private AsyncQueryRequestContext context; + + private OpenSearchAsyncQuerySchedulerConfigComposer composer; + + @BeforeEach + public void setUp() { + composer = new OpenSearchAsyncQuerySchedulerConfigComposer(settings); + } + + @Test + public void testCompose() { + when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) + .thenReturn("true"); + when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) + .thenReturn("10 minutes"); + + composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); + + verify(sparkSubmitParameters) + .setConfigItem("spark.flint.job.externalScheduler.enabled", "true"); + verify(sparkSubmitParameters) + .setConfigItem("spark.flint.job.externalScheduler.interval", "10 minutes"); + } + + @Test + public void testComposeWithDisabledScheduler() { + when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) + .thenReturn("false"); + + composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); + + verify(sparkSubmitParameters) + .setConfigItem("spark.flint.job.externalScheduler.enabled", "false"); + } + + @Test + public void testComposeWithMissingInterval() { + when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) + .thenReturn("true"); + when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) + .thenReturn(""); + + composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); + + verify(sparkSubmitParameters).setConfigItem("spark.flint.job.externalScheduler.interval", ""); + } +} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java index b119c345b9..f211548c7c 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/parser/IntervalScheduleParserTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.scheduler.parser; +import static org.junit.Assert.assertNotNull; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -25,6 +26,13 @@ public void setup() { startTime = Instant.now(); } + @Test + public void testConstructor() { + // Test that the constructor of IntervalScheduleParser can be invoked + IntervalScheduleParser parser = new IntervalScheduleParser(); + assertNotNull(parser); + } + @Test public void testParseValidScheduleString() { verifyParseSchedule(5, "5 minutes"); diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index b6643f3209..0037032d22 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -51,6 +51,10 @@ public enum Key { /** Async query Settings * */ ASYNC_QUERY_ENABLED("plugins.query.executionengine.async_query.enabled"), + ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED( + "plugins.query.executionengine.async_query.external_scheduler.enabled"), + ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL( + "plugins.query.executionengine.async_query.external_scheduler.interval"), STREAMING_JOB_HOUSEKEEPER_INTERVAL( "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval"); diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 236406e2c7..71718d1726 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -639,6 +639,75 @@ Request:: } } +plugins.query.executionengine.async_query.external_scheduler.enabled +===================================================================== + +Description +----------- +This setting controls whether the external scheduler is enabled for async queries. + +* Default Value: true +* Scope: Node-level +* Dynamic Update: Yes, this setting can be updated dynamically. + +To disable the external scheduler, use the following command: + +Request :: + + sh$ curl -sS -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings \ + ... -d '{"transient":{"plugins.query.executionengine.async_query.external_scheduler.enabled":"false"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "plugins": { + "query": { + "executionengine": { + "async_query": { + "external_scheduler": { + "enabled": "false" + } + } + } + } + } + } + } + +plugins.query.executionengine.async_query.external_scheduler.interval +===================================================================== + +Description +----------- +This setting defines the interval at which the external scheduler applies for auto refresh queries. It optimizes Spark applications by allowing them to automatically decide whether to use the Spark scheduler or the external scheduler. + +* Default Value: None (must be explicitly set) +* Format: A string representing a time duration follows Spark `CalendarInterval <https://spark.apache.org/docs/latest/api/java/org/apache/spark/unsafe/types/CalendarInterval.html>`__ format (e.g., ``10 minutes`` for 10 minutes, ``1 hour`` for 1 hour). + +To modify the interval to 10 minutes for example, use this command: + +Request :: + + sh$ curl -sS -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings \ + ... -d '{"transient":{"plugins.query.executionengine.async_query.external_scheduler.interval":"10 minutes"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "plugins": { + "query": { + "executionengine": { + "async_query": { + "external_scheduler": { + "interval": "10 minutes" + } + } + } + } + } + } + } + plugins.query.executionengine.spark.streamingjobs.housekeeper.interval ====================================================================== diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 494b906b55..1083dbd836 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -154,6 +154,19 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting<Boolean> ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING = + Setting.boolSetting( + Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + public static final Setting<String> ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING = + Setting.simpleString( + Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL.getKeyValue(), + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting<String> SPARK_EXECUTION_ENGINE_CONFIG = Setting.simpleString( Key.SPARK_EXECUTION_ENGINE_CONFIG.getKeyValue(), @@ -298,6 +311,18 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.ASYNC_QUERY_ENABLED, ASYNC_QUERY_ENABLED_SETTING, new Updater(Key.ASYNC_QUERY_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED, + ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING, + new Updater(Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL, + ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING, + new Updater(Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)); register( settingBuilder, clusterSettings, @@ -419,6 +444,8 @@ public static List<Setting<?>> pluginSettings() { .add(DATASOURCE_URI_HOSTS_DENY_LIST) .add(DATASOURCE_ENABLED_SETTING) .add(ASYNC_QUERY_ENABLED_SETTING) + .add(ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING) + .add(ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING) .add(SPARK_EXECUTION_ENGINE_CONFIG) .add(SPARK_EXECUTION_SESSION_LIMIT_SETTING) .add(SPARK_EXECUTION_REFRESH_JOB_LIMIT_SETTING) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 84fb705ae0..026f0c6218 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -15,6 +15,8 @@ import static org.mockito.Mockito.when; import static org.opensearch.common.unit.TimeValue.timeValueMinutes; import static org.opensearch.sql.opensearch.setting.LegacyOpenDistroSettings.legacySettings; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.METRICS_ROLLING_INTERVAL_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.METRICS_ROLLING_WINDOW_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_ENABLED_SETTING; @@ -195,4 +197,22 @@ void getSparkExecutionEngineConfigSetting() { .put(SPARK_EXECUTION_ENGINE_CONFIG.getKey(), sparkConfig) .build())); } + + @Test + void getAsyncQueryExternalSchedulerEnabledSetting() { + // Default is true + assertEquals( + true, + ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING.get( + org.opensearch.common.settings.Settings.builder().build())); + } + + @Test + void getAsyncQueryExternalSchedulerIntervalSetting() { + // Default is empty string + assertEquals( + "", + ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING.get( + org.opensearch.common.settings.Settings.builder().build())); + } } From 06c56e7d758a2ba8df9be852f33ce182b7fcb352 Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Thu, 5 Sep 2024 15:27:42 -0700 Subject: [PATCH 40/96] Fix type mismatch (#2990) Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .../config/OpenSearchAsyncQuerySchedulerConfigComposer.java | 4 ++-- .../OpenSearchAsyncQuerySchedulerConfigComposerTest.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java index 6dce09a406..f791b050a1 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java @@ -24,12 +24,12 @@ public void compose( SparkSubmitParameters sparkSubmitParameters, DispatchQueryRequest dispatchQueryRequest, AsyncQueryRequestContext context) { - String externalSchedulerEnabled = + Boolean externalSchedulerEnabled = settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED); String externalSchedulerInterval = settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL); sparkSubmitParameters.setConfigItem( - FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, externalSchedulerEnabled); + FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, String.valueOf(externalSchedulerEnabled)); sparkSubmitParameters.setConfigItem( FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java index 7836c63b7a..1556d4db3f 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java @@ -31,7 +31,7 @@ public void setUp() { @Test public void testCompose() { when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) - .thenReturn("true"); + .thenReturn(true); when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) .thenReturn("10 minutes"); @@ -46,7 +46,7 @@ public void testCompose() { @Test public void testComposeWithDisabledScheduler() { when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) - .thenReturn("false"); + .thenReturn(false); composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); @@ -57,7 +57,7 @@ public void testComposeWithDisabledScheduler() { @Test public void testComposeWithMissingInterval() { when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) - .thenReturn("true"); + .thenReturn(true); when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) .thenReturn(""); From 83e89fb0f6a659b6cf5877d14ef260438b459c61 Mon Sep 17 00:00:00 2001 From: Chen Dai <daichen@amazon.com> Date: Thu, 5 Sep 2024 18:50:16 -0700 Subject: [PATCH 41/96] Delegate Flint index vacuum operation to Spark (#2985) * Remove vacuum dispatch and update UT Signed-off-by: Chen Dai <daichen@amazon.com> * Remove unused code and test Signed-off-by: Chen Dai <daichen@amazon.com> * Fix jacoco test Signed-off-by: Chen Dai <daichen@amazon.com> --------- Signed-off-by: Chen Dai <daichen@amazon.com> --- .../sql/spark/dispatcher/IndexDMLHandler.java | 2 - .../dispatcher/SparkQueryDispatcher.java | 1 - .../flint/operation/FlintIndexOpFactory.java | 9 - .../flint/operation/FlintIndexOpVacuum.java | 66 ---- .../sql/spark/utils/SQLQueryUtils.java | 25 -- .../asyncquery/AsyncQueryCoreIntegTest.java | 42 +-- .../dispatcher/SparkQueryDispatcherTest.java | 311 +++--------------- .../operation/FlintIndexOpFactoryTest.java | 5 - .../operation/FlintIndexOpVacuumTest.java | 261 --------------- .../sql/spark/utils/SQLQueryUtilsTest.java | 6 +- .../asyncquery/IndexQuerySpecVacuumTest.java | 218 ------------ .../flint/OpenSearchFlintIndexClientTest.java | 42 +++ 12 files changed, 98 insertions(+), 890 deletions(-) delete mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java delete mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java delete mode 100644 async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java create mode 100644 async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClientTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index 4698bfcccc..7211da0941 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -138,8 +138,6 @@ private FlintIndexOp getIndexOp( case ALTER: return flintIndexOpFactory.getAlter( indexQueryDetails.getFlintIndexOptions(), dispatchQueryRequest.getDatasource()); - case VACUUM: - return flintIndexOpFactory.getVacuum(dispatchQueryRequest.getDatasource()); default: throw new IllegalStateException( String.format( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 4df2b5450d..50e8403d36 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -150,7 +150,6 @@ private boolean isEligibleForStreamingQuery(IndexQueryDetails indexQueryDetails) private boolean isEligibleForIndexDMLHandling(IndexQueryDetails indexQueryDetails) { return IndexQueryActionType.DROP.equals(indexQueryDetails.getIndexQueryActionType()) - || IndexQueryActionType.VACUUM.equals(indexQueryDetails.getIndexQueryActionType()) || (IndexQueryActionType.ALTER.equals(indexQueryDetails.getIndexQueryActionType()) && (indexQueryDetails .getFlintIndexOptions() diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java index 9f925e0bcf..d82b29e928 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java @@ -36,15 +36,6 @@ public FlintIndexOpAlter getAlter(FlintIndexOptions flintIndexOptions, String da asyncQueryScheduler); } - public FlintIndexOpVacuum getVacuum(String datasource) { - return new FlintIndexOpVacuum( - flintIndexStateModelService, - datasource, - flintIndexClient, - emrServerlessClientFactory, - asyncQueryScheduler); - } - public FlintIndexOpCancel getCancel(String datasource) { return new FlintIndexOpCancel( flintIndexStateModelService, datasource, emrServerlessClientFactory); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java deleted file mode 100644 index 324ddb5720..0000000000 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.flint.operation; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; -import org.opensearch.sql.spark.client.EMRServerlessClientFactory; -import org.opensearch.sql.spark.flint.FlintIndexClient; -import org.opensearch.sql.spark.flint.FlintIndexMetadata; -import org.opensearch.sql.spark.flint.FlintIndexState; -import org.opensearch.sql.spark.flint.FlintIndexStateModel; -import org.opensearch.sql.spark.flint.FlintIndexStateModelService; -import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; - -/** Flint index vacuum operation. */ -public class FlintIndexOpVacuum extends FlintIndexOp { - private static final Logger LOG = LogManager.getLogger(); - - private final AsyncQueryScheduler asyncQueryScheduler; - - /** OpenSearch client. */ - private final FlintIndexClient flintIndexClient; - - public FlintIndexOpVacuum( - FlintIndexStateModelService flintIndexStateModelService, - String datasourceName, - FlintIndexClient flintIndexClient, - EMRServerlessClientFactory emrServerlessClientFactory, - AsyncQueryScheduler asyncQueryScheduler) { - super(flintIndexStateModelService, datasourceName, emrServerlessClientFactory); - this.flintIndexClient = flintIndexClient; - this.asyncQueryScheduler = asyncQueryScheduler; - } - - @Override - boolean validate(FlintIndexState state) { - return state == FlintIndexState.DELETED; - } - - @Override - FlintIndexState transitioningState() { - return FlintIndexState.VACUUMING; - } - - @Override - public void runOp( - FlintIndexMetadata flintIndexMetadata, - FlintIndexStateModel flintIndex, - AsyncQueryRequestContext asyncQueryRequestContext) { - LOG.info("Vacuuming Flint index {}", flintIndexMetadata.getOpensearchIndexName()); - if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { - asyncQueryScheduler.removeJob(flintIndexMetadata.getOpensearchIndexName()); - } - flintIndexClient.deleteIndex(flintIndexMetadata.getOpensearchIndexName()); - } - - @Override - FlintIndexState stableState() { - // Instruct StateStore to purge the index state doc - return FlintIndexState.NONE; - } -} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index ce3bcab06b..b1a8c3d4f6 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -267,31 +267,6 @@ public Void visitDropMaterializedViewStatement( return super.visitDropMaterializedViewStatement(ctx); } - @Override - public Void visitVacuumSkippingIndexStatement( - FlintSparkSqlExtensionsParser.VacuumSkippingIndexStatementContext ctx) { - indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM); - indexQueryDetailsBuilder.indexType(FlintIndexType.SKIPPING); - return super.visitVacuumSkippingIndexStatement(ctx); - } - - @Override - public Void visitVacuumCoveringIndexStatement( - FlintSparkSqlExtensionsParser.VacuumCoveringIndexStatementContext ctx) { - indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM); - indexQueryDetailsBuilder.indexType(FlintIndexType.COVERING); - return super.visitVacuumCoveringIndexStatement(ctx); - } - - @Override - public Void visitVacuumMaterializedViewStatement( - FlintSparkSqlExtensionsParser.VacuumMaterializedViewStatementContext ctx) { - indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM); - indexQueryDetailsBuilder.indexType(FlintIndexType.MATERIALIZED_VIEW); - indexQueryDetailsBuilder.mvName(ctx.mvName.getText()); - return super.visitVacuumMaterializedViewStatement(ctx); - } - @Override public Void visitDescribeCoveringIndexStatement( FlintSparkSqlExtensionsParser.DescribeCoveringIndexStatementContext ctx) { diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index ca4a8736d2..1214935dc6 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -236,32 +236,12 @@ public void createDropIndexQueryWithScheduler() { public void createVacuumIndexQuery() { givenSparkExecutionEngineConfigIsSupplied(); givenValidDataSourceMetadataExist(); + givenSessionExists(); when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); - String indexName = "flint_datasource_name_table_name_index_name_index"; - givenFlintIndexMetadataExists(indexName); - - CreateAsyncQueryResponse response = - asyncQueryExecutorService.createAsyncQuery( - new CreateAsyncQueryRequest( - "VACUUM INDEX index_name ON table_name", DATASOURCE_NAME, LangType.SQL), - asyncQueryRequestContext); - - assertEquals(QUERY_ID, response.getQueryId()); - assertNull(response.getSessionId()); - verifyGetQueryIdCalled(); - verify(flintIndexClient).deleteIndex(indexName); - verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); - } - - @Test - public void createVacuumIndexQueryWithScheduler() { - givenSparkExecutionEngineConfigIsSupplied(); - givenValidDataSourceMetadataExist(); - when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); - - String indexName = "flint_datasource_name_table_name_index_name_index"; - givenFlintIndexMetadataExistsWithExternalScheduler(indexName); + when(sessionIdProvider.getSessionId(any())).thenReturn(SESSION_ID); + givenSessionExists(); // called twice + when(awsemrServerless.startJobRun(any())) + .thenReturn(new StartJobRunResult().withApplicationId(APPLICATION_ID).withJobRunId(JOB_ID)); CreateAsyncQueryResponse response = asyncQueryExecutorService.createAsyncQuery( @@ -270,14 +250,12 @@ public void createVacuumIndexQueryWithScheduler() { asyncQueryRequestContext); assertEquals(QUERY_ID, response.getQueryId()); - assertNull(response.getSessionId()); + assertEquals(SESSION_ID, response.getSessionId()); verifyGetQueryIdCalled(); - - verify(flintIndexClient).deleteIndex(indexName); - verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); - - verify(asyncQueryScheduler).removeJob(indexName); + verifyGetSessionIdCalled(); + verify(leaseManager).borrow(any()); + verifyStartJobRunCalled(); + verifyStoreJobMetadataCalled(JOB_ID, JobType.INTERACTIVE); } @Test diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 5154b71574..8b855c190c 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -163,84 +163,12 @@ void setUp() { @Test void testDispatchSelectQuery() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "select * from my_glue.default.http_logs"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch( - DispatchQueryRequest.builder() - .applicationId(EMRS_APPLICATION_ID) - .query(query) - .datasource(MY_GLUE) - .langType(LangType.SQL) - .executionRoleARN(EMRS_EXECUTION_ROLE) - .clusterName(TEST_CLUSTER_NAME) - .sparkSubmitParameterModifier(sparkSubmitParameterModifier) - .build(), - asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("select * from my_glue.default.http_logs"); } @Test void testDispatchSelectQueryWithBasicAuthIndexStoreDatasource() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "select * from my_glue.default.http_logs"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadataWithBasicAuth(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("select * from my_glue.default.http_logs"); } @Test @@ -354,41 +282,9 @@ void testDispatchCreateAutoRefreshIndexQuery() { @Test void testDispatchCreateManualRefreshIndexQuery() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, "my_glue"); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = + testDispatchBatchQuery( "CREATE INDEX elb_and_requestUri ON my_glue.default.http_logs(l_orderkey, l_quantity) WITH" - + " (auto_refresh = false)"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - "my_glue", asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - Assertions.assertEquals(JobType.BATCH, dispatchQueryResponse.getJobType()); - verifyNoInteractions(flintIndexMetadataService); + + " (auto_refresh = false)"); } @Test @@ -460,84 +356,12 @@ void testDispatchWithSparkUDFQuery() { @Test void testInvalidSQLQueryDispatchToSpark() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "myselect 1"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch( - DispatchQueryRequest.builder() - .applicationId(EMRS_APPLICATION_ID) - .query(query) - .datasource(MY_GLUE) - .langType(LangType.SQL) - .executionRoleARN(EMRS_EXECUTION_ROLE) - .clusterName(TEST_CLUSTER_NAME) - .sparkSubmitParameterModifier(sparkSubmitParameterModifier) - .build(), - asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("myselect 1"); } @Test void testDispatchQueryWithoutATableAndDataSourceName() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "show tables"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("show tables"); } @Test @@ -619,38 +443,7 @@ void testDispatchMaterializedViewQuery() { @Test void testDispatchShowMVQuery() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "SHOW MATERIALIZED VIEW IN mys3.default"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("SHOW MATERIALIZED VIEW IN mys3.default"); } @Test @@ -692,38 +485,7 @@ void testRefreshIndexQuery() { @Test void testDispatchDescribeIndexQuery() { - when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); - when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); - HashMap<String, String> tags = new HashMap<>(); - tags.put(DATASOURCE_TAG_KEY, MY_GLUE); - tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); - tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); - String query = "DESCRIBE SKIPPING INDEX ON mys3.default.http_logs"; - String sparkSubmitParameters = - constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); - StartJobRequest expected = - new StartJobRequest( - "TEST_CLUSTER:batch", - null, - EMRS_APPLICATION_ID, - EMRS_EXECUTION_ROLE, - sparkSubmitParameters, - tags, - false, - "query_execution_result_my_glue"); - when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - MY_GLUE, asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - - DispatchQueryResponse dispatchQueryResponse = - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - - verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - verifyNoInteractions(flintIndexMetadataService); + testDispatchBatchQuery("DESCRIBE SKIPPING INDEX ON mys3.default.http_logs"); } @Test @@ -817,26 +579,7 @@ void testDispatchDropIndexQuery() { @Test void testDispatchVacuumIndexQuery() { - QueryHandlerFactory queryHandlerFactory = mock(QueryHandlerFactory.class); - sparkQueryDispatcher = - new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); - - String query = "VACUUM INDEX elb_and_requestUri ON my_glue.default.http_logs"; - DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); - when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( - "my_glue", asyncQueryRequestContext)) - .thenReturn(dataSourceMetadata); - when(queryHandlerFactory.getIndexDMLHandler()) - .thenReturn( - new IndexDMLHandler( - jobExecutionResponseReader, - flintIndexMetadataService, - indexDMLResultStorageService, - flintIndexOpFactory)); - - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); - verify(queryHandlerFactory, times(1)).getIndexDMLHandler(); + testDispatchBatchQuery("VACUUM INDEX elb_and_requestUri ON my_glue.default.http_logs"); } @Test @@ -1087,6 +830,42 @@ void testDispatchQueryWithExtraSparkSubmitParameters() { } } + private void testDispatchBatchQuery(String query) { + when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); + HashMap<String, String> tags = new HashMap<>(); + tags.put(DATASOURCE_TAG_KEY, MY_GLUE); + tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); + tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); + + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); + StartJobRequest expected = + new StartJobRequest( + "TEST_CLUSTER:batch", + null, + EMRS_APPLICATION_ID, + EMRS_EXECUTION_ROLE, + sparkSubmitParameters, + tags, + false, + "query_execution_result_my_glue"); + when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) + .thenReturn(dataSourceMetadata); + + DispatchQueryResponse dispatchQueryResponse = + sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); + + verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); + Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + Assertions.assertEquals(JobType.BATCH, dispatchQueryResponse.getJobType()); + verifyNoInteractions(flintIndexMetadataService); + } + private String constructExpectedSparkSubmitParameterString(String query) { return constructExpectedSparkSubmitParameterString(query, null, null); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java index 62ac98f1a2..e73c5614ae 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactoryTest.java @@ -41,11 +41,6 @@ void getAlter() { assertNotNull(flintIndexOpFactory.getAlter(new FlintIndexOptions(), DATASOURCE_NAME)); } - @Test - void getVacuum() { - assertNotNull(flintIndexOpFactory.getDrop(DATASOURCE_NAME)); - } - @Test void getCancel() { assertNotNull(flintIndexOpFactory.getDrop(DATASOURCE_NAME)); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java deleted file mode 100644 index 08f8efd488..0000000000 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuumTest.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.flint.operation; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; -import org.opensearch.sql.spark.client.EMRServerlessClientFactory; -import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions; -import org.opensearch.sql.spark.flint.FlintIndexClient; -import org.opensearch.sql.spark.flint.FlintIndexMetadata; -import org.opensearch.sql.spark.flint.FlintIndexState; -import org.opensearch.sql.spark.flint.FlintIndexStateModel; -import org.opensearch.sql.spark.flint.FlintIndexStateModelService; -import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; - -@ExtendWith(MockitoExtension.class) -class FlintIndexOpVacuumTest { - - public static final String DATASOURCE_NAME = "DATASOURCE_NAME"; - public static final String LATEST_ID = "LATEST_ID"; - public static final String INDEX_NAME = "INDEX_NAME"; - - public static final FlintIndexMetadata FLINT_INDEX_METADATA_WITH_LATEST_ID = - createFlintIndexMetadataWithLatestId(); - - public static final FlintIndexMetadata FLINT_INDEX_METADATA_WITHOUT_LATEST_ID = - createFlintIndexMetadataWithoutLatestId(); - - @Mock FlintIndexClient flintIndexClient; - @Mock FlintIndexStateModelService flintIndexStateModelService; - @Mock EMRServerlessClientFactory emrServerlessClientFactory; - @Mock FlintIndexStateModel flintIndexStateModel; - @Mock FlintIndexStateModel transitionedFlintIndexStateModel; - @Mock AsyncQueryRequestContext asyncQueryRequestContext; - @Mock AsyncQueryScheduler asyncQueryScheduler; - - RuntimeException testException = new RuntimeException("Test Exception"); - - FlintIndexOpVacuum flintIndexOpVacuum; - - @BeforeEach - public void setUp() { - flintIndexOpVacuum = - new FlintIndexOpVacuum( - flintIndexStateModelService, - DATASOURCE_NAME, - flintIndexClient, - emrServerlessClientFactory, - asyncQueryScheduler); - } - - private static FlintIndexMetadata createFlintIndexMetadataWithLatestId() { - return FlintIndexMetadata.builder() - .latestId(LATEST_ID) - .opensearchIndexName(INDEX_NAME) - .flintIndexOptions(new FlintIndexOptions()) - .build(); - } - - private static FlintIndexMetadata createFlintIndexMetadataWithoutLatestId() { - return FlintIndexMetadata.builder() - .opensearchIndexName(INDEX_NAME) - .flintIndexOptions(new FlintIndexOptions()) - .build(); - } - - private FlintIndexMetadata createFlintIndexMetadataWithExternalScheduler() { - FlintIndexOptions flintIndexOptions = new FlintIndexOptions(); - flintIndexOptions.setOption(FlintIndexOptions.SCHEDULER_MODE, "external"); - - return FlintIndexMetadata.builder() - .opensearchIndexName(INDEX_NAME) - .flintIndexOptions(flintIndexOptions) - .build(); - } - - @Test - public void testApplyWithEmptyLatestId() { - flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITHOUT_LATEST_ID, asyncQueryRequestContext); - - verify(flintIndexClient).deleteIndex(INDEX_NAME); - } - - @Test - public void testApplyWithFlintIndexStateNotFound() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.empty()); - - assertThrows( - IllegalStateException.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - } - - @Test - public void testApplyWithNotDeletedState() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.ACTIVE); - - assertThrows( - IllegalStateException.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - } - - @Test - public void testApplyWithUpdateFlintIndexStateThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); - when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, - FlintIndexState.VACUUMING, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenThrow(testException); - - assertThrows( - IllegalStateException.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - } - - @Test - public void testApplyWithRunOpThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); - when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, - FlintIndexState.VACUUMING, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenReturn(transitionedFlintIndexStateModel); - doThrow(testException).when(flintIndexClient).deleteIndex(INDEX_NAME); - - assertThrows( - Exception.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - - verify(flintIndexStateModelService) - .updateFlintIndexState( - transitionedFlintIndexStateModel, - FlintIndexState.DELETED, - DATASOURCE_NAME, - asyncQueryRequestContext); - } - - @Test - public void testApplyWithRunOpThrowAndRollbackThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); - when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, - FlintIndexState.VACUUMING, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenReturn(transitionedFlintIndexStateModel); - doThrow(testException).when(flintIndexClient).deleteIndex(INDEX_NAME); - when(flintIndexStateModelService.updateFlintIndexState( - transitionedFlintIndexStateModel, - FlintIndexState.DELETED, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenThrow(testException); - - assertThrows( - Exception.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - } - - @Test - public void testApplyWithDeleteFlintIndexStateModelThrow() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); - when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, - FlintIndexState.VACUUMING, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenReturn(transitionedFlintIndexStateModel); - when(flintIndexStateModelService.deleteFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenThrow(testException); - - assertThrows( - IllegalStateException.class, - () -> - flintIndexOpVacuum.apply( - FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext)); - } - - @Test - public void testApplyHappyPath() { - when(flintIndexStateModelService.getFlintIndexStateModel( - LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext)) - .thenReturn(Optional.of(flintIndexStateModel)); - when(flintIndexStateModel.getIndexState()).thenReturn(FlintIndexState.DELETED); - when(flintIndexStateModelService.updateFlintIndexState( - flintIndexStateModel, - FlintIndexState.VACUUMING, - DATASOURCE_NAME, - asyncQueryRequestContext)) - .thenReturn(transitionedFlintIndexStateModel); - when(transitionedFlintIndexStateModel.getLatestId()).thenReturn(LATEST_ID); - - flintIndexOpVacuum.apply(FLINT_INDEX_METADATA_WITH_LATEST_ID, asyncQueryRequestContext); - - verify(flintIndexStateModelService) - .deleteFlintIndexStateModel(LATEST_ID, DATASOURCE_NAME, asyncQueryRequestContext); - verify(flintIndexClient).deleteIndex(INDEX_NAME); - } - - @Test - public void testRunOpWithExternalScheduler() { - FlintIndexMetadata flintIndexMetadata = createFlintIndexMetadataWithExternalScheduler(); - flintIndexOpVacuum.runOp(flintIndexMetadata, flintIndexStateModel, asyncQueryRequestContext); - - verify(asyncQueryScheduler).removeJob(INDEX_NAME); - verify(flintIndexClient).deleteIndex(INDEX_NAME); - } - - @Test - public void testRunOpWithoutExternalScheduler() { - FlintIndexMetadata flintIndexMetadata = FLINT_INDEX_METADATA_WITHOUT_LATEST_ID; - flintIndexOpVacuum.runOp(flintIndexMetadata, flintIndexStateModel, asyncQueryRequestContext); - - verify(asyncQueryScheduler, never()).removeJob(INDEX_NAME); - verify(flintIndexClient).deleteIndex(INDEX_NAME); - } -} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index 235fe84c70..f1853f2c1e 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -142,7 +142,6 @@ void testExtractionFromFlintSkippingIndexQueries() { + " WHERE elb_status_code = 500 " + " WITH (auto_refresh = true)", "DROP SKIPPING INDEX ON myS3.default.alb_logs", - "VACUUM SKIPPING INDEX ON myS3.default.alb_logs", "ALTER SKIPPING INDEX ON myS3.default.alb_logs WITH (auto_refresh = false)", }; @@ -171,7 +170,6 @@ void testExtractionFromFlintCoveringIndexQueries() { + " WHERE elb_status_code = 500 " + " WITH (auto_refresh = true)", "DROP INDEX elb_and_requestUri ON myS3.default.alb_logs", - "VACUUM INDEX elb_and_requestUri ON myS3.default.alb_logs", "ALTER INDEX elb_and_requestUri ON myS3.default.alb_logs WITH (auto_refresh = false)" }; @@ -203,9 +201,7 @@ void testExtractionFromCreateMVQuery() { @Test void testExtractionFromFlintMVQuery() { String[] mvQueries = { - "DROP MATERIALIZED VIEW mv_1", - "VACUUM MATERIALIZED VIEW mv_1", - "ALTER MATERIALIZED VIEW mv_1 WITH (auto_refresh = false)", + "DROP MATERIALIZED VIEW mv_1", "ALTER MATERIALIZED VIEW mv_1 WITH (auto_refresh = false)", }; for (String query : mvQueries) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java deleted file mode 100644 index e62b60bfd2..0000000000 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.asyncquery; - -import static org.opensearch.sql.spark.flint.FlintIndexState.ACTIVE; -import static org.opensearch.sql.spark.flint.FlintIndexState.CREATING; -import static org.opensearch.sql.spark.flint.FlintIndexState.DELETED; -import static org.opensearch.sql.spark.flint.FlintIndexState.EMPTY; -import static org.opensearch.sql.spark.flint.FlintIndexState.REFRESHING; -import static org.opensearch.sql.spark.flint.FlintIndexState.VACUUMING; -import static org.opensearch.sql.spark.flint.FlintIndexType.COVERING; -import static org.opensearch.sql.spark.flint.FlintIndexType.MATERIALIZED_VIEW; -import static org.opensearch.sql.spark.flint.FlintIndexType.SKIPPING; - -import com.amazonaws.services.emrserverless.model.CancelJobRunResult; -import com.amazonaws.services.emrserverless.model.GetJobRunResult; -import com.amazonaws.services.emrserverless.model.JobRun; -import com.google.common.collect.Lists; -import java.util.Base64; -import java.util.List; -import java.util.function.BiConsumer; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.Test; -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.opensearch.action.delete.DeleteRequest; -import org.opensearch.action.get.GetRequest; -import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; -import org.opensearch.sql.spark.asyncquery.model.MockFlintSparkJob; -import org.opensearch.sql.spark.client.EMRServerlessClientFactory; -import org.opensearch.sql.spark.execution.statestore.OpenSearchStateStoreUtil; -import org.opensearch.sql.spark.flint.FlintIndexState; -import org.opensearch.sql.spark.flint.FlintIndexType; -import org.opensearch.sql.spark.rest.model.CreateAsyncQueryRequest; -import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse; -import org.opensearch.sql.spark.rest.model.LangType; - -@SuppressWarnings({"unchecked", "rawtypes"}) -public class IndexQuerySpecVacuumTest extends AsyncQueryExecutorServiceSpec { - - private static final EMRApiCall DEFAULT_OP = () -> null; - - private final List<FlintDatasetMock> FLINT_TEST_DATASETS = - List.of( - mockDataset( - "VACUUM SKIPPING INDEX ON mys3.default.http_logs", - SKIPPING, - "flint_mys3_default_http_logs_skipping_index"), - mockDataset( - "VACUUM INDEX covering ON mys3.default.http_logs", - COVERING, - "flint_mys3_default_http_logs_covering_index"), - mockDataset( - "VACUUM MATERIALIZED VIEW mys3.default.http_logs_metrics", - MATERIALIZED_VIEW, - "flint_mys3_default_http_logs_metrics"), - mockDataset( - "VACUUM SKIPPING INDEX ON mys3.default.`test ,:\"+/\\|?#><`", - SKIPPING, - "flint_mys3_default_test%20%2c%3a%22%2b%2f%5c%7c%3f%23%3e%3c_skipping_index") - .isSpecialCharacter(true)); - - @Test - public void shouldVacuumIndexInDeletedState() { - List<List<Object>> testCases = - Lists.cartesianProduct( - FLINT_TEST_DATASETS, - List.of(DELETED), - List.of( - Pair.<EMRApiCall, EMRApiCall>of( - DEFAULT_OP, - () -> new GetJobRunResult().withJobRun(new JobRun().withState("Cancelled"))))); - - runVacuumTestSuite( - testCases, - (mockDS, response) -> { - assertEquals("SUCCESS", response.getStatus()); - assertFalse(flintIndexExists(mockDS.indexName)); - assertFalse(indexDocExists(mockDS.latestId)); - }); - } - - @Test - public void shouldNotVacuumIndexInOtherStates() { - List<List<Object>> testCases = - Lists.cartesianProduct( - FLINT_TEST_DATASETS, - List.of(EMPTY, CREATING, ACTIVE, REFRESHING, VACUUMING), - List.of( - Pair.<EMRApiCall, EMRApiCall>of( - () -> { - throw new AssertionError("should not call cancelJobRun"); - }, - () -> { - throw new AssertionError("should not call getJobRunResult"); - }))); - - runVacuumTestSuite( - testCases, - (mockDS, response) -> { - assertEquals("FAILED", response.getStatus()); - assertTrue(flintIndexExists(mockDS.indexName)); - assertTrue(indexDocExists(mockDS.latestId)); - }); - } - - private void runVacuumTestSuite( - List<List<Object>> testCases, - BiConsumer<FlintDatasetMock, AsyncQueryExecutionResponse> assertion) { - testCases.forEach( - params -> { - FlintDatasetMock mockDS = (FlintDatasetMock) params.get(0); - try { - FlintIndexState state = (FlintIndexState) params.get(1); - EMRApiCall cancelJobRun = ((Pair<EMRApiCall, EMRApiCall>) params.get(2)).getLeft(); - EMRApiCall getJobRunResult = ((Pair<EMRApiCall, EMRApiCall>) params.get(2)).getRight(); - - AsyncQueryExecutionResponse response = - runVacuumTest(mockDS, state, cancelJobRun, getJobRunResult); - assertion.accept(mockDS, response); - } finally { - // Clean up because we simulate parameterized test in single unit test method - if (flintIndexExists(mockDS.indexName)) { - mockDS.deleteIndex(); - } - if (indexDocExists(mockDS.latestId)) { - deleteIndexDoc(mockDS.latestId); - } - } - }); - } - - private AsyncQueryExecutionResponse runVacuumTest( - FlintDatasetMock mockDS, - FlintIndexState state, - EMRApiCall<CancelJobRunResult> cancelJobRun, - EMRApiCall<GetJobRunResult> getJobRunResult) { - LocalEMRSClient emrsClient = - new LocalEMRSClient() { - @Override - public CancelJobRunResult cancelJobRun( - String applicationId, String jobId, boolean allowExceptionPropagation) { - if (cancelJobRun == DEFAULT_OP) { - return super.cancelJobRun(applicationId, jobId, allowExceptionPropagation); - } - return cancelJobRun.call(); - } - - @Override - public GetJobRunResult getJobRunResult(String applicationId, String jobId) { - if (getJobRunResult == DEFAULT_OP) { - return super.getJobRunResult(applicationId, jobId); - } - return getJobRunResult.call(); - } - }; - EMRServerlessClientFactory emrServerlessClientFactory = (accountId) -> emrsClient; - AsyncQueryExecutorService asyncQueryExecutorService = - createAsyncQueryExecutorService(emrServerlessClientFactory); - - // Mock Flint index - mockDS.createIndex(); - - // Mock index state doc - MockFlintSparkJob flintIndexJob = - new MockFlintSparkJob(flintIndexStateModelService, mockDS.latestId, "mys3"); - flintIndexJob.transition(state); - - // Vacuum index - CreateAsyncQueryResponse response = - asyncQueryExecutorService.createAsyncQuery( - new CreateAsyncQueryRequest(mockDS.query, MYS3_DATASOURCE, LangType.SQL, null), - asyncQueryRequestContext); - - return asyncQueryExecutorService.getAsyncQueryResults( - response.getQueryId(), asyncQueryRequestContext); - } - - private boolean flintIndexExists(String flintIndexName) { - return client - .admin() - .indices() - .exists(new IndicesExistsRequest(flintIndexName)) - .actionGet() - .isExists(); - } - - private boolean indexDocExists(String docId) { - return client - .get(new GetRequest(OpenSearchStateStoreUtil.getIndexName("mys3"), docId)) - .actionGet() - .isExists(); - } - - private void deleteIndexDoc(String docId) { - client - .delete(new DeleteRequest(OpenSearchStateStoreUtil.getIndexName("mys3"), docId)) - .actionGet(); - } - - private FlintDatasetMock mockDataset(String query, FlintIndexType indexType, String indexName) { - FlintDatasetMock dataset = new FlintDatasetMock(query, "", indexType, indexName); - dataset.latestId(Base64.getEncoder().encodeToString(indexName.getBytes())); - return dataset; - } - - /** - * EMR API call mock interface. - * - * @param <V> API call response type - */ - @FunctionalInterface - public interface EMRApiCall<V> { - V call(); - } -} diff --git a/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClientTest.java b/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClientTest.java new file mode 100644 index 0000000000..d9f2e58dba --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClientTest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.flint; + +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; + +@ExtendWith(MockitoExtension.class) +public class OpenSearchFlintIndexClientTest { + + @Mock(answer = RETURNS_DEEP_STUBS) + private Client client; + + @Mock private AcknowledgedResponse acknowledgedResponse; + + @InjectMocks private OpenSearchFlintIndexClient openSearchFlintIndexClient; + + @Test + public void testDeleteIndex() { + when(client.admin().indices().delete(any(DeleteIndexRequest.class)).actionGet()) + .thenReturn(acknowledgedResponse); + when(acknowledgedResponse.isAcknowledged()).thenReturn(true); + + openSearchFlintIndexClient.deleteIndex("test-index"); + verify(client.admin().indices()).delete(any(DeleteIndexRequest.class)); + verify(acknowledgedResponse).isAcknowledged(); + } +} From cb2c4529123ade101c433e9fc3686a2b6332a4e6 Mon Sep 17 00:00:00 2001 From: Sean Kao <seankao@amazon.com> Date: Fri, 6 Sep 2024 13:08:32 -0700 Subject: [PATCH 42/96] Add release notes for v2.17.0 (#2992) * Add release notes for v2.17.0 Signed-off-by: Sean Kao <seankao@amazon.com> * Update with latest PRs Signed-off-by: Sean Kao <seankao@amazon.com> --------- Signed-off-by: Sean Kao <seankao@amazon.com> --- .../opensearch-sql.release-notes-2.17.0.0.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.17.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.17.0.0.md b/release-notes/opensearch-sql.release-notes-2.17.0.0.md new file mode 100644 index 0000000000..2282d71f94 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.17.0.0.md @@ -0,0 +1,35 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.17.0 + +### Features +* Flint query scheduler part1 - integrate job scheduler plugin ([#2889](https://github.com/opensearch-project/sql/pull/2889)) +* Flint query scheduler part 2 ([#2975](https://github.com/opensearch-project/sql/pull/2975)) +* Add feature flag for async query scheduler ([#2989](https://github.com/opensearch-project/sql/pull/2989)) + +### Enhancements +* Change the default value of plugins.query.size_limit to MAX_RESULT_WINDOW (10000) ([#2877](https://github.com/opensearch-project/sql/pull/2877)) +* Support common format geo point ([#2896](https://github.com/opensearch-project/sql/pull/2896)) +* Add TakeOrderedOperator ([#2906](https://github.com/opensearch-project/sql/pull/2906)) +* IF function should support complex predicates in PPL ([#2970](https://github.com/opensearch-project/sql/pull/2970)) +* Add flags for Iceberg and Lake Formation and Security Lake as a data source type ([#2978](https://github.com/opensearch-project/sql/pull/2978)) +* Adds validation to allow only flint queries and sql SELECT queries to security lake type datasource ([#2977](https://github.com/opensearch-project/sql/pull/2977)) +* Delegate Flint index vacuum operation to Spark ([#2995](https://github.com/opensearch-project/sql/pull/2995)) + +### Bug Fixes +* Restrict UDF functions ([#2884](https://github.com/opensearch-project/sql/pull/2884)) +* Update SqlBaseParser ([#2890](https://github.com/opensearch-project/sql/pull/2890)) +* Boolean function in PPL should be case insensitive ([#2842](https://github.com/opensearch-project/sql/pull/2842)) +* Fix SparkExecutionEngineConfigClusterSetting deserialize issue ([#2972](https://github.com/opensearch-project/sql/pull/2972)) +* Fix jobType for Batch and IndexDML query ([#2982](https://github.com/opensearch-project/sql/pull/2982)) +* Fix handler for existing query ([#2983](https://github.com/opensearch-project/sql/pull/2983)) + +### Infrastructure +* Increment version to 2.17.0-SNAPSHOT ([#2892](https://github.com/opensearch-project/sql/pull/2892)) +* Fix :integ-test:sqlBwcCluster#fullRestartClusterTask ([#2996](https://github.com/opensearch-project/sql/pull/2996)) + +### Refactoring +* Add RequestContext parameter to verifyDataSourceAccessAndGetRawMetada method ([#2872](https://github.com/opensearch-project/sql/pull/2872)) +* Add AsyncQueryRequestContext to QueryIdProvider parameter ([#2887](https://github.com/opensearch-project/sql/pull/2887)) +* Add AsyncQueryRequestContext to FlintIndexMetadataService/FlintIndexStateModelService ([#2885](https://github.com/opensearch-project/sql/pull/2885)) +* Add mvQuery attribute in IndexQueryDetails ([#2951](https://github.com/opensearch-project/sql/pull/2951)) +* Add AsyncQueryRequestContext to update/get in StatementStorageService ([#2953](https://github.com/opensearch-project/sql/pull/2953)) +* Extract validation logic from FlintIndexMetadataServiceImpl ([#2954](https://github.com/opensearch-project/sql/pull/2954)) From b76aa6588972fecbfdbd9bd477116fb0b1bdd5ab Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Fri, 6 Sep 2024 13:31:57 -0700 Subject: [PATCH 43/96] Populate indexName for BatchQuery (#2956) * Populate indexName for BatchQuery Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix test failure due to rebase Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> Signed-off-by: Tomoyuki MORITA <moritato@amazon.com> --- .../spark/dispatcher/BatchQueryHandler.java | 7 ++ .../dispatcher/model/IndexQueryDetails.java | 27 +++-- .../dispatcher/SparkQueryDispatcherTest.java | 103 ++++++++++++------ .../sql/spark/utils/SQLQueryUtilsTest.java | 45 ++++++-- 4 files changed, 135 insertions(+), 47 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index c693656150..33d78b174e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -111,6 +111,13 @@ public DispatchQueryResponse submit( .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.BATCH) + .indexName(getIndexName(context)) .build(); } + + private static String getIndexName(DispatchQueryContext context) { + return context.getIndexQueryDetails() != null + ? context.getIndexQueryDetails().openSearchIndexName() + : null; + } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java index 2ca997f6b0..50ce95ffe0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java @@ -93,24 +93,35 @@ public IndexQueryDetails build() { } public String openSearchIndexName() { + if (getIndexType() == null) { + return null; + } FullyQualifiedTableName fullyQualifiedTableName = getFullyQualifiedTableName(); String indexName = StringUtils.EMPTY; switch (getIndexType()) { case COVERING: - indexName = - "flint_" - + fullyQualifiedTableName.toFlintName() - + "_" - + strip(getIndexName(), STRIP_CHARS) - + "_" - + getIndexType().getSuffix(); + if (getIndexName() != null) { // getIndexName will be null for SHOW INDEX query + indexName = + "flint_" + + fullyQualifiedTableName.toFlintName() + + "_" + + strip(getIndexName(), STRIP_CHARS) + + "_" + + getIndexType().getSuffix(); + } else { + return null; + } break; case SKIPPING: indexName = "flint_" + fullyQualifiedTableName.toFlintName() + "_" + getIndexType().getSuffix(); break; case MATERIALIZED_VIEW: - indexName = "flint_" + new FullyQualifiedTableName(mvName).toFlintName(); + if (mvName != null) { // mvName is not available for SHOW MATERIALIZED VIEW query + indexName = "flint_" + new FullyQualifiedTableName(mvName).toFlintName(); + } else { + return null; + } break; } return percentEncode(indexName).toLowerCase(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 8b855c190c..9f12ddf323 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.dispatcher; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Answers.RETURNS_DEEP_STUBS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -191,8 +192,8 @@ void testDispatchSelectQueryCreateNewSession() { verifyNoInteractions(emrServerlessClient); verify(sessionManager, never()).getSession(any(), any()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - Assertions.assertEquals(MOCK_SESSION_ID, dispatchQueryResponse.getSessionId()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(MOCK_SESSION_ID, dispatchQueryResponse.getSessionId()); } @Test @@ -218,8 +219,8 @@ void testDispatchSelectQueryReuseSession() { verifyNoInteractions(emrServerlessClient); verify(sessionManager, never()).createSession(any(), any()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); - Assertions.assertEquals(MOCK_SESSION_ID, dispatchQueryResponse.getSessionId()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(MOCK_SESSION_ID, dispatchQueryResponse.getSessionId()); } @Test @@ -275,8 +276,8 @@ void testDispatchCreateAutoRefreshIndexQuery() { sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); verifyNoInteractions(flintIndexMetadataService); } @@ -320,8 +321,8 @@ void testDispatchWithPPLQuery() { asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); verifyNoInteractions(flintIndexMetadataService); } @@ -346,7 +347,7 @@ void testDispatchWithSparkUDFQuery() { sparkQueryDispatcher.dispatch( getBaseDispatchQueryRequestBuilder(query).langType(LangType.SQL).build(), asyncQueryRequestContext)); - Assertions.assertEquals( + assertEquals( "Query is not allowed: Creating user-defined functions is not allowed", illegalArgumentException.getMessage()); verifyNoInteractions(emrServerlessClient); @@ -398,8 +399,8 @@ void testDispatchIndexQueryWithoutADatasourceName() { sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); verifyNoInteractions(flintIndexMetadataService); } @@ -436,8 +437,46 @@ void testDispatchMaterializedViewQuery() { sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + verifyNoInteractions(flintIndexMetadataService); + } + + @Test + void testManualRefreshMaterializedViewQuery() { + when(emrServerlessClientFactory.getClient(any())).thenReturn(emrServerlessClient); + when(queryIdProvider.getQueryId(any(), any())).thenReturn(QUERY_ID); + HashMap<String, String> tags = new HashMap<>(); + tags.put(DATASOURCE_TAG_KEY, MY_GLUE); + tags.put(CLUSTER_NAME_TAG_KEY, TEST_CLUSTER_NAME); + tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); + String query = + "CREATE MATERIALIZED VIEW mv_1 AS select * from logs WITH" + " (auto_refresh = false)"; + String sparkSubmitParameters = + constructExpectedSparkSubmitParameterString(query, null, QUERY_ID); + StartJobRequest expected = + new StartJobRequest( + "TEST_CLUSTER:batch", + null, + EMRS_APPLICATION_ID, + EMRS_EXECUTION_ROLE, + sparkSubmitParameters, + tags, + false, + "query_execution_result_my_glue"); + when(emrServerlessClient.startJobRun(expected)).thenReturn(EMR_JOB_ID); + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) + .thenReturn(dataSourceMetadata); + + DispatchQueryResponse dispatchQueryResponse = + sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); + + verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals("flint_mv_1", dispatchQueryResponse.getIndexName()); verifyNoInteractions(flintIndexMetadataService); } @@ -477,8 +516,8 @@ void testRefreshIndexQuery() { sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); Assertions.assertEquals(JobType.REFRESH, dispatchQueryResponse.getJobType()); verifyNoInteractions(flintIndexMetadataService); } @@ -522,8 +561,8 @@ void testDispatchAlterToAutoRefreshIndexQuery() { sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); verify(emrServerlessClient, times(1)).startJobRun(startJobRequestArgumentCaptor.capture()); - Assertions.assertEquals(expected, startJobRequestArgumentCaptor.getValue()); - Assertions.assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); + assertEquals(expected, startJobRequestArgumentCaptor.getValue()); + assertEquals(EMR_JOB_ID, dispatchQueryResponse.getJobId()); verifyNoInteractions(flintIndexMetadataService); } @@ -533,7 +572,6 @@ void testDispatchAlterToManualRefreshIndexQuery() { sparkQueryDispatcher = new SparkQueryDispatcher( dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); - String query = "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = false)"; @@ -550,6 +588,7 @@ void testDispatchAlterToManualRefreshIndexQuery() { flintIndexOpFactory)); sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); + verify(queryHandlerFactory, times(1)).getIndexDMLHandler(); } @@ -559,7 +598,6 @@ void testDispatchDropIndexQuery() { sparkQueryDispatcher = new SparkQueryDispatcher( dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); - String query = "DROP INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( @@ -573,7 +611,9 @@ void testDispatchDropIndexQuery() { indexDMLResultStorageService, flintIndexOpFactory)); - sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); + DispatchQueryResponse response = + sparkQueryDispatcher.dispatch(getBaseDispatchQueryRequest(query), asyncQueryRequestContext); + verify(queryHandlerFactory, times(1)).getIndexDMLHandler(); } @@ -597,7 +637,7 @@ void testDispatchWithUnSupportedDataSourceType() { getBaseDispatchQueryRequestBuilder(query).datasource("my_prometheus").build(), asyncQueryRequestContext)); - Assertions.assertEquals( + assertEquals( "UnSupported datasource type for async queries:: PROMETHEUS", unsupportedOperationException.getMessage()); } @@ -609,7 +649,7 @@ void testCancelJob() { String queryId = sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata(), asyncQueryRequestContext); - Assertions.assertEquals(QUERY_ID, queryId); + assertEquals(QUERY_ID, queryId); } @Test @@ -625,7 +665,7 @@ void testCancelQueryWithSession() { verifyNoInteractions(emrServerlessClient); verify(statement, times(1)).cancel(); - Assertions.assertEquals(MOCK_STATEMENT_ID, queryId); + assertEquals(MOCK_STATEMENT_ID, queryId); } @Test @@ -642,7 +682,7 @@ void testCancelQueryWithInvalidSession() { verifyNoInteractions(emrServerlessClient); verifyNoInteractions(session); - Assertions.assertEquals("no session found. invalid", exception.getMessage()); + assertEquals("no session found. invalid", exception.getMessage()); } @Test @@ -659,8 +699,7 @@ void testCancelQueryWithInvalidStatementId() { verifyNoInteractions(emrServerlessClient); verifyNoInteractions(statement); - Assertions.assertEquals( - "no statement found. " + new StatementId("invalid"), exception.getMessage()); + assertEquals("no statement found. " + new StatementId("invalid"), exception.getMessage()); } @Test @@ -705,7 +744,7 @@ void testGetQueryResponse() { JSONObject result = sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); - Assertions.assertEquals("PENDING", result.get("status")); + assertEquals("PENDING", result.get("status")); } @Test @@ -724,7 +763,7 @@ void testGetQueryResponseWithSession() { asyncQueryRequestContext); verifyNoInteractions(emrServerlessClient); - Assertions.assertEquals("waiting", result.get("status")); + assertEquals("waiting", result.get("status")); } @Test @@ -743,7 +782,7 @@ void testGetQueryResponseWithInvalidSession() { asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); - Assertions.assertEquals("no session found. " + MOCK_SESSION_ID, exception.getMessage()); + assertEquals("no session found. " + MOCK_SESSION_ID, exception.getMessage()); } @Test @@ -763,7 +802,7 @@ void testGetQueryResponseWithStatementNotExist() { asyncQueryRequestContext)); verifyNoInteractions(emrServerlessClient); - Assertions.assertEquals( + assertEquals( "no statement found. " + new StatementId(MOCK_STATEMENT_ID), exception.getMessage()); } @@ -780,7 +819,7 @@ void testGetQueryResponseWithSuccess() { sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); verify(jobExecutionResponseReader, times(1)).getResultWithJobId(EMR_JOB_ID, null); - Assertions.assertEquals( + assertEquals( new HashSet<>(Arrays.asList(DATA_FIELD, STATUS_FIELD, ERROR_FIELD)), result.keySet()); JSONObject dataJson = new JSONObject(); dataJson.put(ERROR_FIELD, ""); @@ -791,7 +830,7 @@ void testGetQueryResponseWithSuccess() { // the same order. // We need similar. Assertions.assertTrue(dataJson.similar(result.get(DATA_FIELD))); - Assertions.assertEquals("SUCCESS", result.get(STATUS_FIELD)); + assertEquals("SUCCESS", result.get(STATUS_FIELD)); verifyNoInteractions(emrServerlessClient); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index f1853f2c1e..4608bce74e 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -154,6 +154,8 @@ void testExtractionFromFlintSkippingIndexQueries() { assertNull(indexQueryDetails.getIndexName()); assertFullyQualifiedTableName("myS3", "default", "alb_logs", fullyQualifiedTableName); + assertEquals( + "flint_mys3_default_alb_logs_skipping_index", indexQueryDetails.openSearchIndexName()); } } @@ -182,6 +184,9 @@ void testExtractionFromFlintCoveringIndexQueries() { assertEquals("elb_and_requestUri", indexQueryDetails.getIndexName()); assertFullyQualifiedTableName("myS3", "default", "alb_logs", fullyQualifiedTableName); + assertEquals( + "flint_mys3_default_alb_logs_elb_and_requesturi_index", + indexQueryDetails.openSearchIndexName()); } } @@ -196,6 +201,7 @@ void testExtractionFromCreateMVQuery() { assertNull(indexQueryDetails.getFullyQualifiedTableName()); assertEquals(mvQuery, indexQueryDetails.getMvQuery()); assertEquals("mv_1", indexQueryDetails.getMvName()); + assertEquals("flint_mv_1", indexQueryDetails.openSearchIndexName()); } @Test @@ -215,61 +221,86 @@ void testExtractionFromFlintMVQuery() { assertNull(fullyQualifiedTableName); assertNull(indexQueryDetails.getMvQuery()); assertEquals("mv_1", indexQueryDetails.getMvName()); + assertEquals("flint_mv_1", indexQueryDetails.openSearchIndexName()); } } @Test void testDescSkippingIndex() { String descSkippingIndex = "DESC SKIPPING INDEX ON mys3.default.http_logs"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(descSkippingIndex)); IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(descSkippingIndex); FullyQualifiedTableName fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + assertNull(indexDetails.getIndexName()); assertNotNull(fullyQualifiedTableName); assertEquals(FlintIndexType.SKIPPING, indexDetails.getIndexType()); assertEquals(IndexQueryActionType.DESCRIBE, indexDetails.getIndexQueryActionType()); + assertEquals("flint_mys3_default_http_logs_skipping_index", indexDetails.openSearchIndexName()); + } + @Test + void testDescCoveringIndex() { String descCoveringIndex = "DESC INDEX cv1 ON mys3.default.http_logs"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(descCoveringIndex)); - indexDetails = SQLQueryUtils.extractIndexDetails(descCoveringIndex); - fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(descCoveringIndex); + FullyQualifiedTableName fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + assertEquals("cv1", indexDetails.getIndexName()); assertNotNull(fullyQualifiedTableName); assertEquals(FlintIndexType.COVERING, indexDetails.getIndexType()); assertEquals(IndexQueryActionType.DESCRIBE, indexDetails.getIndexQueryActionType()); + assertEquals("flint_mys3_default_http_logs_cv1_index", indexDetails.openSearchIndexName()); + } + @Test + void testDescMaterializedView() { String descMv = "DESC MATERIALIZED VIEW mv1"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(descMv)); - indexDetails = SQLQueryUtils.extractIndexDetails(descMv); - fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(descMv); + FullyQualifiedTableName fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + assertNull(indexDetails.getIndexName()); assertEquals("mv1", indexDetails.getMvName()); assertNull(fullyQualifiedTableName); assertEquals(FlintIndexType.MATERIALIZED_VIEW, indexDetails.getIndexType()); assertEquals(IndexQueryActionType.DESCRIBE, indexDetails.getIndexQueryActionType()); + assertEquals("flint_mv1", indexDetails.openSearchIndexName()); } @Test void testShowIndex() { - String showCoveringIndex = " SHOW INDEX ON myS3.default.http_logs"; + String showCoveringIndex = "SHOW INDEX ON myS3.default.http_logs"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(showCoveringIndex)); IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(showCoveringIndex); FullyQualifiedTableName fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + assertNull(indexDetails.getIndexName()); assertNull(indexDetails.getMvName()); assertNotNull(fullyQualifiedTableName); assertEquals(FlintIndexType.COVERING, indexDetails.getIndexType()); assertEquals(IndexQueryActionType.SHOW, indexDetails.getIndexQueryActionType()); + assertNull(indexDetails.openSearchIndexName()); + } + @Test + void testShowMaterializedView() { String showMV = "SHOW MATERIALIZED VIEW IN my_glue.default"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(showMV)); - indexDetails = SQLQueryUtils.extractIndexDetails(showMV); - fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(showMV); + FullyQualifiedTableName fullyQualifiedTableName = indexDetails.getFullyQualifiedTableName(); + assertNull(indexDetails.getIndexName()); assertNull(indexDetails.getMvName()); assertNull(fullyQualifiedTableName); assertEquals(FlintIndexType.MATERIALIZED_VIEW, indexDetails.getIndexType()); assertEquals(IndexQueryActionType.SHOW, indexDetails.getIndexQueryActionType()); + assertNull(indexDetails.openSearchIndexName()); } @Test From 1b1a1b5e97def94504a75ffaedbd5d72ffc9614a Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Mon, 9 Sep 2024 10:22:58 -0700 Subject: [PATCH 44/96] Add query, langType, status, error in AsyncQueryJobMetadata (#2958) * Add query, langType, status, error in AsyncQueryJobMetadata Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix test Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> Signed-off-by: Tomoyuki MORITA <moritato@amazon.com> --- .../AsyncQueryExecutorServiceImpl.java | 4 ++ .../model/AsyncQueryJobMetadata.java | 9 ++++ .../spark/asyncquery/model/QueryState.java | 41 +++++++++++++++++++ .../spark/dispatcher/BatchQueryHandler.java | 2 + .../sql/spark/dispatcher/IndexDMLHandler.java | 4 ++ .../dispatcher/InteractiveQueryHandler.java | 2 + .../spark/dispatcher/RefreshQueryHandler.java | 2 + .../dispatcher/StreamingQueryHandler.java | 2 + .../model/DispatchQueryResponse.java | 3 ++ .../asyncquery/AsyncQueryCoreIntegTest.java | 24 ++++++----- .../AsyncQueryExecutorServiceImplTest.java | 15 +++---- .../asyncquery/model/QueryStateTest.java | 28 +++++++++++++ 12 files changed, 119 insertions(+), 17 deletions(-) create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/QueryState.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/QueryStateTest.java diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java index 5933343ba4..0e9e128896 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java @@ -67,6 +67,10 @@ public CreateAsyncQueryResponse createAsyncQuery( .datasourceName(dispatchQueryResponse.getDatasourceName()) .jobType(dispatchQueryResponse.getJobType()) .indexName(dispatchQueryResponse.getIndexName()) + .query(createAsyncQueryRequest.getQuery()) + .langType(createAsyncQueryRequest.getLang()) + .state(dispatchQueryResponse.getStatus()) + .error(dispatchQueryResponse.getError()) .build(), asyncQueryRequestContext); return new CreateAsyncQueryResponse( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java index 1cfab4832d..46aa8ac898 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java @@ -12,6 +12,7 @@ import lombok.experimental.SuperBuilder; import org.opensearch.sql.spark.dispatcher.model.JobType; import org.opensearch.sql.spark.execution.statestore.StateModel; +import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.utils.SerializeUtils; /** This class models all the metadata required for a job. */ @@ -35,6 +36,10 @@ public class AsyncQueryJobMetadata extends StateModel { private final String datasourceName; // null if JobType is INTERACTIVE or null private final String indexName; + private final String query; + private final LangType langType; + private final QueryState state; + private final String error; @Override public String toString() { @@ -54,6 +59,10 @@ public static AsyncQueryJobMetadata copy( .datasourceName(copy.datasourceName) .jobType(copy.jobType) .indexName(copy.indexName) + .query(copy.query) + .langType(copy.langType) + .state(copy.state) + .error(copy.error) .metadata(metadata) .build(); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/QueryState.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/QueryState.java new file mode 100644 index 0000000000..62bceb6637 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/QueryState.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.asyncquery.model; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.Getter; + +@Getter +public enum QueryState { + WAITING("waiting"), + RUNNING("running"), + SUCCESS("success"), + FAILED("failed"), + TIMEOUT("timeout"), + CANCELLED("cancelled"); + + private final String state; + + QueryState(String state) { + this.state = state; + } + + private static final Map<String, QueryState> STATES = + Arrays.stream(QueryState.values()) + .collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t)); + + public static QueryState fromString(String key) { + for (QueryState ss : QueryState.values()) { + if (ss.getState().toLowerCase(Locale.ROOT).equals(key)) { + return ss; + } + } + throw new IllegalArgumentException("Invalid query state: " + key); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 33d78b174e..5a775aa243 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -17,6 +17,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.client.StartJobRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; @@ -111,6 +112,7 @@ public DispatchQueryResponse submit( .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.BATCH) + .status(QueryState.WAITING) .indexName(getIndexName(context)) .build(); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index 7211da0941..fe848593a7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -18,6 +18,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryResponse; @@ -83,6 +84,7 @@ public DispatchQueryResponse submit( .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.BATCH) + .status(QueryState.SUCCESS) .build(); } catch (Exception e) { LOG.error(e.getMessage()); @@ -101,6 +103,8 @@ public DispatchQueryResponse submit( .resultIndex(dataSourceMetadata.getResultIndex()) .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.BATCH) + .status(QueryState.FAILED) + .error(e.getMessage()) .build(); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java index 7be6809912..75912f3a7c 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java @@ -17,6 +17,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryResponse; @@ -151,6 +152,7 @@ public DispatchQueryResponse submit( .sessionId(session.getSessionId()) .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.INTERACTIVE) + .status(QueryState.WAITING) .build(); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java index cf5a0c6c59..52cd863081 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java @@ -9,6 +9,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @@ -85,6 +86,7 @@ public DispatchQueryResponse submit( .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.REFRESH) .indexName(context.getIndexQueryDetails().openSearchIndexName()) + .status(QueryState.WAITING) .build(); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java index 51e245b57c..58fb5244b4 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java @@ -13,6 +13,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.client.EMRServerlessClient; import org.opensearch.sql.spark.client.StartJobRequest; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext; @@ -102,6 +103,7 @@ public DispatchQueryResponse submit( .datasourceName(dataSourceMetadata.getName()) .jobType(JobType.STREAMING) .indexName(indexQueryDetails.openSearchIndexName()) + .status(QueryState.WAITING) .build(); } } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java index b97d9fd7b0..c484236d6e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java @@ -2,6 +2,7 @@ import lombok.Builder; import lombok.Getter; +import org.opensearch.sql.spark.asyncquery.model.QueryState; @Getter @Builder @@ -13,4 +14,6 @@ public class DispatchQueryResponse { private final String datasourceName; private final JobType jobType; private final String indexName; + private final QueryState status; + private final String error; } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 1214935dc6..52d805dd01 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -46,6 +46,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata.AsyncQueryJobMetadataBuilder; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.client.EMRServerlessClientFactory; import org.opensearch.sql.spark.client.EmrServerlessClientImpl; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; @@ -205,7 +206,7 @@ public void createDropIndexQuery() { verifyGetQueryIdCalled(); verifyCancelJobRunCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); } @Test @@ -227,7 +228,7 @@ public void createDropIndexQueryWithScheduler() { assertNull(response.getSessionId()); verifyGetQueryIdCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); verify(asyncQueryScheduler).unscheduleJob(indexName); } @@ -255,7 +256,7 @@ public void createVacuumIndexQuery() { verifyGetSessionIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.INTERACTIVE); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.INTERACTIVE); } @Test @@ -286,7 +287,7 @@ public void createAlterIndexQuery() { assertFalse(flintIndexOptions.autoRefresh()); verifyCancelJobRunCalled(); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); } @Test @@ -320,7 +321,7 @@ public void createAlterIndexQueryWithScheduler() { verify(asyncQueryScheduler).unscheduleJob(indexName); verifyCreateIndexDMLResultCalled(); - verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); } @Test @@ -345,7 +346,7 @@ public void createStreamingQuery() { verifyGetQueryIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.STREAMING); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.STREAMING); } private void verifyStartJobRunCalled() { @@ -380,7 +381,7 @@ public void createCreateIndexQuery() { assertNull(response.getSessionId()); verifyGetQueryIdCalled(); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.BATCH); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.BATCH); } @Test @@ -402,7 +403,7 @@ public void createRefreshQuery() { verifyGetQueryIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.REFRESH); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.REFRESH); } @Test @@ -428,7 +429,7 @@ public void createInteractiveQuery() { verifyGetSessionIdCalled(); verify(leaseManager).borrow(any()); verifyStartJobRunCalled(); - verifyStoreJobMetadataCalled(JOB_ID, JobType.INTERACTIVE); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.INTERACTIVE); } @Test @@ -644,7 +645,7 @@ private void verifyGetSessionIdCalled() { assertEquals(APPLICATION_ID, createSessionRequest.getApplicationId()); } - private void verifyStoreJobMetadataCalled(String jobId, JobType jobType) { + private void verifyStoreJobMetadataCalled(String jobId, QueryState state, JobType jobType) { verify(asyncQueryJobMetadataStorageService) .storeJobMetadata( asyncQueryJobMetadataArgumentCaptor.capture(), eq(asyncQueryRequestContext)); @@ -652,6 +653,9 @@ private void verifyStoreJobMetadataCalled(String jobId, JobType jobType) { assertEquals(QUERY_ID, asyncQueryJobMetadata.getQueryId()); assertEquals(jobId, asyncQueryJobMetadata.getJobId()); assertEquals(DATASOURCE_NAME, asyncQueryJobMetadata.getDatasourceName()); + assertNull(asyncQueryJobMetadata.getError()); + assertEquals(LangType.SQL, asyncQueryJobMetadata.getLangType()); + assertEquals(state, asyncQueryJobMetadata.getState()); assertEquals(jobType, asyncQueryJobMetadata.getJobType()); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index 1491f0bd61..73850db83d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -47,6 +47,9 @@ @ExtendWith(MockitoExtension.class) public class AsyncQueryExecutorServiceImplTest { + private static final String QUERY = "select * from my_glue.default.http_logs"; + private static final String QUERY_ID = "QUERY_ID"; + @Mock private SparkQueryDispatcher sparkQueryDispatcher; @Mock private AsyncQueryJobMetadataStorageService asyncQueryJobMetadataStorageService; private AsyncQueryExecutorService jobExecutorService; @@ -54,7 +57,6 @@ public class AsyncQueryExecutorServiceImplTest { @Mock private SparkExecutionEngineConfigSupplier sparkExecutionEngineConfigSupplier; @Mock private SparkSubmitParameterModifier sparkSubmitParameterModifier; @Mock private AsyncQueryRequestContext asyncQueryRequestContext; - private final String QUERY_ID = "QUERY_ID"; @BeforeEach void setUp() { @@ -68,8 +70,7 @@ void setUp() { @Test void testCreateAsyncQuery() { CreateAsyncQueryRequest createAsyncQueryRequest = - new CreateAsyncQueryRequest( - "select * from my_glue.default.http_logs", "my_glue", LangType.SQL); + new CreateAsyncQueryRequest(QUERY, "my_glue", LangType.SQL); when(sparkExecutionEngineConfigSupplier.getSparkExecutionEngineConfig(any())) .thenReturn( SparkExecutionEngineConfig.builder() @@ -82,7 +83,7 @@ void testCreateAsyncQuery() { DispatchQueryRequest expectedDispatchQueryRequest = DispatchQueryRequest.builder() .applicationId(EMRS_APPLICATION_ID) - .query("select * from my_glue.default.http_logs") + .query(QUERY) .datasource("my_glue") .langType(LangType.SQL) .executionRoleARN(EMRS_EXECUTION_ROLE) @@ -134,9 +135,7 @@ void testCreateAsyncQueryWithExtraSparkSubmitParameter() { .build()); jobExecutorService.createAsyncQuery( - new CreateAsyncQueryRequest( - "select * from my_glue.default.http_logs", "my_glue", LangType.SQL), - asyncQueryRequestContext); + new CreateAsyncQueryRequest(QUERY, "my_glue", LangType.SQL), asyncQueryRequestContext); verify(sparkQueryDispatcher, times(1)) .dispatch( @@ -237,6 +236,8 @@ private AsyncQueryJobMetadata getAsyncQueryJobMetadata() { .queryId(QUERY_ID) .applicationId(EMRS_APPLICATION_ID) .jobId(EMR_JOB_ID) + .query(QUERY) + .langType(LangType.SQL) .build(); } } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/QueryStateTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/QueryStateTest.java new file mode 100644 index 0000000000..8e86e3b176 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/QueryStateTest.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.asyncquery.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class QueryStateTest { + @Test + public void testFromString() { + assertEquals(QueryState.WAITING, QueryState.fromString("waiting")); + assertEquals(QueryState.RUNNING, QueryState.fromString("running")); + assertEquals(QueryState.SUCCESS, QueryState.fromString("success")); + assertEquals(QueryState.FAILED, QueryState.fromString("failed")); + assertEquals(QueryState.CANCELLED, QueryState.fromString("cancelled")); + assertEquals(QueryState.TIMEOUT, QueryState.fromString("timeout")); + } + + @Test + public void testFromStringWithUnknownState() { + assertThrows(IllegalArgumentException.class, () -> QueryState.fromString("UNKNOWN_STATE")); + } +} From 2c878fbc67de1b18052e1184e10700a2b799defa Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Mon, 9 Sep 2024 14:56:05 -0700 Subject: [PATCH 45/96] Throw exception for RECOVER INDEX JOB query (#2988) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../sql/spark/dispatcher/SparkQueryDispatcher.java | 3 +++ .../spark/dispatcher/model/IndexQueryActionType.java | 3 ++- .../org/opensearch/sql/spark/utils/SQLQueryUtils.java | 7 +++++++ .../sql/spark/dispatcher/SparkQueryDispatcherTest.java | 10 ++++++++++ .../opensearch/sql/spark/utils/SQLQueryUtilsTest.java | 9 +++++++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 50e8403d36..732f5f71ab 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -119,6 +119,9 @@ private AsyncQueryHandler getQueryHandlerForFlintExtensionQuery( } else if (IndexQueryActionType.REFRESH.equals(indexQueryDetails.getIndexQueryActionType())) { // Manual refresh should be handled by batch handler return queryHandlerFactory.getRefreshQueryHandler(dispatchQueryRequest.getAccountId()); + } else if (IndexQueryActionType.RECOVER.equals(indexQueryDetails.getIndexQueryActionType())) { + // RECOVER INDEX JOB should not be executed from async-query-core + throw new IllegalArgumentException("RECOVER INDEX JOB is not allowed."); } else { return getDefaultAsyncQueryHandler(dispatchQueryRequest.getAccountId()); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java index 96e7d159af..51e0832217 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java @@ -13,5 +13,6 @@ public enum IndexQueryActionType { SHOW, DROP, VACUUM, - ALTER + ALTER, + RECOVER } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index b1a8c3d4f6..7550de2f1e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -26,6 +26,7 @@ import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsLexer; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser.MaterializedViewQueryContext; +import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser.RecoverIndexJobStatementContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.IdentifierReferenceContext; @@ -386,6 +387,12 @@ public Void visitMaterializedViewQuery(MaterializedViewQueryContext ctx) { return super.visitMaterializedViewQuery(ctx); } + @Override + public Void visitRecoverIndexJobStatement(RecoverIndexJobStatementContext ctx) { + indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.RECOVER); + return super.visitRecoverIndexJobStatement(ctx); + } + private String propertyKey(FlintSparkSqlExtensionsParser.PropertyKeyContext key) { if (key.STRING() != null) { return key.STRING().getText(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 9f12ddf323..75c0e00337 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -622,6 +622,16 @@ void testDispatchVacuumIndexQuery() { testDispatchBatchQuery("VACUUM INDEX elb_and_requestUri ON my_glue.default.http_logs"); } + @Test + void testDispatchRecoverIndexQuery() { + String query = "RECOVER INDEX JOB `flint_spark_catalog_default_test_skipping_index`"; + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + sparkQueryDispatcher.dispatch( + getBaseDispatchQueryRequest(query), asyncQueryRequestContext)); + } + @Test void testDispatchWithUnSupportedDataSourceType() { when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index 4608bce74e..56cab7ce7f 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -435,6 +435,15 @@ void testAutoRefresh() { .autoRefresh()); } + @Test + void testRecoverIndex() { + String refreshSkippingIndex = + "RECOVER INDEX JOB `flint_spark_catalog_default_test_skipping_index`"; + assertTrue(SQLQueryUtils.isFlintExtensionQuery(refreshSkippingIndex)); + IndexQueryDetails indexDetails = SQLQueryUtils.extractIndexDetails(refreshSkippingIndex); + assertEquals(IndexQueryActionType.RECOVER, indexDetails.getIndexQueryActionType()); + } + @Test void testValidateSparkSqlQuery_ValidQuery() { List<String> errors = From c22929b2eb45e1fd13028354f2e09dcca1c75be5 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Mon, 9 Sep 2024 16:11:20 -0700 Subject: [PATCH 46/96] Throw IllegalArgumentException when cancelJob failed with ValidationException (#2986) * Throw IllegalArgumentException when cancelJob failed with ValidationException Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add jobId to error message Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> --- .../sql/spark/client/EmrServerlessClientImpl.java | 13 +++++++++---- .../spark/client/EmrServerlessClientImplTest.java | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java index c785067398..f6f3633bc7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java @@ -116,11 +116,16 @@ public CancelJobRunResult cancelJobRun( } catch (Throwable t) { if (allowExceptionPropagation) { throw t; - } else { - logger.error("Error while making cancel job request to emr:", t); - metricsService.incrementNumericalMetric(EMR_CANCEL_JOB_REQUEST_FAILURE_COUNT); - throw new RuntimeException(GENERIC_INTERNAL_SERVER_ERROR_MESSAGE); } + + logger.error("Error while making cancel job request to emr: jobId=" + jobId, t); + metricsService.incrementNumericalMetric(EMR_CANCEL_JOB_REQUEST_FAILURE_COUNT); + if (t instanceof ValidationException) { + throw new IllegalArgumentException( + "The input fails to satisfy the constraints specified by AWS EMR" + + " Serverless."); + } + throw new RuntimeException(GENERIC_INTERNAL_SERVER_ERROR_MESSAGE); } }); logger.info(String.format("Job : %s cancelled", cancelJobRunResult.getJobRunId())); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java index 42d703f9ac..b3a2bda36a 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java @@ -172,10 +172,12 @@ void testCancelJobRunWithValidationException() { RuntimeException runtimeException = Assertions.assertThrows( - RuntimeException.class, + IllegalArgumentException.class, () -> emrServerlessClient.cancelJobRun(EMRS_APPLICATION_ID, EMR_JOB_ID, false)); - Assertions.assertEquals("Internal Server Error.", runtimeException.getMessage()); + Assertions.assertEquals( + "The input fails to satisfy the constraints specified by AWS EMR Serverless.", + runtimeException.getMessage()); } @Test From 4303a2ab755d53903094dd94a5100572677a27a1 Mon Sep 17 00:00:00 2001 From: Peng Huo <penghuo@gmail.com> Date: Tue, 10 Sep 2024 07:24:14 -0700 Subject: [PATCH 47/96] Bump BWC version to 2.17 (#3007) Signed-off-by: Peng Huo <penghuo@gmail.com> --- integ-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1acacdb4a5..798a0be536 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -44,7 +44,7 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' -String baseVersion = "2.16.0" +String baseVersion = "2.17.0" String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" From 25064688db2d08e61579aaf7d21ff440c67605a4 Mon Sep 17 00:00:00 2001 From: Peng Huo <penghuo@gmail.com> Date: Fri, 13 Sep 2024 10:53:45 -0700 Subject: [PATCH 48/96] Upgrade org.bouncycastle:bcprov-jdk18on to 1.78.1 (#3004) Signed-off-by: Peng Huo <penghuo@gmail.com> --- datasources/build.gradle | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datasources/build.gradle b/datasources/build.gradle index 9bd233e1f9..9456141e80 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -22,7 +22,12 @@ dependencies { implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' - implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1' + // FIXME. upgrade aws-encryption-sdk-java once the bouncycastle dependency update to 1.78. + implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { + exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' + } + // Use OpenSearch bouncycastle version. https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/version.properties + implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' testImplementation group: 'junit', name: 'junit', version: '4.13.2' From 37188bdd5154f9a67043dcaa74ddc5587dd52858 Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Mon, 16 Sep 2024 10:11:52 -0700 Subject: [PATCH 49/96] Extend scheduler interface for Multitenancy (#3014) Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .../workflows/integ-tests-with-security.yml | 4 +- .github/workflows/sql-pitest.yml | 2 +- .../workflows/sql-test-and-build-workflow.yml | 8 +- .github/workflows/sql-test-workflow.yml | 2 +- .../flint/operation/FlintIndexOpAlter.java | 3 +- .../flint/operation/FlintIndexOpDrop.java | 3 +- .../spark/scheduler/AsyncQueryScheduler.java | 26 +++++- .../model/AsyncQuerySchedulerRequest.java | 2 + .../asyncquery/AsyncQueryCoreIntegTest.java | 5 +- .../OpenSearchAsyncQueryScheduler.java | 34 +++++--- .../model/ScheduledAsyncQueryJobRequest.java | 4 +- ...enSearchScheduleQueryJobRequestParser.java | 2 +- .../OpenSearchAsyncQuerySchedulerTest.java | 85 ++++++++++++------- .../job/ScheduledAsyncQueryJobRunnerTest.java | 6 +- .../ScheduledAsyncQueryJobRequestTest.java | 10 +-- 15 files changed, 127 insertions(+), 69 deletions(-) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 751f3d01db..9d214a45cb 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -47,7 +47,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} @@ -79,7 +79,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-${{ matrix.os }}-${{ matrix.java }} diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index 78eed9e846..d9034cce9f 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -48,7 +48,7 @@ jobs: - name: Upload test reports if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.entry.java }} path: | diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index fd11716b51..6aa674b73f 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -73,7 +73,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: opensearch-sql-ubuntu-latest-${{ matrix.java }} @@ -81,7 +81,7 @@ jobs: - name: Upload test reports if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-ubuntu-latest-${{ matrix.java }} @@ -135,7 +135,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: opensearch-sql-${{ matrix.entry.os }}-${{ matrix.entry.java }} @@ -143,7 +143,7 @@ jobs: - name: Upload test reports if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }} diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index 40a1e2fc8f..f2f22bd44f 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -86,7 +86,7 @@ jobs: - name: Upload test reports if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.entry.java }} path: | diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java index de34803823..596d76c24b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java @@ -62,7 +62,8 @@ void runOp( this.flintIndexMetadataService.updateIndexToManualRefresh( flintIndexMetadata.getOpensearchIndexName(), flintIndexOptions, asyncQueryRequestContext); if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { - asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + asyncQueryScheduler.unscheduleJob( + flintIndexMetadata.getOpensearchIndexName(), asyncQueryRequestContext); } else { cancelStreamingJob(flintIndexStateModel); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java index 3fa5423c10..88aca66fef 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java @@ -54,7 +54,8 @@ void runOp( "Performing drop index operation for index: {}", flintIndexMetadata.getOpensearchIndexName()); if (flintIndexMetadata.getFlintIndexOptions().isExternalScheduler()) { - asyncQueryScheduler.unscheduleJob(flintIndexMetadata.getOpensearchIndexName()); + asyncQueryScheduler.unscheduleJob( + flintIndexMetadata.getOpensearchIndexName(), asyncQueryRequestContext); } else { cancelStreamingJob(flintIndexStateModel); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java index 8ac499081e..6d5350821b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java @@ -1,5 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.spark.scheduler; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; /** Scheduler interface for scheduling asynchronous query jobs. */ @@ -13,10 +19,13 @@ public interface AsyncQueryScheduler { * task * * @param asyncQuerySchedulerRequest The request containing job configuration details + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService * @throws IllegalArgumentException if a job with the same name already exists * @throws RuntimeException if there's an error during job creation */ - void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + void scheduleJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Updates an existing job with new parameters. This method modifies the configuration of an @@ -26,10 +35,13 @@ public interface AsyncQueryScheduler { * scheduled job - Updating resource allocations for a job * * @param asyncQuerySchedulerRequest The request containing updated job configuration + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService * @throws IllegalArgumentException if the job to be updated doesn't exist * @throws RuntimeException if there's an error during the update process */ - void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); + void updateJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Unschedules a job by marking it as disabled and updating its last update time. This method is @@ -41,8 +53,11 @@ public interface AsyncQueryScheduler { * re-enabling of the job in the future * * @param jobId The unique identifier of the job to unschedule + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService + * @throws IllegalArgumentException if the job to be unscheduled doesn't exist + * @throws RuntimeException if there's an error during the unschedule process */ - void unscheduleJob(String jobId); + void unscheduleJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext); /** * Removes a job completely from the scheduler. This method permanently deletes the job and all @@ -52,6 +67,9 @@ public interface AsyncQueryScheduler { * created jobs - Freeing up resources by deleting unused job configurations * * @param jobId The unique identifier of the job to remove + * @param asyncQueryRequestContext The request context passed to AsyncQueryExecutorService + * @throws IllegalArgumentException if the job to be removed doesn't exist + * @throws RuntimeException if there's an error during the remove process */ - void removeJob(String jobId); + void removeJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java index b54e5b30ce..c38d92365a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/model/AsyncQuerySchedulerRequest.java @@ -7,12 +7,14 @@ import java.time.Instant; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.opensearch.sql.spark.rest.model.LangType; /** Represents a job request for a scheduled task. */ @Data +@Builder @NoArgsConstructor @AllArgsConstructor public class AsyncQuerySchedulerRequest { diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 52d805dd01..ddadeb65e2 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -230,7 +230,7 @@ public void createDropIndexQueryWithScheduler() { verifyCreateIndexDMLResultCalled(); verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); - verify(asyncQueryScheduler).unscheduleJob(indexName); + verify(asyncQueryScheduler).unscheduleJob(indexName, asyncQueryRequestContext); } @Test @@ -318,8 +318,7 @@ public void createAlterIndexQueryWithScheduler() { FlintIndexOptions flintIndexOptions = flintIndexOptionsArgumentCaptor.getValue(); assertFalse(flintIndexOptions.autoRefresh()); - verify(asyncQueryScheduler).unscheduleJob(indexName); - + verify(asyncQueryScheduler).unscheduleJob(indexName, asyncQueryRequestContext); verifyCreateIndexDMLResultCalled(); verifyStoreJobMetadataCalled(DML_QUERY_JOB_ID, QueryState.SUCCESS, JobType.BATCH); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java index 9ebde4fe83..59bad14320 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java @@ -8,6 +8,7 @@ import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -35,6 +36,7 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.scheduler.job.ScheduledAsyncQueryJobRunner; import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; @@ -55,7 +57,9 @@ public class OpenSearchAsyncQueryScheduler implements AsyncQueryScheduler { @Override /** Schedules a new job by indexing it into the job index. */ - public void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + public void scheduleJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext) { ScheduledAsyncQueryJobRequest request = ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); if (!this.clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)) { @@ -87,15 +91,18 @@ public void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { /** Unschedules a job by marking it as disabled and updating its last update time. */ @Override - public void unscheduleJob(String jobId) { - ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() - .jobId(jobId) - .enabled(false) - .lastUpdateTime(Instant.now()) - .build(); + public void unscheduleJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext) { + if (Strings.isNullOrEmpty(jobId)) { + throw new IllegalArgumentException("JobId cannot be null or empty"); + } try { - updateJob(request); + AsyncQuerySchedulerRequest request = + ScheduledAsyncQueryJobRequest.builder() + .jobId(jobId) + .enabled(false) + .lastUpdateTime(Instant.now()) + .build(); + updateJob(request, asyncQueryRequestContext); LOG.info("Unscheduled job for jobId: {}", jobId); } catch (IllegalStateException | DocumentMissingException e) { LOG.error("Failed to unschedule job: {}", jobId, e); @@ -105,7 +112,9 @@ public void unscheduleJob(String jobId) { /** Updates an existing job with new parameters. */ @Override @SneakyThrows - public void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { + public void updateJob( + AsyncQuerySchedulerRequest asyncQuerySchedulerRequest, + AsyncQueryRequestContext asyncQueryRequestContext) { ScheduledAsyncQueryJobRequest request = ScheduledAsyncQueryJobRequest.fromAsyncQuerySchedulerRequest(asyncQuerySchedulerRequest); assertIndexExists(); @@ -134,8 +143,11 @@ public void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest) { /** Removes a job by deleting its document from the index. */ @Override - public void removeJob(String jobId) { + public void removeJob(String jobId, AsyncQueryRequestContext asyncQueryRequestContext) { assertIndexExists(); + if (Strings.isNullOrEmpty(jobId)) { + throw new IllegalArgumentException("JobId cannot be null or empty"); + } DeleteRequest deleteRequest = new DeleteRequest(SCHEDULER_INDEX_NAME, jobId); deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); ActionFuture<DeleteResponse> deleteResponseActionFuture = client.delete(deleteRequest); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java index 9b85a11888..48aa52a3ce 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequest.java @@ -38,7 +38,7 @@ public class ScheduledAsyncQueryJobRequest extends AsyncQuerySchedulerRequest public static final String ENABLED_FIELD = "enabled"; private final Schedule schedule; - @Builder + @Builder(builderMethodName = "scheduledAsyncQueryJobRequestBuilder") public ScheduledAsyncQueryJobRequest( String accountId, String jobId, @@ -139,7 +139,7 @@ public static ScheduledAsyncQueryJobRequest fromAsyncQuerySchedulerRequest( AsyncQuerySchedulerRequest request) { Instant updateTime = request.getLastUpdateTime() != null ? request.getLastUpdateTime() : Instant.now(); - return ScheduledAsyncQueryJobRequest.builder() + return ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId(request.getAccountId()) .jobId(request.getJobId()) .dataSource(request.getDataSource()) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java index 9e33ef0248..a824797066 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/parser/OpenSearchScheduleQueryJobRequestParser.java @@ -30,7 +30,7 @@ private static Instant parseInstantValue(XContentParser parser) throws IOExcepti public static ScheduledJobParser getJobParser() { return (parser, id, jobDocVersion) -> { ScheduledAsyncQueryJobRequest.ScheduledAsyncQueryJobRequestBuilder builder = - ScheduledAsyncQueryJobRequest.builder(); + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder(); XContentParserUtils.ensureExpectedToken( XContentParser.Token.START_OBJECT, parser.nextToken(), parser); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java index a4a6eb6471..d6e672f7a2 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java @@ -16,7 +16,6 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME; -import java.io.IOException; import java.time.Instant; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -43,6 +42,8 @@ import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest; import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest; public class OpenSearchAsyncQuerySchedulerTest { @@ -57,6 +58,8 @@ public class OpenSearchAsyncQuerySchedulerTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) private ClusterService clusterService; + @Mock private AsyncQueryRequestContext context; + @Mock private ActionFuture<IndexResponse> indexResponseActionFuture; @Mock private ActionFuture<UpdateResponse> updateResponseActionFuture; @@ -92,12 +95,12 @@ public void testScheduleJob() { when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); - scheduler.scheduleJob(request); + scheduler.scheduleJob(request, context); // Verify index created verify(client.admin().indices(), times(1)).create(ArgumentMatchers.any()); @@ -116,7 +119,7 @@ public void testScheduleJobWithExistingJob() { .thenReturn(Boolean.TRUE); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -127,7 +130,7 @@ public void testScheduleJobWithExistingJob() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.scheduleJob(request); + scheduler.scheduleJob(request, context); }); verify(client, times(1)).index(ArgumentCaptor.forClass(IndexRequest.class).capture()); @@ -145,24 +148,24 @@ public void testScheduleJobWithExceptions() { when(client.index(any(IndexRequest.class))).thenThrow(new RuntimeException("Test exception")); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); - assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request, context)); when(client.index(any(IndexRequest.class))).thenReturn(indexResponseActionFuture); when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); RuntimeException exception = - assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request, context)); assertEquals("Schedule job failed with result : not_found", exception.getMessage()); } @Test - public void testUnscheduleJob() throws IOException { + public void testUnscheduleJob() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); @@ -170,7 +173,7 @@ public void testUnscheduleJob() throws IOException { when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); ArgumentCaptor<UpdateRequest> captor = ArgumentCaptor.forClass(UpdateRequest.class); verify(client).update(captor.capture()); @@ -183,7 +186,7 @@ public void testUnscheduleJob() throws IOException { captor = ArgumentCaptor.forClass(UpdateRequest.class); when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); verify(client, times(2)).update(captor.capture()); capturedRequest = captor.getValue(); @@ -191,20 +194,29 @@ public void testUnscheduleJob() throws IOException { assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, capturedRequest.getRefreshPolicy()); } + @Test + public void testUnscheduleJobInvalidJobId() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> scheduler.unscheduleJob("", context)); + assertEquals("JobId cannot be null or empty", exception.getMessage()); + } + @Test public void testUnscheduleJobWithIndexNotFound() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - scheduler.unscheduleJob(TEST_JOB_ID); + scheduler.unscheduleJob(TEST_JOB_ID, context); // Verify that no update operation was performed verify(client, never()).update(any(UpdateRequest.class)); } @Test - public void testUpdateJob() throws IOException { + public void testUpdateJob() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -216,7 +228,7 @@ public void testUpdateJob() throws IOException { when(client.update(any(UpdateRequest.class))).thenReturn(updateResponseActionFuture); - scheduler.updateJob(request); + scheduler.updateJob(request, context); ArgumentCaptor<UpdateRequest> captor = ArgumentCaptor.forClass(UpdateRequest.class); verify(client).update(captor.capture()); @@ -229,20 +241,20 @@ public void testUpdateJob() throws IOException { @Test public void testUpdateJobWithIndexNotFound() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - assertThrows(IllegalStateException.class, () -> scheduler.updateJob(request)); + assertThrows(IllegalStateException.class, () -> scheduler.updateJob(request, context)); } @Test public void testUpdateJobWithExceptions() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -255,7 +267,7 @@ public void testUpdateJobWithExceptions() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("Job: testJob doesn't exist", exception1.getMessage()); @@ -266,7 +278,7 @@ public void testUpdateJobWithExceptions() { assertThrows( RuntimeException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("java.lang.RuntimeException: Test exception", exception2.getMessage()); @@ -276,7 +288,7 @@ public void testUpdateJobWithExceptions() { when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); RuntimeException exception = - assertThrows(RuntimeException.class, () -> scheduler.updateJob(request)); + assertThrows(RuntimeException.class, () -> scheduler.updateJob(request, context)); assertEquals("Update job failed with result : not_found", exception.getMessage()); } @@ -290,7 +302,7 @@ public void testRemoveJob() { when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); - scheduler.removeJob(TEST_JOB_ID); + scheduler.removeJob(TEST_JOB_ID, context); ArgumentCaptor<DeleteRequest> captor = ArgumentCaptor.forClass(DeleteRequest.class); verify(client).delete(captor.capture()); @@ -304,7 +316,18 @@ public void testRemoveJob() { public void testRemoveJobWithIndexNotFound() { when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(false); - assertThrows(IllegalStateException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + AsyncQuerySchedulerRequest request = + AsyncQuerySchedulerRequest.builder().jobId(TEST_JOB_ID).build(); + assertThrows(IllegalStateException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); + } + + @Test + public void testRemoveJobInvalidJobId() { + when(clusterService.state().routingTable().hasIndex(SCHEDULER_INDEX_NAME)).thenReturn(true); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> scheduler.removeJob("", context)); + assertEquals("JobId cannot be null or empty", exception.getMessage()); } @Test @@ -351,13 +374,14 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { .thenReturn(new CreateIndexResponse(false, false, SCHEDULER_INDEX_NAME)); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); RuntimeException runtimeException = - Assertions.assertThrows(RuntimeException.class, () -> scheduler.scheduleJob(request)); + Assertions.assertThrows( + RuntimeException.class, () -> scheduler.scheduleJob(request, context)); Assertions.assertEquals( "Internal server error while creating .async-query-scheduler index: Index creation is not" + " acknowledged.", @@ -367,7 +391,7 @@ public void testCreateAsyncQuerySchedulerIndexFailure() { @Test public void testUpdateJobNotFound() { ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId(TEST_JOB_ID) .lastUpdateTime(Instant.now()) .build(); @@ -381,7 +405,7 @@ public void testUpdateJobNotFound() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.updateJob(request); + scheduler.updateJob(request, context); }); assertEquals("Job: testJob doesn't exist", exception.getMessage()); @@ -401,7 +425,7 @@ public void testRemoveJobNotFound() { assertThrows( IllegalArgumentException.class, () -> { - scheduler.removeJob(TEST_JOB_ID); + scheduler.removeJob(TEST_JOB_ID, context); }); assertEquals("Job : testJob doesn't exist", exception.getMessage()); @@ -413,7 +437,7 @@ public void testRemoveJobWithExceptions() { when(client.delete(any(DeleteRequest.class))).thenThrow(new RuntimeException("Test exception")); - assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); DeleteResponse deleteResponse = mock(DeleteResponse.class); when(client.delete(any(DeleteRequest.class))).thenReturn(deleteResponseActionFuture); @@ -421,7 +445,8 @@ public void testRemoveJobWithExceptions() { when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); RuntimeException runtimeException = - Assertions.assertThrows(RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID)); + Assertions.assertThrows( + RuntimeException.class, () -> scheduler.removeJob(TEST_JOB_ID, context)); Assertions.assertEquals("Remove job failed with result : noop", runtimeException.getMessage()); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java index cba8d43a2a..fdfb138ddb 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java @@ -72,7 +72,7 @@ public void testRunJobWithCorrectParameter() { spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .lockDurationSeconds(10L) @@ -123,7 +123,7 @@ public void testDoRefreshThrowsException() { spyJobRunner.loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .lockDurationSeconds(10L) @@ -158,7 +158,7 @@ public void testDoRefreshThrowsException() { @Test public void testRunJobWithUninitializedServices() { ScheduledAsyncQueryJobRequest jobParameter = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .jobId("testJob") .lastUpdateTime(Instant.now()) .build(); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java index 85d1948dc3..edf8379195 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/scheduler/model/ScheduledAsyncQueryJobRequestTest.java @@ -28,7 +28,7 @@ public void testBuilderAndGetterMethods() { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest jobRequest = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -62,7 +62,7 @@ public void testToXContent() throws IOException { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest request = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -146,7 +146,7 @@ public void testEqualsAndHashCode() { IntervalSchedule schedule = new IntervalSchedule(now, 1, ChronoUnit.MINUTES); ScheduledAsyncQueryJobRequest request1 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -172,7 +172,7 @@ public void testEqualsAndHashCode() { assertTrue(toString.contains("jitter=0.1")); ScheduledAsyncQueryJobRequest request2 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("testAccount") .jobId("testJob") .dataSource("testDataSource") @@ -190,7 +190,7 @@ public void testEqualsAndHashCode() { assertEquals(request1.hashCode(), request2.hashCode()); ScheduledAsyncQueryJobRequest request3 = - ScheduledAsyncQueryJobRequest.builder() + ScheduledAsyncQueryJobRequest.scheduledAsyncQueryJobRequestBuilder() .accountId("differentAccount") .jobId("testJob") .dataSource("testDataSource") From a87893ac6771cb8739b82cacf8721d0dd7d1cbe3 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Mon, 23 Sep 2024 16:04:49 -0700 Subject: [PATCH 50/96] Implement SQL validation based on grammar element (#3039) * Implement SQL validation based on grammar element Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add function types Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * fix style Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add security lake Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add File support Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Integrate into SparkQueryDispatcher Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix style Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add tests Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Integration Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add comments Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Address comments Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Allow join types for now Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix style Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix coverage check Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- async-query-core/build.gradle | 3 +- .../dispatcher/SparkQueryDispatcher.java | 11 +- .../sql/spark/utils/SQLQueryUtils.java | 67 +- .../DefaultGrammarElementValidator.java | 13 + .../DenyListGrammarElementValidator.java | 19 + .../sql/spark/validator/FunctionType.java | 436 +++++++++++++ .../sql/spark/validator/GrammarElement.java | 89 +++ .../validator/GrammarElementValidator.java | 15 + .../GrammarElementValidatorProvider.java | 22 + .../S3GlueGrammarElementValidator.java | 71 ++ .../validator/SQLQueryValidationVisitor.java | 609 ++++++++++++++++++ .../spark/validator/SQLQueryValidator.java | 39 ++ .../SecurityLakeGrammarElementValidator.java | 113 ++++ .../asyncquery/AsyncQueryCoreIntegTest.java | 15 +- .../dispatcher/SparkQueryDispatcherTest.java | 39 +- .../sql/spark/utils/SQLQueryUtilsTest.java | 102 --- .../sql/spark/validator/FunctionTypeTest.java | 47 ++ .../GrammarElementValidatorProviderTest.java | 39 ++ .../validator/SQLQueryValidatorTest.java | 600 +++++++++++++++++ .../config/AsyncExecutorServiceModule.java | 24 +- ...AsyncQueryExecutorServiceImplSpecTest.java | 2 +- .../AsyncQueryExecutorServiceSpec.java | 12 +- 22 files changed, 2195 insertions(+), 192 deletions(-) create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 1de6cb3105..deba81735d 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -122,7 +122,8 @@ jacocoTestCoverageVerification { 'org.opensearch.sql.spark.flint.*', 'org.opensearch.sql.spark.flint.operation.*', 'org.opensearch.sql.spark.rest.*', - 'org.opensearch.sql.spark.utils.SQLQueryUtils.*' + 'org.opensearch.sql.spark.utils.SQLQueryUtils.*', + 'org.opensearch.sql.spark.validator.SQLQueryValidationVisitor' ] limit { counter = 'LINE' diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 732f5f71ab..ff8c8d1fe8 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -6,7 +6,6 @@ package org.opensearch.sql.spark.dispatcher; import java.util.HashMap; -import java.util.List; import java.util.Map; import lombok.AllArgsConstructor; import org.jetbrains.annotations.NotNull; @@ -24,6 +23,7 @@ import org.opensearch.sql.spark.execution.session.SessionManager; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.utils.SQLQueryUtils; +import org.opensearch.sql.spark.validator.SQLQueryValidator; /** This class takes care of understanding query and dispatching job query to emr serverless. */ @AllArgsConstructor @@ -38,6 +38,7 @@ public class SparkQueryDispatcher { private final SessionManager sessionManager; private final QueryHandlerFactory queryHandlerFactory; private final QueryIdProvider queryIdProvider; + private final SQLQueryValidator sqlQueryValidator; public DispatchQueryResponse dispatch( DispatchQueryRequest dispatchQueryRequest, @@ -54,13 +55,7 @@ public DispatchQueryResponse dispatch( dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } - List<String> validationErrors = - SQLQueryUtils.validateSparkSqlQuery( - dataSourceService.getDataSource(dispatchQueryRequest.getDatasource()), query); - if (!validationErrors.isEmpty()) { - throw new IllegalArgumentException( - "Query is not allowed: " + String.join(", ", validationErrors)); - } + sqlQueryValidator.validate(query, dataSourceMetadata.getConnector()); } return handleDefaultQuery(dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index 7550de2f1e..3ba9c23ed7 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -5,8 +5,6 @@ package org.opensearch.sql.spark.utils; -import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -20,8 +18,6 @@ import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.common.antlr.SyntaxCheckException; -import org.opensearch.sql.datasource.model.DataSource; -import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsBaseVisitor; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsLexer; import org.opensearch.sql.spark.antlr.parser.FlintSparkSqlExtensionsParser; @@ -84,71 +80,12 @@ public static boolean isFlintExtensionQuery(String sqlQuery) { } } - public static List<String> validateSparkSqlQuery(DataSource datasource, String sqlQuery) { + public static SqlBaseParser getBaseParser(String sqlQuery) { SqlBaseParser sqlBaseParser = new SqlBaseParser( new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery)))); sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); - try { - SqlBaseValidatorVisitor sqlParserBaseVisitor = getSparkSqlValidatorVisitor(datasource); - StatementContext statement = sqlBaseParser.statement(); - sqlParserBaseVisitor.visit(statement); - return sqlParserBaseVisitor.getValidationErrors(); - } catch (SyntaxCheckException e) { - logger.error( - String.format( - "Failed to parse sql statement context while validating sql query %s", sqlQuery), - e); - return Collections.emptyList(); - } - } - - private SqlBaseValidatorVisitor getSparkSqlValidatorVisitor(DataSource datasource) { - if (datasource != null - && datasource.getConnectorType() != null - && datasource.getConnectorType().equals(DataSourceType.SECURITY_LAKE)) { - return new SparkSqlSecurityLakeValidatorVisitor(); - } else { - return new SparkSqlValidatorVisitor(); - } - } - - /** - * A base class extending SqlBaseParserBaseVisitor for validating Spark Sql Queries. The class - * supports accumulating validation errors on visiting sql statement - */ - @Getter - private static class SqlBaseValidatorVisitor<T> extends SqlBaseParserBaseVisitor<T> { - private final List<String> validationErrors = new ArrayList<>(); - } - - /** A generic validator impl for Spark Sql Queries */ - private static class SparkSqlValidatorVisitor extends SqlBaseValidatorVisitor<Void> { - @Override - public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { - getValidationErrors().add("Creating user-defined functions is not allowed"); - return super.visitCreateFunction(ctx); - } - } - - /** A validator impl specific to Security Lake for Spark Sql Queries */ - private static class SparkSqlSecurityLakeValidatorVisitor extends SqlBaseValidatorVisitor<Void> { - - public SparkSqlSecurityLakeValidatorVisitor() { - // only select statement allowed. hence we add the validation error to all types of statements - // by default - // and remove the validation error only for select statement. - getValidationErrors() - .add( - "Unsupported sql statement for security lake data source. Only select queries are" - + " allowed"); - } - - @Override - public Void visitStatementDefault(SqlBaseParser.StatementDefaultContext ctx) { - getValidationErrors().clear(); - return super.visitStatementDefault(ctx); - } + return sqlBaseParser; } public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor<Void> { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java new file mode 100644 index 0000000000..ddd0a1d094 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DefaultGrammarElementValidator.java @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +public class DefaultGrammarElementValidator implements GrammarElementValidator { + @Override + public boolean isValid(GrammarElement element) { + return true; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java new file mode 100644 index 0000000000..514e2c8ad8 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/DenyListGrammarElementValidator.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import java.util.Set; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class DenyListGrammarElementValidator implements GrammarElementValidator { + private final Set<GrammarElement> denyList; + + @Override + public boolean isValid(GrammarElement element) { + return !denyList.contains(element); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java new file mode 100644 index 0000000000..da3760efd6 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java @@ -0,0 +1,436 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; + +/** + * Enum for defining and looking up SQL function type based on its name. Unknown one will be + * considered as UDF (User Defined Function) + */ +@AllArgsConstructor +public enum FunctionType { + AGGREGATE("Aggregate"), + WINDOW("Window"), + ARRAY("Array"), + MAP("Map"), + DATE_TIMESTAMP("Date and Timestamp"), + JSON("JSON"), + MATH("Math"), + STRING("String"), + CONDITIONAL("Conditional"), + BITWISE("Bitwise"), + CONVERSION("Conversion"), + PREDICATE("Predicate"), + CSV("CSV"), + MISC("Misc"), + GENERATOR("Generator"), + UDF("User Defined Function"); + + private final String name; + + private static final Map<FunctionType, Set<String>> FUNCTION_TYPE_TO_FUNCTION_NAMES_MAP = + ImmutableMap.<FunctionType, Set<String>>builder() + .put( + AGGREGATE, + Set.of( + "any", + "any_value", + "approx_count_distinct", + "approx_percentile", + "array_agg", + "avg", + "bit_and", + "bit_or", + "bit_xor", + "bitmap_construct_agg", + "bitmap_or_agg", + "bool_and", + "bool_or", + "collect_list", + "collect_set", + "corr", + "count", + "count_if", + "count_min_sketch", + "covar_pop", + "covar_samp", + "every", + "first", + "first_value", + "grouping", + "grouping_id", + "histogram_numeric", + "hll_sketch_agg", + "hll_union_agg", + "kurtosis", + "last", + "last_value", + "max", + "max_by", + "mean", + "median", + "min", + "min_by", + "mode", + "percentile", + "percentile_approx", + "regr_avgx", + "regr_avgy", + "regr_count", + "regr_intercept", + "regr_r2", + "regr_slope", + "regr_sxx", + "regr_sxy", + "regr_syy", + "skewness", + "some", + "std", + "stddev", + "stddev_pop", + "stddev_samp", + "sum", + "try_avg", + "try_sum", + "var_pop", + "var_samp", + "variance")) + .put( + WINDOW, + Set.of( + "cume_dist", + "dense_rank", + "lag", + "lead", + "nth_value", + "ntile", + "percent_rank", + "rank", + "row_number")) + .put( + ARRAY, + Set.of( + "array", + "array_append", + "array_compact", + "array_contains", + "array_distinct", + "array_except", + "array_insert", + "array_intersect", + "array_join", + "array_max", + "array_min", + "array_position", + "array_prepend", + "array_remove", + "array_repeat", + "array_union", + "arrays_overlap", + "arrays_zip", + "flatten", + "get", + "sequence", + "shuffle", + "slice", + "sort_array")) + .put( + MAP, + Set.of( + "element_at", + "map", + "map_concat", + "map_contains_key", + "map_entries", + "map_from_arrays", + "map_from_entries", + "map_keys", + "map_values", + "str_to_map", + "try_element_at")) + .put( + DATE_TIMESTAMP, + Set.of( + "add_months", + "convert_timezone", + "curdate", + "current_date", + "current_timestamp", + "current_timezone", + "date_add", + "date_diff", + "date_format", + "date_from_unix_date", + "date_part", + "date_sub", + "date_trunc", + "dateadd", + "datediff", + "datepart", + "day", + "dayofmonth", + "dayofweek", + "dayofyear", + "extract", + "from_unixtime", + "from_utc_timestamp", + "hour", + "last_day", + "localtimestamp", + "make_date", + "make_dt_interval", + "make_interval", + "make_timestamp", + "make_timestamp_ltz", + "make_timestamp_ntz", + "make_ym_interval", + "minute", + "month", + "months_between", + "next_day", + "now", + "quarter", + "second", + "session_window", + "timestamp_micros", + "timestamp_millis", + "timestamp_seconds", + "to_date", + "to_timestamp", + "to_timestamp_ltz", + "to_timestamp_ntz", + "to_unix_timestamp", + "to_utc_timestamp", + "trunc", + "try_to_timestamp", + "unix_date", + "unix_micros", + "unix_millis", + "unix_seconds", + "unix_timestamp", + "weekday", + "weekofyear", + "window", + "window_time", + "year")) + .put( + JSON, + Set.of( + "from_json", + "get_json_object", + "json_array_length", + "json_object_keys", + "json_tuple", + "schema_of_json", + "to_json")) + .put( + MATH, + Set.of( + "abs", + "acos", + "acosh", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "bin", + "bround", + "cbrt", + "ceil", + "ceiling", + "conv", + "cos", + "cosh", + "cot", + "csc", + "degrees", + "e", + "exp", + "expm1", + "factorial", + "floor", + "greatest", + "hex", + "hypot", + "least", + "ln", + "log", + "log10", + "log1p", + "log2", + "negative", + "pi", + "pmod", + "positive", + "pow", + "power", + "radians", + "rand", + "randn", + "random", + "rint", + "round", + "sec", + "shiftleft", + "sign", + "signum", + "sin", + "sinh", + "sqrt", + "tan", + "tanh", + "try_add", + "try_divide", + "try_multiply", + "try_subtract", + "unhex", + "width_bucket")) + .put( + STRING, + Set.of( + "ascii", + "base64", + "bit_length", + "btrim", + "char", + "char_length", + "character_length", + "chr", + "concat", + "concat_ws", + "contains", + "decode", + "elt", + "encode", + "endswith", + "find_in_set", + "format_number", + "format_string", + "initcap", + "instr", + "lcase", + "left", + "len", + "length", + "levenshtein", + "locate", + "lower", + "lpad", + "ltrim", + "luhn_check", + "mask", + "octet_length", + "overlay", + "position", + "printf", + "regexp_count", + "regexp_extract", + "regexp_extract_all", + "regexp_instr", + "regexp_replace", + "regexp_substr", + "repeat", + "replace", + "right", + "rpad", + "rtrim", + "sentences", + "soundex", + "space", + "split", + "split_part", + "startswith", + "substr", + "substring", + "substring_index", + "to_binary", + "to_char", + "to_number", + "to_varchar", + "translate", + "trim", + "try_to_binary", + "try_to_number", + "ucase", + "unbase64", + "upper")) + .put(CONDITIONAL, Set.of("coalesce", "if", "ifnull", "nanvl", "nullif", "nvl", "nvl2")) + .put( + BITWISE, Set.of("bit_count", "bit_get", "getbit", "shiftright", "shiftrightunsigned")) + .put( + CONVERSION, + Set.of( + "bigint", + "binary", + "boolean", + "cast", + "date", + "decimal", + "double", + "float", + "int", + "smallint", + "string", + "timestamp", + "tinyint")) + .put(PREDICATE, Set.of("isnan", "isnotnull", "isnull", "regexp", "regexp_like", "rlike")) + .put(CSV, Set.of("from_csv", "schema_of_csv", "to_csv")) + .put( + MISC, + Set.of( + "aes_decrypt", + "aes_encrypt", + "assert_true", + "bitmap_bit_position", + "bitmap_bucket_number", + "bitmap_count", + "current_catalog", + "current_database", + "current_schema", + "current_user", + "equal_null", + "hll_sketch_estimate", + "hll_union", + "input_file_block_length", + "input_file_block_start", + "input_file_name", + "java_method", + "monotonically_increasing_id", + "reflect", + "spark_partition_id", + "try_aes_decrypt", + "typeof", + "user", + "uuid", + "version")) + .put( + GENERATOR, + Set.of( + "explode", + "explode_outer", + "inline", + "inline_outer", + "posexplode", + "posexplode_outer", + "stack")) + .build(); + + private static final Map<String, FunctionType> FUNCTION_NAME_TO_FUNCTION_TYPE_MAP = + FUNCTION_TYPE_TO_FUNCTION_NAMES_MAP.entrySet().stream() + .flatMap( + entry -> entry.getValue().stream().map(value -> Map.entry(value, entry.getKey()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + public static FunctionType fromFunctionName(String functionName) { + return FUNCTION_NAME_TO_FUNCTION_TYPE_MAP.getOrDefault(functionName.toLowerCase(), UDF); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java new file mode 100644 index 0000000000..217640bada --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +enum GrammarElement { + ALTER_NAMESPACE("ALTER (DATABASE|TABLE|NAMESPACE)"), + ALTER_VIEW("ALTER VIEW"), + CREATE_NAMESPACE("CREATE (DATABASE|TABLE|NAMESPACE)"), + CREATE_FUNCTION("CREATE FUNCTION"), + CREATE_VIEW("CREATE VIEW"), + DROP_NAMESPACE("DROP (DATABASE|TABLE|NAMESPACE)"), + DROP_FUNCTION("DROP FUNCTION"), + DROP_VIEW("DROP VIEW"), + DROP_TABLE("DROP TABLE"), + REPAIR_TABLE("REPAIR TABLE"), + TRUNCATE_TABLE("TRUNCATE TABLE"), + // DML Statements + INSERT("INSERT"), + LOAD("LOAD"), + + // Data Retrieval Statements + EXPLAIN("EXPLAIN"), + WITH("WITH"), + CLUSTER_BY("CLUSTER BY"), + DISTRIBUTE_BY("DISTRIBUTE BY"), + // GROUP_BY("GROUP BY"), + // HAVING("HAVING"), + HINTS("HINTS"), + INLINE_TABLE("Inline Table(VALUES)"), + FILE("File"), + INNER_JOIN("INNER JOIN"), + CROSS_JOIN("CROSS JOIN"), + LEFT_OUTER_JOIN("LEFT OUTER JOIN"), + LEFT_SEMI_JOIN("LEFT SEMI JOIN"), + RIGHT_OUTER_JOIN("RIGHT OUTER JOIN"), + FULL_OUTER_JOIN("FULL OUTER JOIN"), + LEFT_ANTI_JOIN("LEFT ANTI JOIN"), + TABLESAMPLE("TABLESAMPLE"), + TABLE_VALUED_FUNCTION("Table-valued function"), + LATERAL_VIEW("LATERAL VIEW"), + LATERAL_SUBQUERY("LATERAL SUBQUERY"), + TRANSFORM("TRANSFORM"), + + // Auxiliary Statements + MANAGE_RESOURCE("Resource management statements"), + ANALYZE_TABLE("ANALYZE TABLE(S)"), + CACHE_TABLE("CACHE TABLE"), + CLEAR_CACHE("CLEAR CACHE"), + DESCRIBE_NAMESPACE("DESCRIBE (NAMESPACE|DATABASE|SCHEMA)"), + DESCRIBE_FUNCTION("DESCRIBE FUNCTION"), + DESCRIBE_QUERY("DESCRIBE QUERY"), + DESCRIBE_TABLE("DESCRIBE TABLE"), + REFRESH_RESOURCE("REFRESH"), + REFRESH_TABLE("REFRESH TABLE"), + REFRESH_FUNCTION("REFRESH FUNCTION"), + RESET("RESET"), + SET("SET"), + SHOW_COLUMNS("SHOW COLUMNS"), + SHOW_CREATE_TABLE("SHOW CREATE TABLE"), + SHOW_NAMESPACES("SHOW (DATABASES|SCHEMAS)"), + SHOW_FUNCTIONS("SHOW FUNCTIONS"), + SHOW_PARTITIONS("SHOW PARTITIONS"), + SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED"), + SHOW_TABLES("SHOW TABLES"), + SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES"), + SHOW_VIEWS("SHOW VIEWS"), + UNCACHE_TABLE("UNCACHE TABLE"), + + // Functions + MAP_FUNCTIONS("Map functions"), + CSV_FUNCTIONS("CSV functions"), + MISC_FUNCTIONS("Misc functions"), + + // UDF + UDF("User Defined functions"); + + String description; + + @Override + public String toString() { + return description; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java new file mode 100644 index 0000000000..cc49643772 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidator.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +/** Interface for validator to decide if each GrammarElement is valid or not. */ +public interface GrammarElementValidator { + + /** + * @return true if element is valid (accepted) + */ + boolean isValid(GrammarElement element); +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java new file mode 100644 index 0000000000..9755a1c0b6 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import java.util.Map; +import lombok.AllArgsConstructor; +import org.opensearch.sql.datasource.model.DataSourceType; + +/** Provides GrammarElementValidator based on DataSourceType. */ +@AllArgsConstructor +public class GrammarElementValidatorProvider { + + private final Map<DataSourceType, GrammarElementValidator> validatorMap; + private final GrammarElementValidator defaultValidator; + + public GrammarElementValidator getValidatorForDatasource(DataSourceType dataSourceType) { + return validatorMap.getOrDefault(dataSourceType, defaultValidator); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java new file mode 100644 index 0000000000..e7a0ce1b36 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.SET; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.GrammarElement.UDF; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set<GrammarElement> S3GLUE_DENY_LIST = + ImmutableSet.<GrammarElement>builder() + .add( + ALTER_VIEW, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_VIEW, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + DESCRIBE_FUNCTION, + REFRESH_RESOURCE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_FUNCTIONS, + SHOW_VIEWS, + MISC_FUNCTIONS, + UDF) + .build(); + + public S3GlueGrammarElementValidator() { + super(S3GLUE_DENY_LIST); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java new file mode 100644 index 0000000000..9ec0fb0109 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -0,0 +1,609 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTableColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTablePartitionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterClusterByContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterTableAlterColumnContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewQueryContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewSchemaBindingContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeTablesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CacheTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClearCacheContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClusterBySpecContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableLikeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CtesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeQueryContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DescribeRelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropNamespaceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTableColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTablePartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ExplainContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.FunctionNameContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.HintContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.HiveReplaceColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InlineTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertIntoReplaceWhereContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertIntoTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteDirContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteHiveDirContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.InsertOverwriteTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinRelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinTypeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LateralViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LoadDataContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ManageResourceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.QueryOrganizationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RecoverPartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshResourceContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RefreshTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RelationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTableColumnContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RenameTablePartitionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.RepairTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ReplaceTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ResetConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ResetQuotedConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SampleContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetConfigurationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetNamespaceLocationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetNamespacePropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetTableLocationContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SetTableSerDeContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowColumnsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowCreateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowFunctionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowNamespacesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowPartitionsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTableExtendedContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTablesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowTblPropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ShowViewsContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TableNameContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TableValuedFunctionContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsetNamespacePropertiesContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; + +/** This visitor validate grammar using GrammarElementValidator */ +@AllArgsConstructor +public class SQLQueryValidationVisitor extends SqlBaseParserBaseVisitor<Void> { + private final GrammarElementValidator grammarElementValidator; + + @Override + public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { + validateAllowed(GrammarElement.CREATE_FUNCTION); + return super.visitCreateFunction(ctx); + } + + @Override + public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetNamespaceProperties(ctx); + } + + @Override + public Void visitUnsetNamespaceProperties(UnsetNamespacePropertiesContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitUnsetNamespaceProperties(ctx); + } + + @Override + public Void visitAddTableColumns(AddTableColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAddTableColumns(ctx); + } + + @Override + public Void visitAddTablePartition(AddTablePartitionContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAddTablePartition(ctx); + } + + @Override + public Void visitRenameTableColumn(RenameTableColumnContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRenameTableColumn(ctx); + } + + @Override + public Void visitDropTableColumns(DropTableColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitDropTableColumns(ctx); + } + + @Override + public Void visitAlterTableAlterColumn(AlterTableAlterColumnContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAlterTableAlterColumn(ctx); + } + + @Override + public Void visitHiveReplaceColumns(HiveReplaceColumnsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitHiveReplaceColumns(ctx); + } + + @Override + public Void visitSetTableSerDe(SetTableSerDeContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetTableSerDe(ctx); + } + + @Override + public Void visitRenameTablePartition(RenameTablePartitionContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRenameTablePartition(ctx); + } + + @Override + public Void visitDropTablePartitions(DropTablePartitionsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitDropTablePartitions(ctx); + } + + @Override + public Void visitSetTableLocation(SetTableLocationContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetTableLocation(ctx); + } + + @Override + public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitRecoverPartitions(ctx); + } + + @Override + public Void visitAlterClusterBy(AlterClusterByContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitAlterClusterBy(ctx); + } + + @Override + public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + return super.visitSetNamespaceLocation(ctx); + } + + @Override + public Void visitAlterViewQuery(AlterViewQueryContext ctx) { + validateAllowed(GrammarElement.ALTER_VIEW); + return super.visitAlterViewQuery(ctx); + } + + @Override + public Void visitAlterViewSchemaBinding(AlterViewSchemaBindingContext ctx) { + validateAllowed(GrammarElement.ALTER_VIEW); + return super.visitAlterViewSchemaBinding(ctx); + } + + @Override + public Void visitRenameTable(RenameTableContext ctx) { + if (ctx.VIEW() != null) { + validateAllowed(GrammarElement.ALTER_VIEW); + } else { + validateAllowed(GrammarElement.ALTER_NAMESPACE); + } + + return super.visitRenameTable(ctx); + } + + @Override + public Void visitCreateNamespace(CreateNamespaceContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateNamespace(ctx); + } + + @Override + public Void visitCreateTable(CreateTableContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateTable(ctx); + } + + @Override + public Void visitCreateTableLike(CreateTableLikeContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitCreateTableLike(ctx); + } + + @Override + public Void visitReplaceTable(ReplaceTableContext ctx) { + validateAllowed(GrammarElement.CREATE_NAMESPACE); + return super.visitReplaceTable(ctx); + } + + @Override + public Void visitDropNamespace(DropNamespaceContext ctx) { + validateAllowed(GrammarElement.DROP_NAMESPACE); + return super.visitDropNamespace(ctx); + } + + @Override + public Void visitDropTable(DropTableContext ctx) { + validateAllowed(GrammarElement.DROP_NAMESPACE); + return super.visitDropTable(ctx); + } + + @Override + public Void visitCreateView(CreateViewContext ctx) { + validateAllowed(GrammarElement.CREATE_VIEW); + return super.visitCreateView(ctx); + } + + @Override + public Void visitDropView(DropViewContext ctx) { + validateAllowed(GrammarElement.DROP_VIEW); + return super.visitDropView(ctx); + } + + @Override + public Void visitDropFunction(DropFunctionContext ctx) { + validateAllowed(GrammarElement.DROP_FUNCTION); + return super.visitDropFunction(ctx); + } + + @Override + public Void visitRepairTable(RepairTableContext ctx) { + validateAllowed(GrammarElement.REPAIR_TABLE); + return super.visitRepairTable(ctx); + } + + @Override + public Void visitTruncateTable(TruncateTableContext ctx) { + validateAllowed(GrammarElement.TRUNCATE_TABLE); + return super.visitTruncateTable(ctx); + } + + @Override + public Void visitInsertOverwriteTable(InsertOverwriteTableContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteTable(ctx); + } + + @Override + public Void visitInsertIntoReplaceWhere(InsertIntoReplaceWhereContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertIntoReplaceWhere(ctx); + } + + @Override + public Void visitInsertIntoTable(InsertIntoTableContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertIntoTable(ctx); + } + + @Override + public Void visitInsertOverwriteDir(InsertOverwriteDirContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteDir(ctx); + } + + @Override + public Void visitInsertOverwriteHiveDir(InsertOverwriteHiveDirContext ctx) { + validateAllowed(GrammarElement.INSERT); + return super.visitInsertOverwriteHiveDir(ctx); + } + + @Override + public Void visitLoadData(LoadDataContext ctx) { + validateAllowed(GrammarElement.LOAD); + return super.visitLoadData(ctx); + } + + @Override + public Void visitExplain(ExplainContext ctx) { + validateAllowed(GrammarElement.EXPLAIN); + return super.visitExplain(ctx); + } + + @Override + public Void visitTableName(TableNameContext ctx) { + String reference = ctx.identifierReference().getText(); + if (isFileReference(reference)) { + validateAllowed(GrammarElement.FILE); + } + return super.visitTableName(ctx); + } + + private static final String FILE_REFERENCE_PATTERN = "^[a-zA-Z]+\\.`[^`]+`$"; + + private boolean isFileReference(String reference) { + return reference.matches(FILE_REFERENCE_PATTERN); + } + + @Override + public Void visitCtes(CtesContext ctx) { + validateAllowed(GrammarElement.WITH); + return super.visitCtes(ctx); + } + + @Override + public Void visitClusterBySpec(ClusterBySpecContext ctx) { + validateAllowed(GrammarElement.CLUSTER_BY); + return super.visitClusterBySpec(ctx); + } + + @Override + public Void visitQueryOrganization(QueryOrganizationContext ctx) { + if (ctx.CLUSTER() != null) { + validateAllowed(GrammarElement.CLUSTER_BY); + } else if (ctx.DISTRIBUTE() != null) { + validateAllowed(GrammarElement.DISTRIBUTE_BY); + } + return super.visitQueryOrganization(ctx); + } + + @Override + public Void visitHint(HintContext ctx) { + validateAllowed(GrammarElement.HINTS); + return super.visitHint(ctx); + } + + @Override + public Void visitInlineTable(InlineTableContext ctx) { + validateAllowed(GrammarElement.INLINE_TABLE); + return super.visitInlineTable(ctx); + } + + @Override + public Void visitJoinType(JoinTypeContext ctx) { + if (ctx.CROSS() != null) { + validateAllowed(GrammarElement.CROSS_JOIN); + } else if (ctx.LEFT() != null && ctx.SEMI() != null) { + validateAllowed(GrammarElement.LEFT_SEMI_JOIN); + } else if (ctx.ANTI() != null) { + validateAllowed(GrammarElement.LEFT_ANTI_JOIN); + } else if (ctx.LEFT() != null) { + validateAllowed(GrammarElement.LEFT_OUTER_JOIN); + } else if (ctx.RIGHT() != null) { + validateAllowed(GrammarElement.RIGHT_OUTER_JOIN); + } else if (ctx.FULL() != null) { + validateAllowed(GrammarElement.FULL_OUTER_JOIN); + } else { + validateAllowed(GrammarElement.INNER_JOIN); + } + return super.visitJoinType(ctx); + } + + @Override + public Void visitSample(SampleContext ctx) { + validateAllowed(GrammarElement.TABLESAMPLE); + return super.visitSample(ctx); + } + + @Override + public Void visitTableValuedFunction(TableValuedFunctionContext ctx) { + validateAllowed(GrammarElement.TABLE_VALUED_FUNCTION); + return super.visitTableValuedFunction(ctx); + } + + @Override + public Void visitLateralView(LateralViewContext ctx) { + validateAllowed(GrammarElement.LATERAL_VIEW); + return super.visitLateralView(ctx); + } + + @Override + public Void visitRelation(RelationContext ctx) { + if (ctx.LATERAL() != null) { + validateAllowed(GrammarElement.LATERAL_SUBQUERY); + } + return super.visitRelation(ctx); + } + + @Override + public Void visitJoinRelation(JoinRelationContext ctx) { + if (ctx.LATERAL() != null) { + validateAllowed(GrammarElement.LATERAL_SUBQUERY); + } + return super.visitJoinRelation(ctx); + } + + @Override + public Void visitTransformClause(TransformClauseContext ctx) { + if (ctx.TRANSFORM() != null) { + validateAllowed(GrammarElement.TRANSFORM); + } + return super.visitTransformClause(ctx); + } + + @Override + public Void visitManageResource(ManageResourceContext ctx) { + validateAllowed(GrammarElement.MANAGE_RESOURCE); + return super.visitManageResource(ctx); + } + + @Override + public Void visitAnalyze(AnalyzeContext ctx) { + validateAllowed(GrammarElement.ANALYZE_TABLE); + return super.visitAnalyze(ctx); + } + + @Override + public Void visitAnalyzeTables(AnalyzeTablesContext ctx) { + validateAllowed(GrammarElement.ANALYZE_TABLE); + return super.visitAnalyzeTables(ctx); + } + + @Override + public Void visitCacheTable(CacheTableContext ctx) { + validateAllowed(GrammarElement.CACHE_TABLE); + return super.visitCacheTable(ctx); + } + + @Override + public Void visitClearCache(ClearCacheContext ctx) { + validateAllowed(GrammarElement.CLEAR_CACHE); + return super.visitClearCache(ctx); + } + + @Override + public Void visitDescribeNamespace(DescribeNamespaceContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_NAMESPACE); + return super.visitDescribeNamespace(ctx); + } + + @Override + public Void visitDescribeFunction(DescribeFunctionContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_FUNCTION); + return super.visitDescribeFunction(ctx); + } + + @Override + public Void visitDescribeRelation(DescribeRelationContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_TABLE); + return super.visitDescribeRelation(ctx); + } + + @Override + public Void visitDescribeQuery(DescribeQueryContext ctx) { + validateAllowed(GrammarElement.DESCRIBE_QUERY); + return super.visitDescribeQuery(ctx); + } + + @Override + public Void visitRefreshResource(RefreshResourceContext ctx) { + validateAllowed(GrammarElement.REFRESH_RESOURCE); + return super.visitRefreshResource(ctx); + } + + @Override + public Void visitRefreshTable(RefreshTableContext ctx) { + validateAllowed(GrammarElement.REFRESH_TABLE); + return super.visitRefreshTable(ctx); + } + + @Override + public Void visitRefreshFunction(RefreshFunctionContext ctx) { + validateAllowed(GrammarElement.REFRESH_FUNCTION); + return super.visitRefreshFunction(ctx); + } + + @Override + public Void visitResetConfiguration(ResetConfigurationContext ctx) { + validateAllowed(GrammarElement.RESET); + return super.visitResetConfiguration(ctx); + } + + @Override + public Void visitResetQuotedConfiguration(ResetQuotedConfigurationContext ctx) { + validateAllowed(GrammarElement.RESET); + return super.visitResetQuotedConfiguration(ctx); + } + + @Override + public Void visitSetConfiguration(SetConfigurationContext ctx) { + validateAllowed(GrammarElement.SET); + return super.visitSetConfiguration(ctx); + } + + @Override + public Void visitShowColumns(ShowColumnsContext ctx) { + validateAllowed(GrammarElement.SHOW_COLUMNS); + return super.visitShowColumns(ctx); + } + + @Override + public Void visitShowCreateTable(ShowCreateTableContext ctx) { + validateAllowed(GrammarElement.SHOW_CREATE_TABLE); + return super.visitShowCreateTable(ctx); + } + + @Override + public Void visitShowNamespaces(ShowNamespacesContext ctx) { + validateAllowed(GrammarElement.SHOW_NAMESPACES); + return super.visitShowNamespaces(ctx); + } + + @Override + public Void visitShowFunctions(ShowFunctionsContext ctx) { + validateAllowed(GrammarElement.SHOW_FUNCTIONS); + return super.visitShowFunctions(ctx); + } + + @Override + public Void visitShowPartitions(ShowPartitionsContext ctx) { + validateAllowed(GrammarElement.SHOW_PARTITIONS); + return super.visitShowPartitions(ctx); + } + + @Override + public Void visitShowTableExtended(ShowTableExtendedContext ctx) { + validateAllowed(GrammarElement.SHOW_TABLE_EXTENDED); + return super.visitShowTableExtended(ctx); + } + + @Override + public Void visitShowTables(ShowTablesContext ctx) { + validateAllowed(GrammarElement.SHOW_TABLES); + return super.visitShowTables(ctx); + } + + @Override + public Void visitShowTblProperties(ShowTblPropertiesContext ctx) { + validateAllowed(GrammarElement.SHOW_TBLPROPERTIES); + return super.visitShowTblProperties(ctx); + } + + @Override + public Void visitShowViews(ShowViewsContext ctx) { + validateAllowed(GrammarElement.SHOW_VIEWS); + return super.visitShowViews(ctx); + } + + @Override + public Void visitUncacheTable(UncacheTableContext ctx) { + validateAllowed(GrammarElement.UNCACHE_TABLE); + return super.visitUncacheTable(ctx); + } + + @Override + public Void visitFunctionName(FunctionNameContext ctx) { + validateFunctionAllowed(ctx.qualifiedName().getText()); + return super.visitFunctionName(ctx); + } + + private void validateFunctionAllowed(String function) { + FunctionType type = FunctionType.fromFunctionName(function.toLowerCase()); + switch (type) { + case MAP: + validateAllowed(GrammarElement.MAP_FUNCTIONS); + break; + case CSV: + validateAllowed(GrammarElement.CSV_FUNCTIONS); + break; + case MISC: + validateAllowed(GrammarElement.MISC_FUNCTIONS); + break; + case UDF: + validateAllowed(GrammarElement.UDF); + break; + } + } + + private void validateAllowed(GrammarElement element) { + if (!grammarElementValidator.isValid(element)) { + throw new IllegalArgumentException(element + " is not allowed."); + } + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java new file mode 100644 index 0000000000..f387cbad25 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.utils.SQLQueryUtils; + +/** Validate input SQL query based on the DataSourceType. */ +@AllArgsConstructor +public class SQLQueryValidator { + private static final Logger log = LogManager.getLogger(SQLQueryValidator.class); + + private final GrammarElementValidatorProvider grammarElementValidatorProvider; + + /** + * It will look up validator associated with the DataSourceType, and throw + * IllegalArgumentException if invalid grammar element is found. + * + * @param sqlQuery The query to be validated + * @param datasourceType + */ + public void validate(String sqlQuery, DataSourceType datasourceType) { + GrammarElementValidator grammarElementValidator = + grammarElementValidatorProvider.getValidatorForDatasource(datasourceType); + SQLQueryValidationVisitor visitor = new SQLQueryValidationVisitor(grammarElementValidator); + try { + visitor.visit(SQLQueryUtils.getBaseParser(sqlQuery).singleStatement()); + } catch (IllegalArgumentException e) { + log.error("Query validation failed. DataSourceType=" + datasourceType, e); + throw e; + } + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java new file mode 100644 index 0000000000..ca8f2b5bdd --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.ANALYZE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.CACHE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.CLEAR_CACHE; +import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_QUERY; +import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; +import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.SET; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_NAMESPACES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_PARTITIONS; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLE_EXTENDED; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TBLPROPERTIES; +import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.GrammarElement.TRUNCATE_TABLE; +import static org.opensearch.sql.spark.validator.GrammarElement.UDF; +import static org.opensearch.sql.spark.validator.GrammarElement.UNCACHE_TABLE; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set<GrammarElement> SECURITY_LAKE_DENY_LIST = + ImmutableSet.<GrammarElement>builder() + .add( + ALTER_NAMESPACE, + ALTER_VIEW, + CREATE_NAMESPACE, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_NAMESPACE, + DROP_VIEW, + REPAIR_TABLE, + TRUNCATE_TABLE, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + ANALYZE_TABLE, + CACHE_TABLE, + CLEAR_CACHE, + DESCRIBE_NAMESPACE, + DESCRIBE_FUNCTION, + DESCRIBE_QUERY, + DESCRIBE_TABLE, + REFRESH_RESOURCE, + REFRESH_TABLE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_COLUMNS, + SHOW_CREATE_TABLE, + SHOW_NAMESPACES, + SHOW_FUNCTIONS, + SHOW_PARTITIONS, + SHOW_TABLE_EXTENDED, + SHOW_TABLES, + SHOW_TBLPROPERTIES, + SHOW_VIEWS, + UNCACHE_TABLE, + CSV_FUNCTIONS, + MISC_FUNCTIONS, + UDF) + .build(); + + public SecurityLakeGrammarElementValidator() { + super(SECURITY_LAKE_DENY_LIST); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index ddadeb65e2..57ad4ecf42 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -85,6 +85,10 @@ import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; /** * This tests async-query-core library end-to-end using mocked implementation of extension points. @@ -175,9 +179,18 @@ public void setUp() { emrServerlessClientFactory, metricsService, new SparkSubmitParametersBuilderProvider(collection)); + SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); asyncQueryExecutorService = new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 75c0e00337..1a38b6977f 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -42,6 +42,7 @@ import com.amazonaws.services.emrserverless.model.GetJobRunResult; import com.amazonaws.services.emrserverless.model.JobRun; import com.amazonaws.services.emrserverless.model.JobRunState; +import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -88,6 +89,10 @@ import org.opensearch.sql.spark.response.JobExecutionResponseReader; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; @ExtendWith(MockitoExtension.class) public class SparkQueryDispatcherTest { @@ -111,6 +116,13 @@ public class SparkQueryDispatcherTest { @Mock private AsyncQueryRequestContext asyncQueryRequestContext; @Mock private MetricsService metricsService; @Mock private AsyncQueryScheduler asyncQueryScheduler; + + private final SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); + private DataSourceSparkParameterComposer dataSourceSparkParameterComposer = (datasourceMetadata, sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, "basic"); @@ -159,7 +171,11 @@ void setUp() { sparkSubmitParametersBuilderProvider); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); } @Test @@ -347,19 +363,12 @@ void testDispatchWithSparkUDFQuery() { sparkQueryDispatcher.dispatch( getBaseDispatchQueryRequestBuilder(query).langType(LangType.SQL).build(), asyncQueryRequestContext)); - assertEquals( - "Query is not allowed: Creating user-defined functions is not allowed", - illegalArgumentException.getMessage()); + assertEquals("CREATE FUNCTION is not allowed.", illegalArgumentException.getMessage()); verifyNoInteractions(emrServerlessClient); verifyNoInteractions(flintIndexMetadataService); } } - @Test - void testInvalidSQLQueryDispatchToSpark() { - testDispatchBatchQuery("myselect 1"); - } - @Test void testDispatchQueryWithoutATableAndDataSourceName() { testDispatchBatchQuery("show tables"); @@ -571,7 +580,11 @@ void testDispatchAlterToManualRefreshIndexQuery() { QueryHandlerFactory queryHandlerFactory = mock(QueryHandlerFactory.class); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); String query = "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = false)"; @@ -597,7 +610,11 @@ void testDispatchDropIndexQuery() { QueryHandlerFactory queryHandlerFactory = mock(QueryHandlerFactory.class); sparkQueryDispatcher = new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator); String query = "DROP INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java index 56cab7ce7f..881ad0e56a 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.index; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.mv; import static org.opensearch.sql.spark.utils.SQLQueryUtilsTest.IndexQuery.skippingIndex; @@ -22,7 +21,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.datasource.model.DataSource; -import org.opensearch.sql.datasource.model.DataSourceType; import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName; import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType; import org.opensearch.sql.spark.dispatcher.model.IndexQueryDetails; @@ -444,106 +442,6 @@ void testRecoverIndex() { assertEquals(IndexQueryActionType.RECOVER, indexDetails.getIndexQueryActionType()); } - @Test - void testValidateSparkSqlQuery_ValidQuery() { - List<String> errors = - validateSparkSqlQueryForDataSourceType( - "DELETE FROM Customers WHERE CustomerName='Alfreds Futterkiste'", - DataSourceType.PROMETHEUS); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors"); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake() { - List<String> errors = - validateSparkSqlQueryForDataSourceType( - "SELECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceTypeNull() { - List<String> errors = - validateSparkSqlQueryForDataSourceType("SELECT * FROM users WHERE age > 18", null); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_InvalidQuery_SyntaxCheckFailureSkippedWithoutValidationError() { - List<String> errors = - validateSparkSqlQueryForDataSourceType( - "SEECT * FROM users WHERE age > 18", DataSourceType.SECURITY_LAKE); - - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - @Test - void testValidateSparkSqlQuery_nullDatasource() { - List<String> errors = - SQLQueryUtils.validateSparkSqlQuery(null, "SELECT * FROM users WHERE age > 18"); - assertTrue(errors.isEmpty(), "Valid query should not produce any errors "); - } - - private List<String> validateSparkSqlQueryForDataSourceType( - String query, DataSourceType dataSourceType) { - when(this.dataSource.getConnectorType()).thenReturn(dataSourceType); - - return SQLQueryUtils.validateSparkSqlQuery(this.dataSource, query); - } - - @Test - void testValidateSparkSqlQuery_SelectQuery_DataSourceSecurityLake_ValidationFails() { - List<String> errors = - validateSparkSqlQueryForDataSourceType( - "REFRESH INDEX cv1 ON mys3.default.http_logs", DataSourceType.SECURITY_LAKE); - - assertFalse( - errors.isEmpty(), - "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" - + " queries. Given query was REFRESH sql query"); - assertEquals( - errors.get(0), - "Unsupported sql statement for security lake data source. Only select queries are allowed"); - } - - @Test - void - testValidateSparkSqlQuery_NonSelectStatementContainingSelectClause_DataSourceSecurityLake_ValidationFails() { - String query = - "CREATE TABLE AccountSummaryOrWhatever AS " - + "select taxid, address1, count(address1) from dbo.t " - + "group by taxid, address1;"; - - List<String> errors = - validateSparkSqlQueryForDataSourceType(query, DataSourceType.SECURITY_LAKE); - - assertFalse( - errors.isEmpty(), - "Invalid query as Security Lake datasource supports only flint queries and SELECT sql" - + " queries. Given query was REFRESH sql query"); - assertEquals( - errors.get(0), - "Unsupported sql statement for security lake data source. Only select queries are allowed"); - } - - @Test - void testValidateSparkSqlQuery_InvalidQuery() { - when(dataSource.getConnectorType()).thenReturn(DataSourceType.PROMETHEUS); - String invalidQuery = "CREATE FUNCTION myUDF AS 'com.example.UDF'"; - - List<String> errors = SQLQueryUtils.validateSparkSqlQuery(dataSource, invalidQuery); - - assertFalse(errors.isEmpty(), "Invalid query should produce errors"); - assertEquals(1, errors.size(), "Should have one error"); - assertEquals( - "Creating user-defined functions is not allowed", - errors.get(0), - "Error message should match"); - } - @Getter protected static class IndexQuery { private String query; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java new file mode 100644 index 0000000000..a5f868421c --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class FunctionTypeTest { + @Test + public void test() { + assertEquals(FunctionType.AGGREGATE, FunctionType.fromFunctionName("any")); + assertEquals(FunctionType.AGGREGATE, FunctionType.fromFunctionName("variance")); + assertEquals(FunctionType.WINDOW, FunctionType.fromFunctionName("cume_dist")); + assertEquals(FunctionType.WINDOW, FunctionType.fromFunctionName("row_number")); + assertEquals(FunctionType.ARRAY, FunctionType.fromFunctionName("array")); + assertEquals(FunctionType.ARRAY, FunctionType.fromFunctionName("sort_array")); + assertEquals(FunctionType.MAP, FunctionType.fromFunctionName("element_at")); + assertEquals(FunctionType.MAP, FunctionType.fromFunctionName("try_element_at")); + assertEquals(FunctionType.DATE_TIMESTAMP, FunctionType.fromFunctionName("add_months")); + assertEquals(FunctionType.DATE_TIMESTAMP, FunctionType.fromFunctionName("year")); + assertEquals(FunctionType.JSON, FunctionType.fromFunctionName("from_json")); + assertEquals(FunctionType.JSON, FunctionType.fromFunctionName("to_json")); + assertEquals(FunctionType.MATH, FunctionType.fromFunctionName("abs")); + assertEquals(FunctionType.MATH, FunctionType.fromFunctionName("width_bucket")); + assertEquals(FunctionType.STRING, FunctionType.fromFunctionName("ascii")); + assertEquals(FunctionType.STRING, FunctionType.fromFunctionName("upper")); + assertEquals(FunctionType.CONDITIONAL, FunctionType.fromFunctionName("coalesce")); + assertEquals(FunctionType.CONDITIONAL, FunctionType.fromFunctionName("nvl2")); + assertEquals(FunctionType.BITWISE, FunctionType.fromFunctionName("bit_count")); + assertEquals(FunctionType.BITWISE, FunctionType.fromFunctionName("shiftrightunsigned")); + assertEquals(FunctionType.CONVERSION, FunctionType.fromFunctionName("bigint")); + assertEquals(FunctionType.CONVERSION, FunctionType.fromFunctionName("tinyint")); + assertEquals(FunctionType.PREDICATE, FunctionType.fromFunctionName("isnan")); + assertEquals(FunctionType.PREDICATE, FunctionType.fromFunctionName("rlike")); + assertEquals(FunctionType.CSV, FunctionType.fromFunctionName("from_csv")); + assertEquals(FunctionType.CSV, FunctionType.fromFunctionName("to_csv")); + assertEquals(FunctionType.MISC, FunctionType.fromFunctionName("aes_decrypt")); + assertEquals(FunctionType.MISC, FunctionType.fromFunctionName("version")); + assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("explode")); + assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("stack")); + assertEquals(FunctionType.UDF, FunctionType.fromFunctionName("unknown")); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java new file mode 100644 index 0000000000..7d4b255356 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.datasource.model.DataSourceType; + +class GrammarElementValidatorProviderTest { + S3GlueGrammarElementValidator s3GlueGrammarElementValidator = new S3GlueGrammarElementValidator(); + SecurityLakeGrammarElementValidator securityLakeGrammarElementValidator = + new SecurityLakeGrammarElementValidator(); + DefaultGrammarElementValidator defaultGrammarElementValidator = + new DefaultGrammarElementValidator(); + GrammarElementValidatorProvider grammarElementValidatorProvider = + new GrammarElementValidatorProvider( + ImmutableMap.of( + DataSourceType.S3GLUE, s3GlueGrammarElementValidator, + DataSourceType.SECURITY_LAKE, securityLakeGrammarElementValidator), + defaultGrammarElementValidator); + + @Test + public void test() { + assertEquals( + s3GlueGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.S3GLUE)); + assertEquals( + securityLakeGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.SECURITY_LAKE)); + assertEquals( + defaultGrammarElementValidator, + grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.PROMETHEUS)); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java new file mode 100644 index 0000000000..6726b56994 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -0,0 +1,600 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.antlr.v4.runtime.CommonTokenStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SingleStatementContext; + +@ExtendWith(MockitoExtension.class) +class SQLQueryValidatorTest { + @Mock GrammarElementValidatorProvider mockedProvider; + + @InjectMocks SQLQueryValidator sqlQueryValidator; + + private enum TestElement { + // DDL Statements + ALTER_DATABASE( + "ALTER DATABASE inventory SET DBPROPERTIES ('Edit-date' = '01/01/2001');", + "ALTER DATABASE dbx.tab1 UNSET PROPERTIES ('winner');", + "ALTER DATABASE dbx.tab1 SET LOCATION '/path/to/part/ways';"), + ALTER_TABLE( + "ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');", + "ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');", + "ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);", + "ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);", + "ALTER TABLE StudentInfo RENAME COLUMN name TO FirstName;", + "ALTER TABLE StudentInfo RENAME TO newName;", + "ALTER TABLE StudentInfo DROP columns (LastName, DOB);", + "ALTER TABLE StudentInfo ALTER COLUMN FirstName COMMENT \"new comment\";", + "ALTER TABLE StudentInfo REPLACE COLUMNS (name string, ID int COMMENT 'new comment');", + "ALTER TABLE test_tab SET SERDE 'org.apache.LazyBinaryColumnarSerDe';", + "ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);", + "ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways';", + "ALTER TABLE dbx.tab1 RECOVER PARTITIONS;", + "ALTER TABLE dbx.tab1 CLUSTER BY NONE;", + "ALTER TABLE dbx.tab1 SET LOCATION '/path/to/part/ways';"), + ALTER_VIEW( + "ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;", + "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;", + "ALTER VIEW tempdb1.v2 WITH SCHEMA BINDING"), + CREATE_DATABASE("CREATE DATABASE IF NOT EXISTS customer_db;\n"), + CREATE_FUNCTION("CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar';"), + CREATE_TABLE( + "CREATE TABLE Student_Dupli like Student;", + "CREATE TABLE student (id INT, name STRING, age INT) USING CSV;", + "CREATE TABLE student_copy USING CSV AS SELECT * FROM student;", + "CREATE TABLE student (id INT, name STRING, age INT);", + "REPLACE TABLE student (id INT, name STRING, age INT) USING CSV;"), + CREATE_VIEW( + "CREATE OR REPLACE VIEW experienced_employee" + + " (ID COMMENT 'Unique identification number', Name)" + + " COMMENT 'View for experienced employees'" + + " AS SELECT id, name FROM all_employee" + + " WHERE working_years > 5;"), + DROP_DATABASE("DROP DATABASE inventory_db CASCADE;"), + DROP_FUNCTION("DROP FUNCTION test_avg;"), + DROP_TABLE("DROP TABLE employeetable;"), + DROP_VIEW("DROP VIEW employeeView;"), + REPAIR_TABLE("REPAIR TABLE t1;"), + TRUNCATE_TABLE("TRUNCATE TABLE Student partition(age=10);"), + + // DML Statements + INSERT_TABLE( + "INSERT INTO target_table SELECT * FROM source_table;", + "INSERT INTO persons REPLACE WHERE ssn = 123456789 SELECT * FROM persons2;", + "INSERT OVERWRITE students VALUES ('Ashua Hill', '456 Erica Ct, Cupertino', 111111);"), + INSERT_OVERWRITE_DIRECTORY( + "INSERT OVERWRITE DIRECTORY '/path/to/output' SELECT * FROM source_table;", + "INSERT OVERWRITE DIRECTORY USING myTable SELECT * FROM source_table;", + "INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination' STORED AS orc SELECT * FROM" + + " test_table;"), + LOAD("LOAD DATA INPATH '/path/to/data' INTO TABLE target_table;"), + + // Data Retrieval Statements + SELECT("SELECT 1;"), + EXPLAIN("EXPLAIN SELECT * FROM my_table;"), + COMMON_TABLE_EXPRESSION( + "WITH cte AS (SELECT * FROM my_table WHERE age > 30) SELECT * FROM cte;"), + CLUSTER_BY_CLAUSE( + "SELECT * FROM my_table CLUSTER BY age;", "ALTER TABLE testTable CLUSTER BY (age);"), + DISTRIBUTE_BY_CLAUSE("SELECT * FROM my_table DISTRIBUTE BY name;"), + GROUP_BY_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name;"), + HAVING_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name HAVING count(*) > 1;"), + HINTS("SELECT /*+ BROADCAST(my_table) */ * FROM my_table;"), + INLINE_TABLE("SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS inline_table(id, value);"), + FILE("SELECT * FROM text.`/path/to/file.txt`;"), + INNER_JOIN("SELECT t1.name, t2.age FROM table1 t1 INNER JOIN table2 t2 ON t1.id = t2.id;"), + CROSS_JOIN("SELECT t1.name, t2.age FROM table1 t1 CROSS JOIN table2 t2;"), + LEFT_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 LEFT OUTER JOIN table2 t2 ON t1.id = t2.id;"), + LEFT_SEMI_JOIN("SELECT t1.name FROM table1 t1 LEFT SEMI JOIN table2 t2 ON t1.id = t2.id;"), + RIGHT_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 RIGHT OUTER JOIN table2 t2 ON t1.id = t2.id;"), + FULL_OUTER_JOIN( + "SELECT t1.name, t2.age FROM table1 t1 FULL OUTER JOIN table2 t2 ON t1.id = t2.id;"), + LEFT_ANTI_JOIN("SELECT t1.name FROM table1 t1 LEFT ANTI JOIN table2 t2 ON t1.id = t2.id;"), + LIKE_PREDICATE("SELECT * FROM my_table WHERE name LIKE 'A%';"), + LIMIT_CLAUSE("SELECT * FROM my_table LIMIT 10;"), + OFFSET_CLAUSE("SELECT * FROM my_table OFFSET 5;"), + ORDER_BY_CLAUSE("SELECT * FROM my_table ORDER BY age DESC;"), + SET_OPERATORS("SELECT * FROM table1 UNION SELECT * FROM table2;"), + SORT_BY_CLAUSE("SELECT * FROM my_table SORT BY age DESC;"), + TABLESAMPLE("SELECT * FROM my_table TABLESAMPLE(10 PERCENT);"), + // TABLE_VALUED_FUNCTION("SELECT explode(array(10, 20));"), TODO: Need to handle this case + TABLE_VALUED_FUNCTION("SELECT * FROM explode(array(10, 20));"), + WHERE_CLAUSE("SELECT * FROM my_table WHERE age > 30;"), + AGGREGATE_FUNCTION("SELECT count(*) FROM my_table;"), + WINDOW_FUNCTION("SELECT name, age, rank() OVER (ORDER BY age DESC) FROM my_table;"), + CASE_CLAUSE("SELECT name, CASE WHEN age > 30 THEN 'Adult' ELSE 'Young' END FROM my_table;"), + PIVOT_CLAUSE( + "SELECT * FROM (SELECT name, age, gender FROM my_table) PIVOT (COUNT(*) FOR gender IN ('M'," + + " 'F'));"), + UNPIVOT_CLAUSE( + "SELECT name, value, category FROM (SELECT name, 'M' AS gender, age AS male_age, 0 AS" + + " female_age FROM my_table) UNPIVOT (value FOR category IN (male_age, female_age));"), + LATERAL_VIEW_CLAUSE( + "SELECT name, age, exploded_value FROM my_table LATERAL VIEW OUTER EXPLODE(split(comments," + + " ',')) exploded_table AS exploded_value;"), + LATERAL_SUBQUERY( + "SELECT * FROM t1, LATERAL (SELECT * FROM t2 WHERE t1.c1 = t2.c1);", + "SELECT * FROM t1 JOIN LATERAL (SELECT * FROM t2 WHERE t1.c1 = t2.c1);"), + TRANSFORM_CLAUSE( + "SELECT transform(zip_code, name, age) USING 'cat' AS (a, b, c) FROM my_table;"), + + // Auxiliary Statements + ADD_FILE("ADD FILE /tmp/test.txt;"), + ADD_JAR("ADD JAR /path/to/my.jar;"), + ANALYZE_TABLE( + "ANALYZE TABLE my_table COMPUTE STATISTICS;", + "ANALYZE TABLES IN school_db COMPUTE STATISTICS NOSCAN;"), + CACHE_TABLE("CACHE TABLE my_table;"), + CLEAR_CACHE("CLEAR CACHE;"), + DESCRIBE_DATABASE("DESCRIBE DATABASE my_db;"), + DESCRIBE_FUNCTION("DESCRIBE FUNCTION my_function;"), + DESCRIBE_QUERY("DESCRIBE QUERY SELECT * FROM my_table;"), + DESCRIBE_TABLE("DESCRIBE TABLE my_table;"), + LIST_FILE("LIST FILE '/path/to/files';"), + LIST_JAR("LIST JAR;"), + REFRESH("REFRESH;"), + REFRESH_TABLE("REFRESH TABLE my_table;"), + REFRESH_FUNCTION("REFRESH FUNCTION my_function;"), + RESET("RESET;", "RESET spark.abc;", "RESET `key`;"), + SET( + "SET spark.sql.shuffle.partitions=200;", + "SET -v;", + "SET;", + "SET spark.sql.variable.substitute;"), + SHOW_COLUMNS("SHOW COLUMNS FROM my_table;"), + SHOW_CREATE_TABLE("SHOW CREATE TABLE my_table;"), + SHOW_DATABASES("SHOW DATABASES;"), + SHOW_FUNCTIONS("SHOW FUNCTIONS;"), + SHOW_PARTITIONS("SHOW PARTITIONS my_table;"), + SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED LIKE 'my_table';"), + SHOW_TABLES("SHOW TABLES;"), + SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES my_table;"), + SHOW_VIEWS("SHOW VIEWS;"), + UNCACHE_TABLE("UNCACHE TABLE my_table;"), + + // Functions + ARRAY_FUNCTIONS("SELECT array_contains(array(1, 2, 3), 2);"), + MAP_FUNCTIONS("SELECT map_keys(map('a', 1, 'b', 2));"), + DATE_AND_TIMESTAMP_FUNCTIONS("SELECT date_format(current_date(), 'yyyy-MM-dd');"), + JSON_FUNCTIONS("SELECT json_tuple('{\"a\":1, \"b\":2}', 'a', 'b');"), + MATHEMATICAL_FUNCTIONS("SELECT round(3.1415, 2);"), + STRING_FUNCTIONS("SELECT ascii('Hello');"), + BITWISE_FUNCTIONS("SELECT bit_count(42);"), + CONVERSION_FUNCTIONS("SELECT cast('2023-04-01' as date);"), + CONDITIONAL_FUNCTIONS("SELECT if(1 > 0, 'true', 'false');"), + PREDICATE_FUNCTIONS("SELECT isnotnull(1);"), + CSV_FUNCTIONS("SELECT from_csv(array('a', 'b', 'c'), ',');"), + MISC_FUNCTIONS("SELECT current_user();"), + + // Aggregate-like Functions + AGGREGATE_FUNCTIONS("SELECT count(*), max(age), min(age) FROM my_table;"), + WINDOW_FUNCTIONS("SELECT name, age, rank() OVER (ORDER BY age DESC) FROM my_table;"), + + // Generator Functions + GENERATOR_FUNCTIONS("SELECT explode(array(1, 2, 3));"), + + // UDFs (User-Defined Functions) + SCALAR_USER_DEFINED_FUNCTIONS("SELECT my_udf(name) FROM my_table;"), + USER_DEFINED_AGGREGATE_FUNCTIONS("SELECT my_udaf(age) FROM my_table GROUP BY name;"), + INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS("SELECT my_hive_udf(name) FROM my_table;"); + + @Getter private final String[] queries; + + TestElement(String... queries) { + this.queries = queries; + } + } + + @Test + void testAllowAllByDefault() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new DefaultGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + Arrays.stream(TestElement.values()).forEach(v::ok); + } + + @Test + void testDenyAllValidator() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> false); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + // The elements which doesn't have validation will be accepted. + // That's why there are some ok case + + // DDL Statements + v.ng(TestElement.ALTER_DATABASE); + v.ng(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ng(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ng(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ng(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ng(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ng(TestElement.REPAIR_TABLE); + v.ng(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ng(TestElement.EXPLAIN); + v.ng(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ng(TestElement.INNER_JOIN); + v.ng(TestElement.CROSS_JOIN); + v.ng(TestElement.LEFT_OUTER_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ng(TestElement.LATERAL_VIEW_CLAUSE); + v.ng(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ng(TestElement.ANALYZE_TABLE); + v.ng(TestElement.CACHE_TABLE); + v.ng(TestElement.CLEAR_CACHE); + v.ng(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ng(TestElement.DESCRIBE_QUERY); + v.ng(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ng(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ng(TestElement.SHOW_COLUMNS); + v.ng(TestElement.SHOW_CREATE_TABLE); + v.ng(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ng(TestElement.SHOW_PARTITIONS); + v.ng(TestElement.SHOW_TABLE_EXTENDED); + v.ng(TestElement.SHOW_TABLES); + v.ng(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ng(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ng(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ng(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @Test + void testS3glueQueries() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new S3GlueGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.S3GLUE); + + // DDL Statements + v.ok(TestElement.ALTER_DATABASE); + v.ok(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ok(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ok(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ok(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ok(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ok(TestElement.REPAIR_TABLE); + v.ok(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ok(TestElement.SELECT); + v.ok(TestElement.EXPLAIN); + v.ok(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ok(TestElement.INNER_JOIN); + v.ok(TestElement.CROSS_JOIN); + v.ok(TestElement.LEFT_OUTER_JOIN); + v.ok(TestElement.LEFT_SEMI_JOIN); + v.ok(TestElement.RIGHT_OUTER_JOIN); + v.ok(TestElement.FULL_OUTER_JOIN); + v.ok(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ok(TestElement.LATERAL_VIEW_CLAUSE); + v.ok(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ok(TestElement.ANALYZE_TABLE); + v.ok(TestElement.CACHE_TABLE); + v.ok(TestElement.CLEAR_CACHE); + v.ok(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ok(TestElement.DESCRIBE_QUERY); + v.ok(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ok(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ok(TestElement.SHOW_COLUMNS); + v.ok(TestElement.SHOW_CREATE_TABLE); + v.ok(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ok(TestElement.SHOW_PARTITIONS); + v.ok(TestElement.SHOW_TABLE_EXTENDED); + v.ok(TestElement.SHOW_TABLES); + v.ok(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ok(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ok(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ok(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @Test + void testSecurityLakeQueries() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new SecurityLakeGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SECURITY_LAKE); + + // DDL Statements + v.ng(TestElement.ALTER_DATABASE); + v.ng(TestElement.ALTER_TABLE); + v.ng(TestElement.ALTER_VIEW); + v.ng(TestElement.CREATE_DATABASE); + v.ng(TestElement.CREATE_FUNCTION); + v.ng(TestElement.CREATE_TABLE); + v.ng(TestElement.CREATE_VIEW); + v.ng(TestElement.DROP_DATABASE); + v.ng(TestElement.DROP_FUNCTION); + v.ng(TestElement.DROP_TABLE); + v.ng(TestElement.DROP_VIEW); + v.ng(TestElement.REPAIR_TABLE); + v.ng(TestElement.TRUNCATE_TABLE); + + // DML Statements + v.ng(TestElement.INSERT_TABLE); + v.ng(TestElement.INSERT_OVERWRITE_DIRECTORY); + v.ng(TestElement.LOAD); + + // Data Retrieval + v.ok(TestElement.SELECT); + v.ok(TestElement.EXPLAIN); + v.ok(TestElement.COMMON_TABLE_EXPRESSION); + v.ng(TestElement.CLUSTER_BY_CLAUSE); + v.ng(TestElement.DISTRIBUTE_BY_CLAUSE); + v.ok(TestElement.GROUP_BY_CLAUSE); + v.ok(TestElement.HAVING_CLAUSE); + v.ng(TestElement.HINTS); + v.ng(TestElement.INLINE_TABLE); + v.ng(TestElement.FILE); + v.ok(TestElement.INNER_JOIN); + v.ok(TestElement.CROSS_JOIN); + v.ok(TestElement.LEFT_OUTER_JOIN); + v.ok(TestElement.LEFT_SEMI_JOIN); + v.ok(TestElement.RIGHT_OUTER_JOIN); + v.ok(TestElement.FULL_OUTER_JOIN); + v.ok(TestElement.LEFT_ANTI_JOIN); + v.ok(TestElement.LIKE_PREDICATE); + v.ok(TestElement.LIMIT_CLAUSE); + v.ok(TestElement.OFFSET_CLAUSE); + v.ok(TestElement.ORDER_BY_CLAUSE); + v.ok(TestElement.SET_OPERATORS); + v.ok(TestElement.SORT_BY_CLAUSE); + v.ng(TestElement.TABLESAMPLE); + v.ng(TestElement.TABLE_VALUED_FUNCTION); + v.ok(TestElement.WHERE_CLAUSE); + v.ok(TestElement.AGGREGATE_FUNCTION); + v.ok(TestElement.WINDOW_FUNCTION); + v.ok(TestElement.CASE_CLAUSE); + v.ok(TestElement.PIVOT_CLAUSE); + v.ok(TestElement.UNPIVOT_CLAUSE); + v.ok(TestElement.LATERAL_VIEW_CLAUSE); + v.ok(TestElement.LATERAL_SUBQUERY); + v.ng(TestElement.TRANSFORM_CLAUSE); + + // Auxiliary Statements + v.ng(TestElement.ADD_FILE); + v.ng(TestElement.ADD_JAR); + v.ng(TestElement.ANALYZE_TABLE); + v.ng(TestElement.CACHE_TABLE); + v.ng(TestElement.CLEAR_CACHE); + v.ng(TestElement.DESCRIBE_DATABASE); + v.ng(TestElement.DESCRIBE_FUNCTION); + v.ng(TestElement.DESCRIBE_QUERY); + v.ng(TestElement.DESCRIBE_TABLE); + v.ng(TestElement.LIST_FILE); + v.ng(TestElement.LIST_JAR); + v.ng(TestElement.REFRESH); + v.ng(TestElement.REFRESH_TABLE); + v.ng(TestElement.REFRESH_FUNCTION); + v.ng(TestElement.RESET); + v.ng(TestElement.SET); + v.ng(TestElement.SHOW_COLUMNS); + v.ng(TestElement.SHOW_CREATE_TABLE); + v.ng(TestElement.SHOW_DATABASES); + v.ng(TestElement.SHOW_FUNCTIONS); + v.ng(TestElement.SHOW_PARTITIONS); + v.ng(TestElement.SHOW_TABLE_EXTENDED); + v.ng(TestElement.SHOW_TABLES); + v.ng(TestElement.SHOW_TBLPROPERTIES); + v.ng(TestElement.SHOW_VIEWS); + v.ng(TestElement.UNCACHE_TABLE); + + // Functions + v.ok(TestElement.ARRAY_FUNCTIONS); + v.ok(TestElement.MAP_FUNCTIONS); + v.ok(TestElement.DATE_AND_TIMESTAMP_FUNCTIONS); + v.ok(TestElement.JSON_FUNCTIONS); + v.ok(TestElement.MATHEMATICAL_FUNCTIONS); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.BITWISE_FUNCTIONS); + v.ok(TestElement.CONVERSION_FUNCTIONS); + v.ok(TestElement.CONDITIONAL_FUNCTIONS); + v.ok(TestElement.PREDICATE_FUNCTIONS); + v.ng(TestElement.CSV_FUNCTIONS); + v.ng(TestElement.MISC_FUNCTIONS); + + // Aggregate-like Functions + v.ok(TestElement.AGGREGATE_FUNCTIONS); + v.ok(TestElement.WINDOW_FUNCTIONS); + + // Generator Functions + v.ok(TestElement.GENERATOR_FUNCTIONS); + + // UDFs + v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); + v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS); + v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); + } + + @AllArgsConstructor + private static class VerifyValidator { + private final SQLQueryValidator validator; + private final DataSourceType dataSourceType; + + public void ok(TestElement query) { + runValidate(query.getQueries()); + } + + public void ng(TestElement query) { + assertThrows( + IllegalArgumentException.class, + () -> runValidate(query.getQueries()), + "The query should throw: query=`" + query.toString() + "`"); + } + + void runValidate(String[] queries) { + Arrays.stream(queries).forEach(query -> validator.validate(query, dataSourceType)); + } + + void runValidate(String query) { + validator.validate(query, dataSourceType); + } + + SingleStatementContext getParser(String query) { + SqlBaseParser sqlBaseParser = + new SqlBaseParser( + new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(query)))); + return sqlBaseParser.singleStatement(); + } + } +} diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index c6f6ffcd81..db070182a3 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.spark.execution.statestore.StateStore.ALL_DATASOURCE; +import com.google.common.collect.ImmutableMap; import lombok.RequiredArgsConstructor; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; @@ -64,6 +65,11 @@ import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; +import org.opensearch.sql.spark.validator.SecurityLakeGrammarElementValidator; @RequiredArgsConstructor public class AsyncExecutorServiceModule extends AbstractModule { @@ -101,9 +107,10 @@ public SparkQueryDispatcher sparkQueryDispatcher( DataSourceService dataSourceService, SessionManager sessionManager, QueryHandlerFactory queryHandlerFactory, - QueryIdProvider queryIdProvider) { + QueryIdProvider queryIdProvider, + SQLQueryValidator sqlQueryValidator) { return new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider); + dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider, sqlQueryValidator); } @Provides @@ -174,6 +181,19 @@ public SparkSubmitParametersBuilderProvider sparkSubmitParametersBuilderProvider return new SparkSubmitParametersBuilderProvider(collection); } + @Provides + public SQLQueryValidator sqlQueryValidator() { + GrammarElementValidatorProvider validatorProvider = + new GrammarElementValidatorProvider( + ImmutableMap.of( + DataSourceType.S3GLUE, + new S3GlueGrammarElementValidator(), + DataSourceType.SECURITY_LAKE, + new SecurityLakeGrammarElementValidator()), + new DefaultGrammarElementValidator()); + return new SQLQueryValidator(validatorProvider); + } + @Provides public IndexDMLResultStorageService indexDMLResultStorageService( DataSourceService dataSourceService, StateStore stateStore) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java index db0adfc156..175f9ac914 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java @@ -312,7 +312,7 @@ public void withSessionCreateAsyncQueryFailed() { // 1. create async query. CreateAsyncQueryResponse response = asyncQueryExecutorService.createAsyncQuery( - new CreateAsyncQueryRequest("myselect 1", MYS3_DATASOURCE, LangType.SQL, null), + new CreateAsyncQueryRequest("select 1", MYS3_DATASOURCE, LangType.SQL, null), asyncQueryRequestContext); assertNotNull(response.getSessionId()); Optional<StatementModel> statementModel = diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index 9b897d36b4..72ed17f5aa 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -102,6 +102,10 @@ import org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader; import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; +import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; +import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; +import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.SQLQueryValidator; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.test.OpenSearchIntegTestCase; @@ -308,6 +312,11 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( emrServerlessClientFactory, new OpenSearchMetricsService(), sparkSubmitParametersBuilderProvider); + SQLQueryValidator sqlQueryValidator = + new SQLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( this.dataSourceService, @@ -318,7 +327,8 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( sessionConfigSupplier, sessionIdProvider), queryHandlerFactory, - new DatasourceEmbeddedQueryIdProvider()); + new DatasourceEmbeddedQueryIdProvider(), + sqlQueryValidator); return new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, sparkQueryDispatcher, From ce17d0a385d2d231f405fb8fcd8c27cf046cf2ac Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Mon, 23 Sep 2024 16:15:01 -0700 Subject: [PATCH 51/96] Add missing refactoring of Scroll to PIT API calls for Joins and Pagination query (#2981) * Adding PIT for pagination queries in new SQL engine code paths Signed-off-by: Manasvini B S <manasvis@amazon.com> * Fix legacy code using scroll API instead of PIT for batch physical operator Signed-off-by: Manasvini B S <manasvis@amazon.com> * Fix local debugger issue Signed-off-by: Manasvini B S <manasvis@amazon.com> * Refactor integ-tests data for PIT and fix unit tests Signed-off-by: Manasvini B S <manasvis@amazon.com> * Address feedback comments Signed-off-by: Manasvini B S <manasvis@amazon.com> * Adding test coverage Signed-off-by: Manasvini B S <manasvis@amazon.com> --------- Signed-off-by: Manasvini B S <manasvis@amazon.com> --- .../org/opensearch/sql/legacy/CursorIT.java | 2 +- .../org/opensearch/sql/ppl/StandaloneIT.java | 7 +- .../sql/sql/PaginationFilterIT.java | 35 +- .../sql/sql/StandalonePaginationIT.java | 1 + .../ppl/explain_filter_agg_push.json | 2 +- .../ppl/explain_filter_push.json | 2 +- .../ppl/explain_limit_push.json | 2 +- .../expectedOutput/ppl/explain_output.json | 2 +- .../expectedOutput/ppl/explain_sort_push.json | 2 +- .../executor/format/SelectResultSet.java | 2 +- .../query/planner/logical/node/TableScan.java | 7 + .../query/planner/physical/node/Paginate.java | 144 ++++++++ .../node/{scroll => }/SearchHitRow.java | 6 +- .../node/pointInTime/PointInTime.java | 73 ++++ .../planner/physical/node/scroll/Scroll.java | 157 +-------- .../node/scroll/SearchHitRowTest.java | 1 + .../unittest/planner/QueryPlannerTest.java | 2 + .../opensearch/client/OpenSearchClient.java | 17 + .../client/OpenSearchNodeClient.java | 59 +++- .../client/OpenSearchRestClient.java | 56 ++- .../request/OpenSearchQueryRequest.java | 170 ++++++++- .../request/OpenSearchRequestBuilder.java | 71 +++- .../opensearch/storage/OpenSearchIndex.java | 4 +- .../storage/scan/OpenSearchIndexScan.java | 16 +- .../client/OpenSearchNodeClientTest.java | 69 +++- .../client/OpenSearchRestClientTest.java | 62 ++++ .../OpenSearchExecutionEngineTest.java | 6 +- .../OpenSearchExecutionProtectorTest.java | 11 +- .../request/OpenSearchQueryRequestTest.java | 323 ++++++++++++++++- .../request/OpenSearchRequestBuilderTest.java | 324 ++++++++++++++++-- .../storage/OpenSearchIndexTest.java | 18 +- .../OpenSearchIndexScanPaginationTest.java | 11 +- .../storage/scan/OpenSearchIndexScanTest.java | 108 ++++-- 33 files changed, 1470 insertions(+), 302 deletions(-) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java rename legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/{scroll => }/SearchHitRow.java (97%) create mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index abd2bbbcc2..d0c2f19f42 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -182,7 +182,7 @@ public void validTotalResultWithAndWithoutPaginationOrderBy() throws IOException String selectQuery = StringUtils.format( "SELECT firstname, state FROM %s ORDER BY balance DESC ", TEST_INDEX_ACCOUNT); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 26, false); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 25, false); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index f81e1b6615..66f85b0754 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -149,8 +149,11 @@ public void onFailure(Exception e) { private Settings defaultSettings() { return new Settings() { - private final Map<Key, Integer> defaultSettings = - new ImmutableMap.Builder<Key, Integer>().put(Key.QUERY_SIZE_LIMIT, 200).build(); + private final Map<Key, Object> defaultSettings = + new ImmutableMap.Builder<Key, Object>() + .put(Key.QUERY_SIZE_LIMIT, 200) + .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .build(); @Override public <T> T getSettingValue(Key key) { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java index 038596cf57..9a945ec86f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java @@ -34,25 +34,30 @@ public class PaginationFilterIT extends SQLIntegTestCase { */ private static final Map<String, Integer> STATEMENT_TO_NUM_OF_PAGES = Map.of( - "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT, 1000, + "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT, + 1000, "SELECT * FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " WHERE match(address, 'street')", - 385, + 385, "SELECT * FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(city, 'Ola')", - 1, + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(city, 'Ola')", + 1, "SELECT firstname, lastname, highlight(address) FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(state, 'OH')", - 5, + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(state, 'OH')", + 5, "SELECT firstname, lastname, highlight('*') FROM " - + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE match(address, 'street') AND match(state, 'OH')", - 5, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true", 60, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10", 1, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15", 1, - "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK, 7); + + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE match(address, 'street') AND match(state, 'OH')", + 5, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE true", + 60, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id=10", + 1, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BEER + " WHERE Id + 5=15", + 1, + "SELECT * FROM " + TestsConstants.TEST_INDEX_BANK, + 7); private final String sqlStatement; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index e884734c96..698e185abb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -166,6 +166,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder<Key, Object>() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) + .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) .build(); @Override diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json index 568b397f07..8035822357 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json index 0e7087aa1f..3e92a17b97 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json index 51a627ea4d..0a0b58f17d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json @@ -16,7 +16,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json index 8d45714283..bd7310810e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json @@ -31,7 +31,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json index af2a57e536..e2630e24f9 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json @@ -8,7 +8,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone\u003dfalse)" + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" }, "children": [] } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 261816cddc..9d1862023c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -567,7 +567,7 @@ private void populateDefaultCursor(DefaultCursor cursor) { Integer limit = cursor.getLimit(); long rowsLeft = rowsLeft(cursor.getFetchSize(), cursor.getLimit()); if (rowsLeft <= 0) { - // close the cursor + // Delete Point In Time ID if (LocalClusterState.state().getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { String pitId = cursor.getPitId(); PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java index 16af199ed7..59e6f27216 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/TableScan.java @@ -5,11 +5,15 @@ package org.opensearch.sql.legacy.query.planner.logical.node; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + import java.util.Map; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; import org.opensearch.sql.legacy.query.planner.core.PlanNode; import org.opensearch.sql.legacy.query.planner.logical.LogicalOperator; import org.opensearch.sql.legacy.query.planner.physical.PhysicalOperator; +import org.opensearch.sql.legacy.query.planner.physical.node.pointInTime.PointInTime; import org.opensearch.sql.legacy.query.planner.physical.node.scroll.Scroll; /** Table scan */ @@ -33,6 +37,9 @@ public PlanNode[] children() { @Override public <T> PhysicalOperator[] toPhysical(Map<LogicalOperator, PhysicalOperator<T>> optimalOps) { + if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { + return new PhysicalOperator[] {new PointInTime(request, pageSize)}; + } return new PhysicalOperator[] {new Scroll(request, pageSize)}; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java new file mode 100644 index 0000000000..5bf31bb691 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java @@ -0,0 +1,144 @@ +package org.opensearch.sql.legacy.query.planner.physical.node; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Objects; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.sql.legacy.domain.Where; +import org.opensearch.sql.legacy.exception.SqlParseException; +import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import org.opensearch.sql.legacy.query.maker.QueryMaker; +import org.opensearch.sql.legacy.query.planner.core.ExecuteParams; +import org.opensearch.sql.legacy.query.planner.core.PlanNode; +import org.opensearch.sql.legacy.query.planner.physical.Row; +import org.opensearch.sql.legacy.query.planner.physical.estimation.Cost; +import org.opensearch.sql.legacy.query.planner.resource.ResourceManager; + +public abstract class Paginate extends BatchPhysicalOperator<SearchHit> { + + /** Request to submit to OpenSearch to scan over */ + protected final TableInJoinRequestBuilder request; + + protected final int pageSize; + + protected Client client; + + protected SearchResponse searchResponse; + + protected Integer timeout; + + protected ResourceManager resourceMgr; + + public Paginate(TableInJoinRequestBuilder request, int pageSize) { + this.request = request; + this.pageSize = pageSize; + } + + @Override + public PlanNode[] children() { + return new PlanNode[0]; + } + + @Override + public Cost estimate() { + return new Cost(); + } + + @Override + public void open(ExecuteParams params) throws Exception { + super.open(params); + client = params.get(ExecuteParams.ExecuteParamType.CLIENT); + timeout = params.get(ExecuteParams.ExecuteParamType.TIMEOUT); + resourceMgr = params.get(ExecuteParams.ExecuteParamType.RESOURCE_MANAGER); + + Object filter = params.get(ExecuteParams.ExecuteParamType.EXTRA_QUERY_FILTER); + if (filter instanceof BoolQueryBuilder) { + request + .getRequestBuilder() + .setQuery(generateNewQueryWithExtraFilter((BoolQueryBuilder) filter)); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Received extra query filter, re-build query: {}", + Strings.toString( + XContentType.JSON, request.getRequestBuilder().request().source(), true, true)); + } + } + } + + @Override + protected Collection<Row<SearchHit>> prefetch() { + Objects.requireNonNull(client, "Client connection is not ready"); + Objects.requireNonNull(resourceMgr, "ResourceManager is not set"); + Objects.requireNonNull(timeout, "Time out is not set"); + + if (searchResponse == null) { + loadFirstBatch(); + updateMetaResult(); + } else { + loadNextBatch(); + } + return wrapRowForCurrentBatch(); + } + + protected abstract void loadFirstBatch(); + + protected abstract void loadNextBatch(); + + /** + * Extra filter pushed down from upstream. Re-parse WHERE clause with extra filter because + * OpenSearch RequestBuilder doesn't allow QueryBuilder inside be changed after added. + */ + protected QueryBuilder generateNewQueryWithExtraFilter(BoolQueryBuilder filter) + throws SqlParseException { + Where where = request.getOriginalSelect().getWhere(); + BoolQueryBuilder newQuery; + if (where != null) { + newQuery = QueryMaker.explain(where, false); + newQuery.must(filter); + } else { + newQuery = filter; + } + return newQuery; + } + + protected void updateMetaResult() { + resourceMgr.getMetaResult().addTotalNumOfShards(searchResponse.getTotalShards()); + resourceMgr.getMetaResult().addSuccessfulShards(searchResponse.getSuccessfulShards()); + resourceMgr.getMetaResult().addFailedShards(searchResponse.getFailedShards()); + resourceMgr.getMetaResult().updateTimeOut(searchResponse.isTimedOut()); + } + + @SuppressWarnings("unchecked") + protected Collection<Row<SearchHit>> wrapRowForCurrentBatch() { + SearchHit[] hits = searchResponse.getHits().getHits(); + Row[] rows = new Row[hits.length]; + for (int i = 0; i < hits.length; i++) { + rows[i] = new SearchHitRow(hits[i], request.getAlias()); + } + return Arrays.asList(rows); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [ " + describeTable() + ", pageSize=" + pageSize + " ]"; + } + + protected String describeTable() { + return request.getOriginalSelect().getFrom().get(0).getIndex() + " as " + request.getAlias(); + } + + /********************************************* + * Getters for Explain + *********************************************/ + + public String getRequest() { + return Strings.toString(XContentType.JSON, request.getRequestBuilder().request().source()); + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java similarity index 97% rename from legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java rename to legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java index d03dd5af40..3031429ba8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/SearchHitRow.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.legacy.query.planner.physical.node.scroll; +package org.opensearch.sql.legacy.query.planner.physical.node; import com.google.common.base.Strings; import java.util.HashMap; @@ -36,7 +36,7 @@ * ---------------------------------------------------------------------------------------------------------------------- * </pre> */ -class SearchHitRow implements Row<SearchHit> { +public class SearchHitRow implements Row<SearchHit> { /** Native OpenSearch data object for each row */ private final SearchHit hit; @@ -47,7 +47,7 @@ class SearchHitRow implements Row<SearchHit> { /** Table alias owned the row. Empty if this row comes from combination of two other rows */ private final String tableAlias; - SearchHitRow(SearchHit hit, String tableAlias) { + public SearchHitRow(SearchHit hit, String tableAlias) { this.hit = hit; this.source = hit.getSourceAsMap(); this.tableAlias = tableAlias; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java new file mode 100644 index 0000000000..9ddbde2d29 --- /dev/null +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java @@ -0,0 +1,73 @@ +package org.opensearch.sql.legacy.query.planner.physical.node.pointInTime; + +import org.opensearch.common.unit.TimeValue; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; +import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import org.opensearch.sql.legacy.query.planner.physical.node.Paginate; + +/** OpenSearch Search API with Point in time as physical implementation of TableScan */ +public class PointInTime extends Paginate { + + private String pitId; + private PointInTimeHandlerImpl pit; + + public PointInTime(TableInJoinRequestBuilder request, int pageSize) { + super(request, pageSize); + } + + @Override + public void close() { + if (searchResponse != null) { + LOG.debug("Closing Point In Time (PIT) context"); + // Delete the Point In Time context + pit.delete(); + searchResponse = null; + } else { + LOG.debug("PIT context is already closed or was never opened"); + } + } + + @Override + protected void loadFirstBatch() { + // Create PIT and set to request object + pit = new PointInTimeHandlerImpl(client, request.getOriginalSelect().getIndexArr()); + pit.create(); + pitId = pit.getPitId(); + + LOG.info("Loading first batch of response using Point In Time"); + searchResponse = + request + .getRequestBuilder() + .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) + .setSize(pageSize) + .setTimeout(TimeValue.timeValueSeconds(timeout)) + .setPointInTime(new PointInTimeBuilder(pitId)) + .get(); + } + + @Override + protected void loadNextBatch() { + // Add PIT with search after to fetch next batch of data + if (searchResponse.getHits().getHits() != null + && searchResponse.getHits().getHits().length > 0) { + Object[] sortValues = + searchResponse + .getHits() + .getHits()[searchResponse.getHits().getHits().length - 1] + .getSortValues(); + + LOG.info("Loading next batch of response using Point In Time. - " + pitId); + searchResponse = + request + .getRequestBuilder() + .setSize(pageSize) + .setTimeout(TimeValue.timeValueSeconds(timeout)) + .setPointInTime(new PointInTimeBuilder(pitId)) + .searchAfter(sortValues) + .get(); + } + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java index 40e9860886..5019e9cde8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java @@ -5,138 +5,38 @@ package org.opensearch.sql.legacy.query.planner.physical.node.scroll; -import java.util.Arrays; -import java.util.Collection; -import java.util.Objects; import org.opensearch.action.search.ClearScrollResponse; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.common.Strings; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.search.SearchHit; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.sql.legacy.domain.Where; -import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder; -import org.opensearch.sql.legacy.query.maker.QueryMaker; -import org.opensearch.sql.legacy.query.planner.core.ExecuteParams; -import org.opensearch.sql.legacy.query.planner.core.PlanNode; -import org.opensearch.sql.legacy.query.planner.physical.Row; -import org.opensearch.sql.legacy.query.planner.physical.estimation.Cost; -import org.opensearch.sql.legacy.query.planner.physical.node.BatchPhysicalOperator; -import org.opensearch.sql.legacy.query.planner.resource.ResourceManager; +import org.opensearch.sql.legacy.query.planner.physical.node.Paginate; /** OpenSearch Scroll API as physical implementation of TableScan */ -public class Scroll extends BatchPhysicalOperator<SearchHit> { - - /** Request to submit to OpenSearch to scroll over */ - private final TableInJoinRequestBuilder request; - - /** Page size to scroll over index */ - private final int pageSize; - - /** Client connection to ElasticSearch */ - private Client client; - - /** Currently undergoing Scroll */ - private SearchResponse scrollResponse; - - /** Time out */ - private Integer timeout; - - /** Resource monitor manager */ - private ResourceManager resourceMgr; +public class Scroll extends Paginate { public Scroll(TableInJoinRequestBuilder request, int pageSize) { - this.request = request; - this.pageSize = pageSize; - } - - @Override - public PlanNode[] children() { - return new PlanNode[0]; - } - - @Override - public Cost estimate() { - return new Cost(); - } - - @Override - public void open(ExecuteParams params) throws Exception { - super.open(params); - client = params.get(ExecuteParams.ExecuteParamType.CLIENT); - timeout = params.get(ExecuteParams.ExecuteParamType.TIMEOUT); - resourceMgr = params.get(ExecuteParams.ExecuteParamType.RESOURCE_MANAGER); - - Object filter = params.get(ExecuteParams.ExecuteParamType.EXTRA_QUERY_FILTER); - if (filter instanceof BoolQueryBuilder) { - request - .getRequestBuilder() - .setQuery(generateNewQueryWithExtraFilter((BoolQueryBuilder) filter)); - - if (LOG.isDebugEnabled()) { - LOG.debug( - "Received extra query filter, re-build query: {}", - Strings.toString( - XContentType.JSON, request.getRequestBuilder().request().source(), true, true)); - } - } + super(request, pageSize); } @Override public void close() { - if (scrollResponse != null) { + if (searchResponse != null) { LOG.debug("Closing all scroll resources"); ClearScrollResponse clearScrollResponse = - client.prepareClearScroll().addScrollId(scrollResponse.getScrollId()).get(); + client.prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); if (!clearScrollResponse.isSucceeded()) { LOG.warn("Failed to close scroll: {}", clearScrollResponse.status()); } - scrollResponse = null; + searchResponse = null; } else { LOG.debug("Scroll already be closed"); } } @Override - protected Collection<Row<SearchHit>> prefetch() { - Objects.requireNonNull(client, "Client connection is not ready"); - Objects.requireNonNull(resourceMgr, "ResourceManager is not set"); - Objects.requireNonNull(timeout, "Time out is not set"); - - if (scrollResponse == null) { - loadFirstBatch(); - updateMetaResult(); - } else { - loadNextBatchByScrollId(); - } - return wrapRowForCurrentBatch(); - } - - /** - * Extra filter pushed down from upstream. Re-parse WHERE clause with extra filter because - * OpenSearch RequestBuilder doesn't allow QueryBuilder inside be changed after added. - */ - private QueryBuilder generateNewQueryWithExtraFilter(BoolQueryBuilder filter) - throws SqlParseException { - Where where = request.getOriginalSelect().getWhere(); - BoolQueryBuilder newQuery; - if (where != null) { - newQuery = QueryMaker.explain(where, false); - newQuery.must(filter); - } else { - newQuery = filter; - } - return newQuery; - } - - private void loadFirstBatch() { - scrollResponse = + protected void loadFirstBatch() { + searchResponse = request .getRequestBuilder() .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) @@ -145,45 +45,12 @@ private void loadFirstBatch() { .get(); } - private void updateMetaResult() { - resourceMgr.getMetaResult().addTotalNumOfShards(scrollResponse.getTotalShards()); - resourceMgr.getMetaResult().addSuccessfulShards(scrollResponse.getSuccessfulShards()); - resourceMgr.getMetaResult().addFailedShards(scrollResponse.getFailedShards()); - resourceMgr.getMetaResult().updateTimeOut(scrollResponse.isTimedOut()); - } - - private void loadNextBatchByScrollId() { - scrollResponse = + @Override + protected void loadNextBatch() { + searchResponse = client - .prepareSearchScroll(scrollResponse.getScrollId()) + .prepareSearchScroll(searchResponse.getScrollId()) .setScroll(TimeValue.timeValueSeconds(timeout)) .get(); } - - @SuppressWarnings("unchecked") - private Collection<Row<SearchHit>> wrapRowForCurrentBatch() { - SearchHit[] hits = scrollResponse.getHits().getHits(); - Row[] rows = new Row[hits.length]; - for (int i = 0; i < hits.length; i++) { - rows[i] = new SearchHitRow(hits[i], request.getAlias()); - } - return Arrays.asList(rows); - } - - @Override - public String toString() { - return "Scroll [ " + describeTable() + ", pageSize=" + pageSize + " ]"; - } - - private String describeTable() { - return request.getOriginalSelect().getFrom().get(0).getIndex() + " as " + request.getAlias(); - } - - /********************************************* - * Getters for Explain - *********************************************/ - - public String getRequest() { - return Strings.toString(XContentType.JSON, request.getRequestBuilder().request().source()); - } } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java index dd0fc626c0..f7d2030b0c 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.search.SearchHit; +import org.opensearch.sql.legacy.query.planner.physical.node.SearchHitRow; public class SearchHitRowTest { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java index 521b225893..6ff907ba30 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.domain.JoinSelect; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; @@ -104,6 +105,7 @@ public void init() { // to mock. // In this case, default value in Setting will be returned all the time. doReturn(emptyList()).when(settings).getSettings(); + doReturn(false).when(settings).getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER); LocalClusterState.state().setPluginSettings(settings); ActionFuture mockFuture = mock(ActionFuture.class); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 0a9cc67993..cdc3d4462f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -7,6 +7,8 @@ import java.util.List; import java.util.Map; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.DeletePitRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; @@ -89,4 +91,19 @@ public interface OpenSearchClient { void schedule(Runnable task); NodeClient getNodeClient(); + + /** + * Create PIT for given indices + * + * @param createPitRequest Create Point In Time request + * @return PitId + */ + String createPit(CreatePitRequest createPitRequest); + + /** + * Delete PIT + * + * @param deletePitRequest Delete Point In Time request + */ + void deletePit(DeletePitRequest deletePitRequest); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 993e092534..7a9487ef6a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -11,6 +11,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -21,13 +22,16 @@ import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.action.search.*; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; +import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; /** OpenSearch connection by node client. */ @@ -155,20 +159,32 @@ public List<String> indices() { */ @Override public Map<String, String> meta() { - return ImmutableMap.of(META_CLUSTER_NAME, client.settings().get("cluster.name", "opensearch")); + return ImmutableMap.of( + META_CLUSTER_NAME, + client.settings().get("cluster.name", "opensearch"), + "plugins.sql.pagination.api", + client.settings().get("plugins.sql.pagination.api", "true")); } @Override public void cleanup(OpenSearchRequest request) { - request.clean( - scrollId -> { - try { - client.prepareClearScroll().addScrollId(scrollId).get(); - } catch (Exception e) { - throw new IllegalStateException( - "Failed to clean up resources for search request " + request, e); - } - }); + if (request instanceof OpenSearchScrollRequest) { + request.clean( + scrollId -> { + try { + client.prepareClearScroll().addScrollId(scrollId).get(); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.clean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } } @Override @@ -181,4 +197,27 @@ public void schedule(Runnable task) { public NodeClient getNodeClient() { return client; } + + @Override + public String createPit(CreatePitRequest createPitRequest) { + ActionFuture<CreatePitResponse> execute = + this.client.execute(CreatePitAction.INSTANCE, createPitRequest); + try { + CreatePitResponse pitResponse = execute.get(); + return pitResponse.getId(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } + + @Override + public void deletePit(DeletePitRequest deletePitRequest) { + ActionFuture<DeletePitResponse> execute = + this.client.execute(DeletePitAction.INSTANCE, deletePitRequest); + try { + DeletePitResponse deletePitResponse = execute.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Error occurred while deleting PIT.", e); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index b6106982a7..5cb6a69918 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -19,7 +19,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.opensearch.action.search.ClearScrollRequest; +import org.opensearch.action.search.*; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; import org.opensearch.client.indices.CreateIndexRequest; @@ -32,6 +32,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; +import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; /** @@ -166,6 +167,8 @@ public Map<String, String> meta() { final Settings defaultSettings = client.cluster().getSettings(request, RequestOptions.DEFAULT).getDefaultSettings(); builder.put(META_CLUSTER_NAME, defaultSettings.get("cluster.name", "opensearch")); + builder.put( + "plugins.sql.pagination.api", defaultSettings.get("plugins.sql.pagination.api", "true")); return builder.build(); } catch (IOException e) { throw new IllegalStateException("Failed to get cluster meta info", e); @@ -174,17 +177,25 @@ public Map<String, String> meta() { @Override public void cleanup(OpenSearchRequest request) { - request.clean( - scrollId -> { - try { - ClearScrollRequest clearRequest = new ClearScrollRequest(); - clearRequest.addScrollId(scrollId); - client.clearScroll(clearRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new IllegalStateException( - "Failed to clean up resources for search request " + request, e); - } - }); + if (request instanceof OpenSearchScrollRequest) { + request.clean( + scrollId -> { + try { + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(scrollId); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.clean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } } @Override @@ -196,4 +207,25 @@ public void schedule(Runnable task) { public NodeClient getNodeClient() { throw new UnsupportedOperationException("Unsupported method."); } + + @Override + public String createPit(CreatePitRequest createPitRequest) { + try { + CreatePitResponse createPitResponse = + client.createPit(createPitRequest, RequestOptions.DEFAULT); + return createPitResponse.getId(); + } catch (IOException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } + + @Override + public void deletePit(DeletePitRequest deletePitRequest) { + try { + DeletePitResponse deletePitResponse = + client.deletePit(deletePitRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error occurred while creating PIT for new engine SQL query", e); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 6447a3ff65..fff252f3b4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -5,21 +5,35 @@ package org.opensearch.sql.opensearch.request; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; + import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.action.search.*; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; /** * OpenSearch search request. This has to be stateful because it needs to: @@ -36,7 +50,7 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { private final IndexName indexName; /** Search request source builder. */ - private final SearchSourceBuilder sourceBuilder; + private SearchSourceBuilder sourceBuilder; /** OpenSearchExprValueFactory. */ @EqualsAndHashCode.Exclude @ToString.Exclude @@ -45,9 +59,19 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { /** List of includes expected in the response. */ @EqualsAndHashCode.Exclude @ToString.Exclude private final List<String> includes; + @EqualsAndHashCode.Exclude private boolean needClean = true; + /** Indicate the search already done. */ private boolean searchDone = false; + private String pitId; + + private TimeValue cursorKeepAlive; + + private Object[] searchAfter; + + private SearchResponse searchResponse = null; + /** Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest( String indexName, int size, OpenSearchExprValueFactory factory, List<String> includes) { @@ -78,35 +102,153 @@ public OpenSearchQueryRequest( this.includes = includes; } + /** Constructor of OpenSearchQueryRequest with PIT support. */ + public OpenSearchQueryRequest( + IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List<String> includes, + TimeValue cursorKeepAlive, + String pitId) { + this.indexName = indexName; + this.sourceBuilder = sourceBuilder; + this.exprValueFactory = factory; + this.includes = includes; + this.cursorKeepAlive = cursorKeepAlive; + this.pitId = pitId; + } + + /** + * Constructs OpenSearchQueryRequest from serialized representation. + * + * @param in stream to read data from. + * @param engine OpenSearchSqlEngine to get node-specific context. + * @throws IOException thrown if reading from input {@code in} fails. + */ + public OpenSearchQueryRequest(StreamInput in, OpenSearchStorageEngine engine) throws IOException { + // Deserialize the SearchSourceBuilder from the string representation + String sourceBuilderString = in.readString(); + + NamedXContentRegistry xContentRegistry = + new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, IGNORE_DEPRECATIONS, sourceBuilderString); + this.sourceBuilder = SearchSourceBuilder.fromXContent(parser); + + cursorKeepAlive = in.readTimeValue(); + pitId = in.readString(); + includes = in.readStringList(); + indexName = new IndexName(in); + + int length = in.readVInt(); + this.searchAfter = new Object[length]; + for (int i = 0; i < length; i++) { + this.searchAfter[i] = in.readGenericValue(); + } + + OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); + exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + } + @Override public OpenSearchResponse search( Function<SearchRequest, SearchResponse> searchAction, Function<SearchScrollRequest, SearchResponse> scrollAction) { + if (this.pitId == null) { + // When SearchRequest doesn't contain PitId, fetch single page request + if (searchDone) { + return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + } else { + searchDone = true; + return new OpenSearchResponse( + searchAction.apply( + new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), + exprValueFactory, + includes); + } + } else { + // Search with PIT instead of scroll API + return searchWithPIT(searchAction); + } + } + + public OpenSearchResponse searchWithPIT(Function<SearchRequest, SearchResponse> searchAction) { + OpenSearchResponse openSearchResponse; if (searchDone) { - return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + openSearchResponse = new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); } else { - searchDone = true; - return new OpenSearchResponse( - searchAction.apply( - new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), - exprValueFactory, - includes); + this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); + this.sourceBuilder.timeout(cursorKeepAlive); + // check for search after + if (searchAfter != null) { + this.sourceBuilder.searchAfter(searchAfter); + } + // Set sort field for search_after + if (this.sourceBuilder.sorts() == null) { + this.sourceBuilder.sort(DOC_FIELD_NAME, ASC); + } + SearchRequest searchRequest = new SearchRequest().source(this.sourceBuilder); + this.searchResponse = searchAction.apply(searchRequest); + + openSearchResponse = new OpenSearchResponse(this.searchResponse, exprValueFactory, includes); + + needClean = openSearchResponse.isEmpty(); + searchDone = openSearchResponse.isEmpty(); + SearchHit[] searchHits = this.searchResponse.getHits().getHits(); + if (searchHits != null && searchHits.length > 0) { + searchAfter = searchHits[searchHits.length - 1].getSortValues(); + this.sourceBuilder.searchAfter(searchAfter); + } } + return openSearchResponse; } @Override public void clean(Consumer<String> cleanAction) { - // do nothing. + try { + // clean on the last page only, to prevent deleting the PitId in the middle of paging. + if (this.pitId != null && needClean) { + cleanAction.accept(this.pitId); + searchDone = true; + } + } finally { + this.pitId = null; + } } @Override public boolean hasAnotherBatch() { + if (this.pitId != null) { + return !needClean; + } return false; } @Override public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException( - "OpenSearchQueryRequest serialization is not implemented."); + if (this.pitId != null) { + // Convert SearchSourceBuilder to XContent and write it as a string + out.writeString(sourceBuilder.toString()); + + out.writeTimeValue(sourceBuilder.timeout()); + out.writeString(sourceBuilder.pointInTimeBuilder().getId()); + out.writeStringCollection(includes); + indexName.writeTo(out); + + // Serialize the searchAfter array + if (searchAfter != null) { + out.writeVInt(searchAfter.length); + for (Object obj : searchAfter) { + out.writeGenericValue(obj); + } + } + } else { + // OpenSearch Query request without PIT for single page requests + throw new UnsupportedOperationException( + "OpenSearchQueryRequest serialization is not implemented."); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 1df3dcb183..6fa9b17697 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -26,6 +26,7 @@ import lombok.ToString; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.InnerHitBuilder; @@ -39,9 +40,11 @@ import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -67,10 +70,13 @@ public class OpenSearchRequestBuilder { private int startFrom = 0; + private final Settings settings; + /** Constructor. */ public OpenSearchRequestBuilder( - int requestedTotalSize, OpenSearchExprValueFactory exprValueFactory) { + int requestedTotalSize, OpenSearchExprValueFactory exprValueFactory, Settings settings) { this.requestedTotalSize = requestedTotalSize; + this.settings = settings; this.sourceBuilder = new SearchSourceBuilder() .from(startFrom) @@ -82,18 +88,65 @@ public OpenSearchRequestBuilder( /** * Build DSL request. * - * @return query request or scroll request + * @return query request with PIT or scroll request */ public OpenSearchRequest build( - OpenSearchRequest.IndexName indexName, int maxResultWindow, TimeValue scrollTimeout) { + OpenSearchRequest.IndexName indexName, + int maxResultWindow, + TimeValue cursorKeepAlive, + OpenSearchClient client) { + if (this.settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) { + return buildRequestWithPit(indexName, maxResultWindow, cursorKeepAlive, client); + } else { + return buildRequestWithScroll(indexName, maxResultWindow, cursorKeepAlive); + } + } + + private OpenSearchRequest buildRequestWithPit( + OpenSearchRequest.IndexName indexName, + int maxResultWindow, + TimeValue cursorKeepAlive, + OpenSearchClient client) { + int size = requestedTotalSize; + FetchSourceContext fetchSource = this.sourceBuilder.fetchSource(); + List<String> includes = fetchSource != null ? Arrays.asList(fetchSource.includes()) : List.of(); + + if (pageSize == null) { + if (startFrom + size > maxResultWindow) { + sourceBuilder.size(maxResultWindow - startFrom); + // Search with PIT request + String pitId = createPit(indexName, cursorKeepAlive, client); + return new OpenSearchQueryRequest( + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + } else { + sourceBuilder.from(startFrom); + sourceBuilder.size(requestedTotalSize); + // Search with non-Pit request + return new OpenSearchQueryRequest(indexName, sourceBuilder, exprValueFactory, includes); + } + } else { + if (startFrom != 0) { + throw new UnsupportedOperationException("Non-zero offset is not supported with pagination"); + } + sourceBuilder.size(pageSize); + // Search with PIT request + String pitId = createPit(indexName, cursorKeepAlive, client); + return new OpenSearchQueryRequest( + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + } + } + + private OpenSearchRequest buildRequestWithScroll( + OpenSearchRequest.IndexName indexName, int maxResultWindow, TimeValue cursorKeepAlive) { int size = requestedTotalSize; FetchSourceContext fetchSource = this.sourceBuilder.fetchSource(); List<String> includes = fetchSource != null ? Arrays.asList(fetchSource.includes()) : List.of(); + if (pageSize == null) { if (startFrom + size > maxResultWindow) { sourceBuilder.size(maxResultWindow - startFrom); return new OpenSearchScrollRequest( - indexName, scrollTimeout, sourceBuilder, exprValueFactory, includes); + indexName, cursorKeepAlive, sourceBuilder, exprValueFactory, includes); } else { sourceBuilder.from(startFrom); sourceBuilder.size(requestedTotalSize); @@ -105,10 +158,18 @@ public OpenSearchRequest build( } sourceBuilder.size(pageSize); return new OpenSearchScrollRequest( - indexName, scrollTimeout, sourceBuilder, exprValueFactory, includes); + indexName, cursorKeepAlive, sourceBuilder, exprValueFactory, includes); } } + private String createPit( + OpenSearchRequest.IndexName indexName, TimeValue cursorKeepAlive, OpenSearchClient client) { + // Create PIT ID for request + CreatePitRequest createPitRequest = + new CreatePitRequest(cursorKeepAlive, false, indexName.getIndexNames()); + return client.createPit(createPitRequest); + } + boolean isBoolFilterQuery(QueryBuilder current) { return (current instanceof BoolQueryBuilder); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c6afdb8511..a6fe83c8c4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -163,13 +163,13 @@ public TableScanBuilder createScanBuilder() { final int querySizeLimit = settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT); final TimeValue cursorKeepAlive = settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); - var builder = new OpenSearchRequestBuilder(querySizeLimit, createExprValueFactory()); + var builder = new OpenSearchRequestBuilder(querySizeLimit, createExprValueFactory(), settings); Function<OpenSearchRequestBuilder, OpenSearchIndexScan> createScanOperator = requestBuilder -> new OpenSearchIndexScan( client, requestBuilder.getMaxResponseSize(), - requestBuilder.build(indexName, getMaxResultWindow(), cursorKeepAlive)); + requestBuilder.build(indexName, getMaxResultWindow(), cursorKeepAlive, client)); return new OpenSearchIndexScanBuilder(builder, createScanOperator); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java index b1e4ccc463..b17773cb03 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java @@ -14,10 +14,12 @@ import lombok.ToString; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NoCursorException; import org.opensearch.sql.executor.pagination.PlanSerializer; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -45,6 +47,8 @@ public class OpenSearchIndexScan extends TableScanOperator implements Serializab /** Search response for current batch. */ private Iterator<ExprValue> iterator; + private Settings pluginSettings; + /** Creates index scan based on a provided OpenSearchRequestBuilder. */ public OpenSearchIndexScan( OpenSearchClient client, int maxResponseSize, OpenSearchRequest request) { @@ -121,12 +125,18 @@ public void readExternal(ObjectInput in) throws IOException { (OpenSearchStorageEngine) ((PlanSerializer.CursorDeserializationStream) in).resolveObject("engine"); + client = engine.getClient(); + boolean pointInTimeEnabled = + Boolean.parseBoolean( + client.meta().get(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue())); try (BytesStreamInput bsi = new BytesStreamInput(requestStream)) { - request = new OpenSearchScrollRequest(bsi, engine); + if (pointInTimeEnabled) { + request = new OpenSearchQueryRequest(bsi, engine); + } else { + request = new OpenSearchScrollRequest(bsi, engine); + } } maxResponseSize = in.readInt(); - - client = engine.getClient(); } @Override diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 9da6e05e92..ba0fb85422 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -13,8 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -31,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import lombok.SneakyThrows; import org.apache.commons.lang3.reflect.FieldUtils; @@ -51,15 +51,16 @@ import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.opensearch.action.search.ClearScrollRequestBuilder; -import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.*; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -74,6 +75,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -393,6 +395,65 @@ void cleanup_rethrows_exception() { assertThrows(IllegalStateException.class, () -> client.cleanup(request)); } + @Test + @SneakyThrows + void cleanup_pit_request() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + client.cleanup(request); + verify(nodeClient).execute(any(), any()); + } + + @Test + @SneakyThrows + void cleanup_pit_request_throw_exception() { + DeletePitRequest deletePitRequest = new DeletePitRequest("samplePitId"); + ActionFuture<DeletePitResponse> actionFuture = mock(ActionFuture.class); + when(actionFuture.get()).thenThrow(new ExecutionException("Execution failed", new Throwable())); + when(nodeClient.execute(eq(DeletePitAction.INSTANCE), any(DeletePitRequest.class))) + .thenReturn(actionFuture); + assertThrows(RuntimeException.class, () -> client.deletePit(deletePitRequest)); + } + + @Test + @SneakyThrows + void create_pit() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + ActionFuture<CreatePitResponse> actionFuture = mock(ActionFuture.class); + CreatePitResponse createPitResponse = mock(CreatePitResponse.class); + when(createPitResponse.getId()).thenReturn("samplePitId"); + when(actionFuture.get()).thenReturn(createPitResponse); + when(nodeClient.execute(eq(CreatePitAction.INSTANCE), any(CreatePitRequest.class))) + .thenReturn(actionFuture); + + String pitId = client.createPit(createPitRequest); + assertEquals("samplePitId", pitId); + + verify(nodeClient).execute(CreatePitAction.INSTANCE, createPitRequest); + verify(actionFuture).get(); + } + + @Test + @SneakyThrows + void create_pit_request_throw_exception() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + ActionFuture<CreatePitResponse> actionFuture = mock(ActionFuture.class); + when(actionFuture.get()).thenThrow(new ExecutionException("Execution failed", new Throwable())); + when(nodeClient.execute(eq(CreatePitAction.INSTANCE), any(CreatePitRequest.class))) + .thenReturn(actionFuture); + assertThrows(RuntimeException.class, () -> client.createPit(createPitRequest)); + } + @Test void get_indices() { AliasMetadata aliasMetadata = mock(AliasMetadata.class); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index b83313de07..82d9e74422 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -43,6 +43,8 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; @@ -56,6 +58,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -69,6 +72,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -411,6 +415,64 @@ void cleanup_with_IOException() { assertThrows(IllegalStateException.class, () -> client.cleanup(request)); } + @Test + @SneakyThrows + void cleanup_pit_request() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + client.cleanup(request); + verify(restClient).deletePit(any(), any()); + } + + @Test + @SneakyThrows + void cleanup_pit_request_throw_exception() { + when(restClient.deletePit(any(), any())).thenThrow(new IOException()); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder(), + factory, + List.of(), + TimeValue.timeValueMinutes(1L), + "samplePitId"); + // Enforce cleaning by setting a private field. + FieldUtils.writeField(request, "needClean", true, true); + assertThrows(RuntimeException.class, () -> client.cleanup(request)); + } + + @Test + @SneakyThrows + void create_pit() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + CreatePitResponse createPitResponse = mock(CreatePitResponse.class); + when(createPitResponse.getId()).thenReturn("samplePitId"); + when(restClient.createPit(any(CreatePitRequest.class), any())).thenReturn(createPitResponse); + + String pitId = client.createPit(createPitRequest); + assertEquals("samplePitId", pitId); + + verify(restClient).createPit(createPitRequest, RequestOptions.DEFAULT); + } + + @Test + @SneakyThrows + void create_pit_request_throw_exception() { + CreatePitRequest createPitRequest = + new CreatePitRequest(TimeValue.timeValueMinutes(5), false, Strings.EMPTY_ARRAY); + when(restClient.createPit(any(), any())).thenThrow(new IOException()); + assertThrows(RuntimeException.class, () -> client.createPit(createPitRequest)); + } + @Test void get_indices() throws IOException { when(restClient.indices().get(any(GetIndexRequest.class), any(RequestOptions.class))) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java index 739b70b1b8..e5cf94eb86 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java @@ -174,18 +174,20 @@ void explain_successfully() { new OpenSearchExecutionEngine(client, protector, new PlanSerializer(null)); Settings settings = mock(Settings.class); when(settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE)).thenReturn(TimeValue.timeValueMinutes(1)); + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); OpenSearchExprValueFactory exprValueFactory = mock(OpenSearchExprValueFactory.class); final var name = new OpenSearchRequest.IndexName("test"); final int defaultQuerySize = 100; final int maxResultWindow = 10000; - final var requestBuilder = new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory, settings); PhysicalPlan plan = new OpenSearchIndexScan( mock(OpenSearchClient.class), maxResultWindow, requestBuilder.build( - name, maxResultWindow, settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE))); + name, maxResultWindow, settings.getSettingValue(SQL_CURSOR_KEEP_ALIVE), client)); AtomicReference<ExplainResponse> result = new AtomicReference<>(); executor.explain( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index 5cd11c6cd4..da06c1eb66 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -8,9 +8,7 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -91,6 +89,8 @@ public void setup() { @Test void test_protect_indexScan() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + String indexName = "test"; final int maxResultWindow = 10000; final int querySizeLimit = 200; @@ -114,11 +114,12 @@ void test_protect_indexScan() { final var name = new OpenSearchRequest.IndexName(indexName); final var request = - new OpenSearchRequestBuilder(querySizeLimit, exprValueFactory) + new OpenSearchRequestBuilder(querySizeLimit, exprValueFactory, settings) .build( name, maxResultWindow, - settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)); + settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE), + client); assertEquals( PhysicalPlanDSL.project( PhysicalPlanDSL.limit( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index d2bc5b0641..89b51207b5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -5,37 +5,40 @@ package org.opensearch.sql.opensearch.request; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.opensearch.sql.opensearch.request.OpenSearchRequest.DEFAULT_QUERY_TIMEOUT; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import lombok.SneakyThrows; import org.apache.lucene.search.TotalHits; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; @ExtendWith(MockitoExtension.class) public class OpenSearchQueryRequestTest { @@ -64,6 +67,93 @@ public class OpenSearchQueryRequestTest { private final OpenSearchQueryRequest remoteRequest = new OpenSearchQueryRequest("ccs:test", 200, factory, List.of()); + @Mock private StreamOutput streamOutput; + @Mock private StreamInput streamInput; + @Mock private OpenSearchStorageEngine engine; + @Mock private PointInTimeBuilder pointInTimeBuilder; + + @InjectMocks private OpenSearchQueryRequest serializationRequest; + + private SearchSourceBuilder sourceBuilderForSerializer; + + @BeforeEach + void setup() { + sourceBuilderForSerializer = new SearchSourceBuilder(); + sourceBuilderForSerializer.pointInTimeBuilder(pointInTimeBuilder); + sourceBuilderForSerializer.timeout(TimeValue.timeValueSeconds(30)); + } + + @SneakyThrows + @Test + void testWriteTo() throws IOException { + when(pointInTimeBuilder.getId()).thenReturn("samplePITId"); + sourceBuilderForSerializer.searchAfter(new Object[] {"value1", 123}); + List<String> includes = List.of("field1", "field2"); + serializationRequest = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilderForSerializer, + factory, + includes, + new TimeValue(1000), + "samplePITId"); + + Field searchAfterField = OpenSearchQueryRequest.class.getDeclaredField("searchAfter"); + searchAfterField.setAccessible(true); + searchAfterField.set(serializationRequest, new Object[] {"value1", 123}); + + serializationRequest.writeTo(streamOutput); + + String expectedJson = "{\"timeout\":\"30s\",\"search_after\":[\"value1\",123]}"; + verify(streamOutput).writeString(expectedJson); + verify(streamOutput).writeTimeValue(TimeValue.timeValueSeconds(30)); + verify(streamOutput).writeString("samplePITId"); + verify(streamOutput).writeStringCollection(includes); + + verify(streamOutput).writeVInt(2); + verify(streamOutput).writeGenericValue("value1"); + verify(streamOutput).writeGenericValue(123); + } + + @Test + void testWriteToWithoutSearchAfter() + throws IOException, NoSuchFieldException, IllegalAccessException { + when(pointInTimeBuilder.getId()).thenReturn("samplePITId"); + + List<String> includes = List.of("field1", "field2"); + serializationRequest = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilderForSerializer, + factory, + includes, + new TimeValue(1000), + "samplePITId"); + + serializationRequest.writeTo(streamOutput); + verify(streamOutput).writeString("{\"timeout\":\"30s\"}"); + verify(streamOutput).writeTimeValue(TimeValue.timeValueSeconds(30)); + verify(streamOutput).writeString("samplePITId"); + verify(streamOutput).writeStringCollection(includes); + verify(streamOutput, never()).writeVInt(anyInt()); + verify(streamOutput, never()).writeGenericValue(any()); + } + + @Test + void testWriteToWithoutPIT() { + serializationRequest = new OpenSearchQueryRequest("test", 200, factory, List.of()); + + UnsupportedOperationException exception = + assertThrows( + UnsupportedOperationException.class, + () -> { + request.writeTo(streamOutput); + }); + + assertEquals( + "OpenSearchQueryRequest serialization is not implemented.", exception.getMessage()); + } + @Test void search() { OpenSearchQueryRequest request = @@ -81,6 +171,145 @@ void search() { verify(searchAction, times(1)).apply(any()); } + @Test + void search_with_pit() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + when(searchHit.getSortValues()).thenReturn(new String[] {"sortedValue"}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertFalse(openSearchResponse.isEmpty()); + verify(searchAction, times(1)).apply(any()); + + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchResponse.getAggregations()).thenReturn(null); + when(searchHits.getHits()).thenReturn(null); + openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + verify(searchAction, times(2)).apply(any()); + + openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_hits_null() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertFalse(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_hits_empty() { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {}); + when(sourceBuilder.sorts()).thenReturn(null); + + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(openSearchResponse.isEmpty()); + } + + @Test + void search_with_pit_null() { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "sample"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + + OpenSearchResponse openSearchResponse = request.search(searchAction, scrollAction); + assertFalse(openSearchResponse.isEmpty()); + } + + @Test + void has_another_batch() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "sample"); + assertFalse(request.hasAnotherBatch()); + } + + @Test + void has_another_batch_pid_null() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + null); + assertFalse(request.hasAnotherBatch()); + } + + @Test + void has_another_batch_need_clean() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + assertTrue(request.hasAnotherBatch()); + } + @Test void search_withoutContext() { OpenSearchQueryRequest request = @@ -121,6 +350,68 @@ void clean() { verify(cleanAction, never()).accept(any()); } + @Test + void testCleanConditionTrue() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(null); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + + request.clean(cleanAction); + + verify(cleanAction, times(1)).accept("samplePid"); + assertTrue(request.isSearchDone()); + assertNull(request.getPitId()); + } + + @Test + void testCleanConditionFalse_needCleanFalse() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + "samplePid"); + + when(searchAction.apply(any())).thenReturn(searchResponse); + when(searchResponse.getHits()).thenReturn(searchHits); + when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); + OpenSearchResponse openSearchResponse = request.searchWithPIT(searchAction); + + request.clean(cleanAction); + verify(cleanAction, never()).accept(anyString()); + assertFalse(request.isSearchDone()); + assertNull(request.getPitId()); + } + + @Test + void testCleanConditionFalse_pidNull() { + OpenSearchQueryRequest request = + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + sourceBuilder, + factory, + List.of(), + new TimeValue(1000), + null); + + request.clean(cleanAction); + verify(cleanAction, never()).accept(anyString()); + assertFalse(request.isSearchDone()); + assertNull(request.getPitId()); + } + @Test void searchRequest() { request.getSourceBuilder().query(QueryBuilders.termQuery("name", "John")); @@ -159,6 +450,20 @@ void writeTo_unsupported() { UnsupportedOperationException.class, () -> request.writeTo(mock(StreamOutput.class))); } + @Test + void constructor_serialized() throws IOException { + StreamInput stream = mock(StreamInput.class); + OpenSearchStorageEngine engine = mock(OpenSearchStorageEngine.class); + when(stream.readString()).thenReturn("{}"); + when(stream.readStringArray()).thenReturn(new String[] {"sample"}); + OpenSearchIndex index = mock(OpenSearchIndex.class); + when(engine.getTable(null, "sample")).thenReturn(index); + when(stream.readVInt()).thenReturn(2); + when(stream.readGenericValue()).thenReturn("sampleSearchAfter"); + OpenSearchQueryRequest request = new OpenSearchQueryRequest(stream, engine); + assertNotNull(request); + } + private void assertSearchRequest(SearchRequest expected, OpenSearchQueryRequest request) { Function<SearchRequest, SearchResponse> querySearch = searchRequest -> { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index 742e76cbd0..bf87840b60 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -5,13 +5,10 @@ package org.opensearch.sql.opensearch.request; -import static org.junit.Assert.assertThrows; +import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.index.query.QueryBuilders.nestedQuery; +import static org.mockito.Mockito.*; +import static org.opensearch.index.query.QueryBuilders.*; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -25,21 +22,16 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.join.ScoreMode; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayNameGeneration; -import org.junit.jupiter.api.DisplayNameGenerator; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.query.InnerHitBuilder; -import org.opensearch.index.query.NestedQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.*; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.AggregationBuilder; @@ -47,13 +39,19 @@ import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.ScoreSortBuilder; import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; @@ -76,11 +74,18 @@ class OpenSearchRequestBuilderTest { @Mock private OpenSearchExprValueFactory exprValueFactory; + @Mock private OpenSearchClient client; + + @Mock private Settings settings; + private OpenSearchRequestBuilder requestBuilder; @BeforeEach void setup() { - requestBuilder = new OpenSearchRequestBuilder(DEFAULT_LIMIT, exprValueFactory); + requestBuilder = new OpenSearchRequestBuilder(DEFAULT_LIMIT, exprValueFactory, settings); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Test @@ -100,14 +105,148 @@ void build_query_request() { .trackScores(true), exprValueFactory, List.of()), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void build_query_request_push_down_size() { + Integer limit = 200; + Integer offset = 0; + requestBuilder.pushDownLimit(limit, offset); + requestBuilder.pushDownTrackedScore(true); + + assertNotNull( + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void build_PIT_request_with_correct_size() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + Integer limit = 0; + Integer offset = 0; + requestBuilder.pushDownLimit(limit, offset); + requestBuilder.pushDownPageSize(2); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder().from(offset).size(2).timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + Integer limit = 600; + Integer offset = 0; + int requestedTotalSize = 600; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNull_sizeLessThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true); + Integer limit = 400; + Integer offset = 0; + int requestedTotalSize = 400; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(requestedTotalSize) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNotNull_startFromZero() { + int pageSize = 200; + int offset = 0; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder().from(offset).size(pageSize).timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithPit_pageSizeNotNull_startFromNonZero() { + int pageSize = 200; + int offset = 100; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + assertThrows( + UnsupportedOperationException.class, + () -> { + requestBuilder.build(indexName, 500, TimeValue.timeValueMinutes(1), client); + }); } @Test void build_scroll_request_with_correct_size() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); Integer limit = 800; Integer offset = 10; requestBuilder.pushDownLimit(limit, offset); + requestBuilder.getSourceBuilder().fetchSource("a", "b"); + + assertEquals( + new OpenSearchScrollRequest( + new OpenSearchRequest.IndexName("test"), + TimeValue.timeValueMinutes(1), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNull_sizeGreaterThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + Integer limit = 600; + Integer offset = 0; + int requestedTotalSize = 600; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); assertEquals( new OpenSearchScrollRequest( @@ -119,7 +258,65 @@ void build_scroll_request_with_correct_size() { .timeout(DEFAULT_QUERY_TIMEOUT), exprValueFactory, List.of()), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNull_sizeLessThanMaxResultWindow() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + Integer limit = 400; + Integer offset = 0; + int requestedTotalSize = 400; + requestBuilder = new OpenSearchRequestBuilder(requestedTotalSize, exprValueFactory, settings); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(requestedTotalSize) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNotNull_startFromZero() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + int pageSize = 200; + int offset = 0; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchScrollRequest( + new OpenSearchRequest.IndexName("test"), + TimeValue.timeValueMinutes(1), + new SearchSourceBuilder() + .from(offset) + .size(MAX_RESULT_WINDOW - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + exprValueFactory, + List.of()), + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); + } + + @Test + void buildRequestWithScroll_pageSizeNotNull_startFromNonZero() { + when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false); + int pageSize = 200; + int offset = 100; + int limit = 400; + requestBuilder.pushDownPageSize(pageSize); + requestBuilder.pushDownLimit(limit, offset); + assertThrows( + UnsupportedOperationException.class, + () -> { + requestBuilder.build(indexName, 500, TimeValue.timeValueMinutes(1), client); + }); } @Test @@ -127,7 +324,7 @@ void test_push_down_query() { QueryBuilder query = QueryBuilders.termQuery("intA", 1); requestBuilder.pushDownFilter(query); - var r = requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT); + var r = requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client); Function<SearchRequest, SearchResponse> querySearch = searchRequest -> { assertEquals( @@ -203,6 +400,51 @@ void test_push_down_query_and_sort() { requestBuilder); } + @Test + void test_push_down_query_not_null() { + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + sourceBuilder.query(QueryBuilders.termQuery("name", "John")); + sourceBuilder.sort(DOC_FIELD_NAME, SortOrder.ASC); + + QueryBuilder query = QueryBuilders.termQuery("intA", 1); + requestBuilder.pushDownFilter(query); + + BoolQueryBuilder expectedQuery = + QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("name", "John")).filter(query); + + SearchSourceBuilder expectedSourceBuilder = + new SearchSourceBuilder() + .from(DEFAULT_OFFSET) + .size(DEFAULT_LIMIT) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(expectedQuery) + .sort(DOC_FIELD_NAME, SortOrder.ASC); + + assertSearchSourceBuilder(expectedSourceBuilder, requestBuilder); + } + + @Test + void test_push_down_query_with_bool_filter() { + BoolQueryBuilder initialBoolQuery = + QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("name", "John")); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + sourceBuilder.query(initialBoolQuery); + + QueryBuilder newQuery = QueryBuilders.termQuery("intA", 1); + requestBuilder.pushDownFilter(newQuery); + initialBoolQuery.filter(newQuery); + SearchSourceBuilder expectedSourceBuilder = + new SearchSourceBuilder() + .from(DEFAULT_OFFSET) + .size(DEFAULT_LIMIT) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(initialBoolQuery) + .sort(DOC_FIELD_NAME, SortOrder.ASC); + + assertSearchSourceBuilder(expectedSourceBuilder, requestBuilder); + } + void assertSearchSourceBuilder( SearchSourceBuilder expected, OpenSearchRequestBuilder requestBuilder) throws UnsupportedOperationException { @@ -220,7 +462,7 @@ void assertSearchSourceBuilder( throw new UnsupportedOperationException(); }; requestBuilder - .build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT) + .build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client) .search(querySearch, scrollSearch); } @@ -290,7 +532,7 @@ void test_push_down_project() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -320,7 +562,7 @@ void test_push_down_project_limit() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -350,7 +592,7 @@ void test_push_down_project_limit_and_offset() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test @@ -377,7 +619,7 @@ void test_push_down_nested() { assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(nestedQuery))) + .query(boolQuery().filter(boolQuery().must(nestedQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -411,7 +653,7 @@ void test_push_down_multiple_nested_with_same_path() { true, new String[] {"message.info", "message.from"}, null))); assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(nestedQuery))) + .query(boolQuery().filter(boolQuery().must(nestedQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -444,9 +686,9 @@ void test_push_down_nested_with_filter() { assertSearchSourceBuilder( new SearchSourceBuilder() .query( - QueryBuilders.boolQuery() + boolQuery() .filter( - QueryBuilders.boolQuery() + boolQuery() .must(QueryBuilders.rangeQuery("myNum").gt(3)) .must(nestedQuery))) .from(DEFAULT_OFFSET) @@ -483,7 +725,7 @@ void testPushDownNestedWithNestedFilter() { assertSearchSourceBuilder( new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().filter(QueryBuilders.boolQuery().must(filterQuery))) + .query(boolQuery().filter(boolQuery().must(filterQuery))) .from(DEFAULT_OFFSET) .size(DEFAULT_LIMIT) .timeout(DEFAULT_QUERY_TIMEOUT), @@ -507,6 +749,32 @@ void push_down_highlight_with_repeating_fields() { assertEquals("Duplicate field name in highlight", exception.getMessage()); } + @Test + void test_push_down_highlight_with_pre_tags() { + requestBuilder.pushDownHighlight( + "name", Map.of("pre_tags", new Literal("pre1", DataType.STRING))); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + assertNotNull(sourceBuilder.highlighter()); + assertEquals(1, sourceBuilder.highlighter().fields().size()); + HighlightBuilder.Field field = sourceBuilder.highlighter().fields().get(0); + assertEquals("name", field.name()); + assertEquals("pre1", field.preTags()[0]); + } + + @Test + void test_push_down_highlight_with_post_tags() { + requestBuilder.pushDownHighlight( + "name", Map.of("post_tags", new Literal("post1", DataType.STRING))); + + SearchSourceBuilder sourceBuilder = requestBuilder.getSourceBuilder(); + assertNotNull(sourceBuilder.highlighter()); + assertEquals(1, sourceBuilder.highlighter().fields().size()); + HighlightBuilder.Field field = sourceBuilder.highlighter().fields().get(0); + assertEquals("name", field.name()); + assertEquals("post1", field.postTags()[0]); + } + @Test void push_down_page_size() { requestBuilder.pushDownPageSize(3); @@ -521,7 +789,7 @@ void exception_when_non_zero_offset_and_page_size() { requestBuilder.pushDownLimit(300, 2); assertThrows( UnsupportedOperationException.class, - () -> requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT)); + () -> requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client)); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ca566fac6..ef6b86c42a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -79,6 +79,9 @@ class OpenSearchIndexTest { @BeforeEach void setUp() { this.index = new OpenSearchIndex(client, settings, "test"); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Test @@ -198,10 +201,11 @@ void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); LogicalPlan plan = index.createScanBuilder(); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( new OpenSearchIndexScan( - client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), index.implement(index.optimize(plan))); } @@ -211,10 +215,11 @@ void implementRelationOperatorWithOptimization() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); LogicalPlan plan = index.createScanBuilder(); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( new OpenSearchIndexScan( - client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + client, 200, requestBuilder.build(INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), index.implement(plan)); } @@ -243,7 +248,8 @@ void implementOtherLogicalOperators() { include); Integer maxResultWindow = index.getMaxResultWindow(); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(QUERY_SIZE_LIMIT, exprValueFactory, settings); assertEquals( PhysicalPlanDSL.project( PhysicalPlanDSL.dedupe( @@ -255,7 +261,7 @@ void implementOtherLogicalOperators() { client, QUERY_SIZE_LIMIT, requestBuilder.build( - INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT)), + INDEX_NAME, maxResultWindow, SCROLL_TIMEOUT, client)), mappings), exclude), newEvalField), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index 2085519b12..e6a17aceaf 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -56,6 +56,9 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)) .thenReturn(TimeValue.timeValueMinutes(1)); + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); } @Mock private OpenSearchClient client; @@ -69,12 +72,12 @@ void setup() { @Test void query_empty_result() { mockResponse(client); - var builder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + var builder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (var indexScan = new OpenSearchIndexScan( client, MAX_RESULT_WINDOW, - builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT))) { + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT, client))) { indexScan.open(); assertFalse(indexScan.hasNext()); } @@ -96,13 +99,13 @@ void dont_serialize_if_no_cursor() { OpenSearchRequestBuilder builder = mock(); OpenSearchRequest request = mock(); OpenSearchResponse response = mock(); - when(builder.build(any(), anyInt(), any())).thenReturn(request); + when(builder.build(any(), anyInt(), any(), any())).thenReturn(request); when(client.search(any())).thenReturn(response); try (var indexScan = new OpenSearchIndexScan( client, MAX_RESULT_WINDOW, - builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT))) { + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, SCROLL_TIMEOUT, client))) { indexScan.open(); when(request.hasAnotherBatch()).thenReturn(false); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index f813d8f551..e680c6b3a6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -5,24 +5,19 @@ package org.opensearch.sql.opensearch.storage.scan; +import static org.junit.Assert.assertNotNull; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import java.io.*; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -52,6 +47,7 @@ import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.NoCursorException; @@ -77,6 +73,7 @@ class OpenSearchIndexScanTest { public static final int MAX_RESULT_WINDOW = 10000; public static final TimeValue CURSOR_KEEP_ALIVE = TimeValue.timeValueMinutes(1); @Mock private OpenSearchClient client; + @Mock private Settings settings; private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory( @@ -84,7 +81,11 @@ class OpenSearchIndexScanTest { "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING))); @BeforeEach - void setup() {} + void setup() { + lenient() + .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); + } @Test void explain() { @@ -144,6 +145,49 @@ void serialize(Integer numberOfIncludes) { } } + @SneakyThrows + @ParameterizedTest + @ValueSource(ints = {0, 150}) + void serialize_PIT(Integer numberOfIncludes) { + var searchSourceBuilder = new SearchSourceBuilder().size(4); + + var factory = mock(OpenSearchExprValueFactory.class); + var engine = mock(OpenSearchStorageEngine.class); + var index = mock(OpenSearchIndex.class); + when(engine.getClient()).thenReturn(client); + when(engine.getTable(any(), any())).thenReturn(index); + Map map = mock(Map.class); + when(map.get(any(String.class))).thenReturn("true"); + when(client.meta()).thenReturn(map); + var includes = + Stream.iterate(1, i -> i + 1) + .limit(numberOfIncludes) + .map(i -> "column" + i) + .collect(Collectors.toList()); + var request = + new OpenSearchQueryRequest( + INDEX_NAME, searchSourceBuilder, factory, includes, CURSOR_KEEP_ALIVE, "samplePitId"); + // make a response, so OpenSearchResponse::isEmpty would return true and unset needClean + var response = mock(SearchResponse.class); + when(response.getAggregations()).thenReturn(mock()); + var hits = mock(SearchHits.class); + when(response.getHits()).thenReturn(hits); + SearchHit hit = mock(SearchHit.class); + when(hit.getSortValues()).thenReturn(new String[] {"sample1"}); + when(hits.getHits()).thenReturn(new SearchHit[] {hit}); + request.search((req) -> response, null); + + try (var indexScan = new OpenSearchIndexScan(client, QUERY_SIZE, request)) { + var planSerializer = new PlanSerializer(engine); + var cursor = planSerializer.convertToCursor(indexScan); + var newPlan = planSerializer.convertToPlan(cursor.toString()); + assertNotNull(newPlan); + + verify(client).meta(); + verify(map).get(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER.getKeyValue()); + } + } + @SneakyThrows @Test void throws_io_exception_if_too_short() { @@ -172,10 +216,12 @@ void plan_for_serialization() { void query_empty_result() { mockResponse(client); final var name = new OpenSearchRequest.IndexName("test"); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, QUERY_SIZE, requestBuilder.build(name, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + QUERY_SIZE, + requestBuilder.build(name, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertFalse(indexScan.hasNext()); } @@ -190,10 +236,10 @@ void query_all_results_with_query() { employee(1, "John", "IT"), employee(2, "Smith", "HR"), employee(3, "Allen", "IT") }); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE))) { + client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -218,10 +264,10 @@ void query_all_results_with_scroll() { new ExprValue[] {employee(1, "John", "IT"), employee(2, "Smith", "HR")}, new ExprValue[] {employee(3, "Allen", "IT")}); - final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory); + final var requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE))) { + client, 10, requestBuilder.build(INDEX_NAME, 10000, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -248,10 +294,12 @@ void query_some_results_with_query() { }); final int limit = 3; - OpenSearchRequestBuilder builder = new OpenSearchRequestBuilder(0, exprValueFactory); + OpenSearchRequestBuilder builder = new OpenSearchRequestBuilder(0, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, limit, builder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + limit, + builder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -269,10 +317,12 @@ void query_some_results_with_query() { @Test void query_some_results_with_scroll() { mockTwoPageResponse(client); - final var requestuilder = new OpenSearchRequestBuilder(10, exprValueFactory); + final var requestuilder = new OpenSearchRequestBuilder(10, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( - client, 3, requestuilder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE))) { + client, + 3, + requestuilder.build(INDEX_NAME, MAX_RESULT_WINDOW, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -306,12 +356,13 @@ void query_results_limited_by_query_size() { }); final int defaultQuerySize = 2; - final var requestBuilder = new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory); + final var requestBuilder = + new OpenSearchRequestBuilder(defaultQuerySize, exprValueFactory, settings); try (OpenSearchIndexScan indexScan = new OpenSearchIndexScan( client, defaultQuerySize, - requestBuilder.build(INDEX_NAME, QUERY_SIZE, CURSOR_KEEP_ALIVE))) { + requestBuilder.build(INDEX_NAME, QUERY_SIZE, CURSOR_KEEP_ALIVE, client))) { indexScan.open(); assertAll( @@ -368,7 +419,7 @@ void push_down_highlight_with_arguments() { } private PushDownAssertion assertThat() { - return new PushDownAssertion(client, exprValueFactory); + return new PushDownAssertion(client, exprValueFactory, settings); } private static class PushDownAssertion { @@ -377,9 +428,10 @@ private static class PushDownAssertion { private final OpenSearchResponse response; private final OpenSearchExprValueFactory factory; - public PushDownAssertion(OpenSearchClient client, OpenSearchExprValueFactory valueFactory) { + public PushDownAssertion( + OpenSearchClient client, OpenSearchExprValueFactory valueFactory, Settings settings) { this.client = client; - this.requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, valueFactory); + this.requestBuilder = new OpenSearchRequestBuilder(QUERY_SIZE, valueFactory, settings); this.response = mock(OpenSearchResponse.class); this.factory = valueFactory; @@ -411,7 +463,9 @@ PushDownAssertion shouldQueryHighlight(QueryBuilder query, HighlightBuilder high when(client.search(request)).thenReturn(response); var indexScan = new OpenSearchIndexScan( - client, QUERY_SIZE, requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE)); + client, + QUERY_SIZE, + requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE, client)); indexScan.open(); return this; } @@ -429,7 +483,9 @@ PushDownAssertion shouldQuery(QueryBuilder expected) { when(client.search(request)).thenReturn(response); var indexScan = new OpenSearchIndexScan( - client, 10000, requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE)); + client, + 10000, + requestBuilder.build(EMPLOYEES_INDEX, 10000, CURSOR_KEEP_ALIVE, client)); indexScan.open(); return this; } From ec5c3b754e96eb0c3e223880994fd9b39e22d5a6 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana <manasvis@amazon.com> Date: Thu, 26 Sep 2024 14:51:06 -0700 Subject: [PATCH 52/96] Fix minor issues with SQL pIT refactor (#3045) Signed-off-by: Manasvini B S <manasvis@amazon.com> --- .../format/PrettyFormatRestExecutor.java | 11 ++- .../executor/join/ElasticJoinExecutor.java | 1 + .../format/PrettyFormatRestExecutorTest.java | 89 +++++++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 5f758e7d87..3344829859 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; import java.util.Map; +import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -132,13 +133,11 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction return protocol; } - private boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { + protected boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { - if (searchResponse.getHits().getTotalHits().value < queryAction.getSqlRequest().fetchSize()) { - return false; - } else { - return true; - } + return queryAction.getSqlRequest().fetchSize() != 0 + && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value + >= queryAction.getSqlRequest().fetchSize(); } else { return !Strings.isNullOrEmpty(searchResponse.getScrollId()); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index c589edcf50..e5011d1af8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -99,6 +99,7 @@ public void run() throws IOException, SqlParseException { this.metaResults.setTookImMilli(joinTimeInMilli); } catch (Exception e) { LOG.error("Failed during join query run.", e); + throw new IllegalStateException("Error occurred during join query run", e); } finally { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { try { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java new file mode 100644 index 0000000000..1387412d37 --- /dev/null +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java @@ -0,0 +1,89 @@ +package org.opensearch.sql.legacy.executor.format; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; + +import org.apache.lucene.search.TotalHits; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.sql.legacy.esdomain.LocalClusterState; +import org.opensearch.sql.legacy.query.DefaultQueryAction; +import org.opensearch.sql.legacy.request.SqlRequest; +import org.opensearch.sql.opensearch.setting.OpenSearchSettings; + +@RunWith(MockitoJUnitRunner.class) +public class PrettyFormatRestExecutorTest { + + @Mock private SearchResponse searchResponse; + @Mock private SearchHits searchHits; + @Mock private SearchHit searchHit; + @Mock private DefaultQueryAction queryAction; + @Mock private SqlRequest sqlRequest; + private PrettyFormatRestExecutor executor; + + @Before + public void setUp() { + OpenSearchSettings settings = mock(OpenSearchSettings.class); + LocalClusterState.state().setPluginSettings(settings); + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(true); + when(queryAction.getSqlRequest()).thenReturn(sqlRequest); + executor = new PrettyFormatRestExecutor("jdbc"); + } + + @Test + public void testIsDefaultCursor_fetchSizeZero() { + when(sqlRequest.fetchSize()).thenReturn(0); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_totalHitsLessThanFetchSize() { + when(sqlRequest.fetchSize()).thenReturn(10); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, new TotalHits(5, TotalHits.Relation.EQUAL_TO), 1.0F)); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_totalHitsGreaterThanOrEqualToFetchSize() { + when(sqlRequest.fetchSize()).thenReturn(5); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, new TotalHits(5, TotalHits.Relation.EQUAL_TO), 1.0F)); + + assertTrue(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_PaginationApiDisabled() { + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(false); + when(searchResponse.getScrollId()).thenReturn("someScrollId"); + + assertTrue(executor.isDefaultCursor(searchResponse, queryAction)); + } + + @Test + public void testIsDefaultCursor_PaginationApiDisabled_NoScrollId() { + when(LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) + .thenReturn(false); + when(searchResponse.getScrollId()).thenReturn(null); + + assertFalse(executor.isDefaultCursor(searchResponse, queryAction)); + } +} From ef5d14ae107e6b5b1f76aad2dae9234f34cfab8b Mon Sep 17 00:00:00 2001 From: Vamsi Manohar <reddyvam@amazon.com> Date: Thu, 26 Sep 2024 16:21:51 -0700 Subject: [PATCH 53/96] Add new maintiner @noCharger (#3046) Signed-off-by: Vamsi Manohar <reddyvam@amazon.com> --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e4af57c8a..f6aff8a146 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin \ No newline at end of file +* @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 95aaaa67fe..1bc25b6b03 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -22,6 +22,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | | Tomoyuki Morita | [ykmr1224](https://github.com/ykmr1224) | Amazon | | Lantao Jin | [LantaoJin](https://github.com/LantaoJin) | Amazon | +| Louis Chu | [noCharger](https://github.com/noCharger) | Amazon | | Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | Improving | | Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | Improving | | Andrew Carbonetto | [acarbonetto](https://github.com/acarbonetto) | Improving | From c1e623d6d3464056a4b16ae7e51dca218d73a5b6 Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Fri, 27 Sep 2024 12:19:18 -0700 Subject: [PATCH 54/96] Add job scheduler plugin to test cluster (#2957) Signed-off-by: Louis Chu <clingzhi@amazon.com> --- plugin/build.gradle | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plugin/build.gradle b/plugin/build.gradle index 7ebd0ad2d9..9df3d3dd48 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -1,3 +1,5 @@ +import java.util.concurrent.Callable + /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 @@ -114,6 +116,11 @@ configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" } + +configurations { + zipArchive +} + compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } @@ -162,6 +169,8 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: "${versions.mockito}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' + + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" } test { @@ -276,7 +285,24 @@ afterEvaluate { } } +def getJobSchedulerPlugin() { + provider(new Callable<RegularFile>() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + } + }) +} + testClusters.integTest { + plugin(getJobSchedulerPlugin()) plugin(project.tasks.bundlePlugin.archiveFile) testDistribution = "ARCHIVE" From d7710d09593ee66f5efddb0c546f539fdcaa598a Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Fri, 4 Oct 2024 15:06:39 -0700 Subject: [PATCH 55/96] Refer Antlr file using specific branch (#2893) * Refer Antlr file using specific branch Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix version to current one Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Disable auto download Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Disable auto download Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- async-query-core/README.md | 8 + async-query-core/build.gradle | 16 +- .../src/main/antlr/SqlBaseLexer.g4 | 89 +---- .../src/main/antlr/SqlBaseParser.g4 | 373 +++--------------- 4 files changed, 87 insertions(+), 399 deletions(-) diff --git a/async-query-core/README.md b/async-query-core/README.md index 815088bce6..08301c024d 100644 --- a/async-query-core/README.md +++ b/async-query-core/README.md @@ -32,3 +32,11 @@ Following is the list of extension points where the consumer of the library need - [DataSourceSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java) - [GeneralSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java) - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) To be deprecated in favor of GeneralSparkParameterComposer + +## Update Grammar files +This package uses ANTLR grammar files from `opensearch-spark` and `Spark` repositories. +To update the grammar files, update `build.gradle` file (in `downloadG4Files` task) as needed and run: +``` +./gradlew async-query-core:downloadG4Files +``` +This will overwrite the files under `src/main/antlr`. \ No newline at end of file diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index deba81735d..330b418681 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -21,10 +21,12 @@ tasks.register('downloadG4Files', Exec) { executable 'curl' - args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4' - args '-o', 'src/main/antlr/SparkSqlBase.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4' - args '-o', 'src/main/antlr/SqlBaseParser.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4' - args '-o', 'src/main/antlr/SqlBaseLexer.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4' + def opensearchSparkBranch = "0.5" + def apacheSparkVersionTag = "v3.5.1" + args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4" + args '-o', 'src/main/antlr/SparkSqlBase.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4" + args '-o', 'src/main/antlr/SqlBaseParser.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4" + args '-o', 'src/main/antlr/SqlBaseLexer.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4" } generateGrammarSource { @@ -38,12 +40,6 @@ configurations { } } -// skip download in case of offline build -if (!gradle.startParameter.offline) { - // Make sure the downloadG4File task runs before the generateGrammarSource task - generateGrammarSource.dependsOn downloadG4Files -} - dependencies { antlr "org.antlr:antlr4:4.7.1" diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index acfc0011f5..fb440ef8d3 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -69,35 +69,6 @@ lexer grammar SqlBaseLexer; public void markUnclosedComment() { has_unclosed_bracketed_comment = true; } - - /** - * When greater than zero, it's in the middle of parsing ARRAY/MAP/STRUCT type. - */ - public int complex_type_level_counter = 0; - - /** - * Increase the counter by one when hits KEYWORD 'ARRAY', 'MAP', 'STRUCT'. - */ - public void incComplexTypeLevelCounter() { - complex_type_level_counter++; - } - - /** - * Decrease the counter by one when hits close tag '>' && the counter greater than zero - * which means we are in the middle of complex type parsing. Otherwise, it's a dangling - * GT token and we do nothing. - */ - public void decComplexTypeLevelCounter() { - if (complex_type_level_counter > 0) complex_type_level_counter--; - } - - /** - * If the counter is zero, it's a shift right operator. It can be closing tags of an complex - * type definition, such as MAP<INT, ARRAY<INT>>. - */ - public boolean isShiftRightOperator() { - return complex_type_level_counter == 0 ? true : false; - } } SEMICOLON: ';'; @@ -108,7 +79,6 @@ COMMA: ','; DOT: '.'; LEFT_BRACKET: '['; RIGHT_BRACKET: ']'; -BANG: '!'; // NOTE: If you add a new token in the list below, you should update the list of keywords // and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`, and @@ -129,16 +99,14 @@ ANTI: 'ANTI'; ANY: 'ANY'; ANY_VALUE: 'ANY_VALUE'; ARCHIVE: 'ARCHIVE'; -ARRAY: 'ARRAY' {incComplexTypeLevelCounter();}; +ARRAY: 'ARRAY'; AS: 'AS'; ASC: 'ASC'; AT: 'AT'; AUTHORIZATION: 'AUTHORIZATION'; -BEGIN: 'BEGIN'; BETWEEN: 'BETWEEN'; BIGINT: 'BIGINT'; BINARY: 'BINARY'; -BINDING: 'BINDING'; BOOLEAN: 'BOOLEAN'; BOTH: 'BOTH'; BUCKET: 'BUCKET'; @@ -146,7 +114,6 @@ BUCKETS: 'BUCKETS'; BY: 'BY'; BYTE: 'BYTE'; CACHE: 'CACHE'; -CALLED: 'CALLED'; CASCADE: 'CASCADE'; CASE: 'CASE'; CAST: 'CAST'; @@ -161,7 +128,6 @@ CLUSTER: 'CLUSTER'; CLUSTERED: 'CLUSTERED'; CODEGEN: 'CODEGEN'; COLLATE: 'COLLATE'; -COLLATION: 'COLLATION'; COLLECTION: 'COLLECTION'; COLUMN: 'COLUMN'; COLUMNS: 'COLUMNS'; @@ -169,11 +135,9 @@ COMMENT: 'COMMENT'; COMMIT: 'COMMIT'; COMPACT: 'COMPACT'; COMPACTIONS: 'COMPACTIONS'; -COMPENSATION: 'COMPENSATION'; COMPUTE: 'COMPUTE'; CONCATENATE: 'CONCATENATE'; CONSTRAINT: 'CONSTRAINT'; -CONTAINS: 'CONTAINS'; COST: 'COST'; CREATE: 'CREATE'; CROSS: 'CROSS'; @@ -197,29 +161,24 @@ DATE_DIFF: 'DATE_DIFF'; DBPROPERTIES: 'DBPROPERTIES'; DEC: 'DEC'; DECIMAL: 'DECIMAL'; -DECLARE: 'DECLARE'; DEFAULT: 'DEFAULT'; DEFINED: 'DEFINED'; -DEFINER: 'DEFINER'; DELETE: 'DELETE'; DELIMITED: 'DELIMITED'; DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; -DETERMINISTIC: 'DETERMINISTIC'; DFS: 'DFS'; DIRECTORIES: 'DIRECTORIES'; DIRECTORY: 'DIRECTORY'; DISTINCT: 'DISTINCT'; DISTRIBUTE: 'DISTRIBUTE'; DIV: 'DIV'; -DO: 'DO'; DOUBLE: 'DOUBLE'; DROP: 'DROP'; ELSE: 'ELSE'; END: 'END'; ESCAPE: 'ESCAPE'; ESCAPED: 'ESCAPED'; -EVOLUTION: 'EVOLUTION'; EXCEPT: 'EXCEPT'; EXCHANGE: 'EXCHANGE'; EXCLUDE: 'EXCLUDE'; @@ -257,7 +216,6 @@ HOURS: 'HOURS'; IDENTIFIER_KW: 'IDENTIFIER'; IF: 'IF'; IGNORE: 'IGNORE'; -IMMEDIATE: 'IMMEDIATE'; IMPORT: 'IMPORT'; IN: 'IN'; INCLUDE: 'INCLUDE'; @@ -265,7 +223,6 @@ INDEX: 'INDEX'; INDEXES: 'INDEXES'; INNER: 'INNER'; INPATH: 'INPATH'; -INPUT: 'INPUT'; INPUTFORMAT: 'INPUTFORMAT'; INSERT: 'INSERT'; INTERSECT: 'INTERSECT'; @@ -273,12 +230,10 @@ INTERVAL: 'INTERVAL'; INT: 'INT'; INTEGER: 'INTEGER'; INTO: 'INTO'; -INVOKER: 'INVOKER'; IS: 'IS'; ITEMS: 'ITEMS'; JOIN: 'JOIN'; KEYS: 'KEYS'; -LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LATERAL: 'LATERAL'; LAZY: 'LAZY'; @@ -297,7 +252,7 @@ LOCKS: 'LOCKS'; LOGICAL: 'LOGICAL'; LONG: 'LONG'; MACRO: 'MACRO'; -MAP: 'MAP' {incComplexTypeLevelCounter();}; +MAP: 'MAP'; MATCHED: 'MATCHED'; MERGE: 'MERGE'; MICROSECOND: 'MICROSECOND'; @@ -306,7 +261,6 @@ MILLISECOND: 'MILLISECOND'; MILLISECONDS: 'MILLISECONDS'; MINUTE: 'MINUTE'; MINUTES: 'MINUTES'; -MODIFIES: 'MODIFIES'; MONTH: 'MONTH'; MONTHS: 'MONTHS'; MSCK: 'MSCK'; @@ -317,8 +271,7 @@ NANOSECOND: 'NANOSECOND'; NANOSECONDS: 'NANOSECONDS'; NATURAL: 'NATURAL'; NO: 'NO'; -NONE: 'NONE'; -NOT: 'NOT'; +NOT: 'NOT' | '!'; NULL: 'NULL'; NULLS: 'NULLS'; NUMERIC: 'NUMERIC'; @@ -340,6 +293,8 @@ OVERWRITE: 'OVERWRITE'; PARTITION: 'PARTITION'; PARTITIONED: 'PARTITIONED'; PARTITIONS: 'PARTITIONS'; +PERCENTILE_CONT: 'PERCENTILE_CONT'; +PERCENTILE_DISC: 'PERCENTILE_DISC'; PERCENTLIT: 'PERCENT'; PIVOT: 'PIVOT'; PLACING: 'PLACING'; @@ -352,7 +307,6 @@ PURGE: 'PURGE'; QUARTER: 'QUARTER'; QUERY: 'QUERY'; RANGE: 'RANGE'; -READS: 'READS'; REAL: 'REAL'; RECORDREADER: 'RECORDREADER'; RECORDWRITER: 'RECORDWRITER'; @@ -367,8 +321,6 @@ REPLACE: 'REPLACE'; RESET: 'RESET'; RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; -RETURN: 'RETURN'; -RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; RIGHT: 'RIGHT'; RLIKE: 'RLIKE' | 'REGEXP'; @@ -382,7 +334,6 @@ SECOND: 'SECOND'; SECONDS: 'SECONDS'; SCHEMA: 'SCHEMA'; SCHEMAS: 'SCHEMAS'; -SECURITY: 'SECURITY'; SELECT: 'SELECT'; SEMI: 'SEMI'; SEPARATED: 'SEPARATED'; @@ -394,21 +345,18 @@ SETMINUS: 'MINUS'; SETS: 'SETS'; SHORT: 'SHORT'; SHOW: 'SHOW'; -SINGLE: 'SINGLE'; SKEWED: 'SKEWED'; SMALLINT: 'SMALLINT'; SOME: 'SOME'; SORT: 'SORT'; SORTED: 'SORTED'; SOURCE: 'SOURCE'; -SPECIFIC: 'SPECIFIC'; -SQL: 'SQL'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; STRATIFY: 'STRATIFY'; STRING: 'STRING'; -STRUCT: 'STRUCT' {incComplexTypeLevelCounter();}; +STRUCT: 'STRUCT'; SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; SYNC: 'SYNC'; @@ -423,7 +371,6 @@ TEMPORARY: 'TEMPORARY' | 'TEMP'; TERMINATED: 'TERMINATED'; THEN: 'THEN'; TIME: 'TIME'; -TIMEDIFF: 'TIMEDIFF'; TIMESTAMP: 'TIMESTAMP'; TIMESTAMP_LTZ: 'TIMESTAMP_LTZ'; TIMESTAMP_NTZ: 'TIMESTAMP_NTZ'; @@ -431,7 +378,6 @@ TIMESTAMPADD: 'TIMESTAMPADD'; TIMESTAMPDIFF: 'TIMESTAMPDIFF'; TINYINT: 'TINYINT'; TO: 'TO'; -EXECUTE: 'EXECUTE'; TOUCH: 'TOUCH'; TRAILING: 'TRAILING'; TRANSACTION: 'TRANSACTION'; @@ -457,9 +403,6 @@ USER: 'USER'; USING: 'USING'; VALUES: 'VALUES'; VARCHAR: 'VARCHAR'; -VAR: 'VAR'; -VARIABLE: 'VARIABLE'; -VARIANT: 'VARIANT'; VERSION: 'VERSION'; VIEW: 'VIEW'; VIEWS: 'VIEWS'; @@ -468,7 +411,6 @@ WEEK: 'WEEK'; WEEKS: 'WEEKS'; WHEN: 'WHEN'; WHERE: 'WHERE'; -WHILE: 'WHILE'; WINDOW: 'WINDOW'; WITH: 'WITH'; WITHIN: 'WITHIN'; @@ -486,11 +428,8 @@ NEQ : '<>'; NEQJ: '!='; LT : '<'; LTE : '<=' | '!>'; -GT : '>' {decComplexTypeLevelCounter();}; +GT : '>'; GTE : '>=' | '!<'; -SHIFT_LEFT: '<<'; -SHIFT_RIGHT: '>>' {isShiftRightOperator()}?; -SHIFT_RIGHT_UNSIGNED: '>>>' {isShiftRightOperator()}?; PLUS: '+'; MINUS: '-'; @@ -503,7 +442,6 @@ PIPE: '|'; CONCAT_PIPE: '||'; HAT: '^'; COLON: ':'; -DOUBLE_COLON: '::'; ARROW: '->'; FAT_ARROW : '=>'; HENT_START: '/*+'; @@ -563,13 +501,8 @@ BIGDECIMAL_LITERAL | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}? ; -// Generalize the identifier to give a sensible INVALID_IDENTIFIER error message: -// * Unicode letters rather than a-z and A-Z only -// * URI paths for table references using paths -// We then narrow down to ANSI rules in exitUnquotedIdentifier() in the parser. IDENTIFIER - : (UNICODE_LETTER | DIGIT | '_')+ - | UNICODE_LETTER+ '://' (UNICODE_LETTER | DIGIT | '_' | '/' | '-' | '.' | '?' | '=' | '&' | '#' | '%')+ + : (LETTER | DIGIT | '_')+ ; BACKQUOTED_IDENTIFIER @@ -593,10 +526,6 @@ fragment LETTER : [A-Z] ; -fragment UNICODE_LETTER - : [\p{L}] - ; - SIMPLE_COMMENT : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN) ; @@ -606,7 +535,7 @@ BRACKETED_COMMENT ; WS - : [ \t\n\f\r\u000B\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u202F\u205F\u3000]+ -> channel(HIDDEN) + : [ \r\n\t]+ -> channel(HIDDEN) ; // Catch-all for anything we can't recognize. diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index 5b8805821b..04128216be 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -42,57 +42,8 @@ options { tokenVocab = SqlBaseLexer; } public boolean double_quoted_identifiers = false; } -compoundOrSingleStatement - : singleStatement - | singleCompoundStatement - ; - -singleCompoundStatement - : beginEndCompoundBlock SEMICOLON? EOF - ; - -beginEndCompoundBlock - : beginLabel? BEGIN compoundBody END endLabel? - ; - -compoundBody - : (compoundStatements+=compoundStatement SEMICOLON)* - ; - -compoundStatement - : statement - | setStatementWithOptionalVarKeyword - | beginEndCompoundBlock - | ifElseStatement - | whileStatement - ; - -setStatementWithOptionalVarKeyword - : SET variable? assignmentList #setVariableWithOptionalKeyword - | SET variable? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword - ; - -whileStatement - : beginLabel? WHILE booleanExpression DO compoundBody END WHILE endLabel? - ; - -ifElseStatement - : IF booleanExpression THEN conditionalBodies+=compoundBody - (ELSE IF booleanExpression THEN conditionalBodies+=compoundBody)* - (ELSE elseBody=compoundBody)? END IF - ; - singleStatement - : (statement|setResetStatement) SEMICOLON* EOF - ; - -beginLabel - : multipartIdentifier COLON - ; - -endLabel - : multipartIdentifier + : statement SEMICOLON* EOF ; singleExpression @@ -121,36 +72,33 @@ singleTableSchema statement : query #statementDefault - | executeImmediate #visitExecuteImmediate | ctes? dmlStatementNoWith #dmlStatement | USE identifierReference #use | USE namespace identifierReference #useNamespace - | SET CATALOG (errorCapturingIdentifier | stringLit) #setCatalog - | CREATE namespace (IF errorCapturingNot EXISTS)? identifierReference + | SET CATALOG (identifier | stringLit) #setCatalog + | CREATE namespace (IF NOT EXISTS)? identifierReference (commentSpec | locationSpec | (WITH (DBPROPERTIES | PROPERTIES) propertyList))* #createNamespace | ALTER namespace identifierReference SET (DBPROPERTIES | PROPERTIES) propertyList #setNamespaceProperties - | ALTER namespace identifierReference - UNSET (DBPROPERTIES | PROPERTIES) propertyList #unsetNamespaceProperties | ALTER namespace identifierReference SET locationSpec #setNamespaceLocation | DROP namespace (IF EXISTS)? identifierReference (RESTRICT | CASCADE)? #dropNamespace | SHOW namespaces ((FROM | IN) multipartIdentifier)? (LIKE? pattern=stringLit)? #showNamespaces - | createTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? + | createTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #createTable - | CREATE TABLE (IF errorCapturingNot EXISTS)? target=tableIdentifier + | CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier LIKE source=tableIdentifier (tableProvider | rowFormat | createFileFormat | locationSpec | (TBLPROPERTIES tableProps=propertyList))* #createTableLike - | replaceTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? + | replaceTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #replaceTable | ANALYZE TABLE identifierReference partitionSpec? COMPUTE STATISTICS @@ -192,7 +140,7 @@ statement SET SERDE stringLit (WITH SERDEPROPERTIES propertyList)? #setTableSerDe | ALTER TABLE identifierReference (partitionSpec)? SET SERDEPROPERTIES propertyList #setTableSerDe - | ALTER (TABLE | VIEW) identifierReference ADD (IF errorCapturingNot EXISTS)? + | ALTER (TABLE | VIEW) identifierReference ADD (IF NOT EXISTS)? partitionSpecLocation+ #addTablePartition | ALTER TABLE identifierReference from=partitionSpec RENAME TO to=partitionSpec #renameTablePartition @@ -201,15 +149,12 @@ statement | ALTER TABLE identifierReference (partitionSpec)? SET locationSpec #setTableLocation | ALTER TABLE identifierReference RECOVER PARTITIONS #recoverPartitions - | ALTER TABLE identifierReference - (clusterBySpec | CLUSTER BY NONE) #alterClusterBy | DROP TABLE (IF EXISTS)? identifierReference PURGE? #dropTable | DROP VIEW (IF EXISTS)? identifierReference #dropView | CREATE (OR REPLACE)? (GLOBAL? TEMPORARY)? - VIEW (IF errorCapturingNot EXISTS)? identifierReference + VIEW (IF NOT EXISTS)? identifierReference identifierCommentList? (commentSpec | - schemaBinding | (PARTITIONED ON identifierList) | (TBLPROPERTIES propertyList))* AS query #createView @@ -217,21 +162,12 @@ statement tableIdentifier (LEFT_PAREN colTypeList RIGHT_PAREN)? tableProvider (OPTIONS propertyList)? #createTempViewUsing | ALTER VIEW identifierReference AS? query #alterViewQuery - | ALTER VIEW identifierReference schemaBinding #alterViewSchemaBinding - | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? + | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF NOT EXISTS)? identifierReference AS className=stringLit (USING resource (COMMA resource)*)? #createFunction - | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? - identifierReference LEFT_PAREN parameters=colDefinitionList? RIGHT_PAREN - (RETURNS (dataType | TABLE LEFT_PAREN returnParams=colTypeList RIGHT_PAREN))? - routineCharacteristics - RETURN (query | expression) #createUserDefinedFunction - | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction - | DECLARE (OR REPLACE)? variable? - identifierReference dataType? variableDefaultExpression? #createVariable - | DROP TEMPORARY variable (IF EXISTS)? identifierReference #dropVariable + | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? - (statement|setResetStatement) #explain + statement #explain | SHOW TABLES ((FROM | IN) identifierReference)? (LIKE? pattern=stringLit)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? @@ -270,51 +206,22 @@ statement | (MSCK)? REPAIR TABLE identifierReference (option=(ADD|DROP|SYNC) PARTITIONS)? #repairTable | op=(ADD | LIST) identifier .*? #manageResource - | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? - identifierReference (USING indexType=identifier)? - LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN - (OPTIONS options=propertyList)? #createIndex - | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex - | unsupportedHiveNativeCommands .*? #failNativeCommand - ; - -setResetStatement - : SET COLLATION collationName=identifier #setCollation - | SET ROLE .*? #failSetRole + | SET ROLE .*? #failNativeCommand | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone - | SET variable assignmentList #setVariable - | SET variable LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariable | SET configKey EQ configValue #setQuotedConfiguration | SET configKey (EQ .*?)? #setConfiguration | SET .*? EQ configValue #setQuotedConfiguration | SET .*? #setConfiguration | RESET configKey #resetQuotedConfiguration | RESET .*? #resetConfiguration - ; - -executeImmediate - : EXECUTE IMMEDIATE queryParam=executeImmediateQueryParam (INTO targetVariable=multipartIdentifierList)? executeImmediateUsing? - ; - -executeImmediateUsing - : USING LEFT_PAREN params=namedExpressionSeq RIGHT_PAREN - | USING params=namedExpressionSeq - ; - -executeImmediateQueryParam - : stringLit - | multipartIdentifier - ; - -executeImmediateArgument - : (constant|multipartIdentifier) (AS name=errorCapturingIdentifier)? - ; - -executeImmediateArgumentSeq - : executeImmediateArgument (COMMA executeImmediateArgument)* + | CREATE INDEX (IF NOT EXISTS)? identifier ON TABLE? + identifierReference (USING indexType=identifier)? + LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN + (OPTIONS options=propertyList)? #createIndex + | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex + | unsupportedHiveNativeCommands .*? #failNativeCommand ; timezone @@ -378,17 +285,13 @@ unsupportedHiveNativeCommands ; createTableHeader - : CREATE TEMPORARY? EXTERNAL? TABLE (IF errorCapturingNot EXISTS)? identifierReference + : CREATE TEMPORARY? EXTERNAL? TABLE (IF NOT EXISTS)? identifierReference ; replaceTableHeader : (CREATE OR)? REPLACE TABLE identifierReference ; -clusterBySpec - : CLUSTER BY LEFT_PAREN multipartIdentifierList RIGHT_PAREN - ; - bucketSpec : CLUSTERED BY identifierList (SORTED BY orderedIdentifierList)? @@ -405,10 +308,6 @@ locationSpec : LOCATION stringLit ; -schemaBinding - : WITH SCHEMA (BINDING | COMPENSATION | EVOLUTION | TYPE EVOLUTION) - ; - commentSpec : COMMENT stringLit ; @@ -418,9 +317,9 @@ query ; insertInto - : INSERT OVERWRITE TABLE? identifierReference optionsClause? (partitionSpec (IF errorCapturingNot EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable - | INSERT INTO TABLE? identifierReference optionsClause? partitionSpec? (IF errorCapturingNot EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable - | INSERT INTO TABLE? identifierReference optionsClause? REPLACE whereClause #insertIntoReplaceWhere + : INSERT OVERWRITE TABLE? identifierReference (partitionSpec (IF NOT EXISTS)?)? ((BY NAME) | identifierList)? #insertOverwriteTable + | INSERT INTO TABLE? identifierReference partitionSpec? (IF NOT EXISTS)? ((BY NAME) | identifierList)? #insertIntoTable + | INSERT INTO TABLE? identifierReference REPLACE whereClause #insertIntoReplaceWhere | INSERT OVERWRITE LOCAL? DIRECTORY path=stringLit rowFormat? createFileFormat? #insertOverwriteHiveDir | INSERT OVERWRITE LOCAL? DIRECTORY (path=stringLit)? tableProvider (OPTIONS options=propertyList)? #insertOverwriteDir ; @@ -450,23 +349,16 @@ namespaces | SCHEMAS ; -variable - : VARIABLE - | VAR - ; - describeFuncName : identifierReference | stringLit | comparisonOperator | arithmeticOperator | predicateOperator - | shiftOperator - | BANG ; describeColName - : nameParts+=errorCapturingIdentifier (DOT nameParts+=errorCapturingIdentifier)* + : nameParts+=identifier (DOT nameParts+=identifier)* ; ctes @@ -485,7 +377,6 @@ createTableClauses :((OPTIONS options=expressionPropertyList) | (PARTITIONED BY partitioning=partitionFieldList) | skewSpec | - clusterBySpec | bucketSpec | rowFormat | createFileFormat | @@ -503,7 +394,7 @@ property ; propertyKey - : errorCapturingIdentifier (DOT errorCapturingIdentifier)* + : identifier (DOT identifier)* | stringLit ; @@ -553,7 +444,7 @@ dmlStatementNoWith | fromClause multiInsertQueryBody+ #multiInsertQuery | DELETE FROM identifierReference tableAlias whereClause? #deleteFromTable | UPDATE identifierReference tableAlias setClause whereClause? #updateTable - | MERGE (WITH SCHEMA EVOLUTION)? INTO target=identifierReference targetAlias=tableAlias + | MERGE INTO target=identifierReference targetAlias=tableAlias USING (source=identifierReference | LEFT_PAREN sourceQuery=query RIGHT_PAREN) sourceAlias=tableAlias ON mergeCondition=booleanExpression @@ -661,11 +552,11 @@ matchedClause : WHEN MATCHED (AND matchedCond=booleanExpression)? THEN matchedAction ; notMatchedClause - : WHEN errorCapturingNot MATCHED (BY TARGET)? (AND notMatchedCond=booleanExpression)? THEN notMatchedAction + : WHEN NOT MATCHED (BY TARGET)? (AND notMatchedCond=booleanExpression)? THEN notMatchedAction ; notMatchedBySourceClause - : WHEN errorCapturingNot MATCHED BY SOURCE (AND notMatchedBySourceCond=booleanExpression)? THEN notMatchedBySourceAction + : WHEN NOT MATCHED BY SOURCE (AND notMatchedBySourceCond=booleanExpression)? THEN notMatchedBySourceAction ; matchedAction @@ -685,10 +576,6 @@ notMatchedBySourceAction | UPDATE SET assignmentList ; -exceptClause - : EXCEPT LEFT_PAREN exceptCols=multipartIdentifierList RIGHT_PAREN - ; - assignmentList : assignment (COMMA assignment)* ; @@ -757,18 +644,18 @@ pivotClause ; pivotColumn - : identifiers+=errorCapturingIdentifier - | LEFT_PAREN identifiers+=errorCapturingIdentifier (COMMA identifiers+=errorCapturingIdentifier)* RIGHT_PAREN + : identifiers+=identifier + | LEFT_PAREN identifiers+=identifier (COMMA identifiers+=identifier)* RIGHT_PAREN ; pivotValue - : expression (AS? errorCapturingIdentifier)? + : expression (AS? identifier)? ; unpivotClause : UNPIVOT nullOperator=unpivotNullClause? LEFT_PAREN operator=unpivotOperator - RIGHT_PAREN (AS? errorCapturingIdentifier)? + RIGHT_PAREN (AS? identifier)? ; unpivotNullClause @@ -810,7 +697,7 @@ unpivotColumn ; unpivotAlias - : AS? errorCapturingIdentifier + : AS? identifier ; lateralView @@ -890,37 +777,21 @@ identifierComment relationPrimary : identifierReference temporalClause? - optionsClause? sample? tableAlias #tableName + sample? tableAlias #tableName | LEFT_PAREN query RIGHT_PAREN sample? tableAlias #aliasedQuery | LEFT_PAREN relation RIGHT_PAREN sample? tableAlias #aliasedRelation | inlineTable #inlineTableDefault2 | functionTable #tableValuedFunction ; -optionsClause - : WITH options=propertyList - ; - inlineTable : VALUES expression (COMMA expression)* tableAlias ; functionTableSubqueryArgument - : TABLE identifierReference tableArgumentPartitioning? - | TABLE LEFT_PAREN identifierReference RIGHT_PAREN tableArgumentPartitioning? - | TABLE LEFT_PAREN query RIGHT_PAREN tableArgumentPartitioning? - ; - -tableArgumentPartitioning - : ((WITH SINGLE PARTITION) - | ((PARTITION | DISTRIBUTE) BY - (((LEFT_PAREN partition+=expression (COMMA partition+=expression)* RIGHT_PAREN)) - | (expression (COMMA invalidMultiPartitionExpression=expression)+) - | partition+=expression))) - ((ORDER | SORT) BY - (((LEFT_PAREN sortItem (COMMA sortItem)* RIGHT_PAREN) - | (sortItem (COMMA invalidMultiSortItem=sortItem)+) - | sortItem)))? + : TABLE identifierReference + | TABLE LEFT_PAREN identifierReference RIGHT_PAREN + | TABLE LEFT_PAREN query RIGHT_PAREN ; functionTableNamedArgumentExpression @@ -1027,7 +898,7 @@ expressionSeq ; booleanExpression - : (NOT | BANG) booleanExpression #logicalNot + : NOT booleanExpression #logicalNot | EXISTS LEFT_PAREN query RIGHT_PAREN #exists | valueExpression predicate? #predicated | left=booleanExpression operator=AND right=booleanExpression #logicalBinary @@ -1035,20 +906,15 @@ booleanExpression ; predicate - : errorCapturingNot? kind=BETWEEN lower=valueExpression AND upper=valueExpression - | errorCapturingNot? kind=IN LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN - | errorCapturingNot? kind=IN LEFT_PAREN query RIGHT_PAREN - | errorCapturingNot? kind=RLIKE pattern=valueExpression - | errorCapturingNot? kind=(LIKE | ILIKE) quantifier=(ANY | SOME | ALL) (LEFT_PAREN RIGHT_PAREN | LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN) - | errorCapturingNot? kind=(LIKE | ILIKE) pattern=valueExpression (ESCAPE escapeChar=stringLit)? - | IS errorCapturingNot? kind=NULL - | IS errorCapturingNot? kind=(TRUE | FALSE | UNKNOWN) - | IS errorCapturingNot? kind=DISTINCT FROM right=valueExpression - ; - -errorCapturingNot - : NOT - | BANG + : NOT? kind=BETWEEN lower=valueExpression AND upper=valueExpression + | NOT? kind=IN LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN + | NOT? kind=IN LEFT_PAREN query RIGHT_PAREN + | NOT? kind=RLIKE pattern=valueExpression + | NOT? kind=(LIKE | ILIKE) quantifier=(ANY | SOME | ALL) (LEFT_PAREN RIGHT_PAREN | LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN) + | NOT? kind=(LIKE | ILIKE) pattern=valueExpression (ESCAPE escapeChar=stringLit)? + | IS NOT? kind=NULL + | IS NOT? kind=(TRUE | FALSE | UNKNOWN) + | IS NOT? kind=DISTINCT FROM right=valueExpression ; valueExpression @@ -1056,19 +922,12 @@ valueExpression | operator=(MINUS | PLUS | TILDE) valueExpression #arithmeticUnary | left=valueExpression operator=(ASTERISK | SLASH | PERCENT | DIV) right=valueExpression #arithmeticBinary | left=valueExpression operator=(PLUS | MINUS | CONCAT_PIPE) right=valueExpression #arithmeticBinary - | left=valueExpression shiftOperator right=valueExpression #shiftExpression | left=valueExpression operator=AMPERSAND right=valueExpression #arithmeticBinary | left=valueExpression operator=HAT right=valueExpression #arithmeticBinary | left=valueExpression operator=PIPE right=valueExpression #arithmeticBinary | left=valueExpression comparisonOperator right=valueExpression #comparison ; -shiftOperator - : SHIFT_LEFT - | SHIFT_RIGHT - | SHIFT_RIGHT_UNSIGNED - ; - datetimeUnit : YEAR | QUARTER | MONTH | WEEK | DAY | DAYOFYEAR @@ -1076,27 +935,24 @@ datetimeUnit ; primaryExpression - : name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER | USER | SESSION_USER) #currentLike + : name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER | USER) #currentLike | name=(TIMESTAMPADD | DATEADD | DATE_ADD) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA unitsAmount=valueExpression COMMA timestamp=valueExpression RIGHT_PAREN #timestampadd - | name=(TIMESTAMPDIFF | DATEDIFF | DATE_DIFF | TIMEDIFF) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA startTimestamp=valueExpression COMMA endTimestamp=valueExpression RIGHT_PAREN #timestampdiff + | name=(TIMESTAMPDIFF | DATEDIFF | DATE_DIFF) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA startTimestamp=valueExpression COMMA endTimestamp=valueExpression RIGHT_PAREN #timestampdiff | CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase | name=(CAST | TRY_CAST) LEFT_PAREN expression AS dataType RIGHT_PAREN #cast - | primaryExpression collateClause #collate - | primaryExpression DOUBLE_COLON dataType #castByColon | STRUCT LEFT_PAREN (argument+=namedExpression (COMMA argument+=namedExpression)*)? RIGHT_PAREN #struct | FIRST LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #first | ANY_VALUE LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #any_value | LAST LEFT_PAREN expression (IGNORE NULLS)? RIGHT_PAREN #last | POSITION LEFT_PAREN substr=valueExpression IN str=valueExpression RIGHT_PAREN #position | constant #constantDefault - | ASTERISK exceptClause? #star - | qualifiedName DOT ASTERISK exceptClause? #star + | ASTERISK #star + | qualifiedName DOT ASTERISK #star | LEFT_PAREN namedExpression (COMMA namedExpression)+ RIGHT_PAREN #rowConstructor | LEFT_PAREN query RIGHT_PAREN #subqueryExpression | functionName LEFT_PAREN (setQuantifier? argument+=functionArgument (COMMA argument+=functionArgument)*)? RIGHT_PAREN - (WITHIN GROUP LEFT_PAREN ORDER BY sortItem (COMMA sortItem)* RIGHT_PAREN)? (FILTER LEFT_PAREN WHERE where=booleanExpression RIGHT_PAREN)? (nullsOption=(IGNORE | RESPECT) NULLS)? ( OVER windowSpec)? #functionCall | identifier ARROW expression #lambda @@ -1112,6 +968,9 @@ primaryExpression FROM srcStr=valueExpression RIGHT_PAREN #trim | OVERLAY LEFT_PAREN input=valueExpression PLACING replace=valueExpression FROM position=valueExpression (FOR length=valueExpression)? RIGHT_PAREN #overlay + | name=(PERCENTILE_CONT | PERCENTILE_DISC) LEFT_PAREN percentage=valueExpression RIGHT_PAREN + WITHIN GROUP LEFT_PAREN ORDER BY sortItem RIGHT_PAREN + (FILTER LEFT_PAREN WHERE where=booleanExpression RIGHT_PAREN)? ( OVER windowSpec)? #percentile ; literalType @@ -1188,10 +1047,6 @@ colPosition : position=FIRST | position=AFTER afterCol=errorCapturingIdentifier ; -collateClause - : COLLATE collationName=identifier - ; - type : BOOLEAN | TINYINT | BYTE @@ -1202,14 +1057,13 @@ type | DOUBLE | DATE | TIMESTAMP | TIMESTAMP_NTZ | TIMESTAMP_LTZ - | STRING collateClause? + | STRING | CHARACTER | CHAR | VARCHAR | BINARY | DECIMAL | DEC | NUMERIC | VOID | INTERVAL - | VARIANT | ARRAY | STRUCT | MAP | unsupportedType=identifier ; @@ -1234,7 +1088,7 @@ qualifiedColTypeWithPosition ; colDefinitionDescriptorWithPosition - : errorCapturingNot NULL + : NOT NULL | defaultExpression | commentSpec | colPosition @@ -1244,28 +1098,24 @@ defaultExpression : DEFAULT expression ; -variableDefaultExpression - : (DEFAULT | EQ) expression - ; - colTypeList : colType (COMMA colType)* ; colType - : colName=errorCapturingIdentifier dataType (errorCapturingNot NULL)? commentSpec? + : colName=errorCapturingIdentifier dataType (NOT NULL)? commentSpec? ; -colDefinitionList - : colDefinition (COMMA colDefinition)* +createOrReplaceTableColTypeList + : createOrReplaceTableColType (COMMA createOrReplaceTableColType)* ; -colDefinition +createOrReplaceTableColType : colName=errorCapturingIdentifier dataType colDefinitionOption* ; colDefinitionOption - : errorCapturingNot NULL + : NOT NULL | defaultExpression | generationExpression | commentSpec @@ -1280,49 +1130,9 @@ complexColTypeList ; complexColType - : errorCapturingIdentifier COLON? dataType (errorCapturingNot NULL)? commentSpec? - ; - -routineCharacteristics - : (routineLanguage - | specificName - | deterministic - | sqlDataAccess - | nullCall - | commentSpec - | rightsClause)* - ; - -routineLanguage - : LANGUAGE (SQL | IDENTIFIER) - ; - -specificName - : SPECIFIC specific=errorCapturingIdentifier - ; - -deterministic - : DETERMINISTIC - | errorCapturingNot DETERMINISTIC + : identifier COLON? dataType (NOT NULL)? commentSpec? ; -sqlDataAccess - : access=NO SQL - | access=CONTAINS SQL - | access=READS SQL DATA - | access=MODIFIES SQL DATA - ; - -nullCall - : RETURNS NULL ON NULL INPUT - | CALLED ON NULL INPUT - ; - -rightsClause - : SQL SECURITY INVOKER - | SQL SECURITY DEFINER - ; - whenClause : WHEN condition=expression THEN result=expression ; @@ -1427,7 +1237,7 @@ alterColumnAction : TYPE dataType | commentSpec | colPosition - | setOrDrop=(SET | DROP) errorCapturingNot NULL + | setOrDrop=(SET | DROP) NOT NULL | SET defaultExpression | dropDefault=DROP DEFAULT ; @@ -1470,19 +1280,16 @@ ansiNonReserved | ARRAY | ASC | AT - | BEGIN | BETWEEN | BIGINT | BINARY | BINARY_HEX - | BINDING | BOOLEAN | BUCKET | BUCKETS | BY | BYTE | CACHE - | CALLED | CASCADE | CATALOG | CATALOGS @@ -1499,10 +1306,8 @@ ansiNonReserved | COMMIT | COMPACT | COMPACTIONS - | COMPENSATION | COMPUTE | CONCATENATE - | CONTAINS | COST | CUBE | CURRENT @@ -1520,25 +1325,20 @@ ansiNonReserved | DBPROPERTIES | DEC | DECIMAL - | DECLARE | DEFAULT | DEFINED - | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE - | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE | DIV - | DO | DOUBLE | DROP | ESCAPED - | EVOLUTION | EXCHANGE | EXCLUDE | EXISTS @@ -1564,22 +1364,18 @@ ansiNonReserved | IDENTIFIER_KW | IF | IGNORE - | IMMEDIATE | IMPORT | INCLUDE | INDEX | INDEXES | INPATH - | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL - | INVOKER | ITEMS | KEYS - | LANGUAGE | LAST | LAZY | LIKE @@ -1604,7 +1400,6 @@ ansiNonReserved | MILLISECONDS | MINUTE | MINUTES - | MODIFIES | MONTH | MONTHS | MSCK @@ -1614,7 +1409,6 @@ ansiNonReserved | NANOSECOND | NANOSECONDS | NO - | NONE | NULLS | NUMERIC | OF @@ -1639,7 +1433,6 @@ ansiNonReserved | QUARTER | QUERY | RANGE - | READS | REAL | RECORDREADER | RECORDWRITER @@ -1653,8 +1446,6 @@ ansiNonReserved | RESET | RESPECT | RESTRICT - | RETURN - | RETURNS | REVOKE | RLIKE | ROLE @@ -1667,7 +1458,6 @@ ansiNonReserved | SCHEMAS | SECOND | SECONDS - | SECURITY | SEMI | SEPARATED | SERDE @@ -1677,13 +1467,11 @@ ansiNonReserved | SETS | SHORT | SHOW - | SINGLE | SKEWED | SMALLINT | SORT | SORTED | SOURCE - | SPECIFIC | START | STATISTICS | STORED @@ -1701,7 +1489,6 @@ ansiNonReserved | TBLPROPERTIES | TEMPORARY | TERMINATED - | TIMEDIFF | TIMESTAMP | TIMESTAMP_LTZ | TIMESTAMP_NTZ @@ -1727,16 +1514,12 @@ ansiNonReserved | USE | VALUES | VARCHAR - | VAR - | VARIABLE - | VARIANT | VERSION | VIEW | VIEWS | VOID | WEEK | WEEKS - | WHILE | WINDOW | YEAR | YEARS @@ -1789,12 +1572,10 @@ nonReserved | ASC | AT | AUTHORIZATION - | BEGIN | BETWEEN | BIGINT | BINARY | BINARY_HEX - | BINDING | BOOLEAN | BOTH | BUCKET @@ -1802,7 +1583,6 @@ nonReserved | BY | BYTE | CACHE - | CALLED | CASCADE | CASE | CAST @@ -1817,7 +1597,6 @@ nonReserved | CLUSTERED | CODEGEN | COLLATE - | COLLATION | COLLECTION | COLUMN | COLUMNS @@ -1825,11 +1604,9 @@ nonReserved | COMMIT | COMPACT | COMPACTIONS - | COMPENSATION | COMPUTE | CONCATENATE | CONSTRAINT - | CONTAINS | COST | CREATE | CUBE @@ -1852,32 +1629,26 @@ nonReserved | DBPROPERTIES | DEC | DECIMAL - | DECLARE | DEFAULT | DEFINED - | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE - | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY | DISTINCT | DISTRIBUTE | DIV - | DO | DOUBLE | DROP | ELSE | END | ESCAPE | ESCAPED - | EVOLUTION | EXCHANGE | EXCLUDE - | EXECUTE | EXISTS | EXPLAIN | EXPORT @@ -1910,25 +1681,21 @@ nonReserved | IDENTIFIER_KW | IF | IGNORE - | IMMEDIATE | IMPORT | IN | INCLUDE | INDEX | INDEXES | INPATH - | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL | INTO - | INVOKER | IS | ITEMS | KEYS - | LANGUAGE | LAST | LAZY | LEADING @@ -1955,7 +1722,6 @@ nonReserved | MILLISECONDS | MINUTE | MINUTES - | MODIFIES | MONTH | MONTHS | MSCK @@ -1965,7 +1731,6 @@ nonReserved | NANOSECOND | NANOSECONDS | NO - | NONE | NOT | NULL | NULLS @@ -1987,6 +1752,8 @@ nonReserved | PARTITION | PARTITIONED | PARTITIONS + | PERCENTILE_CONT + | PERCENTILE_DISC | PERCENTLIT | PIVOT | PLACING @@ -1999,7 +1766,6 @@ nonReserved | QUARTER | QUERY | RANGE - | READS | REAL | RECORDREADER | RECORDWRITER @@ -2014,8 +1780,6 @@ nonReserved | RESET | RESPECT | RESTRICT - | RETURN - | RETURNS | REVOKE | RLIKE | ROLE @@ -2028,7 +1792,6 @@ nonReserved | SCHEMAS | SECOND | SECONDS - | SECURITY | SELECT | SEPARATED | SERDE @@ -2038,15 +1801,12 @@ nonReserved | SETS | SHORT | SHOW - | SINGLE | SKEWED | SMALLINT | SOME | SORT | SORTED | SOURCE - | SPECIFIC - | SQL | START | STATISTICS | STORED @@ -2067,7 +1827,6 @@ nonReserved | TERMINATED | THEN | TIME - | TIMEDIFF | TIMESTAMP | TIMESTAMP_LTZ | TIMESTAMP_NTZ @@ -2098,16 +1857,12 @@ nonReserved | USER | VALUES | VARCHAR - | VAR - | VARIABLE - | VARIANT | VERSION | VIEW | VIEWS | VOID | WEEK | WEEKS - | WHILE | WHEN | WHERE | WINDOW From 564ab60ba1225dcdf30b129fbef5685979a3ab8a Mon Sep 17 00:00:00 2001 From: Aparajita Pandey <37636092+aparajita31pandey@users.noreply.github.com> Date: Tue, 8 Oct 2024 20:05:56 +0530 Subject: [PATCH 56/96] Resolve Alias Issues in Legacy SQL with Filters (#2960) * Fix: Pagination of index aliases is not supported Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * fix: remove extra debug log Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * Integration testadded Signed-off-by: Aparajita Pandey<aparajita31pandey@gmail.com> Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * rollback change Signed-off-by: Aparajita Pandey<aparajita31pandey@gmail.com> Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * Integration TestAdded Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * Integration TestAdded Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> * SpotlessCheck Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> --------- Signed-off-by: Aparajita Pandey <aparajita31pandey@gmail.com> --- .../sql/legacy/SQLIntegTestCase.java | 6 +++ .../org/opensearch/sql/sql/PaginationIT.java | 50 +++++++++++++++++++ .../executor/format/SelectResultSet.java | 8 ++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index c6d15a305d..eed4e29c9c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -462,6 +462,12 @@ protected String makeRequest(String query, int fetch_size) { return String.format("{ \"fetch_size\": \"%s\", \"query\": \"%s\" }", fetch_size, query); } + protected String makeRequest(String query, int fetch_size, String filterQuery) { + return String.format( + "{ \"fetch_size\": \"%s\", \"query\": \"%s\", \"filter\" : %s }", + fetch_size, query, filterQuery); + } + protected String makeFetchLessRequest(String query) { return String.format("{\n" + " \"query\": \"%s\"\n" + "}", query); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java index 49ef7c583e..fbe1e378e2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; @@ -18,6 +19,7 @@ import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; @@ -215,4 +217,52 @@ public void testQueryWithoutFrom() { assertEquals(1, response.getInt("total")); assertEquals(1, response.getJSONArray("datarows").getJSONArray(0).getInt(0)); } + + @Test + public void testAlias() throws Exception { + String indexName = Index.ONLINE.getName(); + String aliasName = "alias_ONLINE"; + String filterQuery = "{\n" + " \"term\": {\n" + " \"107\": 72 \n" + " }\n" + "}"; + + // Execute the SQL query with filter + String selectQuery = "SELECT * FROM " + TEST_INDEX_ONLINE; + JSONObject initialResponse = + new JSONObject(executeFetchQuery(selectQuery, 10, "jdbc", filterQuery)); + assertEquals(initialResponse.getInt("size"), 10); + + // Create an alias + String createAliasQuery = + String.format( + "{ \"actions\": [ { \"add\": { \"index\": \"%s\", \"alias\": \"%s\" } } ] }", + indexName, aliasName); + Request createAliasRequest = new Request("POST", "/_aliases"); + createAliasRequest.setJsonEntity(createAliasQuery); + JSONObject aliasResponse = new JSONObject(executeRequest(createAliasRequest)); + + // Assert that alias creation was acknowledged + assertTrue(aliasResponse.getBoolean("acknowledged")); + + // Query using the alias + String aliasSelectQuery = String.format("SELECT * FROM %s", aliasName); + JSONObject aliasQueryResponse = new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc")); + assertEquals(4, aliasQueryResponse.getInt("size")); + + // Query using the alias with filter + JSONObject aliasFilteredResponse = + new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc", filterQuery)); + assertEquals(aliasFilteredResponse.getInt("size"), 4); + } + + private String executeFetchQuery(String query, int fetchSize, String requestType, String filter) + throws IOException { + String endpoint = "/_plugins/_sql?format=" + requestType; + String requestBody = makeRequest(query, fetchSize, filter); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 9d1862023c..bc5c1fb162 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -26,6 +26,8 @@ import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.opensearch.action.search.ClearScrollResponse; @@ -164,7 +166,11 @@ private void populateResultSetFromDefaultCursor(DefaultCursor cursor) { private void loadFromEsState(Query query) { String indexName = fetchIndexName(query); String[] fieldNames = fetchFieldsAsArray(query); - + GetAliasesResponse getAliasesResponse = + client.admin().indices().getAliases(new GetAliasesRequest(indexName)).actionGet(); + if (getAliasesResponse != null && !getAliasesResponse.getAliases().isEmpty()) { + indexName = getAliasesResponse.getAliases().keySet().iterator().next(); + } // Reset boolean in the case of JOIN query where multiple calls to loadFromEsState() are made selectAll = isSimpleQuerySelectAll(query) || isJoinQuerySelectAll(query, fieldNames); From ac8678c60c3e98c7e3791b2abb2b0e71c643256c Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 8 Oct 2024 08:50:48 -0700 Subject: [PATCH 57/96] Disable join types in validators (#3056) * Disable join types in validators Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Removed methods from SQLQueryValidationVisitor due to grammar file change Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../S3GlueGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidationVisitor.java | 28 ------------------ .../SecurityLakeGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidatorTest.java | 29 ++++++++----------- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java index e7a0ce1b36..9ed1fd9e9e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -9,20 +9,25 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; @@ -50,6 +55,11 @@ public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidat HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index 9ec0fb0109..d50503418e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -9,15 +9,12 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTableColumnsContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTablePartitionContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterClusterByContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterTableAlterColumnContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewQueryContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewSchemaBindingContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeTablesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CacheTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClearCacheContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClusterBySpecContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateNamespaceContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableLikeContext; @@ -81,7 +78,6 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsetNamespacePropertiesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; /** This visitor validate grammar using GrammarElementValidator */ @@ -101,12 +97,6 @@ public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { return super.visitSetNamespaceProperties(ctx); } - @Override - public Void visitUnsetNamespaceProperties(UnsetNamespacePropertiesContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitUnsetNamespaceProperties(ctx); - } - @Override public Void visitAddTableColumns(AddTableColumnsContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -173,12 +163,6 @@ public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { return super.visitRecoverPartitions(ctx); } - @Override - public Void visitAlterClusterBy(AlterClusterByContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitAlterClusterBy(ctx); - } - @Override public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -191,12 +175,6 @@ public Void visitAlterViewQuery(AlterViewQueryContext ctx) { return super.visitAlterViewQuery(ctx); } - @Override - public Void visitAlterViewSchemaBinding(AlterViewSchemaBindingContext ctx) { - validateAllowed(GrammarElement.ALTER_VIEW); - return super.visitAlterViewSchemaBinding(ctx); - } - @Override public Void visitRenameTable(RenameTableContext ctx) { if (ctx.VIEW() != null) { @@ -337,12 +315,6 @@ public Void visitCtes(CtesContext ctx) { return super.visitCtes(ctx); } - @Override - public Void visitClusterBySpec(ClusterBySpecContext ctx) { - validateAllowed(GrammarElement.CLUSTER_BY); - return super.visitClusterBySpec(ctx); - } - @Override public Void visitQueryOrganization(QueryOrganizationContext ctx) { if (ctx.CLUSTER() != null) { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java index ca8f2b5bdd..7dd2b0ee89 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; @@ -24,9 +25,12 @@ import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; @@ -35,6 +39,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; @@ -76,6 +81,11 @@ public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementV HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 6726b56994..695a083809 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -34,11 +34,9 @@ private enum TestElement { // DDL Statements ALTER_DATABASE( "ALTER DATABASE inventory SET DBPROPERTIES ('Edit-date' = '01/01/2001');", - "ALTER DATABASE dbx.tab1 UNSET PROPERTIES ('winner');", "ALTER DATABASE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_TABLE( "ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');", - "ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');", "ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);", "ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);", "ALTER TABLE StudentInfo RENAME COLUMN name TO FirstName;", @@ -50,12 +48,10 @@ private enum TestElement { "ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);", "ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways';", "ALTER TABLE dbx.tab1 RECOVER PARTITIONS;", - "ALTER TABLE dbx.tab1 CLUSTER BY NONE;", "ALTER TABLE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_VIEW( "ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;", - "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;", - "ALTER VIEW tempdb1.v2 WITH SCHEMA BINDING"), + "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;"), CREATE_DATABASE("CREATE DATABASE IF NOT EXISTS customer_db;\n"), CREATE_FUNCTION("CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar';"), CREATE_TABLE( @@ -94,8 +90,7 @@ private enum TestElement { EXPLAIN("EXPLAIN SELECT * FROM my_table;"), COMMON_TABLE_EXPRESSION( "WITH cte AS (SELECT * FROM my_table WHERE age > 30) SELECT * FROM cte;"), - CLUSTER_BY_CLAUSE( - "SELECT * FROM my_table CLUSTER BY age;", "ALTER TABLE testTable CLUSTER BY (age);"), + CLUSTER_BY_CLAUSE("SELECT * FROM my_table CLUSTER BY age;"), DISTRIBUTE_BY_CLAUSE("SELECT * FROM my_table DISTRIBUTE BY name;"), GROUP_BY_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name;"), HAVING_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name HAVING count(*) > 1;"), @@ -370,12 +365,12 @@ void testS3glueQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE); @@ -487,12 +482,12 @@ void testSecurityLakeQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE); From 063015c8290551090b9025687fa40078d435adfb Mon Sep 17 00:00:00 2001 From: qianheng <qianheng@amazon.com> Date: Thu, 10 Oct 2024 22:10:11 +0800 Subject: [PATCH 58/96] CVE-2024-6345 fix (#2931) Signed-off-by: Heng Qian <qianheng@amazon.com> --- doctest/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doctest/requirements.txt b/doctest/requirements.txt index 7d178b80ae..7d5e2afa2d 100644 --- a/doctest/requirements.txt +++ b/doctest/requirements.txt @@ -1 +1,2 @@ -zc.customdoctests==1.0.1 \ No newline at end of file +zc.customdoctests==1.0.1 +setuptools>=70.0.0 From 7477fd8e0a1cf46e31ac2b90e094720fe3d92598 Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Thu, 10 Oct 2024 14:09:38 -0700 Subject: [PATCH 59/96] Improve error handling for malformed query cursors (#3066) * Add reproducer for malformed cursor handling Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Handle malformed query cursors Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Move malformed cursor test to correct file Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Apply spotless Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --------- Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .../java/org/opensearch/sql/legacy/CursorIT.java | 11 +++++++++++ .../legacy/executor/cursor/CursorResultExecutor.java | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index d0c2f19f42..2bcb2902a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -440,6 +440,17 @@ public void noPaginationWithNonJDBCFormat() throws IOException { assertThat(rows.length, equalTo(1000)); } + @Test + public void testMalformedCursorGracefullyHandled() throws IOException { + ResponseException result = + assertThrows( + "Expected query with malformed cursor to raise error, but didn't", + ResponseException.class, + () -> executeCursorQuery("d:a11b4db33f")); + assertTrue(result.getMessage().contains("Malformed cursor")); + assertEquals(result.getResponse().getStatusLine().getStatusCode(), 400); + } + public void verifyWithAndWithoutPaginationResponse( String sqlQuery, String cursorQuery, int fetch_size, boolean shouldFallBackToV1) throws IOException { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index 0af3ca243b..4947d06b2f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -20,6 +20,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.search.SearchHit; @@ -36,6 +37,7 @@ import org.opensearch.sql.legacy.pit.PointInTimeHandler; import org.opensearch.sql.legacy.pit.PointInTimeHandlerImpl; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory; public class CursorResultExecutor implements CursorRestExecutor { @@ -58,7 +60,15 @@ public void execute(Client client, Map<String, String> params, RestChannel chann } catch (IllegalArgumentException | JSONException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_CUS).increment(); LOG.error("Error parsing the cursor", e); - channel.sendResponse(new BytesRestResponse(channel, e)); + channel.sendResponse( + new BytesRestResponse( + RestStatus.BAD_REQUEST, + "application/json; charset=UTF-8", + ErrorMessageFactory.createErrorMessage( + new IllegalArgumentException( + "Malformed cursor: unable to extract cursor information"), + RestStatus.BAD_REQUEST.getStatus()) + .toString())); } catch (OpenSearchException e) { int status = (e.status().getStatus()); if (status > 399 && status < 500) { From cf1564b6aeb03e49f19032d14bf69d51d6332984 Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Fri, 11 Oct 2024 09:55:41 -0700 Subject: [PATCH 60/96] Update doctest code for more consistent runs (#3053) Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- docs/user/beyond/partiql.rst | 20 +- docs/user/dql/aggregations.rst | 216 +- docs/user/dql/basics.rst | 98 +- docs/user/dql/complex.rst | 28 +- docs/user/dql/expressions.rst | 150 +- docs/user/dql/functions.rst | 1804 ++++++++--------- docs/user/dql/metadata.rst | 80 +- docs/user/dql/window.rst | 208 +- docs/user/general/comments.rst | 20 +- docs/user/general/datatypes.rst | 60 +- docs/user/general/identifiers.rst | 76 +- docs/user/general/values.rst | 32 +- .../admin/connectors/prometheus_connector.rst | 80 +- docs/user/ppl/admin/cross_cluster_search.rst | 16 +- docs/user/ppl/cmd/ad.rst | 44 +- docs/user/ppl/cmd/dedup.rst | 70 +- docs/user/ppl/cmd/describe.rst | 50 +- docs/user/ppl/cmd/eval.rst | 48 +- docs/user/ppl/cmd/fields.rst | 32 +- docs/user/ppl/cmd/grok.rst | 16 +- docs/user/ppl/cmd/head.rst | 44 +- docs/user/ppl/cmd/information_schema.rst | 30 +- docs/user/ppl/cmd/ml.rst | 44 +- docs/user/ppl/cmd/parse.rst | 14 +- docs/user/ppl/cmd/patterns.rst | 16 +- docs/user/ppl/cmd/rare.rst | 28 +- docs/user/ppl/cmd/rename.rst | 32 +- docs/user/ppl/cmd/search.rst | 28 +- docs/user/ppl/cmd/showdatasources.rst | 10 +- docs/user/ppl/cmd/sort.rst | 80 +- docs/user/ppl/cmd/stats.rst | 250 +-- docs/user/ppl/cmd/top.rst | 34 +- docs/user/ppl/cmd/where.rst | 12 +- docs/user/ppl/functions/condition.rst | 132 +- docs/user/ppl/functions/conversion.rst | 30 +- docs/user/ppl/functions/datetime.rst | 850 ++++---- docs/user/ppl/functions/expressions.rst | 60 +- docs/user/ppl/functions/math.rst | 212 +- docs/user/ppl/functions/relevance.rst | 110 +- docs/user/ppl/functions/string.rst | 130 +- docs/user/ppl/functions/system.rst | 10 +- docs/user/ppl/general/datatypes.rst | 30 +- docs/user/ppl/general/identifiers.rst | 62 +- doctest/build.gradle | 12 +- doctest/test_docs.py | 28 +- 45 files changed, 2734 insertions(+), 2702 deletions(-) diff --git a/docs/user/beyond/partiql.rst b/docs/user/beyond/partiql.rst index 76fec8405d..6a93a55c94 100644 --- a/docs/user/beyond/partiql.rst +++ b/docs/user/beyond/partiql.rst @@ -202,11 +202,11 @@ Selecting top level for object fields, object fields of array value and nested f os> SELECT city, accounts, projects FROM people; fetched rows / total rows = 1/1 - +-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+ - | city | accounts | projects | - |-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | - +-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+ + +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ + | city | accounts | projects | + |-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | + +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ Example 2: Selecting Deeper Levels ---------------------------------- @@ -215,11 +215,11 @@ Selecting at deeper levels for object fields of regular value returns inner fiel os> SELECT city.location, city.location.latitude FROM people; fetched rows / total rows = 1/1 - +--------------------+--------------------------+ - | city.location | city.location.latitude | - |--------------------+--------------------------| - | {'latitude': 10.5} | 10.5 | - +--------------------+--------------------------+ + +--------------------+------------------------+ + | city.location | city.location.latitude | + |--------------------+------------------------| + | {'latitude': 10.5} | 10.5 | + +--------------------+------------------------+ For selecting second level for nested fields, please read on and find more details in the following sections. diff --git a/docs/user/dql/aggregations.rst b/docs/user/dql/aggregations.rst index 42db4cdb4f..adf933c09c 100644 --- a/docs/user/dql/aggregations.rst +++ b/docs/user/dql/aggregations.rst @@ -34,12 +34,12 @@ The group by expression could be identifier:: os> SELECT gender, sum(age) FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Ordinal @@ -49,12 +49,12 @@ The group by expression could be ordinal:: os> SELECT gender, sum(age) FROM accounts GROUP BY 1; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Expression @@ -64,14 +64,14 @@ The group by expression could be expression:: os> SELECT abs(account_number), sum(age) FROM accounts GROUP BY abs(account_number); fetched rows / total rows = 4/4 - +-----------------------+------------+ - | abs(account_number) | sum(age) | - |-----------------------+------------| - | 1 | 32 | - | 13 | 28 | - | 18 | 33 | - | 6 | 36 | - +-----------------------+------------+ + +---------------------+----------+ + | abs(account_number) | sum(age) | + |---------------------+----------| + | 1 | 32 | + | 13 | 28 | + | 18 | 33 | + | 6 | 36 | + +---------------------+----------+ Aggregation @@ -91,12 +91,12 @@ The aggregation could be used select:: os> SELECT gender, sum(age) FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | F | 28 | - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | F | 28 | + | M | 101 | + +--------+----------+ Expression over Aggregation --------------------------- @@ -105,12 +105,12 @@ The aggregation could be used as arguments of expression:: os> SELECT gender, sum(age) * 2 as sum2 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sum2 | - |----------+--------| - | F | 56 | - | M | 202 | - +----------+--------+ + +--------+------+ + | gender | sum2 | + |--------+------| + | F | 56 | + | M | 202 | + +--------+------+ Expression as argument of Aggregation ------------------------------------- @@ -119,12 +119,12 @@ The aggregation could has expression as arguments:: os> SELECT gender, sum(age * 2) as sum2 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sum2 | - |----------+--------| - | F | 56 | - | M | 202 | - +----------+--------+ + +--------+------+ + | gender | sum2 | + |--------+------| + | F | 56 | + | M | 202 | + +--------+------+ COUNT Aggregations ------------------ @@ -150,12 +150,12 @@ Example:: os> SELECT gender, count(*) as countV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+----------+ - | gender | countV | - |----------+----------| - | F | 1 | - | M | 3 | - +----------+----------+ + +--------+--------+ + | gender | countV | + |--------+--------| + | F | 1 | + | M | 3 | + +--------+--------+ SUM --- @@ -169,12 +169,12 @@ Example:: os> SELECT gender, sum(age) as sumV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------+ - | gender | sumV | - |----------+--------| - | F | 28 | - | M | 101 | - +----------+--------+ + +--------+------+ + | gender | sumV | + |--------+------| + | F | 28 | + | M | 101 | + +--------+------+ AVG --- @@ -188,12 +188,12 @@ Example:: os> SELECT gender, avg(age) as avgV FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+--------------------+ - | gender | avgV | - |----------+--------------------| - | F | 28.0 | - | M | 33.666666666666664 | - +----------+--------------------+ + +--------+--------------------+ + | gender | avgV | + |--------+--------------------| + | F | 28.0 | + | M | 33.666666666666664 | + +--------+--------------------+ MAX --- @@ -207,11 +207,11 @@ Example:: os> SELECT max(age) as maxV FROM accounts; fetched rows / total rows = 1/1 - +--------+ - | maxV | - |--------| - | 36 | - +--------+ + +------+ + | maxV | + |------| + | 36 | + +------+ MIN --- @@ -225,11 +225,11 @@ Example:: os> SELECT min(age) as minV FROM accounts; fetched rows / total rows = 1/1 - +--------+ - | minV | - |--------| - | 28 | - +--------+ + +------+ + | minV | + |------| + | 28 | + +------+ VAR_POP ------- @@ -364,11 +364,11 @@ To get the count of distinct values of a field, you can add a keyword ``DISTINCT os> SELECT COUNT(DISTINCT gender), COUNT(gender) FROM accounts; fetched rows / total rows = 1/1 - +--------------------------+-----------------+ - | COUNT(DISTINCT gender) | COUNT(gender) | - |--------------------------+-----------------| - | 2 | 4 | - +--------------------------+-----------------+ + +------------------------+---------------+ + | COUNT(DISTINCT gender) | COUNT(gender) | + |------------------------+---------------| + | 2 | 4 | + +------------------------+---------------+ PERCENTILE or PERCENTILE_APPROX ------------------------------- @@ -382,12 +382,12 @@ Example:: os> SELECT gender, percentile(age, 90) as p90 FROM accounts GROUP BY gender; fetched rows / total rows = 2/2 - +----------+-------+ - | gender | p90 | - |----------+-------| - | F | 28 | - | M | 36 | - +----------+-------+ + +--------+-----+ + | gender | p90 | + |--------+-----| + | F | 28 | + | M | 36 | + +--------+-----+ HAVING Clause ============= @@ -413,11 +413,11 @@ Here is an example for typical use of ``HAVING`` clause:: ... GROUP BY gender ... HAVING sum(age) > 100; fetched rows / total rows = 1/1 - +----------+------------+ - | gender | sum(age) | - |----------+------------| - | M | 101 | - +----------+------------+ + +--------+----------+ + | gender | sum(age) | + |--------+----------| + | M | 101 | + +--------+----------+ Here is another example for using alias in ``HAVING`` condition. Note that if an identifier is ambiguous, for example present both as a select alias and an index field, preference is alias. This means the identifier will be replaced by expression aliased in ``SELECT`` clause:: @@ -427,11 +427,11 @@ Here is another example for using alias in ``HAVING`` condition. Note that if an ... GROUP BY gender ... HAVING s > 100; fetched rows / total rows = 1/1 - +----------+-----+ - | gender | s | - |----------+-----| - | M | 101 | - +----------+-----+ + +--------+-----+ + | gender | s | + |--------+-----| + | M | 101 | + +--------+-----+ HAVING without GROUP BY ----------------------- @@ -443,11 +443,11 @@ Additionally, a ``HAVING`` clause can work without ``GROUP BY`` clause. This is ... FROM accounts ... HAVING sum(age) > 100; fetched rows / total rows = 1/1 - +------------------------+ - | 'Total of age > 100' | - |------------------------| - | Total of age > 100 | - +------------------------+ + +----------------------+ + | 'Total of age > 100' | + |----------------------| + | Total of age > 100 | + +----------------------+ FILTER Clause @@ -465,12 +465,12 @@ The group by aggregation with ``FILTER`` clause can set different conditions for os> SELECT avg(age) FILTER(WHERE balance > 10000) AS filtered, gender FROM accounts GROUP BY gender fetched rows / total rows = 2/2 - +------------+----------+ - | filtered | gender | - |------------+----------| - | 28.0 | F | - | 32.0 | M | - +------------+----------+ + +----------+--------+ + | filtered | gender | + |----------+--------| + | 28.0 | F | + | 32.0 | M | + +----------+--------+ FILTER without GROUP BY ----------------------- @@ -482,11 +482,11 @@ The ``FILTER`` clause can be used in aggregation functions without GROUP BY as w ... count(*) FILTER(WHERE age > 34) AS filtered ... FROM accounts fetched rows / total rows = 1/1 - +--------------+------------+ - | unfiltered | filtered | - |--------------+------------| - | 4 | 1 | - +--------------+------------+ + +------------+----------+ + | unfiltered | filtered | + |------------+----------| + | 4 | 1 | + +------------+----------+ Distinct count aggregate with FILTER ------------------------------------ @@ -495,9 +495,9 @@ The ``FILTER`` clause is also used in distinct count to do the filtering before os> SELECT COUNT(DISTINCT firstname) FILTER(WHERE age > 30) AS distinct_count FROM accounts fetched rows / total rows = 1/1 - +------------------+ - | distinct_count | - |------------------| - | 3 | - +------------------+ + +----------------+ + | distinct_count | + |----------------| + | 3 | + +----------------+ diff --git a/docs/user/dql/basics.rst b/docs/user/dql/basics.rst index a03ac4db70..a59f193086 100644 --- a/docs/user/dql/basics.rst +++ b/docs/user/dql/basics.rst @@ -191,14 +191,14 @@ This produces results like this for example:: os> SELECT firstname, lastname, _index, _sort FROM accounts; fetched rows / total rows = 4/4 - +-------------+------------+----------+---------+ - | firstname | lastname | _index | _sort | - |-------------+------------+----------+---------| - | Amber | Duke | accounts | -2 | - | Hattie | Bond | accounts | -2 | - | Nanette | Bates | accounts | -2 | - | Dale | Adams | accounts | -2 | - +-------------+------------+----------+---------+ + +-----------+----------+----------+-------+ + | firstname | lastname | _index | _sort | + |-----------+----------+----------+-------| + | Amber | Duke | accounts | -2 | + | Hattie | Bond | accounts | -2 | + | Nanette | Bates | accounts | -2 | + | Dale | Adams | accounts | -2 | + +-----------+----------+----------+-------+ Example 3: Using Field Alias ---------------------------- @@ -303,13 +303,13 @@ In fact your can use any expression in a ``DISTINCT`` clause as follows:: os> SELECT DISTINCT SUBSTRING(lastname, 1, 1) FROM accounts; fetched rows / total rows = 3/3 - +-----------------------------+ - | SUBSTRING(lastname, 1, 1) | - |-----------------------------| - | A | - | B | - | D | - +-----------------------------+ + +---------------------------+ + | SUBSTRING(lastname, 1, 1) | + |---------------------------| + | A | + | B | + | D | + +---------------------------+ FROM ==== @@ -988,14 +988,14 @@ Note that the example above is essentially sorting on a predicate expression. In os> SELECT employer FROM accounts ORDER BY employer ASC NULLS LAST; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | Netagy | - | Pyrami | - | Quility | - | null | - +------------+ + +----------+ + | employer | + |----------| + | Netagy | + | Pyrami | + | Quility | + | null | + +----------+ The sorting rule can be summarized as follows: @@ -1010,14 +1010,14 @@ Here is another example for sort in descending order without ``NULLS`` clause:: os> SELECT employer FROM accounts ORDER BY employer DESC; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | Quility | - | Pyrami | - | Netagy | - | null | - +------------+ + +----------+ + | employer | + |----------| + | Quility | + | Pyrami | + | Netagy | + | null | + +----------+ Example 3: Ordering by Aggregate Functions @@ -1027,23 +1027,23 @@ Aggregate functions are allowed to be used in ``ORDER BY`` clause. You can refer os> SELECT gender, MAX(age) FROM accounts GROUP BY gender ORDER BY MAX(age) DESC; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | MAX(age) | - |----------+------------| - | M | 36 | - | F | 28 | - +----------+------------+ + +--------+----------+ + | gender | MAX(age) | + |--------+----------| + | M | 36 | + | F | 28 | + +--------+----------+ Even if it's not present in ``SELECT`` clause, it can be also used as follows:: os> SELECT gender, MIN(age) FROM accounts GROUP BY gender ORDER BY MAX(age) DESC; fetched rows / total rows = 2/2 - +----------+------------+ - | gender | MIN(age) | - |----------+------------| - | M | 32 | - | F | 28 | - +----------+------------+ + +--------+----------+ + | gender | MIN(age) | + |--------+----------| + | M | 32 | + | F | 28 | + +--------+----------+ LIMIT ===== @@ -1147,12 +1147,12 @@ Offset position can be given following the OFFSET keyword as well, here is an ex >od SELECT age FROM accounts ORDER BY age LIMIT 2 OFFSET 1 fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ Limitation diff --git a/docs/user/dql/complex.rst b/docs/user/dql/complex.rst index 17009d712b..906ea21904 100644 --- a/docs/user/dql/complex.rst +++ b/docs/user/dql/complex.rst @@ -247,14 +247,14 @@ Here is another example with aggregation function and GROUP BY in subquery:: ... SELECT AVG(balance) AS avg_balance FROM accounts GROUP BY gender, age ... ) AS a; fetched rows / total rows = 4/4 - +---------------+ - | avg_balance | - |---------------| - | 32838.0 | - | 39225.0 | - | 4180.0 | - | 5686.0 | - +---------------+ + +-------------+ + | avg_balance | + |-------------| + | 32838.0 | + | 39225.0 | + | 4180.0 | + | 5686.0 | + +-------------+ Query with multiple layers of subquery is supported as well, here follows a example:: @@ -265,12 +265,12 @@ Query with multiple layers of subquery is supported as well, here follows a exam ... ) AS accounts WHERE age < 35 ... ) AS accounts fetched rows / total rows = 2/2 - +--------+ - | name | - |--------| - | Duke | - | Adams | - +--------+ + +-------+ + | name | + |-------| + | Duke | + | Adams | + +-------+ JOINs diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 123bba046a..18a5bdce8f 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -34,29 +34,29 @@ Here is an example for different type of literals:: os> SELECT 123, 'hello', false, -4.567, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; fetched rows / total rows = 1/1 - +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ - | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | - |-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------| - | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | - +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ + +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ + | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + |-----+---------+-------+--------+-------------------+-----------------+---------------------------------| + | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ os> SELECT "Hello", 'Hello', "It""s", 'It''s', "It's", '"Its"', 'It\'s', 'It\\\'s', "\I\t\s" fetched rows / total rows = 1/1 - +-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------+ - | "Hello" | 'Hello' | "It""s" | 'It''s' | "It's" | '"Its"' | 'It\'s' | 'It\\\'s' | "\I\t\s" | - |-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------| - | Hello | Hello | It"s | It's | It's | "Its" | It's | It\'s | \I\t\s | - +-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------+ + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ + | "Hello" | 'Hello' | "It""s" | 'It''s' | "It's" | '"Its"' | 'It\'s' | 'It\\\'s' | "\I\t\s" | + |---------+---------+---------+---------+--------+---------+---------+-----------+----------| + | Hello | Hello | It"s | It's | It's | "Its" | It's | It\'s | \I\t\s | + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ os> SELECT {DATE '2020-07-07'}, {D '2020-07-07'}, {TIME '01:01:01'}, {T '01:01:01'}, {TIMESTAMP '2020-07-07 01:01:01'}, {TS '2020-07-07 01:01:01'} fetched rows / total rows = 1/1 - +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ - | {DATE '2020-07-07'} | {D '2020-07-07'} | {TIME '01:01:01'} | {T '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | {TS '2020-07-07 01:01:01'} | - |-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------| - | 2020-07-07 | 2020-07-07 | 01:01:01 | 01:01:01 | 2020-07-07 01:01:01 | 2020-07-07 01:01:01 | - +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ + +---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------+ + | {DATE '2020-07-07'} | {D '2020-07-07'} | {TIME '01:01:01'} | {T '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | {TS '2020-07-07 01:01:01'} | + |---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------| + | 2020-07-07 | 2020-07-07 | 01:01:01 | 01:01:01 | 2020-07-07 01:01:01 | 2020-07-07 01:01:01 | + +---------------------+------------------+-------------------+----------------+-----------------------------------+----------------------------+ Limitations ----------- @@ -102,11 +102,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT 1 + 2, (9 - 1) % 3, 2 * 4 / 3; fetched rows / total rows = 1/1 - +---------+---------------+-------------+ - | 1 + 2 | (9 - 1) % 3 | 2 * 4 / 3 | - |---------+---------------+-------------| - | 3 | 2 | 2 | - +---------+---------------+-------------+ + +-------+-------------+-----------+ + | 1 + 2 | (9 - 1) % 3 | 2 * 4 / 3 | + |-------+-------------+-----------| + | 3 | 2 | 2 | + +-------+-------------+-----------+ Comparison Operators ================================== @@ -162,38 +162,38 @@ Here is an example for different type of comparison operators:: os> SELECT 2 > 1, 2 >= 1, 2 < 1, 2 != 1, 2 <= 1, 2 = 1; fetched rows / total rows = 1/1 - +---------+----------+---------+----------+----------+---------+ - | 2 > 1 | 2 >= 1 | 2 < 1 | 2 != 1 | 2 <= 1 | 2 = 1 | - |---------+----------+---------+----------+----------+---------| - | True | True | False | True | False | False | - +---------+----------+---------+----------+----------+---------+ + +-------+--------+-------+--------+--------+-------+ + | 2 > 1 | 2 >= 1 | 2 < 1 | 2 != 1 | 2 <= 1 | 2 = 1 | + |-------+--------+-------+--------+--------+-------| + | True | True | False | True | False | False | + +-------+--------+-------+--------+--------+-------+ It is possible to compare datetimes. When comparing different datetime types, for example `DATE` and `TIME`, both converted to `TIMESTAMP`. The following rule is applied on coversion: a `TIME` applied to today's date; `DATE` is interpreted at midnight. See example below:: os> SELECT current_time() > current_date() AS `now.time > today`, typeof(current_time()) AS `now.time.type`, typeof(current_date()) AS `now.date.type`; fetched rows / total rows = 1/1 - +--------------------+-----------------+-----------------+ - | now.time > today | now.time.type | now.date.type | - |--------------------+-----------------+-----------------| - | True | TIME | DATE | - +--------------------+-----------------+-----------------+ + +------------------+---------------+---------------+ + | now.time > today | now.time.type | now.date.type | + |------------------+---------------+---------------| + | True | TIME | DATE | + +------------------+---------------+---------------+ os> SELECT current_time() = now() AS `now.time = now`, typeof(current_time()) AS `now.time.type`, typeof(now()) AS `now.type`; fetched rows / total rows = 1/1 - +------------------+-----------------+------------+ - | now.time = now | now.time.type | now.type | - |------------------+-----------------+------------| - | True | TIME | TIMESTAMP | - +------------------+-----------------+------------+ + +----------------+---------------+-----------+ + | now.time = now | now.time.type | now.type | + |----------------+---------------+-----------| + | True | TIME | TIMESTAMP | + +----------------+---------------+-----------+ os> SELECT subtime(now(), current_time()) = current_date() AS `midnight = now.date`, typeof(subtime(now(), current_time())) AS `midnight.type`, typeof(current_date()) AS `now.date.type`; fetched rows / total rows = 1/1 - +-----------------------+-----------------+-----------------+ - | midnight = now.date | midnight.type | now.date.type | - |-----------------------+-----------------+-----------------| - | True | TIMESTAMP | DATE | - +-----------------------+-----------------+-----------------+ + +---------------------+---------------+---------------+ + | midnight = now.date | midnight.type | now.date.type | + |---------------------+---------------+---------------| + | True | TIMESTAMP | DATE | + +---------------------+---------------+---------------+ LIKE @@ -203,11 +203,11 @@ expr LIKE pattern. The expr is string value, pattern is supports literal text, a os> SELECT 'axyzb' LIKE 'a%b', 'acb' LIKE 'A_B', 'axyzb' NOT LIKE 'a%b', 'acb' NOT LIKE 'a_b'; fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------+------------------------+ - | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'A_B' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | - |----------------------+--------------------+--------------------------+------------------------| - | True | True | False | False | - +----------------------+--------------------+--------------------------+------------------------+ + +--------------------+------------------+------------------------+----------------------+ + | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'A_B' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | + |--------------------+------------------+------------------------+----------------------| + | True | True | False | False | + +--------------------+------------------+------------------------+----------------------+ NULL value test --------------- @@ -216,11 +216,11 @@ Here is an example for null value test:: os> SELECT 0 IS NULL, 0 IS NOT NULL, NULL IS NULL, NULL IS NOT NULL; fetched rows / total rows = 1/1 - +-------------+-----------------+----------------+--------------------+ - | 0 IS NULL | 0 IS NOT NULL | NULL IS NULL | NULL IS NOT NULL | - |-------------+-----------------+----------------+--------------------| - | False | True | True | False | - +-------------+-----------------+----------------+--------------------+ + +-----------+---------------+--------------+------------------+ + | 0 IS NULL | 0 IS NOT NULL | NULL IS NULL | NULL IS NOT NULL | + |-----------+---------------+--------------+------------------| + | False | True | True | False | + +-----------+---------------+--------------+------------------+ REGEXP value test @@ -230,11 +230,11 @@ expr REGEXP pattern. The expr is string value, pattern is supports regular expre os> SELECT 'Hello!' REGEXP '.*', 'a' REGEXP 'b'; fetched rows / total rows = 1/1 - +------------------------+------------------+ - | 'Hello!' REGEXP '.*' | 'a' REGEXP 'b' | - |------------------------+------------------| - | 1 | 0 | - +------------------------+------------------+ + +----------------------+----------------+ + | 'Hello!' REGEXP '.*' | 'a' REGEXP 'b' | + |----------------------+----------------| + | 1 | 0 | + +----------------------+----------------+ IN value list test ------------------ @@ -243,11 +243,11 @@ Here is an example for IN value test:: os> SELECT 1 in (1, 2), 3 not in (1, 2); fetched rows / total rows = 1/1 - +---------------+-------------------+ - | 1 in (1, 2) | 3 not in (1, 2) | - |---------------+-------------------| - | True | True | - +---------------+-------------------+ + +-------------+-----------------+ + | 1 in (1, 2) | 3 not in (1, 2) | + |-------------+-----------------| + | True | True | + +-------------+-----------------+ BETWEEN range test ------------------ @@ -259,11 +259,11 @@ Here is an example for range test by BETWEEN expression:: ... 4 BETWEEN 1 AND 3, ... 4 NOT BETWEEN 1 AND 3; fetched rows / total rows = 1/1 - +---------------------+---------------------+-------------------------+ - | 1 BETWEEN 1 AND 3 | 4 BETWEEN 1 AND 3 | 4 NOT BETWEEN 1 AND 3 | - |---------------------+---------------------+-------------------------| - | True | False | True | - +---------------------+---------------------+-------------------------+ + +-------------------+-------------------+-----------------------+ + | 1 BETWEEN 1 AND 3 | 4 BETWEEN 1 AND 3 | 4 NOT BETWEEN 1 AND 3 | + |-------------------+-------------------+-----------------------| + | True | False | True | + +-------------------+-------------------+-----------------------+ Function Call @@ -293,11 +293,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT abs(-1.234), abs(-1 * abs(-5)); fetched rows / total rows = 1/1 - +---------------+---------------------+ - | abs(-1.234) | abs(-1 * abs(-5)) | - |---------------+---------------------| - | 1.234 | 5 | - +---------------+---------------------+ + +-------------+-------------------+ + | abs(-1.234) | abs(-1 * abs(-5)) | + |-------------+-------------------| + | 1.234 | 5 | + +-------------+-------------------+ Date function examples ---------------------- @@ -306,11 +306,11 @@ Here is an example for different type of arithmetic expressions:: os> SELECT dayofmonth(DATE '2020-07-07'); fetched rows / total rows = 1/1 - +---------------------------------+ - | dayofmonth(DATE '2020-07-07') | - |---------------------------------| - | 7 | - +---------------------------------+ + +-------------------------------+ + | dayofmonth(DATE '2020-07-07') | + |-------------------------------| + | 7 | + +-------------------------------+ Limitations ----------- diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index b445fffa63..a347614ba4 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -49,21 +49,21 @@ Cast to string example:: os> SELECT cast(true as string) as cbool, cast(1 as string) as cint, cast(DATE '2012-08-07' as string) as cdate fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | true | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | true | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> SELECT cast(true as int) as cbool, cast('1' as integer) as cstring fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -79,11 +79,11 @@ Cast function can be chained:: os> SELECT cast(cast(true as string) as boolean) as cbool fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ Mathematical Functions @@ -103,11 +103,11 @@ Example:: os> SELECT ABS(0), ABS(10), ABS(-10), ABS(12.34567), ABS(-12.34567) fetched rows / total rows = 1/1 - +----------+-----------+------------+-----------------+------------------+ - | ABS(0) | ABS(10) | ABS(-10) | ABS(12.34567) | ABS(-12.34567) | - |----------+-----------+------------+-----------------+------------------| - | 0 | 10 | 10 | 12.34567 | 12.34567 | - +----------+-----------+------------+-----------------+------------------+ + +--------+---------+----------+---------------+----------------+ + | ABS(0) | ABS(10) | ABS(-10) | ABS(12.34567) | ABS(-12.34567) | + |--------+---------+----------+---------------+----------------| + | 0 | 10 | 10 | 12.34567 | 12.34567 | + +--------+---------+----------+---------------+----------------+ ACOS @@ -151,11 +151,11 @@ Example:: os> SELECT ADD(2, 1), ADD(2.5, 3); fetched rows / total rows = 1/1 - +-------------+---------------+ - | ADD(2, 1) | ADD(2.5, 3) | - |-------------+---------------| - | 3 | 5.5 | - +-------------+---------------+ + +-----------+-------------+ + | ADD(2, 1) | ADD(2.5, 3) | + |-----------+-------------| + | 3 | 5.5 | + +-----------+-------------+ ASIN ---- @@ -173,11 +173,11 @@ Example:: os> SELECT ASIN(0) fetched rows / total rows = 1/1 - +-----------+ - | ASIN(0) | - |-----------| - | 0.0 | - +-----------+ + +---------+ + | ASIN(0) | + |---------| + | 0.0 | + +---------+ ATAN @@ -244,11 +244,11 @@ Example:: os> SELECT CBRT(8), CBRT(9.261), CBRT(-27); fetched rows / total rows = 1/1 - +-----------+---------------+-------------+ - | CBRT(8) | CBRT(9.261) | CBRT(-27) | - |-----------+---------------+-------------| - | 2.0 | 2.1 | -3.0 | - +-----------+---------------+-------------+ + +---------+-------------+-----------+ + | CBRT(8) | CBRT(9.261) | CBRT(-27) | + |---------+-------------+-----------| + | 2.0 | 2.1 | -3.0 | + +---------+-------------+-----------+ CEIL @@ -281,29 +281,29 @@ Example:: os> SELECT CEILING(0), CEILING(50.00005), CEILING(-50.00005); fetched rows / total rows = 1/1 - +--------------+---------------------+----------------------+ - | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | - |--------------+---------------------+----------------------| - | 0 | 51 | -50 | - +--------------+---------------------+----------------------+ + +------------+-------------------+--------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 51 | -50 | + +------------+-------------------+--------------------+ os> SELECT CEILING(3147483647.12345), CEILING(113147483647.12345), CEILING(3147483647.00001); fetched rows / total rows = 1/1 - +-----------------------------+-------------------------------+-----------------------------+ - | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | - |-----------------------------+-------------------------------+-----------------------------| - | 3147483648 | 113147483648 | 3147483648 | - +-----------------------------+-------------------------------+-----------------------------+ + +---------------------------+-----------------------------+---------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +---------------------------+-----------------------------+---------------------------+ Example:: os> SELECT CEIL(0), CEIL(12.34567), CEIL(-12.34567) fetched rows / total rows = 1/1 - +-----------+------------------+-------------------+ - | CEIL(0) | CEIL(12.34567) | CEIL(-12.34567) | - |-----------+------------------+-------------------| - | 0 | 13 | -12 | - +-----------+------------------+-------------------+ + +---------+----------------+-----------------+ + | CEIL(0) | CEIL(12.34567) | CEIL(-12.34567) | + |---------+----------------+-----------------| + | 0 | 13 | -12 | + +---------+----------------+-----------------+ CONV @@ -322,11 +322,11 @@ Example:: os> SELECT CONV('12', 10, 16), CONV('2C', 16, 10), CONV(12, 10, 2), CONV(1111, 2, 10) fetched rows / total rows = 1/1 - +----------------------+----------------------+-------------------+---------------------+ - | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | - |----------------------+----------------------+-------------------+---------------------| - | c | 44 | 1100 | 15 | - +----------------------+----------------------+-------------------+---------------------+ + +--------------------+--------------------+-----------------+-------------------+ + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | + |--------------------+--------------------+-----------------+-------------------| + | c | 44 | 1100 | 15 | + +--------------------+--------------------+-----------------+-------------------+ COS @@ -345,11 +345,11 @@ Example:: os> SELECT COS(0) fetched rows / total rows = 1/1 - +----------+ - | COS(0) | - |----------| - | 1.0 | - +----------+ + +--------+ + | COS(0) | + |--------| + | 1.0 | + +--------+ COSH @@ -414,11 +414,11 @@ Example:: os> SELECT CRC32('MySQL') fetched rows / total rows = 1/1 - +------------------+ - | CRC32('MySQL') | - |------------------| - | 3259397556 | - +------------------+ + +----------------+ + | CRC32('MySQL') | + |----------------| + | 3259397556 | + +----------------+ DEGREES @@ -462,11 +462,11 @@ Example:: os> SELECT DIVIDE(10, 2), DIVIDE(7.5, 3); fetched rows / total rows = 1/1 - +-----------------+------------------+ - | DIVIDE(10, 2) | DIVIDE(7.5, 3) | - |-----------------+------------------| - | 5 | 2.5 | - +-----------------+------------------+ + +---------------+----------------+ + | DIVIDE(10, 2) | DIVIDE(7.5, 3) | + |---------------+----------------| + | 5 | 2.5 | + +---------------+----------------+ E @@ -533,11 +533,11 @@ Example:: os> SELECT EXPM1(-1), EXPM1(0), EXPM1(1), EXPM1(1.5) fetched rows / total rows = 1/1 - +---------------------+------------+-------------------+-------------------+ - | EXPM1(-1) | EXPM1(0) | EXPM1(1) | EXPM1(1.5) | - |---------------------+------------+-------------------+-------------------| - | -0.6321205588285577 | 0.0 | 1.718281828459045 | 3.481689070338065 | - +---------------------+------------+-------------------+-------------------+ + +---------------------+----------+-------------------+-------------------+ + | EXPM1(-1) | EXPM1(0) | EXPM1(1) | EXPM1(1.5) | + |---------------------+----------+-------------------+-------------------| + | -0.6321205588285577 | 0.0 | 1.718281828459045 | 3.481689070338065 | + +---------------------+----------+-------------------+-------------------+ FLOOR @@ -558,27 +558,27 @@ Example:: os> SELECT FLOOR(0), FLOOR(50.00005), FLOOR(-50.00005); fetched rows / total rows = 1/1 - +------------+-------------------+--------------------+ - | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | - |------------+-------------------+--------------------| - | 0 | 50 | -51 | - +------------+-------------------+--------------------+ + +----------+-----------------+------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |----------+-----------------+------------------| + | 0 | 50 | -51 | + +----------+-----------------+------------------+ os> SELECT FLOOR(3147483647.12345), FLOOR(113147483647.12345), FLOOR(3147483647.00001); fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+---------------------------+ - | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | - |---------------------------+-----------------------------+---------------------------| - | 3147483647 | 113147483647 | 3147483647 | - +---------------------------+-----------------------------+---------------------------+ + +-------------------------+---------------------------+-------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |-------------------------+---------------------------+-------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +-------------------------+---------------------------+-------------------------+ os> SELECT FLOOR(282474973688888.022), FLOOR(9223372036854775807.022), FLOOR(9223372036854775807.0000001); fetched rows / total rows = 1/1 - +------------------------------+----------------------------------+--------------------------------------+ - | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | - |------------------------------+----------------------------------+--------------------------------------| - | 282474973688888 | 9223372036854775807 | 9223372036854775807 | - +------------------------------+----------------------------------+--------------------------------------+ + +----------------------------+--------------------------------+------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |----------------------------+--------------------------------+------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +----------------------------+--------------------------------+------------------------------------+ LN @@ -597,11 +597,11 @@ Example:: os> select LN(1), LN(e()), LN(10), LN(12.34567); fetched rows / total rows = 1/1 - +---------+-----------+-------------------+--------------------+ - | LN(1) | LN(e()) | LN(10) | LN(12.34567) | - |---------+-----------+-------------------+--------------------| - | 0.0 | 1.0 | 2.302585092994046 | 2.5133053943094317 | - +---------+-----------+-------------------+--------------------+ + +-------+---------+-------------------+--------------------+ + | LN(1) | LN(e()) | LN(10) | LN(12.34567) | + |-------+---------+-------------------+--------------------| + | 0.0 | 1.0 | 2.302585092994046 | 2.5133053943094317 | + +-------+---------+-------------------+--------------------+ LOG @@ -623,11 +623,11 @@ Example:: os> select LOG(1), LOG(e()), LOG(2, 65536), LOG(10, 10000); fetched rows / total rows = 1/1 - +----------+------------+-----------------+------------------+ - | LOG(1) | LOG(e()) | LOG(2, 65536) | LOG(10, 10000) | - |----------+------------+-----------------+------------------| - | 0.0 | 1.0 | 16.0 | 4.0 | - +----------+------------+-----------------+------------------+ + +--------+----------+---------------+----------------+ + | LOG(1) | LOG(e()) | LOG(2, 65536) | LOG(10, 10000) | + |--------+----------+---------------+----------------| + | 0.0 | 1.0 | 16.0 | 4.0 | + +--------+----------+---------------+----------------+ LOG2 @@ -646,11 +646,11 @@ Example:: os> select LOG2(1), LOG2(8), LOG2(65536), LOG2(8.8245); fetched rows / total rows = 1/1 - +-----------+-----------+---------------+--------------------+ - | LOG2(1) | LOG2(8) | LOG2(65536) | LOG2(8.8245) | - |-----------+-----------+---------------+--------------------| - | 0.0 | 3.0 | 16.0 | 3.1415145369723745 | - +-----------+-----------+---------------+--------------------+ + +---------+---------+-------------+--------------------+ + | LOG2(1) | LOG2(8) | LOG2(65536) | LOG2(8.8245) | + |---------+---------+-------------+--------------------| + | 0.0 | 3.0 | 16.0 | 3.1415145369723745 | + +---------+---------+-------------+--------------------+ LOG10 @@ -669,11 +669,11 @@ Example:: os> select LOG10(1), LOG10(8), LOG10(1000), LOG10(8.8245); fetched rows / total rows = 1/1 - +------------+--------------------+---------------+--------------------+ - | LOG10(1) | LOG10(8) | LOG10(1000) | LOG10(8.8245) | - |------------+--------------------+---------------+--------------------| - | 0.0 | 0.9030899869919435 | 3.0 | 0.9456901074431278 | - +------------+--------------------+---------------+--------------------+ + +----------+--------------------+-------------+--------------------+ + | LOG10(1) | LOG10(8) | LOG10(1000) | LOG10(8.8245) | + |----------+--------------------+-------------+--------------------| + | 0.0 | 0.9030899869919435 | 3.0 | 0.9456901074431278 | + +----------+--------------------+-------------+--------------------+ MOD @@ -694,11 +694,11 @@ Example:: os> SELECT MOD(3, 2), MOD(3.1, 2) fetched rows / total rows = 1/1 - +-------------+---------------+ - | MOD(3, 2) | MOD(3.1, 2) | - |-------------+---------------| - | 1 | 1.1 | - +-------------+---------------+ + +-----------+-------------+ + | MOD(3, 2) | MOD(3.1, 2) | + |-----------+-------------| + | 1 | 1.1 | + +-----------+-------------+ MODULUS ------- @@ -718,11 +718,11 @@ Example:: os> SELECT MODULUS(3, 2), MODULUS(3.1, 2) fetched rows / total rows = 1/1 - +-----------------+-------------------+ - | MODULUS(3, 2) | MODULUS(3.1, 2) | - |-----------------+-------------------| - | 1 | 1.1 | - +-----------------+-------------------+ + +---------------+-----------------+ + | MODULUS(3, 2) | MODULUS(3.1, 2) | + |---------------+-----------------| + | 1 | 1.1 | + +---------------+-----------------+ MULTIPLY @@ -743,11 +743,11 @@ Example:: os> SELECT MULTIPLY(1, 2), MULTIPLY(-2, 1), MULTIPLY(1.5, 2); fetched rows / total rows = 1/1 - +------------------+-------------------+--------------------+ - | MULTIPLY(1, 2) | MULTIPLY(-2, 1) | MULTIPLY(1.5, 2) | - |------------------+-------------------+--------------------| - | 2 | -2 | 3.0 | - +------------------+-------------------+--------------------+ + +----------------+-----------------+------------------+ + | MULTIPLY(1, 2) | MULTIPLY(-2, 1) | MULTIPLY(1.5, 2) | + |----------------+-----------------+------------------| + | 2 | -2 | 3.0 | + +----------------+-----------------+------------------+ PI @@ -789,11 +789,11 @@ Example:: os> SELECT POW(3, 2), POW(-3, 2), POW(3, -2) fetched rows / total rows = 1/1 - +-------------+--------------+--------------------+ - | POW(3, 2) | POW(-3, 2) | POW(3, -2) | - |-------------+--------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +-------------+--------------+--------------------+ + +-----------+------------+--------------------+ + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | + |-----------+------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-----------+------------+--------------------+ POWER @@ -814,11 +814,11 @@ Example:: os> SELECT POWER(3, 2), POWER(-3, 2), POWER(3, -2) fetched rows / total rows = 1/1 - +---------------+----------------+--------------------+ - | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | - |---------------+----------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +---------------+----------------+--------------------+ + +-------------+--------------+--------------------+ + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | + |-------------+--------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-------------+--------------+--------------------+ RADIANS @@ -883,11 +883,11 @@ Example:: os> SELECT RINT(1.7); fetched rows / total rows = 1/1 - +-------------+ - | RINT(1.7) | - |-------------| - | 2.0 | - +-------------+ + +-----------+ + | RINT(1.7) | + |-----------| + | 2.0 | + +-----------+ ROUND @@ -910,11 +910,11 @@ Example:: os> SELECT ROUND(12.34), ROUND(12.34, 1), ROUND(12.34, -1), ROUND(12, 1) fetched rows / total rows = 1/1 - +----------------+-------------------+--------------------+----------------+ - | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | - |----------------+-------------------+--------------------+----------------| - | 12.0 | 12.3 | 10.0 | 12 | - +----------------+-------------------+--------------------+----------------+ + +--------------+-----------------+------------------+--------------+ + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | + |--------------+-----------------+------------------+--------------| + | 12.0 | 12.3 | 10.0 | 12 | + +--------------+-----------------+------------------+--------------+ SIGN @@ -933,11 +933,11 @@ Example:: os> SELECT SIGN(1), SIGN(0), SIGN(-1.1) fetched rows / total rows = 1/1 - +-----------+-----------+--------------+ - | SIGN(1) | SIGN(0) | SIGN(-1.1) | - |-----------+-----------+--------------| - | 1 | 0 | -1 | - +-----------+-----------+--------------+ + +---------+---------+------------+ + | SIGN(1) | SIGN(0) | SIGN(-1.1) | + |---------+---------+------------| + | 1 | 0 | -1 | + +---------+---------+------------+ SIGNUM @@ -958,11 +958,11 @@ Example:: os> SELECT SIGNUM(1), SIGNUM(0), SIGNUM(-1.1) fetched rows / total rows = 1/1 - +-------------+-------------+----------------+ - | SIGNUM(1) | SIGNUM(0) | SIGNUM(-1.1) | - |-------------+-------------+----------------| - | 1 | 0 | -1 | - +-------------+-------------+----------------+ + +-----------+-----------+--------------+ + | SIGNUM(1) | SIGNUM(0) | SIGNUM(-1.1) | + |-----------+-----------+--------------| + | 1 | 0 | -1 | + +-----------+-----------+--------------+ SIN @@ -981,11 +981,11 @@ Example:: os> select sin(0), sin(1), sin(pi()), abs(sin(pi())) < 0.0001; fetched rows / total rows = 1/1 - +----------+--------------------+------------------------+---------------------------+ - | sin(0) | sin(1) | sin(pi()) | abs(sin(pi())) < 0.0001 | - |----------+--------------------+------------------------+---------------------------| - | 0.0 | 0.8414709848078965 | 1.2246467991473532e-16 | True | - +----------+--------------------+------------------------+---------------------------+ + +--------+--------------------+------------------------+-------------------------+ + | sin(0) | sin(1) | sin(pi()) | abs(sin(pi())) < 0.0001 | + |--------+--------------------+------------------------+-------------------------| + | 0.0 | 0.8414709848078965 | 1.2246467991473532e-16 | True | + +--------+--------------------+------------------------+-------------------------+ SINH @@ -1030,11 +1030,11 @@ Example:: os> SELECT SQRT(4), SQRT(4.41) fetched rows / total rows = 1/1 - +-----------+--------------+ - | SQRT(4) | SQRT(4.41) | - |-----------+--------------| - | 2.0 | 2.1 | - +-----------+--------------+ + +---------+------------+ + | SQRT(4) | SQRT(4.41) | + |---------+------------| + | 2.0 | 2.1 | + +---------+------------+ STRCMP @@ -1053,11 +1053,11 @@ Example:: os> SELECT STRCMP('hello', 'world'), STRCMP('hello', 'hello') fetched rows / total rows = 1/1 - +----------------------------+----------------------------+ - | STRCMP('hello', 'world') | STRCMP('hello', 'hello') | - |----------------------------+----------------------------| - | -1 | 0 | - +----------------------------+----------------------------+ + +--------------------------+--------------------------+ + | STRCMP('hello', 'world') | STRCMP('hello', 'hello') | + |--------------------------+--------------------------| + | -1 | 0 | + +--------------------------+--------------------------+ SUBTRACT @@ -1078,11 +1078,11 @@ Example:: os> SELECT SUBTRACT(2, 1), SUBTRACT(2.5, 3); fetched rows / total rows = 1/1 - +------------------+--------------------+ - | SUBTRACT(2, 1) | SUBTRACT(2.5, 3) | - |------------------+--------------------| - | 1 | -0.5 | - +------------------+--------------------+ + +----------------+------------------+ + | SUBTRACT(2, 1) | SUBTRACT(2.5, 3) | + |----------------+------------------| + | 1 | -0.5 | + +----------------+------------------+ TAN @@ -1101,11 +1101,11 @@ Example:: os> SELECT TAN(0) fetched rows / total rows = 1/1 - +----------+ - | TAN(0) | - |----------| - | 0.0 | - +----------+ + +--------+ + | TAN(0) | + |--------| + | 0.0 | + +--------+ TRUNCATE @@ -1126,11 +1126,11 @@ FLOAT/DOUBLE -> DOUBLE Example:: fetched rows / total rows = 1/1 - +----------------------+-----------------------+-------------------+ - | TRUNCATE(56.78, 1) | TRUNCATE(56.78, -1) | TRUNCATE(56, 1) | - |----------------------+-----------------------+-------------------| - | 56.7 | 50 | 56 | - +----------------------+-----------------------+-------------------+ + +--------------------+---------------------+-----------------+ + | TRUNCATE(56.78, 1) | TRUNCATE(56.78, -1) | TRUNCATE(56, 1) | + |--------------------+---------------------+-----------------| + | 56.7 | 50 | 56 | + +--------------------+---------------------+-----------------+ Date and Time Functions @@ -1163,11 +1163,11 @@ Example:: os> SELECT ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR) AS `'2020-08-26' + 1h`, ADDDATE(DATE('2020-08-26'), 1) AS `'2020-08-26' + 1`, ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) AS `ts '2020-08-26 01:01:01' + 1` fetched rows / total rows = 1/1 - +---------------------+--------------------+--------------------------------+ - | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | - |---------------------+--------------------+--------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | - +---------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | + |---------------------+------------------+------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +---------------------+------------------+------------------------------+ ADDTIME @@ -1200,35 +1200,35 @@ Example:: os> SELECT ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' + 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' + 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' + 0 | + |----------------| + | 23:59:59 | + +----------------+ os> SELECT ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' + '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' + '23:59:59' | - |-----------------------------| - | 2004-01-01 23:59:59 | - +-----------------------------+ - - os> SELECT ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' + '00:05:42' | + | '2004-01-01' + '23:59:59' | |---------------------------| - | 10:26:12 | + | 2004-01-01 23:59:59 | +---------------------------+ + os> SELECT ADDTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' + '00:05:42' | + |-------------------------| + | 10:26:12 | + +-------------------------+ + os> SELECT ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), TIMESTAMP('2002-03-04 20:40:50')) AS `'2007-02-28 10:20:30' + '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-02-28 10:20:30' + '20:40:50' | - |--------------------------------------| - | 2007-03-01 07:01:20 | - +--------------------------------------+ + +------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |------------------------------------| + | 2007-03-01 07:01:20 | + +------------------------------------+ CONVERT_TZ @@ -1247,86 +1247,86 @@ Example:: os> SELECT CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') fetched rows / total rows = 1/1 - +----------------------------------------------------------------------+ - | CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') | - |----------------------------------------------------------------------| - | 2008-12-24 21:30:00 | - +----------------------------------------------------------------------+ + +--------------------------------------------------------------------+ + | CONVERT_TZ('2008-12-25 05:30:00', '+00:00', 'America/Los_Angeles') | + |--------------------------------------------------------------------| + | 2008-12-24 21:30:00 | + +--------------------------------------------------------------------+ os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") | - |---------------------------------------------------------| - | 2010-10-09 23:10:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-10:00") | + |-------------------------------------------------------| + | 2010-10-09 23:10:10 | + +-------------------------------------------------------+ When the datedate, or either of the two time zone fields are invalid format, then the result is null. In this example any timestamp that is not <yyyy-MM-dd HH:mm:ss> will result in null. Example:: os> SELECT CONVERT_TZ("test", "+01:00", "-10:00") fetched rows / total rows = 1/1 - +------------------------------------------+ - | CONVERT_TZ("test", "+01:00", "-10:00") | - |------------------------------------------| - | null | - +------------------------------------------+ + +----------------------------------------+ + | CONVERT_TZ("test", "+01:00", "-10:00") | + |----------------------------------------| + | null | + +----------------------------------------+ When the timestamp, or either of the two time zone fields are invalid format, then the result is null. In this example any timezone that is not <+HH:mm> or <-HH:mm> will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "test", "-10:00") | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") | - |---------------------------------------------------------| - | 2010-10-10 23:10:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:00") | + |-------------------------------------------------------| + | 2010-10-10 23:10:10 | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") | - |---------------------------------------------------------| - | null | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "+14:01") | + |-------------------------------------------------------| + | null | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") | - |---------------------------------------------------------| - | 2010-10-09 19:11:10 | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-13:59") | + |-------------------------------------------------------| + | 2010-10-09 19:11:10 | + +-------------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. Example:: os> SELECT CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") fetched rows / total rows = 1/1 - +---------------------------------------------------------+ - | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") | - |---------------------------------------------------------| - | null | - +---------------------------------------------------------+ + +-------------------------------------------------------+ + | CONVERT_TZ("2010-10-10 10:10:10", "+01:00", "-14:00") | + |-------------------------------------------------------| + | null | + +-------------------------------------------------------+ CURDATE @@ -1346,11 +1346,11 @@ Example:: > SELECT CURDATE(); fetched rows / total rows = 1/1 - +-------------+ - | CURDATE() | - |-------------| - | 2022-08-02 | - +-------------+ + +------------+ + | CURDATE() | + |------------| + | 2022-08-02 | + +------------+ CURRENT_DATE @@ -1365,11 +1365,11 @@ Example:: > SELECT CURRENT_DATE(); fetched rows / total rows = 1/1 - +------------------+ - | CURRENT_DATE() | - |------------------+ - | 2022-08-02 | - +------------------+ + +----------------+ + | CURRENT_DATE() | + |----------------+ + | 2022-08-02 | + +----------------+ CURRENT_TIME @@ -1384,11 +1384,11 @@ Example:: > SELECT CURRENT_TIME(); fetched rows / total rows = 1/1 - +-----------------+ - | CURRENT_TIME() | - |-----------------+ - | 15:39:05 | - +-----------------+ + +----------------+ + | CURRENT_TIME() | + |----------------+ + | 15:39:05 | + +----------------+ CURRENT_TIMESTAMP @@ -1403,11 +1403,11 @@ Example:: > SELECT CURRENT_TIMESTAMP(); fetched rows / total rows = 1/1 - +-----------------------+ - | CURRENT_TIMESTAMP() | - |-----------------------+ - | 2022-08-02 15:54:19 | - +-----------------------+ + +---------------------+ + | CURRENT_TIMESTAMP() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ CURTIME @@ -1427,11 +1427,11 @@ Example:: > SELECT CURTIME() as value_1, CURTIME() as value_2; fetched rows / total rows = 1/1 - +-----------+-----------+ - | value_1 | value_2 | - |-----------+-----------| - | 15:39:05 | 15:39:05 | - +-----------+-----------+ + +----------+----------+ + | value_1 | value_2 | + |----------+----------| + | 15:39:05 | 15:39:05 | + +----------+----------+ DATE @@ -1450,11 +1450,11 @@ Example:: os> SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')), DATE('2020-08-26 13:49:00'), DATE('2020-08-26 13:49') fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+-------------------------------+----------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | - |----------------------+------------------------------------------+-------------------------------+----------------------------| - | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | - +----------------------+------------------------------------------+-------------------------------+----------------------------+ + +--------------------+----------------------------------------+-----------------------------+--------------------------+ + | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | + |--------------------+----------------------------------------+-----------------------------+--------------------------| + | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | + +--------------------+----------------------------------------+-----------------------------+--------------------------+ DATETIME @@ -1477,44 +1477,44 @@ Example:: os> SELECT DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') fetched rows / total rows = 1/1 - +----------------------------------------------------------------+ - | DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') | - |----------------------------------------------------------------| - | 2008-12-24 21:30:00 | - +----------------------------------------------------------------+ + +--------------------------------------------------------------+ + | DATETIME('2008-12-25 05:30:00+00:00', 'America/Los_Angeles') | + |--------------------------------------------------------------| + | 2008-12-24 21:30:00 | + +--------------------------------------------------------------+ This example converts from -10:00 timezone to +10:00 timezone. Example:: os> SELECT DATETIME('2004-02-28 23:00:00-10:00', '+10:00') fetched rows / total rows = 1/1 - +---------------------------------------------------+ - | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | - |---------------------------------------------------| - | 2004-02-29 19:00:00 | - +---------------------------------------------------+ + +-------------------------------------------------+ + | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | + |-------------------------------------------------| + | 2004-02-29 19:00:00 | + +-------------------------------------------------+ This example uses the timezone -14:00, which is outside of the range -13:59 and +14:00. This results in a null value. Example:: os> SELECT DATETIME('2008-01-01 02:00:00', '-14:00') fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-01-01 02:00:00', '-14:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-01-01 02:00:00', '-14:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ February 30th is not a day, so it returns null. Example:: os> SELECT DATETIME('2008-02-30 02:00:00', '-00:00') fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-02-30 02:00:00', '-00:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-02-30 02:00:00', '-00:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ DATETIME(datetime) examples @@ -1523,33 +1523,33 @@ Example:: os> SELECT DATETIME('2008-02-10 02:00:00') fetched rows / total rows = 1/1 - +-----------------------------------+ - | DATETIME('2008-02-10 02:00:00') | - |-----------------------------------| - | 2008-02-10 02:00:00 | - +-----------------------------------+ + +---------------------------------+ + | DATETIME('2008-02-10 02:00:00') | + |---------------------------------| + | 2008-02-10 02:00:00 | + +---------------------------------+ February 30th is not a day, so it returns null. Example:: os> SELECT DATETIME('2008-02-30 02:00:00') fetched rows / total rows = 1/1 - +-----------------------------------+ - | DATETIME('2008-02-30 02:00:00') | - |-----------------------------------| - | null | - +-----------------------------------+ + +---------------------------------+ + | DATETIME('2008-02-30 02:00:00') | + |---------------------------------| + | null | + +---------------------------------+ DATETIME with a datetime and no seperate timezone to convert to returns the datetime object without a timezone. Example:: os> SELECT DATETIME('2008-02-10 02:00:00+04:00') fetched rows / total rows = 1/1 - +-----------------------------------------+ - | DATETIME('2008-02-10 02:00:00+04:00') | - |-----------------------------------------| - | 2008-02-10 02:00:00 | - +-----------------------------------------+ + +---------------------------------------+ + | DATETIME('2008-02-10 02:00:00+04:00') | + |---------------------------------------| + | 2008-02-10 02:00:00 | + +---------------------------------------+ DATE_ADD @@ -1572,11 +1572,11 @@ Example:: os> SELECT DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR) AS `'2020-08-26' + 1h`, DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 DAY) as `ts '2020-08-26 01:01:01' + 1d` fetched rows / total rows = 1/1 - +---------------------+---------------------------------+ - | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | - |---------------------+---------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | - +---------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | + |---------------------+-------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | + +---------------------+-------------------------------+ DATE_FORMAT @@ -1671,11 +1671,11 @@ Example:: os> SELECT DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f'), DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') fetched rows / total rows = 1/1 - +------------------------------------------------------+-----------------------------------------------------------------------+ - | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | - |------------------------------------------------------+-----------------------------------------------------------------------| - | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | - +------------------------------------------------------+-----------------------------------------------------------------------+ + +----------------------------------------------------+---------------------------------------------------------------------+ + | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | + |----------------------------------------------------+---------------------------------------------------------------------| + | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | + +----------------------------------------------------+---------------------------------------------------------------------+ DATE_SUB @@ -1698,11 +1698,11 @@ Example:: os> SELECT DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY) AS `'2008-01-02' - 31d`, DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 HOUR) AS `ts '2020-08-26 01:01:01' + 1h` fetched rows / total rows = 1/1 - +----------------------+---------------------------------+ - | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | - |----------------------+---------------------------------| - | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | - +----------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | + |---------------------+-------------------------------| + | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | + +---------------------+-------------------------------+ DATEDIFF @@ -1718,11 +1718,11 @@ Example:: os> SELECT DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')) AS `'2000-01-02' - '2000-01-01'`, DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')) AS `'2001-02-01' - '2004-01-01'`, DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) AS `today - today` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+-----------------+ - | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | - |-------------------------------+-------------------------------+-----------------| - | 1 | -1064 | 0 | - +-------------------------------+-------------------------------+-----------------+ + +-----------------------------+-----------------------------+---------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-----------------------------+-----------------------------+---------------| + | 1 | -1064 | 0 | + +-----------------------------+-----------------------------+---------------+ DAY @@ -1743,11 +1743,11 @@ Example:: os> SELECT DAY(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------+ - | DAY(DATE('2020-08-26')) | - |---------------------------| - | 26 | - +---------------------------+ + +-------------------------+ + | DAY(DATE('2020-08-26')) | + |-------------------------| + | 26 | + +-------------------------+ DAYNAME @@ -1766,11 +1766,11 @@ Example:: os> SELECT DAYNAME(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------+ - | DAYNAME(DATE('2020-08-26')) | - |-------------------------------| - | Wednesday | - +-------------------------------+ + +-----------------------------+ + | DAYNAME(DATE('2020-08-26')) | + |-----------------------------| + | Wednesday | + +-----------------------------+ DAYOFMONTH @@ -1791,11 +1791,11 @@ Example:: os> SELECT DAYOFMONTH(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +----------------------------------+ - | DAYOFMONTH(DATE('2020-08-26')) | - |----------------------------------| - | 26 | - +----------------------------------+ + +--------------------------------+ + | DAYOFMONTH(DATE('2020-08-26')) | + |--------------------------------| + | 26 | + +--------------------------------+ DAY_OF_MONTH @@ -1816,11 +1816,11 @@ Example:: os> SELECT DAY_OF_MONTH('2020-08-26') fetched rows / total rows = 1/1 - +------------------------------+ - | DAY_OF_MONTH('2020-08-26') | - |------------------------------| - | 26 | - +------------------------------+ + +----------------------------+ + | DAY_OF_MONTH('2020-08-26') | + |----------------------------| + | 26 | + +----------------------------+ DAYOFWEEK @@ -1841,11 +1841,11 @@ Example:: os> SELECT DAYOFWEEK('2020-08-26'), DAY_OF_WEEK('2020-08-26') fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+ - | DAYOFWEEK('2020-08-26') | DAY_OF_WEEK('2020-08-26') | - |---------------------------+-----------------------------| - | 4 | 4 | - +---------------------------+-----------------------------+ + +-------------------------+---------------------------+ + | DAYOFWEEK('2020-08-26') | DAY_OF_WEEK('2020-08-26') | + |-------------------------+---------------------------| + | 4 | 4 | + +-------------------------+---------------------------+ DAYOFYEAR @@ -1866,19 +1866,19 @@ Example:: os> SELECT DAYOFYEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFYEAR(DATE('2020-08-26')) | - |---------------------------------| - | 239 | - +---------------------------------+ + +-------------------------------+ + | DAYOFYEAR(DATE('2020-08-26')) | + |-------------------------------| + | 239 | + +-------------------------------+ os> SELECT DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +-----------------------------------------------+ - | DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) | - |-----------------------------------------------| - | 239 | - +-----------------------------------------------+ + +---------------------------------------------+ + | DAYOFYEAR(TIMESTAMP('2020-08-26 00:00:00')) | + |---------------------------------------------| + | 239 | + +---------------------------------------------+ DAY_OF_YEAR @@ -1898,19 +1898,19 @@ Example:: os> SELECT DAY_OF_YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_YEAR(DATE('2020-08-26')) | - |-----------------------------------| - | 239 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_YEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ os> SELECT DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) fetched rows / total rows = 1/1 - +-------------------------------------------------+ - | DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) | - |-------------------------------------------------| - | 239 | - +-------------------------------------------------+ + +-----------------------------------------------+ + | DAY_OF_YEAR(TIMESTAMP('2020-08-26 00:00:00')) | + |-----------------------------------------------| + | 239 | + +-----------------------------------------------+ EXTRACT @@ -1978,11 +1978,11 @@ Example:: os> SELECT extract(YEAR_MONTH FROM "2023-02-07 10:11:12"); fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | - |--------------------------------------------------| - | 202302 | - +--------------------------------------------------+ + +------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |------------------------------------------------| + | 202302 | + +------------------------------------------------+ FROM_DAYS @@ -2001,11 +2001,11 @@ Example:: os> SELECT FROM_DAYS(733687) fetched rows / total rows = 1/1 - +---------------------+ - | FROM_DAYS(733687) | - |---------------------| - | 2008-10-07 | - +---------------------+ + +-------------------+ + | FROM_DAYS(733687) | + |-------------------| + | 2008-10-07 | + +-------------------+ FROM_UNIXTIME @@ -2030,19 +2030,19 @@ Examples:: os> select FROM_UNIXTIME(1220249547) fetched rows / total rows = 1/1 - +-----------------------------+ - | FROM_UNIXTIME(1220249547) | - |-----------------------------| - | 2008-09-01 06:12:27 | - +-----------------------------+ + +---------------------------+ + | FROM_UNIXTIME(1220249547) | + |---------------------------| + | 2008-09-01 06:12:27 | + +---------------------------+ os> select FROM_UNIXTIME(1220249547, '%T') fetched rows / total rows = 1/1 - +-----------------------------------+ - | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------------| - | 06:12:27 | - +-----------------------------------+ + +---------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |---------------------------------| + | 06:12:27 | + +---------------------------------+ GET_FORMAT @@ -2061,11 +2061,11 @@ Examples:: os> select GET_FORMAT(DATE, 'USA'); fetched rows / total rows = 1/1 - +---------------------------+ - | GET_FORMAT(DATE, 'USA') | - |---------------------------| - | %m.%d.%Y | - +---------------------------+ + +-------------------------+ + | GET_FORMAT(DATE, 'USA') | + |-------------------------| + | %m.%d.%Y | + +-------------------------+ HOUR @@ -2085,11 +2085,11 @@ Example:: os> SELECT HOUR('01:02:03'), HOUR_OF_DAY('01:02:03') fetched rows / total rows = 1/1 - +--------------------+---------------------------+ - | HOUR('01:02:03') | HOUR_OF_DAY('01:02:03') | - |--------------------+---------------------------| - | 1 | 1 | - +--------------------+---------------------------+ + +------------------+-------------------------+ + | HOUR('01:02:03') | HOUR_OF_DAY('01:02:03') | + |------------------+-------------------------| + | 1 | 1 | + +------------------+-------------------------+ LAST_DAY @@ -2105,11 +2105,11 @@ Example:: os> SELECT last_day('2023-02-06'); fetched rows / total rows = 1/1 - +--------------------------+ - | last_day('2023-02-06') | - |--------------------------| - | 2023-02-28 | - +--------------------------+ + +------------------------+ + | last_day('2023-02-06') | + |------------------------| + | 2023-02-28 | + +------------------------+ LOCALTIMESTAMP @@ -2177,11 +2177,11 @@ Example:: os> select MAKEDATE(1945, 5.9), MAKEDATE(1984, 1984) fetched rows / total rows = 1/1 - +-----------------------+------------------------+ - | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | - |-----------------------+------------------------| - | 1945-01-06 | 1989-06-06 | - +-----------------------+------------------------+ + +---------------------+----------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |---------------------+----------------------| + | 1945-01-06 | 1989-06-06 | + +---------------------+----------------------+ MAKETIME @@ -2209,11 +2209,11 @@ Example:: os> select MAKETIME(20, 30, 40), MAKETIME(20.2, 49.5, 42.100502) fetched rows / total rows = 1/1 - +------------------------+-----------------------------------+ - | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | - |------------------------+-----------------------------------| - | 20:30:40 | 20:50:42.100502 | - +------------------------+-----------------------------------+ + +----------------------+---------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |----------------------+---------------------------------| + | 20:30:40 | 20:50:42.100502 | + +----------------------+---------------------------------+ MICROSECOND @@ -2232,11 +2232,11 @@ Example:: os> SELECT MICROSECOND((TIME '01:02:03.123456')) fetched rows / total rows = 1/1 - +-----------------------------------------+ - | MICROSECOND((TIME '01:02:03.123456')) | - |-----------------------------------------| - | 123456 | - +-----------------------------------------+ + +---------------------------------------+ + | MICROSECOND((TIME '01:02:03.123456')) | + |---------------------------------------| + | 123456 | + +---------------------------------------+ MINUTE @@ -2256,11 +2256,11 @@ Example:: os> SELECT MINUTE(time('01:02:03')), MINUTE_OF_HOUR(time('01:02:03')) fetched rows / total rows = 1/1 - +----------------------------+------------------------------------+ - | MINUTE(time('01:02:03')) | MINUTE_OF_HOUR(time('01:02:03')) | - |----------------------------+------------------------------------| - | 2 | 2 | - +----------------------------+------------------------------------+ + +--------------------------+----------------------------------+ + | MINUTE(time('01:02:03')) | MINUTE_OF_HOUR(time('01:02:03')) | + |--------------------------+----------------------------------| + | 2 | 2 | + +--------------------------+----------------------------------+ MINUTE_OF_DAY @@ -2279,11 +2279,11 @@ Example:: os> SELECT MINUTE_OF_DAY((TIME '01:02:03')) fetched rows / total rows = 1/1 - +------------------------------------+ - | MINUTE_OF_DAY((TIME '01:02:03')) | - |------------------------------------| - | 62 | - +------------------------------------+ + +----------------------------------+ + | MINUTE_OF_DAY((TIME '01:02:03')) | + |----------------------------------| + | 62 | + +----------------------------------+ MONTH @@ -2304,20 +2304,20 @@ Example:: os> SELECT MONTH(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-----------------------------+ - | MONTH(DATE('2020-08-26')) | - |-----------------------------| - | 8 | - +-----------------------------+ + +---------------------------+ + | MONTH(DATE('2020-08-26')) | + |---------------------------| + | 8 | + +---------------------------+ os> SELECT MONTH_OF_YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------------+ - | MONTH_OF_YEAR(DATE('2020-08-26')) | - |-------------------------------------| - | 8 | - +-------------------------------------+ + +-----------------------------------+ + | MONTH_OF_YEAR(DATE('2020-08-26')) | + |-----------------------------------| + | 8 | + +-----------------------------------+ MONTHNAME @@ -2336,11 +2336,11 @@ Example:: os> SELECT MONTHNAME(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +---------------------------------+ - | MONTHNAME(DATE('2020-08-26')) | - |---------------------------------| - | August | - +---------------------------------+ + +-------------------------------+ + | MONTHNAME(DATE('2020-08-26')) | + |-------------------------------| + | August | + +-------------------------------+ NOW @@ -2383,11 +2383,11 @@ Example:: os> SELECT PERIOD_ADD(200801, 2), PERIOD_ADD(200801, -12) fetched rows / total rows = 1/1 - +-------------------------+---------------------------+ - | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | - |-------------------------+---------------------------| - | 200803 | 200701 | - +-------------------------+---------------------------+ + +-----------------------+-------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-----------------------+-------------------------| + | 200803 | 200701 | + +-----------------------+-------------------------+ PERIOD_DIFF @@ -2406,11 +2406,11 @@ Example:: os> SELECT PERIOD_DIFF(200802, 200703), PERIOD_DIFF(200802, 201003) fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | - |-------------------------------+-------------------------------| - | 11 | -25 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-----------------------------+-----------------------------| + | 11 | -25 | + +-----------------------------+-----------------------------+ QUARTER @@ -2429,11 +2429,11 @@ Example:: os> SELECT QUARTER(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +-------------------------------+ - | QUARTER(DATE('2020-08-26')) | - |-------------------------------| - | 3 | - +-------------------------------+ + +-----------------------------+ + | QUARTER(DATE('2020-08-26')) | + |-----------------------------| + | 3 | + +-----------------------------+ SEC_TO_TIME @@ -2455,27 +2455,27 @@ Example:: os> SELECT SEC_TO_TIME(3601) fetched rows / total rows = 1/1 - +---------------------+ - | SEC_TO_TIME(3601) | - |---------------------| - | 01:00:01 | - +---------------------+ + +-------------------+ + | SEC_TO_TIME(3601) | + |-------------------| + | 01:00:01 | + +-------------------+ os> SELECT sec_to_time(1234.123); fetched rows / total rows = 1/1 - +-------------------------+ - | sec_to_time(1234.123) | - |-------------------------| - | 00:20:34.123 | - +-------------------------+ + +-----------------------+ + | sec_to_time(1234.123) | + |-----------------------| + | 00:20:34.123 | + +-----------------------+ os> SELECT sec_to_time(NULL); fetched rows / total rows = 1/1 - +---------------------+ - | sec_to_time(NULL) | - |---------------------| - | null | - +---------------------+ + +-------------------+ + | sec_to_time(NULL) | + |-------------------| + | null | + +-------------------+ SECOND @@ -2495,19 +2495,19 @@ Example:: os> SELECT SECOND((TIME '01:02:03')) fetched rows / total rows = 1/1 - +-----------------------------+ - | SECOND((TIME '01:02:03')) | - |-----------------------------| - | 3 | - +-----------------------------+ + +---------------------------+ + | SECOND((TIME '01:02:03')) | + |---------------------------| + | 3 | + +---------------------------+ os> SELECT SECOND_OF_MINUTE(time('01:02:03')) fetched rows / total rows = 1/1 - +--------------------------------------+ - | SECOND_OF_MINUTE(time('01:02:03')) | - |--------------------------------------| - | 3 | - +--------------------------------------+ + +------------------------------------+ + | SECOND_OF_MINUTE(time('01:02:03')) | + |------------------------------------| + | 3 | + +------------------------------------+ STR_TO_DATE @@ -2529,11 +2529,11 @@ Example:: OS> SELECT str_to_date("01,5,2013", "%d,%m,%Y") fetched rows / total rows = 1/1 - +----------------------------------------+ - | str_to_date("01,5,2013", "%d,%m,%Y") | - |----------------------------------------| - | 2013-05-01 00:00:00 | - +----------------------------------------+ + +--------------------------------------+ + | str_to_date("01,5,2013", "%d,%m,%Y") | + |--------------------------------------| + | 2013-05-01 00:00:00 | + +--------------------------------------+ SUBDATE @@ -2563,11 +2563,11 @@ Example:: os> SELECT SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY) AS `'2008-01-02' - 31d`, SUBDATE(DATE('2020-08-26'), 1) AS `'2020-08-26' - 1`, SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) AS `ts '2020-08-26 01:01:01' - 1` fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------------+ - | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | - |----------------------+--------------------+--------------------------------| - | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | - +----------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | + |---------------------+------------------+------------------------------| + | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | + +---------------------+------------------+------------------------------+ SUBTIME @@ -2600,35 +2600,35 @@ Example:: os> SELECT SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) AS `'23:59:59' - 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' - 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' - 0 | + |----------------| + | 23:59:59 | + +----------------+ os> SELECT SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) AS `'2004-01-01' - '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' - '23:59:59' | - |-----------------------------| - | 2003-12-31 00:00:01 | - +-----------------------------+ - - os> SELECT SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' - '00:05:42' | + | '2004-01-01' - '23:59:59' | |---------------------------| - | 10:14:48 | + | 2003-12-31 00:00:01 | +---------------------------+ + os> SELECT SUBTIME(TIME('10:20:30'), TIME('00:05:42')) AS `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' - '00:05:42' | + |-------------------------| + | 10:14:48 | + +-------------------------+ + os> SELECT SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), TIMESTAMP('2002-03-04 20:40:50')) AS `'2007-03-01 10:20:30' - '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-03-01 10:20:30' - '20:40:50' | - |--------------------------------------| - | 2007-02-28 13:39:40 | - +--------------------------------------+ + +------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |------------------------------------| + | 2007-02-28 13:39:40 | + +------------------------------------+ SYSDATE @@ -2674,11 +2674,11 @@ Example:: os> SELECT TIME('13:49:00'), TIME('13:49'), TIME(TIMESTAMP('2020-08-26 13:49:00')), TIME('2020-08-26 13:49:00') fetched rows / total rows = 1/1 - +--------------------+-----------------+------------------------------------------+-------------------------------+ - | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | - |--------------------+-----------------+------------------------------------------+-------------------------------| - | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | - +--------------------+-----------------+------------------------------------------+-------------------------------+ + +------------------+---------------+----------------------------------------+-----------------------------+ + | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | + |------------------+---------------+----------------------------------------+-----------------------------| + | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | + +------------------+---------------+----------------------------------------+-----------------------------+ TIME_FORMAT ----------- @@ -2728,11 +2728,11 @@ Example:: os> SELECT TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') fetched rows / total rows = 1/1 - +------------------------------------------------------------------------------+ - | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | - |------------------------------------------------------------------------------| - | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | - +------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | + |----------------------------------------------------------------------------| + | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | + +----------------------------------------------------------------------------+ TIME_TO_SEC @@ -2751,11 +2751,11 @@ Example:: os> SELECT TIME_TO_SEC(TIME '22:23:00') fetched rows / total rows = 1/1 - +--------------------------------+ - | TIME_TO_SEC(TIME '22:23:00') | - |--------------------------------| - | 80580 | - +--------------------------------+ + +------------------------------+ + | TIME_TO_SEC(TIME '22:23:00') | + |------------------------------| + | 80580 | + +------------------------------+ TIMEDIFF @@ -2774,11 +2774,11 @@ Example:: os> SELECT TIMEDIFF('23:59:59', '13:00:00') fetched rows / total rows = 1/1 - +------------------------------------+ - | TIMEDIFF('23:59:59', '13:00:00') | - |------------------------------------| - | 10:59:59 | - +------------------------------------+ + +----------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |----------------------------------| + | 10:59:59 | + +----------------------------------+ TIMESTAMP @@ -2802,11 +2802,11 @@ Example:: os> SELECT TIMESTAMP('2020-08-26 13:49:00'), TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) fetched rows / total rows = 1/1 - +------------------------------------+------------------------------------------------------+ - | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | - |------------------------------------+------------------------------------------------------| - | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | - +------------------------------------+------------------------------------------------------+ + +----------------------------------+----------------------------------------------------+ + | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | + |----------------------------------+----------------------------------------------------| + | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | + +----------------------------------+----------------------------------------------------+ TIMESTAMPADD @@ -2826,11 +2826,11 @@ Examples:: os> SELECT TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00'), TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') fetched rows / total rows = 1/1 - +------------------------------------------------+----------------------------------------------------+ - | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | - |------------------------------------------------+----------------------------------------------------| - | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | - +------------------------------------------------+----------------------------------------------------+ + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ TIMESTAMPDIFF @@ -2851,11 +2851,11 @@ Examples:: os> SELECT TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00'), TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) fetched rows / total rows = 1/1 - +---------------------------------------------------------------------+-------------------------------------------------------------+ - | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | - |---------------------------------------------------------------------+-------------------------------------------------------------| - | 4 | -23 | - +---------------------------------------------------------------------+-------------------------------------------------------------+ + +-------------------------------------------------------------------+-----------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | + |-------------------------------------------------------------------+-----------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-----------------------------------------------------------+ TO_DAYS @@ -2874,11 +2874,11 @@ Example:: os> SELECT TO_DAYS(DATE '2008-10-07') fetched rows / total rows = 1/1 - +------------------------------+ - | TO_DAYS(DATE '2008-10-07') | - |------------------------------| - | 733687 | - +------------------------------+ + +----------------------------+ + | TO_DAYS(DATE '2008-10-07') | + |----------------------------| + | 733687 | + +----------------------------+ TO_SECONDS @@ -2898,11 +2898,11 @@ Example:: os> SELECT TO_SECONDS(DATE '2008-10-07'), TO_SECONDS(950228) fetched rows / total rows = 1/1 - +---------------------------------+----------------------+ - | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | - |---------------------------------+----------------------| - | 63390556800 | 62961148800 | - +---------------------------------+----------------------+ + +-------------------------------+--------------------+ + | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | + |-------------------------------+--------------------| + | 63390556800 | 62961148800 | + +-------------------------------+--------------------+ UNIX_TIMESTAMP @@ -2924,19 +2924,19 @@ Examples:: os> select UNIX_TIMESTAMP(20771122143845) fetched rows / total rows = 1/1 - +----------------------------------+ - | UNIX_TIMESTAMP(20771122143845) | - |----------------------------------| - | 3404817525.0 | - +----------------------------------+ + +--------------------------------+ + | UNIX_TIMESTAMP(20771122143845) | + |--------------------------------| + | 3404817525.0 | + +--------------------------------+ os> select UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) fetched rows / total rows = 1/1 - +----------------------------------------------------+ - | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | - |----------------------------------------------------| - | 848077542.0 | - +----------------------------------------------------+ + +--------------------------------------------------+ + | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | + |--------------------------------------------------| + | 848077542.0 | + +--------------------------------------------------+ UTC_DATE @@ -2955,11 +2955,11 @@ Example:: > SELECT UTC_DATE(); fetched rows / total rows = 1/1 - +--------------+ - | utc_date() | - |--------------| - | 2022-10-03 | - +--------------+ + +------------+ + | utc_date() | + |------------| + | 2022-10-03 | + +------------+ UTC_TIME @@ -2978,11 +2978,11 @@ Example:: > SELECT UTC_TIME(); fetched rows / total rows = 1/1 - +--------------+ - | utc_time() | - |--------------| - | 17:54:27 | - +--------------+ + +------------+ + | utc_time() | + |------------| + | 17:54:27 | + +------------+ UTC_TIMESTAMP @@ -3067,11 +3067,11 @@ Example:: os> SELECT WEEK(DATE('2008-02-20')), WEEK(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +----------------------------+-------------------------------+ - | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | - |----------------------------+-------------------------------| - | 7 | 8 | - +----------------------------+-------------------------------+ + +--------------------------+-----------------------------+ + | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | + |--------------------------+-----------------------------| + | 7 | 8 | + +--------------------------+-----------------------------+ WEEKDAY @@ -3092,11 +3092,11 @@ Example:: os> SELECT weekday('2020-08-26'), weekday('2020-08-27') fetched rows / total rows = 1/1 - +-------------------------+-------------------------+ - | weekday('2020-08-26') | weekday('2020-08-27') | - |-------------------------+-------------------------| - | 2 | 3 | - +-------------------------+-------------------------+ + +-----------------------+-----------------------+ + | weekday('2020-08-26') | weekday('2020-08-27') | + |-----------------------+-----------------------| + | 2 | 3 | + +-----------------------+-----------------------+ WEEK_OF_YEAR @@ -3116,11 +3116,11 @@ Example:: os> SELECT WEEK_OF_YEAR(DATE('2008-02-20')), WEEK_OF_YEAR(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +------------------------------------+---------------------------------------+ - | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | - |------------------------------------+---------------------------------------| - | 7 | 8 | - +------------------------------------+---------------------------------------+ + +----------------------------------+-------------------------------------+ + | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | + |----------------------------------+-------------------------------------| + | 7 | 8 | + +----------------------------------+-------------------------------------+ WEEKOFYEAR @@ -3140,11 +3140,11 @@ Example:: os> SELECT WEEKOFYEAR(DATE('2008-02-20')), WEEKOFYEAR(DATE('2008-02-20'), 1) fetched rows / total rows = 1/1 - +----------------------------------+-------------------------------------+ - | WEEKOFYEAR(DATE('2008-02-20')) | WEEKOFYEAR(DATE('2008-02-20'), 1) | - |----------------------------------+-------------------------------------| - | 7 | 8 | - +----------------------------------+-------------------------------------+ + +--------------------------------+-----------------------------------+ + | WEEKOFYEAR(DATE('2008-02-20')) | WEEKOFYEAR(DATE('2008-02-20'), 1) | + |--------------------------------+-----------------------------------| + | 7 | 8 | + +--------------------------------+-----------------------------------+ YEAR @@ -3163,11 +3163,11 @@ Example:: os> SELECT YEAR(DATE('2020-08-26')) fetched rows / total rows = 1/1 - +----------------------------+ - | YEAR(DATE('2020-08-26')) | - |----------------------------| - | 2020 | - +----------------------------+ + +--------------------------+ + | YEAR(DATE('2020-08-26')) | + |--------------------------| + | 2020 | + +--------------------------+ YEARWEEK @@ -3186,11 +3186,11 @@ Example:: os> SELECT YEARWEEK('2020-08-26'), YEARWEEK('2019-01-05', 0) fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 0) | - |--------------------------+-----------------------------| - | 202034 | 201852 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 0) | + |------------------------+---------------------------| + | 202034 | 201852 | + +------------------------+---------------------------+ String Functions @@ -3212,11 +3212,11 @@ Example:: os> SELECT ASCII('hello') fetched rows / total rows = 1/1 - +------------------+ - | ASCII('hello') | - |------------------| - | 104 | - +------------------+ + +----------------+ + | ASCII('hello') | + |----------------| + | 104 | + +----------------+ CONCAT @@ -3235,11 +3235,11 @@ Example:: os> SELECT CONCAT('hello ', 'whole ', 'world', '!'), CONCAT('hello', 'world'), CONCAT('hello', null) fetched rows / total rows = 1/1 - +--------------------------------------------+----------------------------+-------------------------+ - | CONCAT('hello ', 'whole ', 'world', '!') | CONCAT('hello', 'world') | CONCAT('hello', null) | - |--------------------------------------------+----------------------------+-------------------------| - | hello whole world! | helloworld | null | - +--------------------------------------------+----------------------------+-------------------------+ + +------------------------------------------+--------------------------+-----------------------+ + | CONCAT('hello ', 'whole ', 'world', '!') | CONCAT('hello', 'world') | CONCAT('hello', null) | + |------------------------------------------+--------------------------+-----------------------| + | hello whole world! | helloworld | null | + +------------------------------------------+--------------------------+-----------------------+ CONCAT_WS @@ -3258,11 +3258,11 @@ Example:: os> SELECT CONCAT_WS(',', 'hello', 'world') fetched rows / total rows = 1/1 - +------------------------------------+ - | CONCAT_WS(',', 'hello', 'world') | - |------------------------------------| - | hello,world | - +------------------------------------+ + +----------------------------------+ + | CONCAT_WS(',', 'hello', 'world') | + |----------------------------------| + | hello,world | + +----------------------------------+ LEFT @@ -3278,11 +3278,11 @@ Example:: os> SELECT LEFT('helloworld', 5), LEFT('HELLOWORLD', 0) fetched rows / total rows = 1/1 - +-------------------------+-------------------------+ - | LEFT('helloworld', 5) | LEFT('HELLOWORLD', 0) | - |-------------------------+-------------------------| - | hello | | - +-------------------------+-------------------------+ + +-----------------------+-----------------------+ + | LEFT('helloworld', 5) | LEFT('HELLOWORLD', 0) | + |-----------------------+-----------------------| + | hello | | + +-----------------------+-----------------------+ LENGTH @@ -3301,11 +3301,11 @@ Example:: os> SELECT LENGTH('helloworld') fetched rows / total rows = 1/1 - +------------------------+ - | LENGTH('helloworld') | - |------------------------| - | 10 | - +------------------------+ + +----------------------+ + | LENGTH('helloworld') | + |----------------------| + | 10 | + +----------------------+ LOCATE @@ -3327,11 +3327,11 @@ Example:: os> SELECT LOCATE('world', 'helloworld'), LOCATE('world', 'helloworldworld', 7) fetched rows / total rows = 1/1 - +---------------------------------+-----------------------------------------+ - | LOCATE('world', 'helloworld') | LOCATE('world', 'helloworldworld', 7) | - |---------------------------------+-----------------------------------------| - | 6 | 11 | - +---------------------------------+-----------------------------------------+ + +-------------------------------+---------------------------------------+ + | LOCATE('world', 'helloworld') | LOCATE('world', 'helloworldworld', 7) | + |-------------------------------+---------------------------------------| + | 6 | 11 | + +-------------------------------+---------------------------------------+ LOWER @@ -3350,11 +3350,11 @@ Example:: os> SELECT LOWER('helloworld'), LOWER('HELLOWORLD') fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | LOWER('helloworld') | LOWER('HELLOWORLD') | - |-----------------------+-----------------------| - | helloworld | helloworld | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | LOWER('helloworld') | LOWER('HELLOWORLD') | + |---------------------+---------------------| + | helloworld | helloworld | + +---------------------+---------------------+ LTRIM @@ -3373,11 +3373,11 @@ Example:: os> SELECT LTRIM(' hello'), LTRIM('hello ') fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LTRIM(' hello') | LTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | LTRIM(' hello') | LTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ POSITION @@ -3398,11 +3398,11 @@ Example:: os> SELECT POSITION('world' IN 'helloworld'), POSITION('invalid' IN 'helloworld'); fetched rows / total rows = 1/1 - +-------------------------------------+---------------------------------------+ - | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | - |-------------------------------------+---------------------------------------| - | 6 | 0 | - +-------------------------------------+---------------------------------------+ + +-----------------------------------+-------------------------------------+ + | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | + |-----------------------------------+-------------------------------------| + | 6 | 0 | + +-----------------------------------+-------------------------------------+ REPLACE @@ -3421,11 +3421,11 @@ Example:: os> SELECT REPLACE('Hello World!', 'World', 'OpenSearch') fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | REPLACE('Hello World!', 'World', 'OpenSearch') | - |--------------------------------------------------| - | Hello OpenSearch! | - +--------------------------------------------------+ + +------------------------------------------------+ + | REPLACE('Hello World!', 'World', 'OpenSearch') | + |------------------------------------------------| + | Hello OpenSearch! | + +------------------------------------------------+ REVERSE @@ -3444,11 +3444,11 @@ Example:: os> SELECT REVERSE('abcde'), REVERSE(null) fetched rows / total rows = 1/1 - +--------------------+-----------------+ - | REVERSE('abcde') | REVERSE(null) | - |--------------------+-----------------| - | edcba | null | - +--------------------+-----------------+ + +------------------+---------------+ + | REVERSE('abcde') | REVERSE(null) | + |------------------+---------------| + | edcba | null | + +------------------+---------------+ RIGHT @@ -3467,11 +3467,11 @@ Example:: os> SELECT RIGHT('helloworld', 5), RIGHT('HELLOWORLD', 0) fetched rows / total rows = 1/1 - +--------------------------+--------------------------+ - | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | - |--------------------------+--------------------------| - | world | | - +--------------------------+--------------------------+ + +------------------------+------------------------+ + | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | + |------------------------+------------------------| + | world | | + +------------------------+------------------------+ RTRIM @@ -3490,11 +3490,11 @@ Example:: os> SELECT RTRIM(' hello'), RTRIM('hello ') fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | RTRIM(' hello') | RTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | RTRIM(' hello') | RTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ SUBSTRING @@ -3515,11 +3515,11 @@ Example:: os> SELECT SUBSTRING('helloworld', 5), SUBSTRING('helloworld', 5, 3) fetched rows / total rows = 1/1 - +------------------------------+---------------------------------+ - | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | - |------------------------------+---------------------------------| - | oworld | owo | - +------------------------------+---------------------------------+ + +----------------------------+-------------------------------+ + | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | + |----------------------------+-------------------------------| + | oworld | owo | + +----------------------------+-------------------------------+ TRIM @@ -3536,11 +3536,11 @@ Example:: os> SELECT TRIM(' hello'), TRIM('hello ') fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | TRIM(' hello') | TRIM('hello ') | - |--------------------+--------------------| - | hello | hello | - +--------------------+--------------------+ + +------------------+------------------+ + | TRIM(' hello') | TRIM('hello ') | + |------------------+------------------| + | hello | hello | + +------------------+------------------+ UPPER @@ -3559,11 +3559,11 @@ Example:: os> SELECT UPPER('helloworld'), UPPER('HELLOWORLD') fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | UPPER('helloworld') | UPPER('HELLOWORLD') | - |-----------------------+-----------------------| - | HELLOWORLD | HELLOWORLD | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | UPPER('helloworld') | UPPER('HELLOWORLD') | + |---------------------+---------------------| + | HELLOWORLD | HELLOWORLD | + +---------------------+---------------------+ @@ -3586,31 +3586,31 @@ Example One:: os> SELECT IFNULL(123, 321), IFNULL(321, 123) fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | IFNULL(123, 321) | IFNULL(321, 123) | - |--------------------+--------------------| - | 123 | 321 | - +--------------------+--------------------+ + +------------------+------------------+ + | IFNULL(123, 321) | IFNULL(321, 123) | + |------------------+------------------| + | 123 | 321 | + +------------------+------------------+ Example Two:: os> SELECT IFNULL(321, 1/0), IFNULL(1/0, 123) fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | IFNULL(321, 1/0) | IFNULL(1/0, 123) | - |--------------------+--------------------| - | 321 | 123 | - +--------------------+--------------------+ + +------------------+------------------+ + | IFNULL(321, 1/0) | IFNULL(1/0, 123) | + |------------------+------------------| + | 321 | 123 | + +------------------+------------------+ Example Three:: os> SELECT IFNULL(1/0, 1/0) fetched rows / total rows = 1/1 - +--------------------+ - | IFNULL(1/0, 1/0) | - |--------------------| - | null | - +--------------------+ + +------------------+ + | IFNULL(1/0, 1/0) | + |------------------| + | null | + +------------------+ NULLIF @@ -3629,11 +3629,11 @@ Example:: os> SELECT NULLIF(123, 123), NULLIF(321, 123), NULLIF(1/0, 321), NULLIF(321, 1/0), NULLIF(1/0, 1/0) fetched rows / total rows = 1/1 - +--------------------+--------------------+--------------------+--------------------+--------------------+ - | NULLIF(123, 123) | NULLIF(321, 123) | NULLIF(1/0, 321) | NULLIF(321, 1/0) | NULLIF(1/0, 1/0) | - |--------------------+--------------------+--------------------+--------------------+--------------------| - | null | 321 | null | 321 | null | - +--------------------+--------------------+--------------------+--------------------+--------------------+ + +------------------+------------------+------------------+------------------+------------------+ + | NULLIF(123, 123) | NULLIF(321, 123) | NULLIF(1/0, 321) | NULLIF(321, 1/0) | NULLIF(1/0, 1/0) | + |------------------+------------------+------------------+------------------+------------------| + | null | 321 | null | 321 | null | + +------------------+------------------+------------------+------------------+------------------+ ISNULL @@ -3652,11 +3652,11 @@ Example:: os> SELECT ISNULL(1/0), ISNULL(123) fetched rows / total rows = 1/1 - +---------------+---------------+ - | ISNULL(1/0) | ISNULL(123) | - |---------------+---------------| - | True | False | - +---------------+---------------+ + +-------------+-------------+ + | ISNULL(1/0) | ISNULL(123) | + |-------------+-------------| + | True | False | + +-------------+-------------+ IF @@ -3677,19 +3677,19 @@ Example:: os> SELECT IF(100 > 200, '100', '200') fetched rows / total rows = 1/1 - +-------------------------------+ - | IF(100 > 200, '100', '200') | - |-------------------------------| - | 200 | - +-------------------------------+ + +-----------------------------+ + | IF(100 > 200, '100', '200') | + |-----------------------------| + | 200 | + +-----------------------------+ os> SELECT IF(200 > 100, '100', '200') fetched rows / total rows = 1/1 - +-------------------------------+ - | IF(200 > 100, '100', '200') | - |-------------------------------| - | 100 | - +-------------------------------+ + +-----------------------------+ + | IF(200 > 100, '100', '200') | + |-----------------------------| + | 100 | + +-----------------------------+ CASE @@ -3744,11 +3744,11 @@ Here are examples for simple case syntax:: ... ELSE TRIM(' Absolute three ') ... END AS func_result; fetched rows / total rows = 1/1 - +---------------+-------------------+----------------+ - | simple_case | func_case_value | func_result | - |---------------+-------------------+----------------| - | One | Absolute two | Absolute three | - +---------------+-------------------+----------------+ + +-------------+-----------------+----------------+ + | simple_case | func_case_value | func_result | + |-------------+-----------------+----------------| + | One | Absolute two | Absolute three | + +-------------+-----------------+----------------+ Here are examples for searched case syntax:: @@ -3764,11 +3764,11 @@ Here are examples for searched case syntax:: ... WHEN 'hello' = 'world' THEN 'Hello' ... END AS no_else; fetched rows / total rows = 1/1 - +-----------------+------------------+-----------+ - | single_search | multi_searches | no_else | - |-----------------+------------------+-----------| - | One | Hello | null | - +-----------------+------------------+-----------+ + +---------------+----------------+---------+ + | single_search | multi_searches | no_else | + |---------------+----------------+---------| + | One | Hello | null | + +---------------+----------------+---------+ RELEVANCE @@ -3803,22 +3803,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE match(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE match(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ MATCHQUERY @@ -3833,32 +3833,32 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE matchquery(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE matchquery(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ The matchquery function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = matchquery('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_QUERY @@ -3873,32 +3873,32 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT lastname, address FROM accounts WHERE match_query(address, 'Street'); fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT lastname FROM accounts WHERE match_query(firstname, 'Hattie', operator='AND', boost=2.0); fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ The match_query function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = match_query('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_PHRASE @@ -3943,19 +3943,19 @@ The match_phrase function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = match_phrase('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ os> SELECT firstname FROM accounts WHERE firstname = matchphrase('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ MATCH_BOOL_PREFIX @@ -3982,22 +3982,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> SELECT firstname, address FROM accounts WHERE match_bool_prefix(address, 'Bristol Stre'); fetched rows / total rows = 2/2 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - | Nanette | 789 Madison Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + | Nanette | 789 Madison Street | + +-----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> SELECT firstname, address FROM accounts WHERE match_bool_prefix(address, 'Bristol Street', minimum_should_match=2); fetched rows / total rows = 1/1 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + +-----------+--------------------+ MATCH_PHRASE_PREFIX @@ -4084,40 +4084,40 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where multi_match(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where multi_match(['title'], 'Pooh House', operator='AND', analyzer=default); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ The multi_match function also supports an alternative syntax:: os> SELECT firstname FROM accounts WHERE firstname = multi_match('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ os> SELECT firstname FROM accounts WHERE firstname = multimatch('Hattie'); fetched rows / total rows = 1/1 - +-------------+ - | firstname | - |-------------| - | Hattie | - +-------------+ + +-----------+ + | firstname | + |-----------| + | Hattie | + +-----------+ SIMPLE_QUERY_STRING @@ -4154,22 +4154,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where simple_query_string(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where simple_query_string(['title'], 'Pooh House', flags='ALL', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ QUERY_STRING @@ -4216,22 +4216,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> select id, title, author from books where query_string(['title'], 'Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where query_string(['title'], 'Pooh House', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ QUERY @@ -4278,22 +4278,22 @@ Example with only ``query_expressions``, and all other parameters are set defaul os> select id, title, author from books where query('title:Pooh House'); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> select id, title, author from books where query('title:Pooh House', default_operator='AND'); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ SCORE @@ -4323,20 +4323,20 @@ Example boosting score:: os> select id, title, author, _score from books where score(query('title:Pooh House', default_operator='AND'), 2.0); fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+-----------+ - | id | title | author | _score | - |------+--------------------------+----------------------+-----------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | 1.5884793 | - +------+--------------------------+----------------------+-----------+ + +----+--------------------------+----------------------+-----------+ + | id | title | author | _score | + |----+--------------------------+----------------------+-----------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | 1.5884793 | + +----+--------------------------+----------------------+-----------+ os> select id, title, author, _score from books where score(query('title:Pooh House', default_operator='AND'), 5.0) OR score(query('title:Winnie', default_operator='AND'), 1.5); fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+-----------+ - | id | title | author | _score | - |------+--------------------------+----------------------+-----------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | 3.9711983 | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | 1.1581701 | - +------+--------------------------+----------------------+-----------+ + +----+--------------------------+----------------------+-----------+ + | id | title | author | _score | + |----+--------------------------+----------------------+-----------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | 3.9711983 | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | 1.1581701 | + +----+--------------------------+----------------------+-----------+ HIGHLIGHT @@ -4435,45 +4435,45 @@ Example with ``field`` and ``path`` parameters:: os> SELECT nested(message.info, message) FROM nested; fetched rows / total rows = 2/2 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | a | - | b | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | a | + | b | + +-------------------------------+ Example with ``field.*`` used in SELECT clause:: os> SELECT nested(message.*) FROM nested; fetched rows / total rows = 2/2 - +--------------------------+-----------------------------+------------------------+ - | nested(message.author) | nested(message.dayOfWeek) | nested(message.info) | - |--------------------------+-----------------------------+------------------------| - | e | 1 | a | - | f | 2 | b | - +--------------------------+-----------------------------+------------------------+ + +------------------------+---------------------------+----------------------+ + | nested(message.author) | nested(message.dayOfWeek) | nested(message.info) | + |------------------------+---------------------------+----------------------| + | e | 1 | a | + | f | 2 | b | + +------------------------+---------------------------+----------------------+ Example with ``field`` and ``path`` parameters in the SELECT and WHERE clause:: os> SELECT nested(message.info, message) FROM nested WHERE nested(message.info, message) = 'b'; fetched rows / total rows = 1/1 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | b | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | b | + +-------------------------------+ Example with ``field`` and ``path`` parameters in the SELECT and ORDER BY clause:: os> SELECT nested(message.info, message) FROM nested ORDER BY nested(message.info, message) DESC; fetched rows / total rows = 2/2 - +---------------------------------+ - | nested(message.info, message) | - |---------------------------------| - | b | - | a | - +---------------------------------+ + +-------------------------------+ + | nested(message.info, message) | + |-------------------------------| + | b | + | a | + +-------------------------------+ System Functions @@ -4495,9 +4495,9 @@ Example:: os> select typeof(DATE('2008-04-14')) as `typeof(date)`, typeof(1) as `typeof(int)`, typeof(now()) as `typeof(now())`, typeof(accounts) as `typeof(column)` from people fetched rows / total rows = 1/1 - +----------------+---------------+-----------------+------------------+ - | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | - |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | TIMESTAMP | OBJECT | - +----------------+---------------+-----------------+------------------+ + +--------------+-------------+---------------+----------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |--------------+-------------+---------------+----------------| + | DATE | INTEGER | TIMESTAMP | OBJECT | + +--------------+-------------+---------------+----------------+ diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index a02bcf096a..fa233020a3 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -36,19 +36,19 @@ SQL query:: os> SHOW TABLES LIKE '%' fetched rows / total rows = 9/9 - +----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | - |----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------| - | docTestCluster | null | .ql-datasources | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | - +----------------+---------------+-----------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ + +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| + | docTestCluster | null | .ql-datasources | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | + +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ Example 2: Show Specific Index Information ------------------------------------------ @@ -59,12 +59,12 @@ SQL query:: os> SHOW TABLES LIKE "acc%" fetched rows / total rows = 2/2 - +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | - |----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------| - | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | - +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ + +----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------| + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + +----------------+-------------+------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ Example 3: Describe Index Fields Information -------------------------------------------- @@ -75,21 +75,21 @@ SQL query:: os> DESCRIBE TABLES LIKE 'accounts' fetched rows / total rows = 11/11 - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | - |----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| - | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ Example 4: Describe Index With Fields Filter -------------------------------------------- @@ -100,9 +100,9 @@ SQL query:: os> DESCRIBE TABLES LIKE "accounts" COLUMNS LIKE "%name" fetched rows / total rows = 2/2 - +----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | - |----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| - | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | - +----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + +----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ diff --git a/docs/user/dql/window.rst b/docs/user/dql/window.rst index feb2aaa44e..f0c53da055 100644 --- a/docs/user/dql/window.rst +++ b/docs/user/dql/window.rst @@ -53,14 +53,14 @@ Here is an example for ``COUNT`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 1 | - | M | 4180 | 1 | - | M | 5686 | 2 | - | M | 39225 | 3 | - +----------+-----------+-------+ + +--------+---------+-----+ + | gender | balance | cnt | + |--------+---------+-----| + | F | 32838 | 1 | + | M | 4180 | 1 | + | M | 5686 | 2 | + | M | 39225 | 3 | + +--------+---------+-----+ MIN --- @@ -74,14 +74,14 @@ Here is an example for ``MIN`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 4180 | - | M | 39225 | 4180 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 4180 | + | M | 39225 | 4180 | + +--------+---------+-------+ MAX --- @@ -95,14 +95,14 @@ Here is an example for ``MAX`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 5686 | - | M | 39225 | 39225 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 5686 | + | M | 39225 | 39225 | + +--------+---------+-------+ AVG --- @@ -116,14 +116,14 @@ Here is an example for ``AVG`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | cnt | - |----------+-----------+--------------------| - | F | 32838 | 32838.0 | - | M | 4180 | 4180.0 | - | M | 5686 | 4933.0 | - | M | 39225 | 16363.666666666666 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | cnt | + |--------+---------+--------------------| + | F | 32838 | 32838.0 | + | M | 4180 | 4180.0 | + | M | 5686 | 4933.0 | + | M | 39225 | 16363.666666666666 | + +--------+---------+--------------------+ SUM --- @@ -137,14 +137,14 @@ Here is an example for ``SUM`` function:: ... ) AS cnt ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | cnt | - |----------+-----------+-------| - | F | 32838 | 32838 | - | M | 4180 | 4180 | - | M | 5686 | 9866 | - | M | 39225 | 49091 | - +----------+-----------+-------+ + +--------+---------+-------+ + | gender | balance | cnt | + |--------+---------+-------| + | F | 32838 | 32838 | + | M | 4180 | 4180 | + | M | 5686 | 9866 | + | M | 39225 | 49091 | + +--------+---------+-------+ STDDEV_POP ---------- @@ -158,14 +158,14 @@ Here is an example for ``STDDEV_POP`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 753.0 | - | M | 39225 | 16177.091422406222 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 753.0 | + | M | 39225 | 16177.091422406222 | + +--------+---------+--------------------+ STDDEV_SAMP ----------- @@ -179,14 +179,14 @@ Here is an example for ``STDDEV_SAMP`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 1064.9028124669405 | - | M | 39225 | 19812.809753624886 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 1064.9028124669405 | + | M | 39225 | 19812.809753624886 | + +--------+---------+--------------------+ VAR_POP ------- @@ -200,14 +200,14 @@ Here is an example for ``SUM`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+--------------------+ - | gender | balance | val | - |----------+-----------+--------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 567009.0 | - | M | 39225 | 261698286.88888893 | - +----------+-----------+--------------------+ + +--------+---------+--------------------+ + | gender | balance | val | + |--------+---------+--------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 567009.0 | + | M | 39225 | 261698286.88888893 | + +--------+---------+--------------------+ VAR_SAMP -------- @@ -221,14 +221,14 @@ Here is an example for ``SUM`` function:: ... ) AS val ... FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------------------+ - | gender | balance | val | - |----------+-----------+-------------------| - | F | 32838 | 0.0 | - | M | 4180 | 0.0 | - | M | 5686 | 1134018.0 | - | M | 39225 | 392547430.3333334 | - +----------+-----------+-------------------+ + +--------+---------+-------------------+ + | gender | balance | val | + |--------+---------+-------------------| + | F | 32838 | 0.0 | + | M | 4180 | 0.0 | + | M | 5686 | 1134018.0 | + | M | 39225 | 392547430.3333334 | + +--------+---------+-------------------+ Ranking Functions @@ -248,14 +248,14 @@ ROW_NUMBER os> SELECT gender, balance, ROW_NUMBER() OVER(PARTITION BY gender ORDER BY balance) AS num FROM accounts; fetched rows / total rows = 4/4 - +----------+-----------+-------+ - | gender | balance | num | - |----------+-----------+-------| - | F | 32838 | 1 | - | M | 4180 | 1 | - | M | 5686 | 2 | - | M | 39225 | 3 | - +----------+-----------+-------+ + +--------+---------+-----+ + | gender | balance | num | + |--------+---------+-----| + | F | 32838 | 1 | + | M | 4180 | 1 | + | M | 5686 | 2 | + | M | 39225 | 3 | + +--------+---------+-----+ Similarly as regular ``ORDER BY`` clause, you can specify null ordering by ``NULLS FIRST`` or ``NULLS LAST`` which has exactly same behavior:: @@ -267,14 +267,14 @@ Similarly as regular ``ORDER BY`` clause, you can specify null ordering by ``NUL ... FROM accounts ... ORDER BY employer NULLS LAST; fetched rows / total rows = 4/4 - +------------+-------+ - | employer | num | - |------------+-------| - | Netagy | 1 | - | Pyrami | 2 | - | Quility | 3 | - | null | 4 | - +------------+-------+ + +----------+-----+ + | employer | num | + |----------+-----| + | Netagy | 1 | + | Pyrami | 2 | + | Quility | 3 | + | null | 4 | + +----------+-----+ RANK ---- @@ -283,14 +283,14 @@ RANK os> SELECT gender, RANK() OVER(ORDER BY gender DESC) AS rnk FROM accounts; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | rnk | - |----------+-------| - | M | 1 | - | M | 1 | - | M | 1 | - | F | 4 | - +----------+-------+ + +--------+-----+ + | gender | rnk | + |--------+-----| + | M | 1 | + | M | 1 | + | M | 1 | + | F | 4 | + +--------+-----+ DENSE_RANK @@ -300,12 +300,12 @@ Similarly as ``RANK``, ``DENSE_RANK`` function also assigns a rank to each row. os> SELECT gender, DENSE_RANK() OVER(ORDER BY gender DESC) AS rnk FROM accounts; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | rnk | - |----------+-------| - | M | 1 | - | M | 1 | - | M | 1 | - | F | 2 | - +----------+-------+ + +--------+-----+ + | gender | rnk | + |--------+-----| + | M | 1 | + | M | 1 | + | M | 1 | + | F | 2 | + +--------+-----+ diff --git a/docs/user/general/comments.rst b/docs/user/general/comments.rst index ab959da342..536843695e 100644 --- a/docs/user/general/comments.rst +++ b/docs/user/general/comments.rst @@ -26,11 +26,11 @@ A single-line comment starts with either ``#`` or ``--``. All characters in the ... -- comments ... 123; -- comments fetched rows / total rows = 1/1 - +-------+ - | 123 | - |-------| - | 123 | - +-------+ + +-----+ + | 123 | + |-----| + | 123 | + +-----+ Note that double-dash style requires at least one whitespace followed. @@ -48,10 +48,10 @@ A block comment is enclosed within ``/*`` and ``*/`` across one or multiple line ... /* comments */ ... 123; fetched rows / total rows = 1/1 - +-------+ - | 123 | - |-------| - | 123 | - +-------+ + +-----+ + | 123 | + |-----| + | 123 | + +-----+ Additionally, ``/*! ... */`` is supported though ignored for now. This may be used to support optimization hints in future. diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 042e97396e..f18b5b716a 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -188,11 +188,11 @@ Here are a few examples for implicit type conversion:: ... 'True' = true, ... DATE('2021-06-10') < '2021-06-11'; fetched rows / total rows = 1/1 - +-----------+-----------------+-------------------------------------+ - | 1 = 1.0 | 'True' = true | DATE('2021-06-10') < '2021-06-11' | - |-----------+-----------------+-------------------------------------| - | True | True | True | - +-----------+-----------------+-------------------------------------+ + +---------+---------------+-----------------------------------+ + | 1 = 1.0 | 'True' = true | DATE('2021-06-10') < '2021-06-11' | + |---------+---------------+-----------------------------------| + | True | True | True | + +---------+---------------+-----------------------------------+ Here are a few examples for explicit type conversion:: @@ -201,11 +201,11 @@ Here are a few examples for explicit type conversion:: ... CAST(1.2 AS STRING), ... CAST('2021-06-10 00:00:00' AS TIMESTAMP); fetched rows / total rows = 1/1 - +---------------------+-----------------------+--------------------------------------------+ - | CAST(true AS INT) | CAST(1.2 AS STRING) | CAST('2021-06-10 00:00:00' AS TIMESTAMP) | - |---------------------+-----------------------+--------------------------------------------| - | 1 | 1.2 | 2021-06-10 00:00:00 | - +---------------------+-----------------------+--------------------------------------------+ + +-------------------+---------------------+------------------------------------------+ + | CAST(true AS INT) | CAST(1.2 AS STRING) | CAST('2021-06-10 00:00:00' AS TIMESTAMP) | + |-------------------+---------------------+------------------------------------------| + | 1 | 1.2 | 2021-06-10 00:00:00 | + +-------------------+---------------------+------------------------------------------+ Undefined Data Type =================== @@ -216,11 +216,11 @@ Here are examples for NULL literal and expressions with NULL literal involved:: os> SELECT NULL, NULL = NULL, 1 + NULL, LENGTH(NULL); fetched rows / total rows = 1/1 - +--------+---------------+------------+----------------+ - | NULL | NULL = NULL | 1 + NULL | LENGTH(NULL) | - |--------+---------------+------------+----------------| - | null | null | null | null | - +--------+---------------+------------+----------------+ + +------+-------------+----------+--------------+ + | NULL | NULL = NULL | 1 + NULL | LENGTH(NULL) | + |------+-------------+----------+--------------| + | null | null | null | null | + +------+-------------+----------+--------------+ Numeric Data Types @@ -318,11 +318,11 @@ A string can also represent and be converted to date and time types (except to i ... '2021-06-18' < DATE('2021-06-17'), ... '10:20:00' <= TIME('11:00:00'); fetched rows / total rows = 1/1 - +------------------------------------------------------------+-------------------------------------+----------------------------------+ - | TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00' | '2021-06-18' < DATE('2021-06-17') | '10:20:00' <= TIME('11:00:00') | - |------------------------------------------------------------+-------------------------------------+----------------------------------| - | True | False | True | - +------------------------------------------------------------+-------------------------------------+----------------------------------+ + +----------------------------------------------------------+-----------------------------------+--------------------------------+ + | TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00' | '2021-06-18' < DATE('2021-06-17') | '10:20:00' <= TIME('11:00:00') | + |----------------------------------------------------------+-----------------------------------+--------------------------------| + | True | False | True | + +----------------------------------------------------------+-----------------------------------+--------------------------------+ Please, see `more examples here <../dql/expressions.rst#toc-entry-15>`_. @@ -449,11 +449,11 @@ A string is a sequence of characters enclosed in either single or double quotes. os> SELECT 'hello', "world", '"hello"', "'world'", '''hello''', """world""", 'I\'m', 'I''m', "I\"m" fetched rows / total rows = 1/1 - +-----------+-----------+-------------+-------------+---------------+---------------+----------+----------+----------+ - | 'hello' | "world" | '"hello"' | "'world'" | '''hello''' | """world""" | 'I\'m' | 'I''m' | "I\"m" | - |-----------+-----------+-------------+-------------+---------------+---------------+----------+----------+----------| - | hello | world | "hello" | 'world' | 'hello' | "world" | I'm | I'm | I"m | - +-----------+-----------+-------------+-------------+---------------+---------------+----------+----------+----------+ + +---------+---------+-----------+-----------+-------------+-------------+--------+--------+--------+ + | 'hello' | "world" | '"hello"' | "'world'" | '''hello''' | """world""" | 'I\'m' | 'I''m' | "I\"m" | + |---------+---------+-----------+-----------+-------------+-------------+--------+--------+--------| + | hello | world | "hello" | 'world' | 'hello' | "world" | I'm | I'm | I"m | + +---------+---------+-----------+-----------+-------------+-------------+--------+--------+--------+ Boolean Data Types ================== @@ -464,8 +464,8 @@ A boolean can be represented by constant value ``TRUE`` or ``FALSE``. Besides, c ... true, FALSE, ... CAST('TRUE' AS boolean), CAST('false' AS boolean); fetched rows / total rows = 1/1 - +--------+---------+---------------------------+----------------------------+ - | true | FALSE | CAST('TRUE' AS boolean) | CAST('false' AS boolean) | - |--------+---------+---------------------------+----------------------------| - | True | False | True | False | - +--------+---------+---------------------------+----------------------------+ + +------+-------+-------------------------+--------------------------+ + | true | FALSE | CAST('TRUE' AS boolean) | CAST('false' AS boolean) | + |------+-------+-------------------------+--------------------------| + | True | False | True | False | + +------+-------+-------------------------+--------------------------+ diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index fad2fa4b23..562bf38526 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -40,14 +40,14 @@ Here are examples for using index pattern directly without quotes:: os> SELECT * FROM *cc*nts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Delimited Identifiers @@ -76,14 +76,14 @@ Here are examples for quoting an index name by back ticks:: os> SELECT * FROM `accounts`; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Case Sensitivity @@ -121,23 +121,23 @@ The first example is to show a column name qualified by full table name original os> SELECT city, accounts.age, ABS(accounts.balance) FROM accounts WHERE accounts.age < 30; fetched rows / total rows = 1/1 - +--------+-------+-------------------------+ - | city | age | ABS(accounts.balance) | - |--------+-------+-------------------------| - | Nogal | 28 | 32838 | - +--------+-------+-------------------------+ + +-------+-----+-----------------------+ + | city | age | ABS(accounts.balance) | + |-------+-----+-----------------------| + | Nogal | 28 | 32838 | + +-------+-----+-----------------------+ The second example is to show a field name qualified by index alias specified. Similarly, the alias qualifier is optional in this case:: os> SELECT city, acc.age, ABS(acc.balance) FROM accounts AS acc WHERE acc.age > 30; fetched rows / total rows = 3/3 - +--------+-------+--------------------+ - | city | age | ABS(acc.balance) | - |--------+-------+--------------------| - | Brogan | 32 | 39225 | - | Dante | 36 | 5686 | - | Orick | 33 | 4180 | - +--------+-------+--------------------+ + +--------+-----+------------------+ + | city | age | ABS(acc.balance) | + |--------+-----+------------------| + | Brogan | 32 | 39225 | + | Dante | 36 | 5686 | + | Orick | 33 | 4180 | + +--------+-----+------------------+ Note that in both examples above, the qualifier is removed in response. This happens only when identifiers selected is a simple field name. In other cases, expressions rather than an atom field, the column name in response is exactly the same as the text in ``SELECT``clause. @@ -160,22 +160,22 @@ Query wildcard indices:: os> SELECT count(*) as cnt FROM acc*; fetched rows / total rows = 1/1 - +-------+ - | cnt | - |-------| - | 5 | - +-------+ + +-----+ + | cnt | + |-----| + | 5 | + +-----+ Query delimited multiple indices seperated by ``,``:: os> SELECT count(*) as cnt FROM `accounts,account2`; fetched rows / total rows = 1/1 - +-------+ - | cnt | - |-------| - | 5 | - +-------+ + +-----+ + | cnt | + |-----| + | 5 | + +-----+ diff --git a/docs/user/general/values.rst b/docs/user/general/values.rst index 178609f175..f675b42e75 100644 --- a/docs/user/general/values.rst +++ b/docs/user/general/values.rst @@ -19,14 +19,14 @@ Here is an example, Nanette doesn't have email field and Dail has employer filed os> SELECT firstname, employer, email FROM accounts; fetched rows / total rows = 4/4 - +-------------+------------+-----------------------+ - | firstname | employer | email | - |-------------+------------+-----------------------| - | Amber | Pyrami | amberduke@pyrami.com | - | Hattie | Netagy | hattiebond@netagy.com | - | Nanette | Quility | null | - | Dale | null | daleadams@boink.com | - +-------------+------------+-----------------------+ + +-----------+----------+-----------------------+ + | firstname | employer | email | + |-----------+----------+-----------------------| + | Amber | Pyrami | amberduke@pyrami.com | + | Hattie | Netagy | hattiebond@netagy.com | + | Nanette | Quility | null | + | Dale | null | daleadams@boink.com | + +-----------+----------+-----------------------+ General NULL and MISSING Values Handling @@ -37,14 +37,14 @@ Here is an example:: os> SELECT firstname, employer LIKE 'Quility', email LIKE '%com' FROM accounts; fetched rows / total rows = 4/4 - +-------------+---------------------------+---------------------+ - | firstname | employer LIKE 'Quility' | email LIKE '%com' | - |-------------+---------------------------+---------------------| - | Amber | False | True | - | Hattie | False | True | - | Nanette | True | null | - | Dale | null | True | - +-------------+---------------------------+---------------------+ + +-----------+-------------------------+-------------------+ + | firstname | employer LIKE 'Quility' | email LIKE '%com' | + |-----------+-------------------------+-------------------| + | Amber | False | True | + | Hattie | False | True | + | Nanette | True | null | + | Dale | null | True | + +-----------+-------------------------+-------------------+ Special NULL and MISSING Values Handling ---------------------------------------- diff --git a/docs/user/ppl/admin/connectors/prometheus_connector.rst b/docs/user/ppl/admin/connectors/prometheus_connector.rst index 1dfe6cda22..812df4f894 100644 --- a/docs/user/ppl/admin/connectors/prometheus_connector.rst +++ b/docs/user/ppl/admin/connectors/prometheus_connector.rst @@ -87,16 +87,16 @@ Sample Example:: > source = my_prometheus.prometheus_http_requests_total; - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ @@ -119,30 +119,30 @@ Example queries 1. Metric Selection Query:: > source = my_prometheus.prometheus_http_requests_total - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ 2. Metric Selecting Query with specific dimensions:: > source = my_prometheus.prometheus_http_requests_total | where handler='/-/ready' and code='200' - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" | "/-/ready" | 200 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + +--------+-----------------------+------------+------+------------+------------+ 3. Average aggregation on a metric:: @@ -199,16 +199,16 @@ PromQL Support for prometheus Connector Example:: > source=my_prometheus.query_range('prometheus_http_requests_total', 1686694425, 1686700130, 14) - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ - | @value | @timestamp | handler | code | instance | job | - |------------+------------------------+--------------------------------+---------------+-------------+-------------| - | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | - | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | - | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | - | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | - +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +--------+-----------------------+--------------+------+------------+------------+ + | @value | @timestamp | handler | code | instance | job | + |--------+-----------------------+--------------+------+------------+------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | 192.15.2.1 | prometheus | + +--------+-----------------------+--------------+------+------------+------------+ Prometheus Connector Table Functions diff --git a/docs/user/ppl/admin/cross_cluster_search.rst b/docs/user/ppl/admin/cross_cluster_search.rst index f57ea288e8..4b267a9340 100644 --- a/docs/user/ppl/admin/cross_cluster_search.rst +++ b/docs/user/ppl/admin/cross_cluster_search.rst @@ -40,14 +40,14 @@ Example PPL query:: os> source=my_remote_cluster:accounts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Limitation diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index 103c7f7483..5d7a572c96 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -50,11 +50,11 @@ PPL query:: > source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------------------+---------+-----------------+ - | value | timestamp | score | anomaly_grade | - |---------+---------------------+---------+-----------------| - | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - +---------+---------------------+---------+-----------------+ + +---------+---------------------+-------+---------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+-------+---------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+-------+---------------+ Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category ============================================================================================================================ @@ -65,12 +65,12 @@ PPL query:: > source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------------------+---------+-----------------+ - | category | value | timestamp | score | anomaly_grade | - |------------+---------+---------------------+---------+-----------------| - | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | - +------------+---------+---------------------+---------+-----------------+ + +----------+---------+---------------------+-------+---------------+ + | category | value | timestamp | score | anomaly_grade | + |----------+---------+---------------------+-------+---------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +----------+---------+---------------------+-------+---------------+ Example 3: Detecting events in New York City from taxi ridership data with non-time-series data @@ -82,11 +82,11 @@ PPL query:: > source=nyc_taxi | fields value | AD | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------+-------------+ - | value | score | anomalous | - |---------+---------+-------------| - | 10844.0 | 0.0 | False | - +---------+---------+-------------+ + +---------+-------+-----------+ + | value | score | anomalous | + |---------+-------+-----------| + | 10844.0 | 0.0 | False | + +---------+-------+-----------+ Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category ================================================================================================================================ @@ -97,9 +97,9 @@ PPL query:: > source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------+-------------+ - | category | value | score | anomalous | - |------------+---------+---------+-------------| - | night | 10844.0 | 0.0 | False | - | day | 6526.0 | 0.0 | False | - +------------+---------+---------+-------------+ + +----------+---------+-------+-----------+ + | category | value | score | anomalous | + |----------+---------+-------+-----------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +----------+---------+-------+-----------+ diff --git a/docs/user/ppl/cmd/dedup.rst b/docs/user/ppl/cmd/dedup.rst index ebceb9e0bd..362d1637f7 100644 --- a/docs/user/ppl/cmd/dedup.rst +++ b/docs/user/ppl/cmd/dedup.rst @@ -34,12 +34,12 @@ PPL query:: os> source=accounts | dedup gender | fields account_number, gender; fetched rows / total rows = 2/2 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + +----------------+--------+ Example 2: Keep 2 duplicates documents ====================================== @@ -50,13 +50,13 @@ PPL query:: os> source=accounts | dedup 2 gender | fields account_number, gender; fetched rows / total rows = 3/3 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 6 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 6 | M | + | 13 | F | + +----------------+--------+ Example 3: Keep or Ignore the empty field by default ============================================ @@ -67,14 +67,14 @@ PPL query:: os> source=accounts | dedup email keepempty=true | fields account_number, email; fetched rows / total rows = 4/4 - +------------------+-----------------------+ - | account_number | email | - |------------------+-----------------------| - | 1 | amberduke@pyrami.com | - | 6 | hattiebond@netagy.com | - | 13 | null | - | 18 | daleadams@boink.com | - +------------------+-----------------------+ + +----------------+-----------------------+ + | account_number | email | + |----------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 13 | null | + | 18 | daleadams@boink.com | + +----------------+-----------------------+ The example show dedup the document by ignore the empty value field. @@ -83,13 +83,13 @@ PPL query:: os> source=accounts | dedup email | fields account_number, email; fetched rows / total rows = 3/3 - +------------------+-----------------------+ - | account_number | email | - |------------------+-----------------------| - | 1 | amberduke@pyrami.com | - | 6 | hattiebond@netagy.com | - | 18 | daleadams@boink.com | - +------------------+-----------------------+ + +----------------+-----------------------+ + | account_number | email | + |----------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 18 | daleadams@boink.com | + +----------------+-----------------------+ Example 4: Dedup in consecutive document @@ -101,13 +101,13 @@ PPL query:: os> source=accounts | dedup gender consecutive=true | fields account_number, gender; fetched rows / total rows = 3/3 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - | 18 | M | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + | 18 | M | + +----------------+--------+ Limitation ========== diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index a0ecbd3169..2b03ceda57 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -33,21 +33,21 @@ PPL query:: os> describe accounts; fetched rows / total rows = 11/11 - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ - | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | - |----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| - | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | - | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | - +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------| + | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+-------------+------------+----------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+ Example 2: Fetch metadata with condition and filter =================================================== @@ -76,13 +76,13 @@ PPL query:: os> describe my_prometheus.prometheus_http_requests_total; fetched rows / total rows = 6/6 - +-----------------+----------------+--------------------------------+---------------+-------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | - |-----------------+----------------+--------------------------------+---------------+-------------| - | my_prometheus | default | prometheus_http_requests_total | handler | keyword | - | my_prometheus | default | prometheus_http_requests_total | code | keyword | - | my_prometheus | default | prometheus_http_requests_total | instance | keyword | - | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | - | my_prometheus | default | prometheus_http_requests_total | @value | double | - | my_prometheus | default | prometheus_http_requests_total | job | keyword | - +-----------------+----------------+--------------------------------+---------------+-------------+ + +---------------+--------------+--------------------------------+-------------+-----------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | + |---------------+--------------+--------------------------------+-------------+-----------| + | my_prometheus | default | prometheus_http_requests_total | handler | keyword | + | my_prometheus | default | prometheus_http_requests_total | code | keyword | + | my_prometheus | default | prometheus_http_requests_total | instance | keyword | + | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | + | my_prometheus | default | prometheus_http_requests_total | @value | double | + | my_prometheus | default | prometheus_http_requests_total | job | keyword | + +---------------+--------------+--------------------------------+-------------+-----------+ diff --git a/docs/user/ppl/cmd/eval.rst b/docs/user/ppl/cmd/eval.rst index 48a14ae0a8..c950028674 100644 --- a/docs/user/ppl/cmd/eval.rst +++ b/docs/user/ppl/cmd/eval.rst @@ -30,14 +30,14 @@ PPL query:: os> source=accounts | eval doubleAge = age * 2 | fields age, doubleAge ; fetched rows / total rows = 4/4 - +-------+-------------+ - | age | doubleAge | - |-------+-------------| - | 32 | 64 | - | 36 | 72 | - | 28 | 56 | - | 33 | 66 | - +-------+-------------+ + +-----+-----------+ + | age | doubleAge | + |-----+-----------| + | 32 | 64 | + | 36 | 72 | + | 28 | 56 | + | 33 | 66 | + +-----+-----------+ Example 2: Override the existing field @@ -49,14 +49,14 @@ PPL query:: os> source=accounts | eval age = age + 1 | fields age ; fetched rows / total rows = 4/4 - +-------+ - | age | - |-------| - | 33 | - | 37 | - | 29 | - | 34 | - +-------+ + +-----+ + | age | + |-----| + | 33 | + | 37 | + | 29 | + | 34 | + +-----+ Example 3: Create the new field with field defined in eval ========================================================== @@ -67,14 +67,14 @@ PPL query:: os> source=accounts | eval doubleAge = age * 2, ddAge = doubleAge * 2 | fields age, doubleAge, ddAge ; fetched rows / total rows = 4/4 - +-------+-------------+---------+ - | age | doubleAge | ddAge | - |-------+-------------+---------| - | 32 | 64 | 128 | - | 36 | 72 | 144 | - | 28 | 56 | 112 | - | 33 | 66 | 132 | - +-------+-------------+---------+ + +-----+-----------+-------+ + | age | doubleAge | ddAge | + |-----+-----------+-------| + | 32 | 64 | 128 | + | 36 | 72 | 144 | + | 28 | 56 | 112 | + | 33 | 66 | 132 | + +-----+-----------+-------+ Limitation ========== diff --git a/docs/user/ppl/cmd/fields.rst b/docs/user/ppl/cmd/fields.rst index dbae5b20a4..32c3a665d7 100644 --- a/docs/user/ppl/cmd/fields.rst +++ b/docs/user/ppl/cmd/fields.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | fields account_number, firstname, lastname; fetched rows / total rows = 4/4 - +------------------+-------------+------------+ - | account_number | firstname | lastname | - |------------------+-------------+------------| - | 1 | Amber | Duke | - | 6 | Hattie | Bond | - | 13 | Nanette | Bates | - | 18 | Dale | Adams | - +------------------+-------------+------------+ + +----------------+-----------+----------+ + | account_number | firstname | lastname | + |----------------+-----------+----------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +----------------+-----------+----------+ Example 2: Remove specified fields from result ============================================== @@ -49,12 +49,12 @@ PPL query:: os> source=accounts | fields account_number, firstname, lastname | fields - account_number ; fetched rows / total rows = 4/4 - +-------------+------------+ - | firstname | lastname | - |-------------+------------| - | Amber | Duke | - | Hattie | Bond | - | Nanette | Bates | - | Dale | Adams | - +-------------+------------+ + +-----------+----------+ + | firstname | lastname | + |-----------+----------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +-----------+----------+ diff --git a/docs/user/ppl/cmd/grok.rst b/docs/user/ppl/cmd/grok.rst index 6a121c7431..35f3b0c846 100644 --- a/docs/user/ppl/cmd/grok.rst +++ b/docs/user/ppl/cmd/grok.rst @@ -72,14 +72,14 @@ PPL query:: os> source=apache | grok message '%{COMMONAPACHELOG}' | fields COMMONAPACHELOG, timestamp, response, bytes ; fetched rows / total rows = 4/4 - +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------+ - | COMMONAPACHELOG | timestamp | response | bytes | - |-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------| - | 177.95.8.74 - upton5450 [28/Sep/2022:10:15:57 -0700] "HEAD /e-business/mindshare HTTP/1.0" 404 19927 | 28/Sep/2022:10:15:57 -0700 | 404 | 19927 | - | 127.45.152.6 - pouros8756 [28/Sep/2022:10:15:57 -0700] "GET /architectures/convergence/niches/mindshare HTTP/1.0" 100 28722 | 28/Sep/2022:10:15:57 -0700 | 100 | 28722 | - | 118.223.210.105 - - [28/Sep/2022:10:15:57 -0700] "PATCH /strategize/out-of-the-box HTTP/1.0" 401 27439 | 28/Sep/2022:10:15:57 -0700 | 401 | 27439 | - | 210.204.15.104 - - [28/Sep/2022:10:15:57 -0700] "POST /users HTTP/1.1" 301 9481 | 28/Sep/2022:10:15:57 -0700 | 301 | 9481 | - +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+------------+---------+ + +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------+ + | COMMONAPACHELOG | timestamp | response | bytes | + |-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------| + | 177.95.8.74 - upton5450 [28/Sep/2022:10:15:57 -0700] "HEAD /e-business/mindshare HTTP/1.0" 404 19927 | 28/Sep/2022:10:15:57 -0700 | 404 | 19927 | + | 127.45.152.6 - pouros8756 [28/Sep/2022:10:15:57 -0700] "GET /architectures/convergence/niches/mindshare HTTP/1.0" 100 28722 | 28/Sep/2022:10:15:57 -0700 | 100 | 28722 | + | 118.223.210.105 - - [28/Sep/2022:10:15:57 -0700] "PATCH /strategize/out-of-the-box HTTP/1.0" 401 27439 | 28/Sep/2022:10:15:57 -0700 | 401 | 27439 | + | 210.204.15.104 - - [28/Sep/2022:10:15:57 -0700] "POST /users HTTP/1.1" 301 9481 | 28/Sep/2022:10:15:57 -0700 | 301 | 9481 | + +-----------------------------------------------------------------------------------------------------------------------------+----------------------------+----------+-------+ Limitations =========== diff --git a/docs/user/ppl/cmd/head.rst b/docs/user/ppl/cmd/head.rst index 1b4599f5de..cd4aed5a54 100644 --- a/docs/user/ppl/cmd/head.rst +++ b/docs/user/ppl/cmd/head.rst @@ -30,14 +30,14 @@ PPL query:: os> source=accounts | fields firstname, age | head; fetched rows / total rows = 4/4 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Amber | 32 | - | Hattie | 36 | - | Nanette | 28 | - | Dale | 33 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + | Dale | 33 | + +-----------+-----+ Example 2: Get first N results =========================================== @@ -48,13 +48,13 @@ PPL query:: os> source=accounts | fields firstname, age | head 3; fetched rows / total rows = 3/3 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Amber | 32 | - | Hattie | 36 | - | Nanette | 28 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + +-----------+-----+ Example 3: Get first N results after offset M ============================================= @@ -65,13 +65,13 @@ PPL query:: os> source=accounts | fields firstname, age | head 3 from 1; fetched rows / total rows = 3/3 - +-------------+-------+ - | firstname | age | - |-------------+-------| - | Hattie | 36 | - | Nanette | 28 | - | Dale | 33 | - +-------------+-------+ + +-----------+-----+ + | firstname | age | + |-----------+-----| + | Hattie | 36 | + | Nanette | 28 | + | Dale | 33 | + +-----------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst index 26341d6972..4210502eda 100644 --- a/docs/user/ppl/cmd/information_schema.rst +++ b/docs/user/ppl/cmd/information_schema.rst @@ -29,11 +29,11 @@ PPL query for fetching PROMETHEUS TABLES with where clause:: os> source = my_prometheus.information_schema.tables | where TABLE_NAME='prometheus_http_requests_total' fetched rows / total rows = 1/1 - +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |-----------------+----------------+--------------------------------+--------------+--------+---------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ + +---------------+--------------+--------------------------------+------------+------+---------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |---------------+--------------+--------------------------------+------------+------+---------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + +---------------+--------------+--------------------------------+------------+------+---------------------------+ Example 2: Search tables in prometheus datasource. @@ -45,13 +45,13 @@ PPL query for searching PROMETHEUS TABLES:: os> source = my_prometheus.information_schema.tables | where LIKE(TABLE_NAME, "%http%"); fetched rows / total rows = 6/6 - +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | - | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | - | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | - | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | - | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | - +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ + +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | + | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | + | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | + | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | + | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | + +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ diff --git a/docs/user/ppl/cmd/ml.rst b/docs/user/ppl/cmd/ml.rst index 2e04674c1e..a48c1ec589 100644 --- a/docs/user/ppl/cmd/ml.rst +++ b/docs/user/ppl/cmd/ml.rst @@ -56,11 +56,11 @@ PPL query:: os> source=nyc_taxi | fields value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------------------+---------+-----------------+ - | value | timestamp | score | anomaly_grade | - |---------+---------------------+---------+-----------------| - | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - +---------+---------------------+---------+-----------------+ + +---------+---------------------+-------+---------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+-------+---------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+-------+---------------+ Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category ============================================================================================================================ @@ -71,12 +71,12 @@ PPL query:: os> source=nyc_taxi | fields category, value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------------------+---------+-----------------+ - | category | value | timestamp | score | anomaly_grade | - |------------+---------+---------------------+---------+-----------------| - | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | - | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | - +------------+---------+---------------------+---------+-----------------+ + +----------+---------+---------------------+-------+---------------+ + | category | value | timestamp | score | anomaly_grade | + |----------+---------+---------------------+-------+---------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +----------+---------+---------------------+-------+---------------+ Example 3: Detecting events in New York City from taxi ridership data with non-time-series data @@ -88,11 +88,11 @@ PPL query:: os> source=nyc_taxi | fields value | ml action='train' algorithm='rcf' | where value=10844.0 fetched rows / total rows = 1/1 - +---------+---------+-------------+ - | value | score | anomalous | - |---------+---------+-------------| - | 10844.0 | 0.0 | False | - +---------+---------+-------------+ + +---------+-------+-----------+ + | value | score | anomalous | + |---------+-------+-----------| + | 10844.0 | 0.0 | False | + +---------+-------+-----------+ Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category ================================================================================================================================ @@ -103,12 +103,12 @@ PPL query:: os> source=nyc_taxi | fields category, value | ml action='train' algorithm='rcf' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 - +------------+---------+---------+-------------+ - | category | value | score | anomalous | - |------------+---------+---------+-------------| - | night | 10844.0 | 0.0 | False | - | day | 6526.0 | 0.0 | False | - +------------+---------+---------+-------------+ + +----------+---------+-------+-----------+ + | category | value | score | anomalous | + |----------+---------+-------+-----------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +----------+---------+-------+-----------+ KMEANS ====== diff --git a/docs/user/ppl/cmd/parse.rst b/docs/user/ppl/cmd/parse.rst index 82eff8ee85..d1015cccb9 100644 --- a/docs/user/ppl/cmd/parse.rst +++ b/docs/user/ppl/cmd/parse.rst @@ -72,13 +72,13 @@ PPL query:: os> source=accounts | parse address '(?<streetNumber>\d+) (?<street>.+)' | where cast(streetNumber as int) > 500 | sort num(streetNumber) | fields streetNumber, street ; fetched rows / total rows = 3/3 - +----------------+----------------+ - | streetNumber | street | - |----------------+----------------| - | 671 | Bristol Street | - | 789 | Madison Street | - | 880 | Holmes Lane | - +----------------+----------------+ + +--------------+----------------+ + | streetNumber | street | + |--------------+----------------| + | 671 | Bristol Street | + | 789 | Madison Street | + | 880 | Holmes Lane | + +--------------+----------------+ Limitations =========== diff --git a/docs/user/ppl/cmd/patterns.rst b/docs/user/ppl/cmd/patterns.rst index 370404ecb6..13f08d0aa6 100644 --- a/docs/user/ppl/cmd/patterns.rst +++ b/docs/user/ppl/cmd/patterns.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | patterns email | fields email, patterns_field ; fetched rows / total rows = 4/4 - +-----------------------+------------------+ - | email | patterns_field | - |-----------------------+------------------| - | amberduke@pyrami.com | @. | - | hattiebond@netagy.com | @. | - | null | | - | daleadams@boink.com | @. | - +-----------------------+------------------+ + +-----------------------+----------------+ + | email | patterns_field | + |-----------------------+----------------| + | amberduke@pyrami.com | @. | + | hattiebond@netagy.com | @. | + | null | | + | daleadams@boink.com | @. | + +-----------------------+----------------+ Example 2: Extract log patterns =============================== diff --git a/docs/user/ppl/cmd/rare.rst b/docs/user/ppl/cmd/rare.rst index 35b660daa7..f6013711ae 100644 --- a/docs/user/ppl/cmd/rare.rst +++ b/docs/user/ppl/cmd/rare.rst @@ -32,12 +32,12 @@ PPL query:: os> source=accounts | rare gender; fetched rows / total rows = 2/2 - +----------+ - | gender | - |----------| - | F | - | M | - +----------+ + +--------+ + | gender | + |--------| + | F | + | M | + +--------+ Example 2: Find the least common values organized by gender @@ -49,14 +49,14 @@ PPL query:: os> source=accounts | rare age by gender; fetched rows / total rows = 4/4 - +----------+-------+ - | gender | age | - |----------+-------| - | F | 28 | - | M | 32 | - | M | 33 | - | M | 36 | - +----------+-------+ + +--------+-----+ + | gender | age | + |--------+-----| + | F | 28 | + | M | 32 | + | M | 33 | + | M | 36 | + +--------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index a4383a9f5f..c942884248 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -31,14 +31,14 @@ PPL query:: os> source=accounts | rename account_number as an | fields an; fetched rows / total rows = 4/4 - +------+ - | an | - |------| - | 1 | - | 6 | - | 13 | - | 18 | - +------+ + +----+ + | an | + |----| + | 1 | + | 6 | + | 13 | + | 18 | + +----+ Example 2: Rename multiple fields @@ -50,14 +50,14 @@ PPL query:: os> source=accounts | rename account_number as an, employer as emp | fields an, emp; fetched rows / total rows = 4/4 - +------+---------+ - | an | emp | - |------+---------| - | 1 | Pyrami | - | 6 | Netagy | - | 13 | Quility | - | 18 | null | - +------+---------+ + +----+---------+ + | an | emp | + |----+---------| + | 1 | Pyrami | + | 6 | Netagy | + | 13 | Quility | + | 18 | null | + +----+---------+ Limitation ========== diff --git a/docs/user/ppl/cmd/search.rst b/docs/user/ppl/cmd/search.rst index 5299f9f78a..9e55daddeb 100644 --- a/docs/user/ppl/cmd/search.rst +++ b/docs/user/ppl/cmd/search.rst @@ -37,14 +37,14 @@ PPL query:: os> source=accounts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ Example 2: Fetch data with condition ==================================== @@ -55,10 +55,10 @@ PPL query:: os> source=accounts account_number=1 or gender="F"; fetched rows / total rows = 2/2 - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + +----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + +----------------+-----------+--------------------+---------+--------+--------+----------+-------+-----+----------------------+----------+ diff --git a/docs/user/ppl/cmd/showdatasources.rst b/docs/user/ppl/cmd/showdatasources.rst index f7c6beb82f..d954ef0c04 100644 --- a/docs/user/ppl/cmd/showdatasources.rst +++ b/docs/user/ppl/cmd/showdatasources.rst @@ -28,9 +28,9 @@ PPL query for all PROMETHEUS DATASOURCES:: os> show datasources | where CONNECTOR_TYPE='PROMETHEUS'; fetched rows / total rows = 1/1 - +-------------------+------------------+ - | DATASOURCE_NAME | CONNECTOR_TYPE | - |-------------------+------------------| - | my_prometheus | PROMETHEUS | - +-------------------+------------------+ + +-----------------+----------------+ + | DATASOURCE_NAME | CONNECTOR_TYPE | + |-----------------+----------------| + | my_prometheus | PROMETHEUS | + +-----------------+----------------+ diff --git a/docs/user/ppl/cmd/sort.rst b/docs/user/ppl/cmd/sort.rst index 6a00ebc24c..377f0a5e01 100644 --- a/docs/user/ppl/cmd/sort.rst +++ b/docs/user/ppl/cmd/sort.rst @@ -32,14 +32,14 @@ PPL query:: os> source=accounts | sort age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 13 | 28 | - | 1 | 32 | - | 18 | 33 | - | 6 | 36 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +----------------+-----+ Example 2: Sort by one field return all the result @@ -51,14 +51,14 @@ PPL query:: os> source=accounts | sort age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 13 | 28 | - | 1 | 32 | - | 18 | 33 | - | 6 | 36 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +----------------+-----+ Example 3: Sort by one field in descending order @@ -70,14 +70,14 @@ PPL query:: os> source=accounts | sort - age | fields account_number, age; fetched rows / total rows = 4/4 - +------------------+-------+ - | account_number | age | - |------------------+-------| - | 6 | 36 | - | 18 | 33 | - | 1 | 32 | - | 13 | 28 | - +------------------+-------+ + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +----------------+-----+ Example 4: Sort by multiple field ============================= @@ -88,14 +88,14 @@ PPL query:: os> source=accounts | sort + gender, - age | fields account_number, gender, age; fetched rows / total rows = 4/4 - +------------------+----------+-------+ - | account_number | gender | age | - |------------------+----------+-------| - | 13 | F | 28 | - | 6 | M | 36 | - | 18 | M | 33 | - | 1 | M | 32 | - +------------------+----------+-------+ + +----------------+--------+-----+ + | account_number | gender | age | + |----------------+--------+-----| + | 13 | F | 28 | + | 6 | M | 36 | + | 18 | M | 33 | + | 1 | M | 32 | + +----------------+--------+-----+ Example 4: Sort by field include null value =========================================== @@ -106,11 +106,11 @@ PPL query:: os> source=accounts | sort employer | fields employer; fetched rows / total rows = 4/4 - +------------+ - | employer | - |------------| - | null | - | Netagy | - | Pyrami | - | Quility | - +------------+ + +----------+ + | employer | + |----------| + | null | + | Netagy | + | Pyrami | + | Quility | + +----------+ diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index 19f5069bba..7d5da804ce 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -86,11 +86,11 @@ Example:: os> source=accounts | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 4 | - +-----------+ + +---------+ + | count() | + |---------| + | 4 | + +---------+ SUM --- @@ -104,12 +104,12 @@ Example:: os> source=accounts | stats sum(age) by gender; fetched rows / total rows = 2/2 - +------------+----------+ - | sum(age) | gender | - |------------+----------| - | 28 | F | - | 101 | M | - +------------+----------+ + +----------+--------+ + | sum(age) | gender | + |----------+--------| + | 28 | F | + | 101 | M | + +----------+--------+ AVG --- @@ -123,12 +123,12 @@ Example:: os> source=accounts | stats avg(age) by gender; fetched rows / total rows = 2/2 - +--------------------+----------+ - | avg(age) | gender | - |--------------------+----------| - | 28.0 | F | - | 33.666666666666664 | M | - +--------------------+----------+ + +--------------------+--------+ + | avg(age) | gender | + |--------------------+--------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+--------+ MAX --- @@ -142,11 +142,11 @@ Example:: os> source=accounts | stats max(age); fetched rows / total rows = 1/1 - +------------+ - | max(age) | - |------------| - | 36 | - +------------+ + +----------+ + | max(age) | + |----------| + | 36 | + +----------+ MIN --- @@ -160,11 +160,11 @@ Example:: os> source=accounts | stats min(age); fetched rows / total rows = 1/1 - +------------+ - | min(age) | - |------------| - | 28 | - +------------+ + +----------+ + | min(age) | + |----------| + | 28 | + +----------+ VAR_SAMP -------- @@ -196,11 +196,11 @@ Example:: os> source=accounts | stats var_pop(age); fetched rows / total rows = 1/1 - +----------------+ - | var_pop(age) | - |----------------| - | 8.1875 | - +----------------+ + +--------------+ + | var_pop(age) | + |--------------| + | 8.1875 | + +--------------+ STDDEV_SAMP ----------- @@ -214,11 +214,11 @@ Example:: os> source=accounts | stats stddev_samp(age); fetched rows / total rows = 1/1 - +--------------------+ - | stddev_samp(age) | - |--------------------| - | 3.304037933599835 | - +--------------------+ + +-------------------+ + | stddev_samp(age) | + |-------------------| + | 3.304037933599835 | + +-------------------+ STDDEV_POP ---------- @@ -273,12 +273,12 @@ Example:: os> source=accounts | stats percentile(age, 90) by gender; fetched rows / total rows = 2/2 - +-----------------------+----------+ - | percentile(age, 90) | gender | - |-----------------------+----------| - | 28 | F | - | 36 | M | - +-----------------------+----------+ + +---------------------+--------+ + | percentile(age, 90) | gender | + |---------------------+--------| + | 28 | F | + | 36 | M | + +---------------------+--------+ Example 1: Calculate the count of events ======================================== @@ -289,11 +289,11 @@ PPL query:: os> source=accounts | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 4 | - +-----------+ + +---------+ + | count() | + |---------| + | 4 | + +---------+ Example 2: Calculate the average of a field @@ -305,11 +305,11 @@ PPL query:: os> source=accounts | stats avg(age); fetched rows / total rows = 1/1 - +------------+ - | avg(age) | - |------------| - | 32.25 | - +------------+ + +----------+ + | avg(age) | + |----------| + | 32.25 | + +----------+ Example 3: Calculate the average of a field by group @@ -321,12 +321,12 @@ PPL query:: os> source=accounts | stats avg(age) by gender; fetched rows / total rows = 2/2 - +--------------------+----------+ - | avg(age) | gender | - |--------------------+----------| - | 28.0 | F | - | 33.666666666666664 | M | - +--------------------+----------+ + +--------------------+--------+ + | avg(age) | gender | + |--------------------+--------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+--------+ Example 4: Calculate the average, sum and count of a field by group @@ -338,12 +338,12 @@ PPL query:: os> source=accounts | stats avg(age), sum(age), count() by gender; fetched rows / total rows = 2/2 - +--------------------+------------+-----------+----------+ - | avg(age) | sum(age) | count() | gender | - |--------------------+------------+-----------+----------| - | 28.0 | 28 | 1 | F | - | 33.666666666666664 | 101 | 3 | M | - +--------------------+------------+-----------+----------+ + +--------------------+----------+---------+--------+ + | avg(age) | sum(age) | count() | gender | + |--------------------+----------+---------+--------| + | 28.0 | 28 | 1 | F | + | 33.666666666666664 | 101 | 3 | M | + +--------------------+----------+---------+--------+ Example 5: Calculate the maximum of a field =========================================== @@ -354,11 +354,11 @@ PPL query:: os> source=accounts | stats max(age); fetched rows / total rows = 1/1 - +------------+ - | max(age) | - |------------| - | 36 | - +------------+ + +----------+ + | max(age) | + |----------| + | 36 | + +----------+ Example 6: Calculate the maximum and minimum of a field by group ================================================================ @@ -369,12 +369,12 @@ PPL query:: os> source=accounts | stats max(age), min(age) by gender; fetched rows / total rows = 2/2 - +------------+------------+----------+ - | max(age) | min(age) | gender | - |------------+------------+----------| - | 28 | 28 | F | - | 36 | 32 | M | - +------------+------------+----------+ + +----------+----------+--------+ + | max(age) | min(age) | gender | + |----------+----------+--------| + | 28 | 28 | F | + | 36 | 32 | M | + +----------+----------+--------+ Example 7: Calculate the distinct count of a field ================================================== @@ -385,11 +385,11 @@ PPL query:: os> source=accounts | stats count(gender), distinct_count(gender); fetched rows / total rows = 1/1 - +-----------------+--------------------------+ - | count(gender) | distinct_count(gender) | - |-----------------+--------------------------| - | 4 | 2 | - +-----------------+--------------------------+ + +---------------+------------------------+ + | count(gender) | distinct_count(gender) | + |---------------+------------------------| + | 4 | 2 | + +---------------+------------------------+ Example 8: Calculate the count by a span ======================================== @@ -400,12 +400,12 @@ PPL query:: os> source=accounts | stats count(age) by span(age, 10) as age_span fetched rows / total rows = 2/2 - +--------------+------------+ - | count(age) | age_span | - |--------------+------------| - | 1 | 20 | - | 3 | 30 | - +--------------+------------+ + +------------+----------+ + | count(age) | age_span | + |------------+----------| + | 1 | 20 | + | 3 | 30 | + +------------+----------+ Example 9: Calculate the count by a gender and span =================================================== @@ -416,13 +416,13 @@ PPL query:: os> source=accounts | stats count() as cnt by span(age, 5) as age_span, gender fetched rows / total rows = 3/3 - +-------+------------+----------+ - | cnt | age_span | gender | - |-------+------------+----------| - | 1 | 25 | F | - | 2 | 30 | M | - | 1 | 35 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | cnt | age_span | gender | + |-----+----------+--------| + | 1 | 25 | F | + | 2 | 30 | M | + | 1 | 35 | M | + +-----+----------+--------+ Span will always be the first grouping key whatever order you specify. @@ -430,13 +430,13 @@ PPL query:: os> source=accounts | stats count() as cnt by gender, span(age, 5) as age_span fetched rows / total rows = 3/3 - +-------+------------+----------+ - | cnt | age_span | gender | - |-------+------------+----------| - | 1 | 25 | F | - | 2 | 30 | M | - | 1 | 35 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | cnt | age_span | gender | + |-----+----------+--------| + | 1 | 25 | F | + | 2 | 30 | M | + | 1 | 35 | M | + +-----+----------+--------+ Example 10: Calculate the count and get email list by a gender and span ======================================================================= @@ -447,13 +447,13 @@ PPL query:: os> source=accounts | stats count() as cnt, take(email, 5) by span(age, 5) as age_span, gender fetched rows / total rows = 3/3 - +-------+--------------------------------------------+------------+----------+ - | cnt | take(email, 5) | age_span | gender | - |-------+--------------------------------------------+------------+----------| - | 1 | [] | 25 | F | - | 2 | [amberduke@pyrami.com,daleadams@boink.com] | 30 | M | - | 1 | [hattiebond@netagy.com] | 35 | M | - +-------+--------------------------------------------+------------+----------+ + +-----+--------------------------------------------+----------+--------+ + | cnt | take(email, 5) | age_span | gender | + |-----+--------------------------------------------+----------+--------| + | 1 | [] | 25 | F | + | 2 | [amberduke@pyrami.com,daleadams@boink.com] | 30 | M | + | 1 | [hattiebond@netagy.com] | 35 | M | + +-----+--------------------------------------------+----------+--------+ Example 11: Calculate the percentile of a field =============================================== @@ -464,11 +464,11 @@ PPL query:: os> source=accounts | stats percentile(age, 90); fetched rows / total rows = 1/1 - +-----------------------+ - | percentile(age, 90) | - |-----------------------| - | 36 | - +-----------------------+ + +---------------------+ + | percentile(age, 90) | + |---------------------| + | 36 | + +---------------------+ Example 12: Calculate the percentile of a field by group @@ -480,12 +480,12 @@ PPL query:: os> source=accounts | stats percentile(age, 90) by gender; fetched rows / total rows = 2/2 - +-----------------------+----------+ - | percentile(age, 90) | gender | - |-----------------------+----------| - | 28 | F | - | 36 | M | - +-----------------------+----------+ + +---------------------+--------+ + | percentile(age, 90) | gender | + |---------------------+--------| + | 28 | F | + | 36 | M | + +---------------------+--------+ Example 13: Calculate the percentile by a gender and span ========================================================= @@ -496,10 +496,10 @@ PPL query:: os> source=accounts | stats percentile(age, 90) as p90 by span(age, 10) as age_span, gender fetched rows / total rows = 2/2 - +-------+------------+----------+ - | p90 | age_span | gender | - |-------+------------+----------| - | 28 | 20 | F | - | 36 | 30 | M | - +-------+------------+----------+ + +-----+----------+--------+ + | p90 | age_span | gender | + |-----+----------+--------| + | 28 | 20 | F | + | 36 | 30 | M | + +-----+----------+--------+ diff --git a/docs/user/ppl/cmd/top.rst b/docs/user/ppl/cmd/top.rst index cbab675d09..6fa4d9cdb0 100644 --- a/docs/user/ppl/cmd/top.rst +++ b/docs/user/ppl/cmd/top.rst @@ -32,12 +32,12 @@ PPL query:: os> source=accounts | top gender; fetched rows / total rows = 2/2 - +----------+ - | gender | - |----------| - | M | - | F | - +----------+ + +--------+ + | gender | + |--------| + | M | + | F | + +--------+ Example 2: Find the most common values in a field =========================================== @@ -48,11 +48,11 @@ PPL query:: os> source=accounts | top 1 gender; fetched rows / total rows = 1/1 - +----------+ - | gender | - |----------| - | M | - +----------+ + +--------+ + | gender | + |--------| + | M | + +--------+ Example 2: Find the most common values organized by gender ==================================================== @@ -63,12 +63,12 @@ PPL query:: os> source=accounts | top 1 age by gender; fetched rows / total rows = 2/2 - +----------+-------+ - | gender | age | - |----------+-------| - | F | 28 | - | M | 32 | - +----------+-------+ + +--------+-----+ + | gender | age | + |--------+-----| + | F | 28 | + | M | 32 | + +--------+-----+ Limitation ========== diff --git a/docs/user/ppl/cmd/where.rst b/docs/user/ppl/cmd/where.rst index 4d8718d69f..115bffe7de 100644 --- a/docs/user/ppl/cmd/where.rst +++ b/docs/user/ppl/cmd/where.rst @@ -29,10 +29,10 @@ PPL query:: os> source=accounts | where account_number=1 or gender="F" | fields account_number, gender; fetched rows / total rows = 2/2 - +------------------+----------+ - | account_number | gender | - |------------------+----------| - | 1 | M | - | 13 | F | - +------------------+----------+ + +----------------+--------+ + | account_number | gender | + |----------------+--------| + | 1 | M | + | 13 | F | + +----------------+--------+ diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index e48d4cb75c..96c3e64e72 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -24,14 +24,14 @@ Example:: os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | False | Pyrami | Amber | - | False | Netagy | Hattie | - | False | Quility | Nanette | - | True | null | Dale | - +----------+------------+-------------+ + +--------+----------+-----------+ + | result | employer | firstname | + |--------+----------+-----------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +--------+----------+-----------+ ISNOTNULL --------- @@ -49,11 +49,11 @@ Example:: os> source=accounts | where not isnotnull(employer) | fields account_number, employer fetched rows / total rows = 1/1 - +------------------+------------+ - | account_number | employer | - |------------------+------------| - | 18 | null | - +------------------+------------+ + +----------------+----------+ + | account_number | employer | + |----------------+----------| + | 18 | null | + +----------------+----------+ EXISTS ------ @@ -64,11 +64,11 @@ Example, the account 13 doesn't have email field:: os> source=accounts | where isnull(email) | fields account_number, email fetched rows / total rows = 1/1 - +------------------+---------+ - | account_number | email | - |------------------+---------| - | 13 | null | - +------------------+---------+ + +----------------+-------+ + | account_number | email | + |----------------+-------| + | 13 | null | + +----------------+-------+ IFNULL ------ @@ -86,14 +86,14 @@ Example:: os> source=accounts | eval result = ifnull(employer, 'default') | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | Pyrami | Pyrami | Amber | - | Netagy | Netagy | Hattie | - | Quility | Quility | Nanette | - | default | null | Dale | - +----------+------------+-------------+ + +---------+----------+-----------+ + | result | employer | firstname | + |---------+----------+-----------| + | Pyrami | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | default | null | Dale | + +---------+----------+-----------+ NULLIF ------ @@ -111,14 +111,14 @@ Example:: os> source=accounts | eval result = nullif(employer, 'Pyrami') | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | null | Pyrami | Amber | - | Netagy | Netagy | Hattie | - | Quility | Quility | Nanette | - | null | null | Dale | - +----------+------------+-------------+ + +---------+----------+-----------+ + | result | employer | firstname | + |---------+----------+-----------| + | null | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | null | null | Dale | + +---------+----------+-----------+ ISNULL @@ -137,14 +137,14 @@ Example:: os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname fetched rows / total rows = 4/4 - +----------+------------+-------------+ - | result | employer | firstname | - |----------+------------+-------------| - | False | Pyrami | Amber | - | False | Netagy | Hattie | - | False | Quility | Nanette | - | True | null | Dale | - +----------+------------+-------------+ + +--------+----------+-----------+ + | result | employer | firstname | + |--------+----------+-----------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +--------+----------+-----------+ IF ------ @@ -162,33 +162,33 @@ Example:: os> source=accounts | eval result = if(true, firstname, lastname) | fields result, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | result | firstname | lastname | - |----------+-------------+------------| - | Amber | Amber | Duke | - | Hattie | Hattie | Bond | - | Nanette | Nanette | Bates | - | Dale | Dale | Adams | - +----------+-------------+------------+ + +---------+-----------+----------+ + | result | firstname | lastname | + |---------+-----------+----------| + | Amber | Amber | Duke | + | Hattie | Hattie | Bond | + | Nanette | Nanette | Bates | + | Dale | Dale | Adams | + +---------+-----------+----------+ os> source=accounts | eval result = if(false, firstname, lastname) | fields result, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | result | firstname | lastname | - |----------+-------------+------------| - | Duke | Amber | Duke | - | Bond | Hattie | Bond | - | Bates | Nanette | Bates | - | Adams | Dale | Adams | - +----------+-------------+------------+ + +--------+-----------+----------+ + | result | firstname | lastname | + |--------+-----------+----------| + | Duke | Amber | Duke | + | Bond | Hattie | Bond | + | Bates | Nanette | Bates | + | Adams | Dale | Adams | + +--------+-----------+----------+ os> source=accounts | eval is_vip = if(age > 30 AND isnotnull(employer), true, false) | fields is_vip, firstname, lastname fetched rows / total rows = 4/4 - +----------+-------------+------------+ - | is_vip | firstname | lastname | - |----------+-------------+------------| - | True | Amber | Duke | - | True | Hattie | Bond | - | False | Nanette | Bates | - | False | Dale | Adams | - +----------+-------------+------------+ + +--------+-----------+----------+ + | is_vip | firstname | lastname | + |--------+-----------+----------| + | True | Amber | Duke | + | True | Hattie | Bond | + | False | Nanette | Bates | + | False | Dale | Adams | + +--------+-----------+----------+ diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index a4a59a6cd1..31fb3e3cdf 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -38,21 +38,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | true | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | true | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -68,8 +68,8 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 9e75e41136..c0d42297ac 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -35,11 +35,11 @@ Example:: os> source=people | eval `'2020-08-26' + 1h` = ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR), `'2020-08-26' + 1` = ADDDATE(DATE('2020-08-26'), 1), `ts '2020-08-26 01:01:01' + 1` = ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields `'2020-08-26' + 1h`, `'2020-08-26' + 1`, `ts '2020-08-26 01:01:01' + 1` fetched rows / total rows = 1/1 - +---------------------+--------------------+--------------------------------+ - | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | - |---------------------+--------------------+--------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | - +---------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2020-08-26' + 1h | '2020-08-26' + 1 | ts '2020-08-26 01:01:01' + 1 | + |---------------------+------------------+------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +---------------------+------------------+------------------------------+ @@ -73,35 +73,35 @@ Example:: os> source=people | eval `'23:59:59' + 0` = ADDTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' + 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' + 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' + 0 | + |----------------| + | 23:59:59 | + +----------------+ os> source=people | eval `'2004-01-01' + '23:59:59'` = ADDTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' + '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' + '23:59:59' | - |-----------------------------| - | 2004-01-01 23:59:59 | - +-----------------------------+ - - os> source=people | eval `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' + '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' + '00:05:42' | + | '2004-01-01' + '23:59:59' | |---------------------------| - | 10:26:12 | + | 2004-01-01 23:59:59 | +---------------------------+ + os> source=people | eval `'10:20:30' + '00:05:42'` = ADDTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' + '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' + '00:05:42' | + |-------------------------| + | 10:26:12 | + +-------------------------+ + os> source=people | eval `'2007-02-28 10:20:30' + '20:40:50'` = ADDTIME(TIMESTAMP('2007-02-28 10:20:30'), TIMESTAMP('2002-03-04 20:40:50')) | fields `'2007-02-28 10:20:30' + '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-02-28 10:20:30' + '20:40:50' | - |--------------------------------------| - | 2007-03-01 07:01:20 | - +--------------------------------------+ + +------------------------------------+ + | '2007-02-28 10:20:30' + '20:40:50' | + |------------------------------------| + | 2007-03-01 07:01:20 | + +------------------------------------+ CONVERT_TZ @@ -121,121 +121,121 @@ Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+00:00','+10:00')` = convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | fields `convert_tz('2008-05-15 12:00:00','+00:00','+10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | - |-------------------------------------------------------| - | 2008-05-15 22:00:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+00:00','+10:00') | + |-----------------------------------------------------| + | 2008-05-15 22:00:00 | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +15:00 in this example will return null. Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+00:00','+15:00')` = convert_tz('2008-05-15 12:00:00','+00:00','+15:00')| fields `convert_tz('2008-05-15 12:00:00','+00:00','+15:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+00:00','+15:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+00:00','+15:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ Conversion from a positive timezone to a negative timezone that goes over date line. Example:: os> source=people | eval `convert_tz('2008-05-15 12:00:00','+03:30','-10:00')` = convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-05-15 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | 2008-05-14 22:30:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-05-15 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | 2008-05-14 22:30:00 | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as April 31st (not a date in the Gregorian calendar) will result in null. Example:: os> source=people | eval `convert_tz('2008-04-31 12:00:00','+03:30','-10:00')` = convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-04-31 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-04-31 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as February 30th (not a date in the Gregorian calendar) will result in null. Example:: os> source=people | eval `convert_tz('2008-02-30 12:00:00','+03:30','-10:00')` = convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-02-30 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-30 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ February 29th 2008 is a valid date because it is a leap year. Example:: os> source=people | eval `convert_tz('2008-02-29 12:00:00','+03:30','-10:00')` = convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | fields `convert_tz('2008-02-29 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | 2008-02-28 22:30:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-29 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | 2008-02-28 22:30:00 | + +-----------------------------------------------------+ Valid dates are required in convert_tz, invalid dates such as February 29th 2007 (2007 is not a leap year) will result in null. Example:: os> source=people | eval `convert_tz('2007-02-29 12:00:00','+03:30','-10:00')` = convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | fields `convert_tz('2007-02-29 12:00:00','+03:30','-10:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2007-02-29 12:00:00','+03:30','-10:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +14:01 in this example will return null. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','+14:01','+00:00')` = convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | fields `convert_tz('2008-02-01 12:00:00','+14:01','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','+14:01','+00:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as +14:00 in this example will return a correctly converted date time object. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','+14:00','+00:00')` = convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | fields `convert_tz('2008-02-01 12:00:00','+14:00','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | - |-------------------------------------------------------| - | 2008-01-31 22:00:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','+14:00','+00:00') | + |-----------------------------------------------------| + | 2008-01-31 22:00:00 | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range, such as -14:00 will result in null Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','-14:00','+00:00')` = convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | fields `convert_tz('2008-02-01 12:00:00','-14:00','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | - |-------------------------------------------------------| - | null | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','-14:00','+00:00') | + |-----------------------------------------------------| + | null | + +-----------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. This timezone is within range so it is valid and will convert the time. Example:: os> source=people | eval `convert_tz('2008-02-01 12:00:00','-13:59','+00:00')` = convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | fields `convert_tz('2008-02-01 12:00:00','-13:59','+00:00')` fetched rows / total rows = 1/1 - +-------------------------------------------------------+ - | convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | - |-------------------------------------------------------| - | 2008-02-02 01:59:00 | - +-------------------------------------------------------+ + +-----------------------------------------------------+ + | convert_tz('2008-02-01 12:00:00','-13:59','+00:00') | + |-----------------------------------------------------| + | 2008-02-02 01:59:00 | + +-----------------------------------------------------+ CURDATE @@ -255,11 +255,11 @@ Example:: > source=people | eval `CURDATE()` = CURDATE() | fields `CURDATE()` fetched rows / total rows = 1/1 - +-------------+ - | CURDATE() | - |-------------| - | 2022-08-02 | - +-------------+ + +------------+ + | CURDATE() | + |------------| + | 2022-08-02 | + +------------+ CURRENT_DATE @@ -336,11 +336,11 @@ Example:: > source=people | eval `value_1` = CURTIME(), `value_2` = CURTIME() | fields `value_1`, `value_2` fetched rows / total rows = 1/1 - +-----------+-----------+ - | value_1 | value_2 | - |-----------+-----------| - | 15:39:05 | 15:39:05 | - +-----------+-----------+ + +----------+----------+ + | value_1 | value_2 | + |----------+----------| + | 15:39:05 | 15:39:05 | + +----------+----------+ DATE @@ -359,35 +359,35 @@ Example:: os> source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26') | fields `DATE('2020-08-26')` fetched rows / total rows = 1/1 - +----------------------+ - | DATE('2020-08-26') | - |----------------------| - | 2020-08-26 | - +----------------------+ + +--------------------+ + | DATE('2020-08-26') | + |--------------------| + | 2020-08-26 | + +--------------------+ os> source=people | eval `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE(TIMESTAMP('2020-08-26 13:49:00'))` fetched rows / total rows = 1/1 - +------------------------------------------+ - | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |------------------------------------------| - | 2020-08-26 | - +------------------------------------------+ + +----------------------------------------+ + | DATE(TIMESTAMP('2020-08-26 13:49:00')) | + |----------------------------------------| + | 2020-08-26 | + +----------------------------------------+ os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | DATE('2020-08-26 13:49') | - |----------------------------| - | 2020-08-26 | - +----------------------------+ + +--------------------------+ + | DATE('2020-08-26 13:49') | + |--------------------------| + | 2020-08-26 | + +--------------------------+ os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | DATE('2020-08-26 13:49') | - |----------------------------| - | 2020-08-26 | - +----------------------------+ + +--------------------------+ + | DATE('2020-08-26 13:49') | + |--------------------------| + | 2020-08-26 | + +--------------------------+ DATE_ADD @@ -410,11 +410,11 @@ Example:: os> source=people | eval `'2020-08-26' + 1h` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR), `ts '2020-08-26 01:01:01' + 1d` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 DAY) | fields `'2020-08-26' + 1h`, `ts '2020-08-26 01:01:01' + 1d` fetched rows / total rows = 1/1 - +---------------------+---------------------------------+ - | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | - |---------------------+---------------------------------| - | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | - +---------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2020-08-26' + 1h | ts '2020-08-26 01:01:01' + 1d | + |---------------------+-------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 01:01:01 | + +---------------------+-------------------------------+ DATE_FORMAT @@ -509,11 +509,11 @@ Example:: os> source=people | eval `DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')` = DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f'), `DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')` = DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | fields `DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')`, `DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')` fetched rows / total rows = 1/1 - +------------------------------------------------------+-----------------------------------------------------------------------+ - | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | - |------------------------------------------------------+-----------------------------------------------------------------------| - | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | - +------------------------------------------------------+-----------------------------------------------------------------------+ + +----------------------------------------------------+---------------------------------------------------------------------+ + | DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f') | DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | + |----------------------------------------------------+---------------------------------------------------------------------| + | 13:14:15.012345 | 1998-Jan-31st 01:14:15 PM | + +----------------------------------------------------+---------------------------------------------------------------------+ DATETIME @@ -538,11 +538,11 @@ Example:: os> source=people | eval `DATETIME('2004-02-28 23:00:00-10:00', '+10:00')` = DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | fields `DATETIME('2004-02-28 23:00:00-10:00', '+10:00')` fetched rows / total rows = 1/1 - +---------------------------------------------------+ - | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | - |---------------------------------------------------| - | 2004-02-29 19:00:00 | - +---------------------------------------------------+ + +-------------------------------------------------+ + | DATETIME('2004-02-28 23:00:00-10:00', '+10:00') | + |-------------------------------------------------| + | 2004-02-29 19:00:00 | + +-------------------------------------------------+ The valid timezone range for convert_tz is (-13:59, +14:00) inclusive. Timezones outside of the range will result in null. @@ -550,11 +550,11 @@ Example:: os> source=people | eval `DATETIME('2008-01-01 02:00:00', '-14:00')` = DATETIME('2008-01-01 02:00:00', '-14:00') | fields `DATETIME('2008-01-01 02:00:00', '-14:00')` fetched rows / total rows = 1/1 - +---------------------------------------------+ - | DATETIME('2008-01-01 02:00:00', '-14:00') | - |---------------------------------------------| - | null | - +---------------------------------------------+ + +-------------------------------------------+ + | DATETIME('2008-01-01 02:00:00', '-14:00') | + |-------------------------------------------| + | null | + +-------------------------------------------+ DATE_SUB @@ -577,11 +577,11 @@ Example:: os> source=people | eval `'2008-01-02' - 31d` = DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY), `ts '2020-08-26 01:01:01' + 1h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 1 HOUR) | fields `'2008-01-02' - 31d`, `ts '2020-08-26 01:01:01' + 1h` fetched rows / total rows = 1/1 - +----------------------+---------------------------------+ - | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | - |----------------------+---------------------------------| - | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | - +----------------------+---------------------------------+ + +---------------------+-------------------------------+ + | '2008-01-02' - 31d | ts '2020-08-26 01:01:01' + 1h | + |---------------------+-------------------------------| + | 2007-12-02 00:00:00 | 2020-08-26 00:01:01 | + +---------------------+-------------------------------+ DATEDIFF @@ -597,11 +597,11 @@ Example:: os> source=people | eval `'2000-01-02' - '2000-01-01'` = DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')), `'2001-02-01' - '2004-01-01'` = DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')), `today - today` = DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) | fields `'2000-01-02' - '2000-01-01'`, `'2001-02-01' - '2004-01-01'`, `today - today` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+-----------------+ - | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | - |-------------------------------+-------------------------------+-----------------| - | 1 | -1064 | 0 | - +-------------------------------+-------------------------------+-----------------+ + +-----------------------------+-----------------------------+---------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-----------------------------+-----------------------------+---------------| + | 1 | -1064 | 0 | + +-----------------------------+-----------------------------+---------------+ DAY @@ -622,11 +622,11 @@ Example:: os> source=people | eval `DAY(DATE('2020-08-26'))` = DAY(DATE('2020-08-26')) | fields `DAY(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------+ - | DAY(DATE('2020-08-26')) | - |---------------------------| - | 26 | - +---------------------------+ + +-------------------------+ + | DAY(DATE('2020-08-26')) | + |-------------------------| + | 26 | + +-------------------------+ DAYNAME @@ -645,11 +645,11 @@ Example:: os> source=people | eval `DAYNAME(DATE('2020-08-26'))` = DAYNAME(DATE('2020-08-26')) | fields `DAYNAME(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | DAYNAME(DATE('2020-08-26')) | - |-------------------------------| - | Wednesday | - +-------------------------------+ + +-----------------------------+ + | DAYNAME(DATE('2020-08-26')) | + |-----------------------------| + | Wednesday | + +-----------------------------+ DAYOFMONTH @@ -670,11 +670,11 @@ Example:: os> source=people | eval `DAYOFMONTH(DATE('2020-08-26'))` = DAYOFMONTH(DATE('2020-08-26')) | fields `DAYOFMONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +----------------------------------+ - | DAYOFMONTH(DATE('2020-08-26')) | - |----------------------------------| - | 26 | - +----------------------------------+ + +--------------------------------+ + | DAYOFMONTH(DATE('2020-08-26')) | + |--------------------------------| + | 26 | + +--------------------------------+ DAY_OF_MONTH @@ -695,11 +695,11 @@ Example:: os> source=people | eval `DAY_OF_MONTH(DATE('2020-08-26'))` = DAY_OF_MONTH(DATE('2020-08-26')) | fields `DAY_OF_MONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +------------------------------------+ - | DAY_OF_MONTH(DATE('2020-08-26')) | - |------------------------------------| - | 26 | - +------------------------------------+ + +----------------------------------+ + | DAY_OF_MONTH(DATE('2020-08-26')) | + |----------------------------------| + | 26 | + +----------------------------------+ DAYOFWEEK @@ -720,11 +720,11 @@ Example:: os> source=people | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | fields `DAYOFWEEK(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFWEEK(DATE('2020-08-26')) | - |---------------------------------| - | 4 | - +---------------------------------+ + +-------------------------------+ + | DAYOFWEEK(DATE('2020-08-26')) | + |-------------------------------| + | 4 | + +-------------------------------+ DAY_OF_WEEK @@ -745,11 +745,11 @@ Example:: os> source=people | eval `DAY_OF_WEEK(DATE('2020-08-26'))` = DAY_OF_WEEK(DATE('2020-08-26')) | fields `DAY_OF_WEEK(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_WEEK(DATE('2020-08-26')) | - |-----------------------------------| - | 4 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_WEEK(DATE('2020-08-26')) | + |---------------------------------| + | 4 | + +---------------------------------+ DAYOFYEAR @@ -770,11 +770,11 @@ Example:: os> source=people | eval `DAYOFYEAR(DATE('2020-08-26'))` = DAYOFYEAR(DATE('2020-08-26')) | fields `DAYOFYEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFYEAR(DATE('2020-08-26')) | - |---------------------------------| - | 239 | - +---------------------------------+ + +-------------------------------+ + | DAYOFYEAR(DATE('2020-08-26')) | + |-------------------------------| + | 239 | + +-------------------------------+ DAY_OF_YEAR @@ -795,11 +795,11 @@ Example:: os> source=people | eval `DAY_OF_YEAR(DATE('2020-08-26'))` = DAY_OF_YEAR(DATE('2020-08-26')) | fields `DAY_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | DAY_OF_YEAR(DATE('2020-08-26')) | - |-----------------------------------| - | 239 | - +-----------------------------------+ + +---------------------------------+ + | DAY_OF_YEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ EXTRACT @@ -866,11 +866,11 @@ Example:: os> source=people | eval `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` = extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | fields `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` fetched rows / total rows = 1/1 - +--------------------------------------------------+ - | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | - |--------------------------------------------------| - | 202302 | - +--------------------------------------------------+ + +------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |------------------------------------------------| + | 202302 | + +------------------------------------------------+ FROM_DAYS @@ -889,11 +889,11 @@ Example:: os> source=people | eval `FROM_DAYS(733687)` = FROM_DAYS(733687) | fields `FROM_DAYS(733687)` fetched rows / total rows = 1/1 - +---------------------+ - | FROM_DAYS(733687) | - |---------------------| - | 2008-10-07 | - +---------------------+ + +-------------------+ + | FROM_DAYS(733687) | + |-------------------| + | 2008-10-07 | + +-------------------+ FROM_UNIXTIME @@ -917,19 +917,19 @@ Examples:: os> source=people | eval `FROM_UNIXTIME(1220249547)` = FROM_UNIXTIME(1220249547) | fields `FROM_UNIXTIME(1220249547)` fetched rows / total rows = 1/1 - +-----------------------------+ - | FROM_UNIXTIME(1220249547) | - |-----------------------------| - | 2008-09-01 06:12:27 | - +-----------------------------+ + +---------------------------+ + | FROM_UNIXTIME(1220249547) | + |---------------------------| + | 2008-09-01 06:12:27 | + +---------------------------+ os> source=people | eval `FROM_UNIXTIME(1220249547, '%T')` = FROM_UNIXTIME(1220249547, '%T') | fields `FROM_UNIXTIME(1220249547, '%T')` fetched rows / total rows = 1/1 - +-----------------------------------+ - | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------------| - | 06:12:27 | - +-----------------------------------+ + +---------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |---------------------------------| + | 06:12:27 | + +---------------------------------+ GET_FORMAT @@ -947,11 +947,11 @@ Examples:: os> source=people | eval `GET_FORMAT(DATE, 'USA')` = GET_FORMAT(DATE, 'USA') | fields `GET_FORMAT(DATE, 'USA')` fetched rows / total rows = 1/1 - +---------------------------+ - | GET_FORMAT(DATE, 'USA') | - |---------------------------| - | %m.%d.%Y | - +---------------------------+ + +-------------------------+ + | GET_FORMAT(DATE, 'USA') | + |-------------------------| + | %m.%d.%Y | + +-------------------------+ HOUR @@ -972,11 +972,11 @@ Example:: os> source=people | eval `HOUR(TIME('01:02:03'))` = HOUR(TIME('01:02:03')) | fields `HOUR(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +--------------------------+ - | HOUR(TIME('01:02:03')) | - |--------------------------| - | 1 | - +--------------------------+ + +------------------------+ + | HOUR(TIME('01:02:03')) | + |------------------------| + | 1 | + +------------------------+ HOUR_OF_DAY @@ -997,11 +997,11 @@ Example:: os> source=people | eval `HOUR_OF_DAY(TIME('01:02:03'))` = HOUR_OF_DAY(TIME('01:02:03')) | fields `HOUR_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | HOUR_OF_DAY(TIME('01:02:03')) | - |---------------------------------| - | 1 | - +---------------------------------+ + +-------------------------------+ + | HOUR_OF_DAY(TIME('01:02:03')) | + |-------------------------------| + | 1 | + +-------------------------------+ LAST_DAY @@ -1017,11 +1017,11 @@ Example:: os> source=people | eval `last_day('2023-02-06')` = last_day('2023-02-06') | fields `last_day('2023-02-06')` fetched rows / total rows = 1/1 - +--------------------------+ - | last_day('2023-02-06') | - |--------------------------| - | 2023-02-28 | - +--------------------------+ + +------------------------+ + | last_day('2023-02-06') | + |------------------------| + | 2023-02-28 | + +------------------------+ LOCALTIMESTAMP @@ -1089,11 +1089,11 @@ Example:: os> source=people | eval `MAKEDATE(1945, 5.9)` = MAKEDATE(1945, 5.9), `MAKEDATE(1984, 1984)` = MAKEDATE(1984, 1984) | fields `MAKEDATE(1945, 5.9)`, `MAKEDATE(1984, 1984)` fetched rows / total rows = 1/1 - +-----------------------+------------------------+ - | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | - |-----------------------+------------------------| - | 1945-01-06 | 1989-06-06 | - +-----------------------+------------------------+ + +---------------------+----------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |---------------------+----------------------| + | 1945-01-06 | 1989-06-06 | + +---------------------+----------------------+ MAKETIME @@ -1121,11 +1121,11 @@ Example:: os> source=people | eval `MAKETIME(20, 30, 40)` = MAKETIME(20, 30, 40), `MAKETIME(20.2, 49.5, 42.100502)` = MAKETIME(20.2, 49.5, 42.100502) | fields `MAKETIME(20, 30, 40)`, `MAKETIME(20.2, 49.5, 42.100502)` fetched rows / total rows = 1/1 - +------------------------+-----------------------------------+ - | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | - |------------------------+-----------------------------------| - | 20:30:40 | 20:50:42.100502 | - +------------------------+-----------------------------------+ + +----------------------+---------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |----------------------+---------------------------------| + | 20:30:40 | 20:50:42.100502 | + +----------------------+---------------------------------+ MICROSECOND @@ -1144,11 +1144,11 @@ Example:: os> source=people | eval `MICROSECOND(TIME('01:02:03.123456'))` = MICROSECOND(TIME('01:02:03.123456')) | fields `MICROSECOND(TIME('01:02:03.123456'))` fetched rows / total rows = 1/1 - +----------------------------------------+ - | MICROSECOND(TIME('01:02:03.123456')) | - |----------------------------------------| - | 123456 | - +----------------------------------------+ + +--------------------------------------+ + | MICROSECOND(TIME('01:02:03.123456')) | + |--------------------------------------| + | 123456 | + +--------------------------------------+ MINUTE @@ -1169,11 +1169,11 @@ Example:: os> source=people | eval `MINUTE(TIME('01:02:03'))` = MINUTE(TIME('01:02:03')) | fields `MINUTE(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +----------------------------+ - | MINUTE(TIME('01:02:03')) | - |----------------------------| - | 2 | - +----------------------------+ + +--------------------------+ + | MINUTE(TIME('01:02:03')) | + |--------------------------| + | 2 | + +--------------------------+ MINUTE_OF_DAY @@ -1192,11 +1192,11 @@ Example:: os> source=people | eval `MINUTE_OF_DAY(TIME('01:02:03'))` = MINUTE_OF_DAY(TIME('01:02:03')) | fields `MINUTE_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +-----------------------------------+ - | MINUTE_OF_DAY(TIME('01:02:03')) | - |-----------------------------------| - | 62 | - +-----------------------------------+ + +---------------------------------+ + | MINUTE_OF_DAY(TIME('01:02:03')) | + |---------------------------------| + | 62 | + +---------------------------------+ MINUTE_OF_HOUR @@ -1217,11 +1217,11 @@ Example:: os> source=people | eval `MINUTE_OF_HOUR(TIME('01:02:03'))` = MINUTE_OF_HOUR(TIME('01:02:03')) | fields `MINUTE_OF_HOUR(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +------------------------------------+ - | MINUTE_OF_HOUR(TIME('01:02:03')) | - |------------------------------------| - | 2 | - +------------------------------------+ + +----------------------------------+ + | MINUTE_OF_HOUR(TIME('01:02:03')) | + |----------------------------------| + | 2 | + +----------------------------------+ MONTH @@ -1242,11 +1242,11 @@ Example:: os> source=people | eval `MONTH(DATE('2020-08-26'))` = MONTH(DATE('2020-08-26')) | fields `MONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------+ - | MONTH(DATE('2020-08-26')) | - |-----------------------------| - | 8 | - +-----------------------------+ + +---------------------------+ + | MONTH(DATE('2020-08-26')) | + |---------------------------| + | 8 | + +---------------------------+ MONTH_OF_YEAR @@ -1267,11 +1267,11 @@ Example:: os> source=people | eval `MONTH_OF_YEAR(DATE('2020-08-26'))` = MONTH_OF_YEAR(DATE('2020-08-26')) | fields `MONTH_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------------+ - | MONTH_OF_YEAR(DATE('2020-08-26')) | - |-------------------------------------| - | 8 | - +-------------------------------------+ + +-----------------------------------+ + | MONTH_OF_YEAR(DATE('2020-08-26')) | + |-----------------------------------| + | 8 | + +-----------------------------------+ MONTHNAME @@ -1290,11 +1290,11 @@ Example:: os> source=people | eval `MONTHNAME(DATE('2020-08-26'))` = MONTHNAME(DATE('2020-08-26')) | fields `MONTHNAME(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | MONTHNAME(DATE('2020-08-26')) | - |---------------------------------| - | August | - +---------------------------------+ + +-------------------------------+ + | MONTHNAME(DATE('2020-08-26')) | + |-------------------------------| + | August | + +-------------------------------+ NOW @@ -1337,11 +1337,11 @@ Example:: os> source=people | eval `PERIOD_ADD(200801, 2)` = PERIOD_ADD(200801, 2), `PERIOD_ADD(200801, -12)` = PERIOD_ADD(200801, -12) | fields `PERIOD_ADD(200801, 2)`, `PERIOD_ADD(200801, -12)` fetched rows / total rows = 1/1 - +-------------------------+---------------------------+ - | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | - |-------------------------+---------------------------| - | 200803 | 200701 | - +-------------------------+---------------------------+ + +-----------------------+-------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-----------------------+-------------------------| + | 200803 | 200701 | + +-----------------------+-------------------------+ PERIOD_DIFF @@ -1360,11 +1360,11 @@ Example:: os> source=people | eval `PERIOD_DIFF(200802, 200703)` = PERIOD_DIFF(200802, 200703), `PERIOD_DIFF(200802, 201003)` = PERIOD_DIFF(200802, 201003) | fields `PERIOD_DIFF(200802, 200703)`, `PERIOD_DIFF(200802, 201003)` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | - |-------------------------------+-------------------------------| - | 11 | -25 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-----------------------------+-----------------------------| + | 11 | -25 | + +-----------------------------+-----------------------------+ QUARTER @@ -1383,11 +1383,11 @@ Example:: os> source=people | eval `QUARTER(DATE('2020-08-26'))` = QUARTER(DATE('2020-08-26')) | fields `QUARTER(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | QUARTER(DATE('2020-08-26')) | - |-------------------------------| - | 3 | - +-------------------------------+ + +-----------------------------+ + | QUARTER(DATE('2020-08-26')) | + |-----------------------------| + | 3 | + +-----------------------------+ SEC_TO_TIME @@ -1409,11 +1409,11 @@ Example:: os> source=people | eval `SEC_TO_TIME(3601)` = SEC_TO_TIME(3601) | eval `SEC_TO_TIME(1234.123)` = SEC_TO_TIME(1234.123) | fields `SEC_TO_TIME(3601)`, `SEC_TO_TIME(1234.123)` fetched rows / total rows = 1/1 - +---------------------+-------------------------+ - | SEC_TO_TIME(3601) | SEC_TO_TIME(1234.123) | - |---------------------+-------------------------| - | 01:00:01 | 00:20:34.123 | - +---------------------+-------------------------+ + +-------------------+-----------------------+ + | SEC_TO_TIME(3601) | SEC_TO_TIME(1234.123) | + |-------------------+-----------------------| + | 01:00:01 | 00:20:34.123 | + +-------------------+-----------------------+ SECOND @@ -1434,11 +1434,11 @@ Example:: os> source=people | eval `SECOND(TIME('01:02:03'))` = SECOND(TIME('01:02:03')) | fields `SECOND(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +----------------------------+ - | SECOND(TIME('01:02:03')) | - |----------------------------| - | 3 | - +----------------------------+ + +--------------------------+ + | SECOND(TIME('01:02:03')) | + |--------------------------| + | 3 | + +--------------------------+ SECOND_OF_MINUTE @@ -1459,11 +1459,11 @@ Example:: os> source=people | eval `SECOND_OF_MINUTE(TIME('01:02:03'))` = SECOND_OF_MINUTE(TIME('01:02:03')) | fields `SECOND_OF_MINUTE(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +--------------------------------------+ - | SECOND_OF_MINUTE(TIME('01:02:03')) | - |--------------------------------------| - | 3 | - +--------------------------------------+ + +------------------------------------+ + | SECOND_OF_MINUTE(TIME('01:02:03')) | + |------------------------------------| + | 3 | + +------------------------------------+ STR_TO_DATE @@ -1485,11 +1485,11 @@ Example:: OS> source=people | eval `str_to_date("01,5,2013", "%d,%m,%Y")` = str_to_date("01,5,2013", "%d,%m,%Y") | fields = `str_to_date("01,5,2013", "%d,%m,%Y")` fetched rows / total rows = 1/1 - +----------------------------------------+ - | str_to_date("01,5,2013", "%d,%m,%Y") | - |----------------------------------------| - | 2013-05-01 00:00:00 | - +----------------------------------------+ + +--------------------------------------+ + | str_to_date("01,5,2013", "%d,%m,%Y") | + |--------------------------------------| + | 2013-05-01 00:00:00 | + +--------------------------------------+ SUBDATE @@ -1519,11 +1519,11 @@ Example:: os> source=people | eval `'2008-01-02' - 31d` = SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY), `'2020-08-26' - 1` = SUBDATE(DATE('2020-08-26'), 1), `ts '2020-08-26 01:01:01' - 1` = SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields `'2008-01-02' - 31d`, `'2020-08-26' - 1`, `ts '2020-08-26 01:01:01' - 1` fetched rows / total rows = 1/1 - +----------------------+--------------------+--------------------------------+ - | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | - |----------------------+--------------------+--------------------------------| - | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | - +----------------------+--------------------+--------------------------------+ + +---------------------+------------------+------------------------------+ + | '2008-01-02' - 31d | '2020-08-26' - 1 | ts '2020-08-26 01:01:01' - 1 | + |---------------------+------------------+------------------------------| + | 2007-12-02 00:00:00 | 2020-08-25 | 2020-08-25 01:01:01 | + +---------------------+------------------+------------------------------+ SUBTIME @@ -1556,35 +1556,35 @@ Example:: os> source=people | eval `'23:59:59' - 0` = SUBTIME(TIME('23:59:59'), DATE('2004-01-01')) | fields `'23:59:59' - 0` fetched rows / total rows = 1/1 - +------------------+ - | '23:59:59' - 0 | - |------------------| - | 23:59:59 | - +------------------+ + +----------------+ + | '23:59:59' - 0 | + |----------------| + | 23:59:59 | + +----------------+ os> source=people | eval `'2004-01-01' - '23:59:59'` = SUBTIME(DATE('2004-01-01'), TIME('23:59:59')) | fields `'2004-01-01' - '23:59:59'` fetched rows / total rows = 1/1 - +-----------------------------+ - | '2004-01-01' - '23:59:59' | - |-----------------------------| - | 2003-12-31 00:00:01 | - +-----------------------------+ - - os> source=people | eval `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' - '00:05:42'` - fetched rows / total rows = 1/1 +---------------------------+ - | '10:20:30' - '00:05:42' | + | '2004-01-01' - '23:59:59' | |---------------------------| - | 10:14:48 | + | 2003-12-31 00:00:01 | +---------------------------+ + os> source=people | eval `'10:20:30' - '00:05:42'` = SUBTIME(TIME('10:20:30'), TIME('00:05:42')) | fields `'10:20:30' - '00:05:42'` + fetched rows / total rows = 1/1 + +-------------------------+ + | '10:20:30' - '00:05:42' | + |-------------------------| + | 10:14:48 | + +-------------------------+ + os> source=people | eval `'2007-03-01 10:20:30' - '20:40:50'` = SUBTIME(TIMESTAMP('2007-03-01 10:20:30'), TIMESTAMP('2002-03-04 20:40:50')) | fields `'2007-03-01 10:20:30' - '20:40:50'` fetched rows / total rows = 1/1 - +--------------------------------------+ - | '2007-03-01 10:20:30' - '20:40:50' | - |--------------------------------------| - | 2007-02-28 13:39:40 | - +--------------------------------------+ + +------------------------------------+ + | '2007-03-01 10:20:30' - '20:40:50' | + |------------------------------------| + | 2007-02-28 13:39:40 | + +------------------------------------+ SYSDATE @@ -1630,35 +1630,35 @@ Example:: os> source=people | eval `TIME('13:49:00')` = TIME('13:49:00') | fields `TIME('13:49:00')` fetched rows / total rows = 1/1 - +--------------------+ - | TIME('13:49:00') | - |--------------------| - | 13:49:00 | - +--------------------+ + +------------------+ + | TIME('13:49:00') | + |------------------| + | 13:49:00 | + +------------------+ os> source=people | eval `TIME('13:49')` = TIME('13:49') | fields `TIME('13:49')` fetched rows / total rows = 1/1 - +-----------------+ - | TIME('13:49') | - |-----------------| - | 13:49:00 | - +-----------------+ + +---------------+ + | TIME('13:49') | + |---------------| + | 13:49:00 | + +---------------+ os> source=people | eval `TIME('2020-08-26 13:49:00')` = TIME('2020-08-26 13:49:00') | fields `TIME('2020-08-26 13:49:00')` fetched rows / total rows = 1/1 - +-------------------------------+ - | TIME('2020-08-26 13:49:00') | - |-------------------------------| - | 13:49:00 | - +-------------------------------+ + +-----------------------------+ + | TIME('2020-08-26 13:49:00') | + |-----------------------------| + | 13:49:00 | + +-----------------------------+ os> source=people | eval `TIME('2020-08-26 13:49')` = TIME('2020-08-26 13:49') | fields `TIME('2020-08-26 13:49')` fetched rows / total rows = 1/1 - +----------------------------+ - | TIME('2020-08-26 13:49') | - |----------------------------| - | 13:49:00 | - +----------------------------+ + +--------------------------+ + | TIME('2020-08-26 13:49') | + |--------------------------| + | 13:49:00 | + +--------------------------+ TIME_FORMAT @@ -1709,11 +1709,11 @@ Example:: os> source=people | eval `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` = TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | fields `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` fetched rows / total rows = 1/1 - +------------------------------------------------------------------------------+ - | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | - |------------------------------------------------------------------------------| - | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | - +------------------------------------------------------------------------------+ + +----------------------------------------------------------------------------+ + | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | + |----------------------------------------------------------------------------| + | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | + +----------------------------------------------------------------------------+ TIME_TO_SEC @@ -1732,11 +1732,11 @@ Example:: os> source=people | eval `TIME_TO_SEC(TIME('22:23:00'))` = TIME_TO_SEC(TIME('22:23:00')) | fields `TIME_TO_SEC(TIME('22:23:00'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | TIME_TO_SEC(TIME('22:23:00')) | - |---------------------------------| - | 80580 | - +---------------------------------+ + +-------------------------------+ + | TIME_TO_SEC(TIME('22:23:00')) | + |-------------------------------| + | 80580 | + +-------------------------------+ TIMEDIFF @@ -1755,11 +1755,11 @@ Example:: os> source=people | eval `TIMEDIFF('23:59:59', '13:00:00')` = TIMEDIFF('23:59:59', '13:00:00') | fields `TIMEDIFF('23:59:59', '13:00:00')` fetched rows / total rows = 1/1 - +------------------------------------+ - | TIMEDIFF('23:59:59', '13:00:00') | - |------------------------------------| - | 10:59:59 | - +------------------------------------+ + +----------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |----------------------------------| + | 10:59:59 | + +----------------------------------+ TIMESTAMP @@ -1783,11 +1783,11 @@ Example:: os> source=people | eval `TIMESTAMP('2020-08-26 13:49:00')` = TIMESTAMP('2020-08-26 13:49:00'), `TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42'))` = TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | fields `TIMESTAMP('2020-08-26 13:49:00')`, `TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42'))` fetched rows / total rows = 1/1 - +------------------------------------+------------------------------------------------------+ - | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | - |------------------------------------+------------------------------------------------------| - | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | - +------------------------------------+------------------------------------------------------+ + +----------------------------------+----------------------------------------------------+ + | TIMESTAMP('2020-08-26 13:49:00') | TIMESTAMP('2020-08-26 13:49:00', TIME('12:15:42')) | + |----------------------------------+----------------------------------------------------| + | 2020-08-26 13:49:00 | 2020-08-27 02:04:42 | + +----------------------------------+----------------------------------------------------+ TIMESTAMPADD @@ -1808,11 +1808,11 @@ Examples:: os> source=people | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` fetched rows / total rows = 1/1 - +------------------------------------------------+----------------------------------------------------+ - | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | - |------------------------------------------------+----------------------------------------------------| - | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | - +------------------------------------------------+----------------------------------------------------+ + +----------------------------------------------+--------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |----------------------------------------------+--------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +----------------------------------------------+--------------------------------------------------+ TIMESTAMPDIFF @@ -1834,11 +1834,11 @@ Examples:: os> source=people | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | eval `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` = TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` fetched rows / total rows = 1/1 - +---------------------------------------------------------------------+-------------------------------------------------------------+ - | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | - |---------------------------------------------------------------------+-------------------------------------------------------------| - | 4 | -23 | - +---------------------------------------------------------------------+-------------------------------------------------------------+ + +-------------------------------------------------------------------+-----------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | + |-------------------------------------------------------------------+-----------------------------------------------------------| + | 4 | -23 | + +-------------------------------------------------------------------+-----------------------------------------------------------+ TO_DAYS @@ -1857,11 +1857,11 @@ Example:: os> source=people | eval `TO_DAYS(DATE('2008-10-07'))` = TO_DAYS(DATE('2008-10-07')) | fields `TO_DAYS(DATE('2008-10-07'))` fetched rows / total rows = 1/1 - +-------------------------------+ - | TO_DAYS(DATE('2008-10-07')) | - |-------------------------------| - | 733687 | - +-------------------------------+ + +-----------------------------+ + | TO_DAYS(DATE('2008-10-07')) | + |-----------------------------| + | 733687 | + +-----------------------------+ TO_SECONDS @@ -1881,11 +1881,11 @@ Example:: os> source=people | eval `TO_SECONDS(DATE('2008-10-07'))` = TO_SECONDS(DATE('2008-10-07')) | eval `TO_SECONDS(950228)` = TO_SECONDS(950228) | fields `TO_SECONDS(DATE('2008-10-07'))`, `TO_SECONDS(950228)` fetched rows / total rows = 1/1 - +----------------------------------+----------------------+ - | TO_SECONDS(DATE('2008-10-07')) | TO_SECONDS(950228) | - |----------------------------------+----------------------| - | 63390556800 | 62961148800 | - +----------------------------------+----------------------+ + +--------------------------------+--------------------+ + | TO_SECONDS(DATE('2008-10-07')) | TO_SECONDS(950228) | + |--------------------------------+--------------------| + | 63390556800 | 62961148800 | + +--------------------------------+--------------------+ UNIX_TIMESTAMP @@ -1907,11 +1907,11 @@ Example:: os> source=people | eval `UNIX_TIMESTAMP(double)` = UNIX_TIMESTAMP(20771122143845), `UNIX_TIMESTAMP(timestamp)` = UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | fields `UNIX_TIMESTAMP(double)`, `UNIX_TIMESTAMP(timestamp)` fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | UNIX_TIMESTAMP(double) | UNIX_TIMESTAMP(timestamp) | - |--------------------------+-----------------------------| - | 3404817525.0 | 848077542.0 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | UNIX_TIMESTAMP(double) | UNIX_TIMESTAMP(timestamp) | + |------------------------+---------------------------| + | 3404817525.0 | 848077542.0 | + +------------------------+---------------------------+ UTC_DATE @@ -1930,11 +1930,11 @@ Example:: > source=people | eval `UTC_DATE()` = UTC_DATE() | fields `UTC_DATE()` fetched rows / total rows = 1/1 - +--------------+ - | UTC_DATE() | - |--------------| - | 2022-10-03 | - +--------------+ + +------------+ + | UTC_DATE() | + |------------| + | 2022-10-03 | + +------------+ UTC_TIME @@ -1953,11 +1953,11 @@ Example:: > source=people | eval `UTC_TIME()` = UTC_TIME() | fields `UTC_TIME()` fetched rows / total rows = 1/1 - +--------------+ - | UTC_TIME() | - |--------------| - | 17:54:27 | - +--------------+ + +------------+ + | UTC_TIME() | + |------------| + | 17:54:27 | + +------------+ UTC_TIMESTAMP @@ -2042,11 +2042,11 @@ Example:: os> source=people | eval `WEEK(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK(DATE('2008-02-20'), 1)` = WEEK(DATE('2008-02-20'), 1) | fields `WEEK(DATE('2008-02-20'))`, `WEEK(DATE('2008-02-20'), 1)` fetched rows / total rows = 1/1 - +----------------------------+-------------------------------+ - | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | - |----------------------------+-------------------------------| - | 7 | 8 | - +----------------------------+-------------------------------+ + +--------------------------+-----------------------------+ + | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | + |--------------------------+-----------------------------| + | 7 | 8 | + +--------------------------+-----------------------------+ WEEKDAY @@ -2067,11 +2067,11 @@ Example:: os> source=people | eval `weekday(DATE('2020-08-26'))` = weekday(DATE('2020-08-26')) | eval `weekday(DATE('2020-08-27'))` = weekday(DATE('2020-08-27')) | fields `weekday(DATE('2020-08-26'))`, `weekday(DATE('2020-08-27'))` fetched rows / total rows = 1/1 - +-------------------------------+-------------------------------+ - | weekday(DATE('2020-08-26')) | weekday(DATE('2020-08-27')) | - |-------------------------------+-------------------------------| - | 2 | 3 | - +-------------------------------+-------------------------------+ + +-----------------------------+-----------------------------+ + | weekday(DATE('2020-08-26')) | weekday(DATE('2020-08-27')) | + |-----------------------------+-----------------------------| + | 2 | 3 | + +-----------------------------+-----------------------------+ WEEK_OF_YEAR @@ -2133,11 +2133,11 @@ Example:: os> source=people | eval `WEEK_OF_YEAR(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` = WEEK_OF_YEAR(DATE('2008-02-20'), 1) | fields `WEEK_OF_YEAR(DATE('2008-02-20'))`, `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` fetched rows / total rows = 1/1 - +------------------------------------+---------------------------------------+ - | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | - |------------------------------------+---------------------------------------| - | 7 | 8 | - +------------------------------------+---------------------------------------+ + +----------------------------------+-------------------------------------+ + | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | + |----------------------------------+-------------------------------------| + | 7 | 8 | + +----------------------------------+-------------------------------------+ YEAR @@ -2156,11 +2156,11 @@ Example:: os> source=people | eval `YEAR(DATE('2020-08-26'))` = YEAR(DATE('2020-08-26')) | fields `YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +----------------------------+ - | YEAR(DATE('2020-08-26')) | - |----------------------------| - | 2020 | - +----------------------------+ + +--------------------------+ + | YEAR(DATE('2020-08-26')) | + |--------------------------| + | 2020 | + +--------------------------+ YEARWEEK @@ -2179,10 +2179,10 @@ Example:: os> source=people | eval `YEARWEEK('2020-08-26')` = YEARWEEK('2020-08-26') | eval `YEARWEEK('2019-01-05', 1)` = YEARWEEK('2019-01-05', 1) | fields `YEARWEEK('2020-08-26')`, `YEARWEEK('2019-01-05', 1)` fetched rows / total rows = 1/1 - +--------------------------+-----------------------------+ - | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 1) | - |--------------------------+-----------------------------| - | 202034 | 201901 | - +--------------------------+-----------------------------+ + +------------------------+---------------------------+ + | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 1) | + |------------------------+---------------------------| + | 202034 | 201901 | + +------------------------+---------------------------+ diff --git a/docs/user/ppl/functions/expressions.rst b/docs/user/ppl/functions/expressions.rst index ac48324680..d25063d559 100644 --- a/docs/user/ppl/functions/expressions.rst +++ b/docs/user/ppl/functions/expressions.rst @@ -48,13 +48,13 @@ Here is an example for different type of arithmetic expressions:: os> source=accounts | where age > (25 + 5) | fields age ; fetched rows / total rows = 3/3 - +-------+ - | age | - |-------| - | 32 | - | 36 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 36 | + | 33 | + +-----+ Predicate Operators =================== @@ -108,11 +108,11 @@ Here is an example for comparison operators:: os> source=accounts | where age > 33 | fields age ; fetched rows / total rows = 1/1 - +-------+ - | age | - |-------| - | 36 | - +-------+ + +-----+ + | age | + |-----| + | 36 | + +-----+ IN @@ -122,12 +122,12 @@ IN operator test field in value lists:: os> source=accounts | where age in (32, 33) | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ OR @@ -137,12 +137,12 @@ OR operator :: os> source=accounts | where age = 32 OR age = 33 | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 32 | - | 33 | - +-------+ + +-----+ + | age | + |-----| + | 32 | + | 33 | + +-----+ NOT @@ -152,10 +152,10 @@ NOT operator :: os> source=accounts | where not age in (32, 33) | fields age ; fetched rows / total rows = 2/2 - +-------+ - | age | - |-------| - | 36 | - | 28 | - +-------+ + +-----+ + | age | + |-----| + | 36 | + | 28 | + +-----+ diff --git a/docs/user/ppl/functions/math.rst b/docs/user/ppl/functions/math.rst index c5eb07b5da..65f544461b 100644 --- a/docs/user/ppl/functions/math.rst +++ b/docs/user/ppl/functions/math.rst @@ -25,11 +25,11 @@ Example:: os> source=people | eval `ABS(-1)` = ABS(-1) | fields `ABS(-1)` fetched rows / total rows = 1/1 - +-----------+ - | ABS(-1) | - |-----------| - | 1 | - +-----------+ + +---------+ + | ABS(-1) | + |---------| + | 1 | + +---------+ ACOS @@ -71,11 +71,11 @@ Example:: os> source=people | eval `ASIN(0)` = ASIN(0) | fields `ASIN(0)` fetched rows / total rows = 1/1 - +-----------+ - | ASIN(0) | - |-----------| - | 0.0 | - +-----------+ + +---------+ + | ASIN(0) | + |---------| + | 0.0 | + +---------+ ATAN @@ -150,19 +150,19 @@ Example:: os> source=people | eval `CEILING(0)` = CEILING(0), `CEILING(50.00005)` = CEILING(50.00005), `CEILING(-50.00005)` = CEILING(-50.00005) | fields `CEILING(0)`, `CEILING(50.00005)`, `CEILING(-50.00005)` fetched rows / total rows = 1/1 - +--------------+---------------------+----------------------+ - | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | - |--------------+---------------------+----------------------| - | 0 | 51 | -50 | - +--------------+---------------------+----------------------+ + +------------+-------------------+--------------------+ + | CEILING(0) | CEILING(50.00005) | CEILING(-50.00005) | + |------------+-------------------+--------------------| + | 0 | 51 | -50 | + +------------+-------------------+--------------------+ os> source=people | eval `CEILING(3147483647.12345)` = CEILING(3147483647.12345), `CEILING(113147483647.12345)` = CEILING(113147483647.12345), `CEILING(3147483647.00001)` = CEILING(3147483647.00001) | fields `CEILING(3147483647.12345)`, `CEILING(113147483647.12345)`, `CEILING(3147483647.00001)` fetched rows / total rows = 1/1 - +-----------------------------+-------------------------------+-----------------------------+ - | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | - |-----------------------------+-------------------------------+-----------------------------| - | 3147483648 | 113147483648 | 3147483648 | - +-----------------------------+-------------------------------+-----------------------------+ + +---------------------------+-----------------------------+---------------------------+ + | CEILING(3147483647.12345) | CEILING(113147483647.12345) | CEILING(3147483647.00001) | + |---------------------------+-----------------------------+---------------------------| + | 3147483648 | 113147483648 | 3147483648 | + +---------------------------+-----------------------------+---------------------------+ CONV @@ -181,11 +181,11 @@ Example:: os> source=people | eval `CONV('12', 10, 16)` = CONV('12', 10, 16), `CONV('2C', 16, 10)` = CONV('2C', 16, 10), `CONV(12, 10, 2)` = CONV(12, 10, 2), `CONV(1111, 2, 10)` = CONV(1111, 2, 10) | fields `CONV('12', 10, 16)`, `CONV('2C', 16, 10)`, `CONV(12, 10, 2)`, `CONV(1111, 2, 10)` fetched rows / total rows = 1/1 - +----------------------+----------------------+-------------------+---------------------+ - | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | - |----------------------+----------------------+-------------------+---------------------| - | c | 44 | 1100 | 15 | - +----------------------+----------------------+-------------------+---------------------+ + +--------------------+--------------------+-----------------+-------------------+ + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | + |--------------------+--------------------+-----------------+-------------------| + | c | 44 | 1100 | 15 | + +--------------------+--------------------+-----------------+-------------------+ COS @@ -204,11 +204,11 @@ Example:: os> source=people | eval `COS(0)` = COS(0) | fields `COS(0)` fetched rows / total rows = 1/1 - +----------+ - | COS(0) | - |----------| - | 1.0 | - +----------+ + +--------+ + | COS(0) | + |--------| + | 1.0 | + +--------+ COT @@ -250,11 +250,11 @@ Example:: os> source=people | eval `CRC32('MySQL')` = CRC32('MySQL') | fields `CRC32('MySQL')` fetched rows / total rows = 1/1 - +------------------+ - | CRC32('MySQL') | - |------------------| - | 3259397556 | - +------------------+ + +----------------+ + | CRC32('MySQL') | + |----------------| + | 3259397556 | + +----------------+ DEGREES @@ -342,27 +342,27 @@ Example:: os> source=people | eval `FLOOR(0)` = FLOOR(0), `FLOOR(50.00005)` = FLOOR(50.00005), `FLOOR(-50.00005)` = FLOOR(-50.00005) | fields `FLOOR(0)`, `FLOOR(50.00005)`, `FLOOR(-50.00005)` fetched rows / total rows = 1/1 - +------------+-------------------+--------------------+ - | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | - |------------+-------------------+--------------------| - | 0 | 50 | -51 | - +------------+-------------------+--------------------+ + +----------+-----------------+------------------+ + | FLOOR(0) | FLOOR(50.00005) | FLOOR(-50.00005) | + |----------+-----------------+------------------| + | 0 | 50 | -51 | + +----------+-----------------+------------------+ os> source=people | eval `FLOOR(3147483647.12345)` = FLOOR(3147483647.12345), `FLOOR(113147483647.12345)` = FLOOR(113147483647.12345), `FLOOR(3147483647.00001)` = FLOOR(3147483647.00001) | fields `FLOOR(3147483647.12345)`, `FLOOR(113147483647.12345)`, `FLOOR(3147483647.00001)` fetched rows / total rows = 1/1 - +---------------------------+-----------------------------+---------------------------+ - | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | - |---------------------------+-----------------------------+---------------------------| - | 3147483647 | 113147483647 | 3147483647 | - +---------------------------+-----------------------------+---------------------------+ + +-------------------------+---------------------------+-------------------------+ + | FLOOR(3147483647.12345) | FLOOR(113147483647.12345) | FLOOR(3147483647.00001) | + |-------------------------+---------------------------+-------------------------| + | 3147483647 | 113147483647 | 3147483647 | + +-------------------------+---------------------------+-------------------------+ os> source=people | eval `FLOOR(282474973688888.022)` = FLOOR(282474973688888.022), `FLOOR(9223372036854775807.022)` = FLOOR(9223372036854775807.022), `FLOOR(9223372036854775807.0000001)` = FLOOR(9223372036854775807.0000001) | fields `FLOOR(282474973688888.022)`, `FLOOR(9223372036854775807.022)`, `FLOOR(9223372036854775807.0000001)` fetched rows / total rows = 1/1 - +------------------------------+----------------------------------+--------------------------------------+ - | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | - |------------------------------+----------------------------------+--------------------------------------| - | 282474973688888 | 9223372036854775807 | 9223372036854775807 | - +------------------------------+----------------------------------+--------------------------------------+ + +----------------------------+--------------------------------+------------------------------------+ + | FLOOR(282474973688888.022) | FLOOR(9223372036854775807.022) | FLOOR(9223372036854775807.0000001) | + |----------------------------+--------------------------------+------------------------------------| + | 282474973688888 | 9223372036854775807 | 9223372036854775807 | + +----------------------------+--------------------------------+------------------------------------+ LN @@ -406,11 +406,11 @@ Example:: os> source=people | eval `LOG(2)` = LOG(2), `LOG(2, 8)` = LOG(2, 8) | fields `LOG(2)`, `LOG(2, 8)` fetched rows / total rows = 1/1 - +--------------------+-------------+ - | LOG(2) | LOG(2, 8) | - |--------------------+-------------| - | 0.6931471805599453 | 3.0 | - +--------------------+-------------+ + +--------------------+-----------+ + | LOG(2) | LOG(2, 8) | + |--------------------+-----------| + | 0.6931471805599453 | 3.0 | + +--------------------+-----------+ LOG2 @@ -431,11 +431,11 @@ Example:: os> source=people | eval `LOG2(8)` = LOG2(8) | fields `LOG2(8)` fetched rows / total rows = 1/1 - +-----------+ - | LOG2(8) | - |-----------| - | 3.0 | - +-----------+ + +---------+ + | LOG2(8) | + |---------| + | 3.0 | + +---------+ LOG10 @@ -456,11 +456,11 @@ Example:: os> source=people | eval `LOG10(100)` = LOG10(100) | fields `LOG10(100)` fetched rows / total rows = 1/1 - +--------------+ - | LOG10(100) | - |--------------| - | 2.0 | - +--------------+ + +------------+ + | LOG10(100) | + |------------| + | 2.0 | + +------------+ MOD @@ -479,11 +479,11 @@ Example:: os> source=people | eval `MOD(3, 2)` = MOD(3, 2), `MOD(3.1, 2)` = MOD(3.1, 2) | fields `MOD(3, 2)`, `MOD(3.1, 2)` fetched rows / total rows = 1/1 - +-------------+---------------+ - | MOD(3, 2) | MOD(3.1, 2) | - |-------------+---------------| - | 1 | 1.1 | - +-------------+---------------+ + +-----------+-------------+ + | MOD(3, 2) | MOD(3.1, 2) | + |-----------+-------------| + | 1 | 1.1 | + +-----------+-------------+ PI @@ -525,11 +525,11 @@ Example:: os> source=people | eval `POW(3, 2)` = POW(3, 2), `POW(-3, 2)` = POW(-3, 2), `POW(3, -2)` = POW(3, -2) | fields `POW(3, 2)`, `POW(-3, 2)`, `POW(3, -2)` fetched rows / total rows = 1/1 - +-------------+--------------+--------------------+ - | POW(3, 2) | POW(-3, 2) | POW(3, -2) | - |-------------+--------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +-------------+--------------+--------------------+ + +-----------+------------+--------------------+ + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | + |-----------+------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-----------+------------+--------------------+ POWER @@ -550,11 +550,11 @@ Example:: os> source=people | eval `POWER(3, 2)` = POWER(3, 2), `POWER(-3, 2)` = POWER(-3, 2), `POWER(3, -2)` = POWER(3, -2) | fields `POWER(3, 2)`, `POWER(-3, 2)`, `POWER(3, -2)` fetched rows / total rows = 1/1 - +---------------+----------------+--------------------+ - | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | - |---------------+----------------+--------------------| - | 9.0 | 9.0 | 0.1111111111111111 | - +---------------+----------------+--------------------+ + +-------------+--------------+--------------------+ + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | + |-------------+--------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-------------+--------------+--------------------+ RADIANS @@ -622,11 +622,11 @@ Example:: os> source=people | eval `ROUND(12.34)` = ROUND(12.34), `ROUND(12.34, 1)` = ROUND(12.34, 1), `ROUND(12.34, -1)` = ROUND(12.34, -1), `ROUND(12, 1)` = ROUND(12, 1) | fields `ROUND(12.34)`, `ROUND(12.34, 1)`, `ROUND(12.34, -1)`, `ROUND(12, 1)` fetched rows / total rows = 1/1 - +----------------+-------------------+--------------------+----------------+ - | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | - |----------------+-------------------+--------------------+----------------| - | 12.0 | 12.3 | 10.0 | 12 | - +----------------+-------------------+--------------------+----------------+ + +--------------+-----------------+------------------+--------------+ + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | + |--------------+-----------------+------------------+--------------| + | 12.0 | 12.3 | 10.0 | 12 | + +--------------+-----------------+------------------+--------------+ SIGN @@ -645,11 +645,11 @@ Example:: os> source=people | eval `SIGN(1)` = SIGN(1), `SIGN(0)` = SIGN(0), `SIGN(-1.1)` = SIGN(-1.1) | fields `SIGN(1)`, `SIGN(0)`, `SIGN(-1.1)` fetched rows / total rows = 1/1 - +-----------+-----------+--------------+ - | SIGN(1) | SIGN(0) | SIGN(-1.1) | - |-----------+-----------+--------------| - | 1 | 0 | -1 | - +-----------+-----------+--------------+ + +---------+---------+------------+ + | SIGN(1) | SIGN(0) | SIGN(-1.1) | + |---------+---------+------------| + | 1 | 0 | -1 | + +---------+---------+------------+ SIN @@ -668,11 +668,11 @@ Example:: os> source=people | eval `SIN(0)` = SIN(0) | fields `SIN(0)` fetched rows / total rows = 1/1 - +----------+ - | SIN(0) | - |----------| - | 0.0 | - +----------+ + +--------+ + | SIN(0) | + |--------| + | 0.0 | + +--------+ SQRT @@ -694,11 +694,11 @@ Example:: os> source=people | eval `SQRT(4)` = SQRT(4), `SQRT(4.41)` = SQRT(4.41) | fields `SQRT(4)`, `SQRT(4.41)` fetched rows / total rows = 1/1 - +-----------+--------------+ - | SQRT(4) | SQRT(4.41) | - |-----------+--------------| - | 2.0 | 2.1 | - +-----------+--------------+ + +---------+------------+ + | SQRT(4) | SQRT(4.41) | + |---------+------------| + | 2.0 | 2.1 | + +---------+------------+ CBRT @@ -719,9 +719,9 @@ Example:: opensearchsql> source=location | eval `CBRT(8)` = CBRT(8), `CBRT(9.261)` = CBRT(9.261), `CBRT(-27)` = CBRT(-27) | fields `CBRT(8)`, `CBRT(9.261)`, `CBRT(-27)`; fetched rows / total rows = 2/2 - +-----------+---------------+-------------+ - | CBRT(8) | CBRT(9.261) | CBRT(-27) | - |-----------+---------------+-------------| - | 2.0 | 2.1 | -3.0 | - | 2.0 | 2.1 | -3.0 | - +-----------+---------------+-------------+ + +---------+-------------+-----------+ + | CBRT(8) | CBRT(9.261) | CBRT(-27) | + |---------+-------------+-----------| + | 2.0 | 2.1 | -3.0 | + | 2.0 | 2.1 | -3.0 | + +---------+-------------+-----------+ diff --git a/docs/user/ppl/functions/relevance.rst b/docs/user/ppl/functions/relevance.rst index fb31edb0d2..a1f240ee05 100644 --- a/docs/user/ppl/functions/relevance.rst +++ b/docs/user/ppl/functions/relevance.rst @@ -37,12 +37,12 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> source=accounts | where match(address, 'Street') | fields lastname, address; fetched rows / total rows = 2/2 - +------------+--------------------+ - | lastname | address | - |------------+--------------------| - | Bond | 671 Bristol Street | - | Bates | 789 Madison Street | - +------------+--------------------+ + +----------+--------------------+ + | lastname | address | + |----------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +----------+--------------------+ @@ -50,11 +50,11 @@ Another example to show how to set custom values for the optional parameters:: os> source=accounts | where match(firstname, 'Hattie', operator='AND', boost=2.0) | fields lastname; fetched rows / total rows = 1/1 - +------------+ - | lastname | - |------------| - | Bond | - +------------+ + +----------+ + | lastname | + |----------| + | Bond | + +----------+ MATCH_PHRASE @@ -175,22 +175,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where multi_match(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where multi_match(['title'], 'Pooh House', operator='AND', analyzer=default) | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ SIMPLE_QUERY_STRING @@ -228,22 +228,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where simple_query_string(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where simple_query_string(['title'], 'Pooh House', flags='ALL', default_operator='AND') | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ MATCH_BOOL_PREFIX @@ -270,22 +270,22 @@ Example with only ``field`` and ``query`` expressions, and all other parameters os> source=accounts | where match_bool_prefix(address, 'Bristol Stre') | fields firstname, address fetched rows / total rows = 2/2 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - | Nanette | 789 Madison Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + | Nanette | 789 Madison Street | + +-----------+--------------------+ Another example to show how to set custom values for the optional parameters:: os> source=accounts | where match_bool_prefix(address, 'Bristol Stre', minimum_should_match = 2) | fields firstname, address fetched rows / total rows = 1/1 - +-------------+--------------------+ - | firstname | address | - |-------------+--------------------| - | Hattie | 671 Bristol Street | - +-------------+--------------------+ + +-----------+--------------------+ + | firstname | address | + |-----------+--------------------| + | Hattie | 671 Bristol Street | + +-----------+--------------------+ QUERY_STRING @@ -335,22 +335,22 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters os> source=books | where query_string(['title'], 'Pooh House') | fields id, title, author; fetched rows / total rows = 2/2 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - | 2 | Winnie-the-Pooh | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +----+--------------------------+----------------------+ Another example to show how to set custom values for the optional parameters:: os> source=books | where query_string(['title'], 'Pooh House', default_operator='AND') | fields id, title, author; fetched rows / total rows = 1/1 - +------+--------------------------+----------------------+ - | id | title | author | - |------+--------------------------+----------------------| - | 1 | The House at Pooh Corner | Alan Alexander Milne | - +------+--------------------------+----------------------+ + +----+--------------------------+----------------------+ + | id | title | author | + |----+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +----+--------------------------+----------------------+ Limitations >>>>>>>>>>> diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index edf5220f4f..0dbb09cbb8 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -24,11 +24,11 @@ Example:: os> source=people | eval `CONCAT('hello', 'world')` = CONCAT('hello', 'world'), `CONCAT('hello ', 'whole ', 'world', '!')` = CONCAT('hello ', 'whole ', 'world', '!') | fields `CONCAT('hello', 'world')`, `CONCAT('hello ', 'whole ', 'world', '!')` fetched rows / total rows = 1/1 - +----------------------------+--------------------------------------------+ - | CONCAT('hello', 'world') | CONCAT('hello ', 'whole ', 'world', '!') | - |----------------------------+--------------------------------------------| - | helloworld | hello whole world! | - +----------------------------+--------------------------------------------+ + +--------------------------+------------------------------------------+ + | CONCAT('hello', 'world') | CONCAT('hello ', 'whole ', 'world', '!') | + |--------------------------+------------------------------------------| + | helloworld | hello whole world! | + +--------------------------+------------------------------------------+ CONCAT_WS @@ -47,11 +47,11 @@ Example:: os> source=people | eval `CONCAT_WS(',', 'hello', 'world')` = CONCAT_WS(',', 'hello', 'world') | fields `CONCAT_WS(',', 'hello', 'world')` fetched rows / total rows = 1/1 - +------------------------------------+ - | CONCAT_WS(',', 'hello', 'world') | - |------------------------------------| - | hello,world | - +------------------------------------+ + +----------------------------------+ + | CONCAT_WS(',', 'hello', 'world') | + |----------------------------------| + | hello,world | + +----------------------------------+ LENGTH @@ -74,11 +74,11 @@ Example:: os> source=people | eval `LENGTH('helloworld')` = LENGTH('helloworld') | fields `LENGTH('helloworld')` fetched rows / total rows = 1/1 - +------------------------+ - | LENGTH('helloworld') | - |------------------------| - | 10 | - +------------------------+ + +----------------------+ + | LENGTH('helloworld') | + |----------------------| + | 10 | + +----------------------+ LIKE @@ -98,11 +98,11 @@ Example:: os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ELLO%') | fields `LIKE('hello world', '_ello%')` fetched rows / total rows = 1/1 - +---------------------------------+ - | LIKE('hello world', '_ello%') | - |---------------------------------| - | True | - +---------------------------------+ + +-------------------------------+ + | LIKE('hello world', '_ello%') | + |-------------------------------| + | True | + +-------------------------------+ LOWER @@ -121,11 +121,11 @@ Example:: os> source=people | eval `LOWER('helloworld')` = LOWER('helloworld'), `LOWER('HELLOWORLD')` = LOWER('HELLOWORLD') | fields `LOWER('helloworld')`, `LOWER('HELLOWORLD')` fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | LOWER('helloworld') | LOWER('HELLOWORLD') | - |-----------------------+-----------------------| - | helloworld | helloworld | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | LOWER('helloworld') | LOWER('HELLOWORLD') | + |---------------------+---------------------| + | helloworld | helloworld | + +---------------------+---------------------+ LTRIM @@ -144,11 +144,11 @@ Example:: os> source=people | eval `LTRIM(' hello')` = LTRIM(' hello'), `LTRIM('hello ')` = LTRIM('hello ') | fields `LTRIM(' hello')`, `LTRIM('hello ')` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LTRIM(' hello') | LTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | LTRIM(' hello') | LTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ POSITION @@ -169,11 +169,11 @@ Example:: os> source=people | eval `POSITION('world' IN 'helloworld')` = POSITION('world' IN 'helloworld'), `POSITION('invalid' IN 'helloworld')`= POSITION('invalid' IN 'helloworld') | fields `POSITION('world' IN 'helloworld')`, `POSITION('invalid' IN 'helloworld')` fetched rows / total rows = 1/1 - +-------------------------------------+---------------------------------------+ - | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | - |-------------------------------------+---------------------------------------| - | 6 | 0 | - +-------------------------------------+---------------------------------------+ + +-----------------------------------+-------------------------------------+ + | POSITION('world' IN 'helloworld') | POSITION('invalid' IN 'helloworld') | + |-----------------------------------+-------------------------------------| + | 6 | 0 | + +-----------------------------------+-------------------------------------+ REVERSE @@ -192,11 +192,11 @@ Example:: os> source=people | eval `REVERSE('abcde')` = REVERSE('abcde') | fields `REVERSE('abcde')` fetched rows / total rows = 1/1 - +--------------------+ - | REVERSE('abcde') | - |--------------------| - | edcba | - +--------------------+ + +------------------+ + | REVERSE('abcde') | + |------------------| + | edcba | + +------------------+ RIGHT @@ -215,11 +215,11 @@ Example:: os> source=people | eval `RIGHT('helloworld', 5)` = RIGHT('helloworld', 5), `RIGHT('HELLOWORLD', 0)` = RIGHT('HELLOWORLD', 0) | fields `RIGHT('helloworld', 5)`, `RIGHT('HELLOWORLD', 0)` fetched rows / total rows = 1/1 - +--------------------------+--------------------------+ - | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | - |--------------------------+--------------------------| - | world | | - +--------------------------+--------------------------+ + +------------------------+------------------------+ + | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | + |------------------------+------------------------| + | world | | + +------------------------+------------------------+ RTRIM @@ -238,11 +238,11 @@ Example:: os> source=people | eval `RTRIM(' hello')` = RTRIM(' hello'), `RTRIM('hello ')` = RTRIM('hello ') | fields `RTRIM(' hello')`, `RTRIM('hello ')` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | RTRIM(' hello') | RTRIM('hello ') | - |---------------------+---------------------| - | hello | hello | - +---------------------+---------------------+ + +-------------------+-------------------+ + | RTRIM(' hello') | RTRIM('hello ') | + |-------------------+-------------------| + | hello | hello | + +-------------------+-------------------+ SUBSTRING @@ -263,11 +263,11 @@ Example:: os> source=people | eval `SUBSTRING('helloworld', 5)` = SUBSTRING('helloworld', 5), `SUBSTRING('helloworld', 5, 3)` = SUBSTRING('helloworld', 5, 3) | fields `SUBSTRING('helloworld', 5)`, `SUBSTRING('helloworld', 5, 3)` fetched rows / total rows = 1/1 - +------------------------------+---------------------------------+ - | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | - |------------------------------+---------------------------------| - | oworld | owo | - +------------------------------+---------------------------------+ + +----------------------------+-------------------------------+ + | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | + |----------------------------+-------------------------------| + | oworld | owo | + +----------------------------+-------------------------------+ TRIM @@ -284,11 +284,11 @@ Example:: os> source=people | eval `TRIM(' hello')` = TRIM(' hello'), `TRIM('hello ')` = TRIM('hello ') | fields `TRIM(' hello')`, `TRIM('hello ')` fetched rows / total rows = 1/1 - +--------------------+--------------------+ - | TRIM(' hello') | TRIM('hello ') | - |--------------------+--------------------| - | hello | hello | - +--------------------+--------------------+ + +------------------+------------------+ + | TRIM(' hello') | TRIM('hello ') | + |------------------+------------------| + | hello | hello | + +------------------+------------------+ UPPER @@ -307,8 +307,8 @@ Example:: os> source=people | eval `UPPER('helloworld')` = UPPER('helloworld'), `UPPER('HELLOWORLD')` = UPPER('HELLOWORLD') | fields `UPPER('helloworld')`, `UPPER('HELLOWORLD')` fetched rows / total rows = 1/1 - +-----------------------+-----------------------+ - | UPPER('helloworld') | UPPER('HELLOWORLD') | - |-----------------------+-----------------------| - | HELLOWORLD | HELLOWORLD | - +-----------------------+-----------------------+ + +---------------------+---------------------+ + | UPPER('helloworld') | UPPER('HELLOWORLD') | + |---------------------+---------------------| + | HELLOWORLD | HELLOWORLD | + +---------------------+---------------------+ diff --git a/docs/user/ppl/functions/system.rst b/docs/user/ppl/functions/system.rst index cfe0414c49..698933a3c4 100644 --- a/docs/user/ppl/functions/system.rst +++ b/docs/user/ppl/functions/system.rst @@ -24,8 +24,8 @@ Example:: os> source=people | eval `typeof(date)` = typeof(DATE('2008-04-14')), `typeof(int)` = typeof(1), `typeof(now())` = typeof(now()), `typeof(column)` = typeof(accounts) | fields `typeof(date)`, `typeof(int)`, `typeof(now())`, `typeof(column)` fetched rows / total rows = 1/1 - +----------------+---------------+-----------------+------------------+ - | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | - |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | TIMESTAMP | OBJECT | - +----------------+---------------+-----------------+------------------+ + +--------------+-------------+---------------+----------------+ + | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | + |--------------+-------------+---------------+----------------| + | DATE | INTEGER | TIMESTAMP | OBJECT | + +--------------+-------------+---------------+----------------+ diff --git a/docs/user/ppl/general/datatypes.rst b/docs/user/ppl/general/datatypes.rst index 18555dec3d..a205626dbd 100644 --- a/docs/user/ppl/general/datatypes.rst +++ b/docs/user/ppl/general/datatypes.rst @@ -356,11 +356,11 @@ PPL query:: os> source=people | fields city, city.name, city.location.latitude; fetched rows / total rows = 1/1 - +-----------------------------------------------------+-------------+--------------------------+ - | city | city.name | city.location.latitude | - |-----------------------------------------------------+-------------+--------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | Seattle | 10.5 | - +-----------------------------------------------------+-------------+--------------------------+ + +-----------------------------------------------------+-----------+------------------------+ + | city | city.name | city.location.latitude | + |-----------------------------------------------------+-----------+------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | Seattle | 10.5 | + +-----------------------------------------------------+-----------+------------------------+ Example 2: Group by struct inner attribute @@ -372,11 +372,11 @@ PPL query:: os> source=people | stats count() by city.name; fetched rows / total rows = 1/1 - +-----------+-------------+ - | count() | city.name | - |-----------+-------------| - | 1 | Seattle | - +-----------+-------------+ + +---------+-----------+ + | count() | city.name | + |---------+-----------| + | 1 | Seattle | + +---------+-----------+ Example 3: Selecting Field of Array Value ----------------------------------------- @@ -385,8 +385,8 @@ Select deeper level for object fields of array value which returns the first ele os> source = people | fields accounts, accounts.id; fetched rows / total rows = 1/1 - +------------+---------------+ - | accounts | accounts.id | - |------------+---------------| - | {'id': 1} | 1 | - +------------+---------------+ \ No newline at end of file + +-----------+-------------+ + | accounts | accounts.id | + |-----------+-------------| + | {'id': 1} | 1 | + +-----------+-------------+ \ No newline at end of file diff --git a/docs/user/ppl/general/identifiers.rst b/docs/user/ppl/general/identifiers.rst index 51fc36c40f..bab540ffdd 100644 --- a/docs/user/ppl/general/identifiers.rst +++ b/docs/user/ppl/general/identifiers.rst @@ -39,14 +39,14 @@ Here are examples for using index pattern directly without quotes:: os> source=accounts | fields account_number, firstname, lastname; fetched rows / total rows = 4/4 - +------------------+-------------+------------+ - | account_number | firstname | lastname | - |------------------+-------------+------------| - | 1 | Amber | Duke | - | 6 | Hattie | Bond | - | 13 | Nanette | Bates | - | 18 | Dale | Adams | - +------------------+-------------+------------+ + +----------------+-----------+----------+ + | account_number | firstname | lastname | + |----------------+-----------+----------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +----------------+-----------+----------+ Delimited Identifiers @@ -73,14 +73,14 @@ Here are examples for quoting an index name by back ticks:: os> source=`accounts` | fields `account_number`; fetched rows / total rows = 4/4 - +------------------+ - | account_number | - |------------------| - | 1 | - | 6 | - | 13 | - | 18 | - +------------------+ + +----------------+ + | account_number | + |----------------| + | 1 | + | 6 | + | 13 | + | 18 | + +----------------+ Cross-Cluster Index Identifiers @@ -135,29 +135,29 @@ Query wildcard indices:: os> source=acc* | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ Query multiple indices seperated by ``,``:: os> source=accounts, account2 | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ Query delimited multiple indices seperated by ``,``:: os> source=`accounts,account2` | stats count(); fetched rows / total rows = 1/1 - +-----------+ - | count() | - |-----------| - | 5 | - +-----------+ + +---------+ + | count() | + |---------| + | 5 | + +---------+ diff --git a/doctest/build.gradle b/doctest/build.gradle index a125a4f336..91d54c9cb2 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -22,8 +22,16 @@ def path = project(':').projectDir def plugin_path = project(':doctest').projectDir task cloneSqlCli(type: Exec) { - // clone the sql-cli repo locally - commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git' + def repoDir = new File("${project.projectDir}/sql-cli") + + if (repoDir.exists()) { + // Repository already exists, fetch and checkout latest + commandLine 'git', '-C', repoDir.absolutePath, 'fetch', 'origin', 'main' + commandLine 'git', '-C', repoDir.absolutePath, 'checkout', 'origin/main' + } else { + // Repository doesn't exist, clone it + commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git', repoDir.absolutePath + } } task bootstrap(type: Exec, dependsOn: ['cloneSqlCli', 'spotlessJava']) { diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 1fedbdf49e..881078a9bd 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -48,10 +48,34 @@ def process(self, statement): click.echo(output) +""" +For _explain requests, there are several additional request fields that will inconsistently +appear/change depending on underlying cluster state. This method normalizes these responses in-place +to make _explain doctests more consistent. + +If the passed response is not an _explain response, the input is left unmodified. +""" +def normalize_explain_response(data): + if "root" in data: + data = data["root"] + + if (request := data.get("description", {}).get("request", None)) and request.startswith("OpenSearchQueryRequest("): + for filter_field in ["needClean", "pitId", "cursorKeepAlive", "searchAfter", "searchResponse"]: + request = re.sub(f", {filter_field}=\\w+", "", request) + data["description"]["request"] = request + + for child in data.get("children", []): + normalize_explain_response(child) + + return data + + def pretty_print(s): try: - d = json.loads(s) - print(json.dumps(d, indent=2)) + data = json.loads(s) + normalize_explain_response(data) + + print(json.dumps(data, indent=2)) except json.decoder.JSONDecodeError: print(s) From 4c44f56f3eed5926430acf2c6a12151cbf182162 Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Fri, 11 Oct 2024 20:25:13 -0700 Subject: [PATCH 61/96] Unify internal OpenSearch exceptions with returned status code (#3068) Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .../src/test/java/org/opensearch/sql/legacy/CursorIT.java | 2 +- .../sql/legacy/executor/format/OpenSearchErrorMessage.java | 4 ++-- .../sql/opensearch/response/error/OpenSearchErrorMessage.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index 2bcb2902a2..e4ba593844 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -393,7 +393,7 @@ public void invalidCursorIdNotDecodable() throws IOException { JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/type"), equalTo("illegal_argument_exception")); + assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); } /** diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java index 8117d241b1..09c09919ec 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/OpenSearchErrorMessage.java @@ -12,8 +12,8 @@ public class OpenSearchErrorMessage extends ErrorMessage<OpenSearchException> { - OpenSearchErrorMessage(OpenSearchException exception, int status) { - super(exception, status); + OpenSearchErrorMessage(OpenSearchException exception, int defaultStatus) { + super(exception, exception.status() != null ? exception.status().getStatus() : defaultStatus); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java index 87a374d353..a712ceaedf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessage.java @@ -13,8 +13,8 @@ /** OpenSearch Error Message. */ public class OpenSearchErrorMessage extends ErrorMessage { - OpenSearchErrorMessage(OpenSearchException exception, int status) { - super(exception, status); + OpenSearchErrorMessage(OpenSearchException exception, int defaultStatus) { + super(exception, exception.status() != null ? exception.status().getStatus() : defaultStatus); } @Override From e838e46f20c164fe00a907725034ea9896b90f93 Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Sat, 12 Oct 2024 19:08:33 -0700 Subject: [PATCH 62/96] Fix record skipping when querying paginated data across shards (#3061) * Add reproducer for pagination skipping bug Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix the bug Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Add additional sorts to other locations Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Use constant for ID field Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix broken cursor test Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Apply spotless Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --------- Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .../sql/legacy/SQLIntegTestCase.java | 9 ++ .../sql/sql/PaginationWindowIT.java | 42 +++++++- .../calcs_with_shards_index_mappings.json | 99 +++++++++++++++++++ .../legacy/executor/ElasticHitsExecutor.java | 3 + .../legacy/executor/join/ElasticUtils.java | 2 + .../sql/legacy/query/DefaultQueryAction.java | 2 + .../node/pointInTime/PointInTime.java | 3 + .../planner/physical/node/scroll/Scroll.java | 3 + .../request/OpenSearchQueryRequest.java | 4 + 9 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 integ-test/src/test/resources/indexDefinitions/calcs_with_shards_index_mappings.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index eed4e29c9c..5b956fb5d3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -707,6 +707,15 @@ public enum Index { "calcs", getMappingFile("calcs_index_mappings.json"), "src/test/resources/calcs.json"), + // Calcs has enough records for shards to be interesting, but updating the existing mapping with + // shards in-place + // breaks existing tests. Aside from introducing a primary shard setting > 1, this index is + // identical to CALCS. + CALCS_WITH_SHARDS( + TestsConstants.TEST_INDEX_CALCS, + "calcs", + getMappingFile("calcs_with_shards_index_mappings.json"), + "src/test/resources/calcs.json"), DATE_FORMATS( TestsConstants.TEST_INDEX_DATE_FORMATS, "date_formats", diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java index 246cbfc4a0..4c387e2c17 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java @@ -5,9 +5,11 @@ package org.opensearch.sql.sql; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PHRASE; +import static org.opensearch.sql.legacy.TestsConstants.*; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.json.JSONObject; import org.junit.After; import org.junit.Test; @@ -18,6 +20,7 @@ public class PaginationWindowIT extends SQLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.PHRASE); + loadIndex(Index.CALCS_WITH_SHARDS); } @After @@ -92,4 +95,41 @@ public void testFetchSizeLargerThanResultWindowFails() throws IOException { () -> executeQueryTemplate("SELECT * FROM %s", TEST_INDEX_PHRASE, window + 1)); resetMaxResultWindow(TEST_INDEX_PHRASE); } + + @Test + public void testMultiShardPagesEqualsActualData() throws IOException { + // A bug made it so when pulling unordered data from an index with multiple shards, data gets + // lost if the fetchSize + // is not a multiple of the shard count. This tests that, for an index with 4 shards, pulling + // one page of 10 records + // is equivalent to pulling two pages of 5 records. + + var query = "SELECT key from " + TEST_INDEX_CALCS; + + var expectedResponse = new JSONObject(executeFetchQuery(query, 10, "jdbc")); + var expectedRows = expectedResponse.getJSONArray("datarows"); + + List<String> expectedKeys = new ArrayList<>(); + for (int i = 0; i < expectedRows.length(); i++) { + expectedKeys.add(expectedRows.getJSONArray(i).getString(0)); + } + + var actualPage1 = new JSONObject(executeFetchQuery(query, 5, "jdbc")); + + var actualRows1 = actualPage1.getJSONArray("datarows"); + var cursor = actualPage1.getString("cursor"); + var actualPage2 = executeCursorQuery(cursor); + + var actualRows2 = actualPage2.getJSONArray("datarows"); + + List<String> actualKeys = new ArrayList<>(); + for (int i = 0; i < actualRows1.length(); i++) { + actualKeys.add(actualRows1.getJSONArray(i).getString(0)); + } + for (int i = 0; i < actualRows2.length(); i++) { + actualKeys.add(actualRows2.getJSONArray(i).getString(0)); + } + + assertEquals(expectedKeys, actualKeys); + } } diff --git a/integ-test/src/test/resources/indexDefinitions/calcs_with_shards_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/calcs_with_shards_index_mappings.json new file mode 100644 index 0000000000..560e1d55e6 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/calcs_with_shards_index_mappings.json @@ -0,0 +1,99 @@ +{ + "mappings" : { + "properties" : { + "key" : { + "type" : "keyword" + }, + "num0" : { + "type" : "double" + }, + "num1" : { + "type" : "double" + }, + "num2" : { + "type" : "double" + }, + "num3" : { + "type" : "double" + }, + "num4" : { + "type" : "double" + }, + "str0" : { + "type" : "keyword" + }, + "str1" : { + "type" : "keyword" + }, + "str2" : { + "type" : "keyword" + }, + "str3" : { + "type" : "keyword" + }, + "int0" : { + "type" : "integer" + }, + "int1" : { + "type" : "integer" + }, + "int2" : { + "type" : "integer" + }, + "int3" : { + "type" : "integer" + }, + "bool0" : { + "type" : "boolean" + }, + "bool1" : { + "type" : "boolean" + }, + "bool2" : { + "type" : "boolean" + }, + "bool3" : { + "type" : "boolean" + }, + "date0" : { + "type" : "date", + "format": "year_month_day" + }, + "date1" : { + "type" : "date", + "format": "year_month_day" + }, + "date2" : { + "type" : "date", + "format": "year_month_day" + }, + "date3" : { + "type" : "date", + "format": "year_month_day" + }, + "time0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "time1" : { + "type" : "date", + "format": "hour_minute_second" + }, + "datetime0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "datetime1" : { + "type" : "date" + }, + "zzz" : { + "type" : "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": 4 + } + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java index 2b80575e1e..052cdb7b36 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java @@ -9,6 +9,7 @@ import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE; import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; import java.io.IOException; import org.apache.logging.log4j.LogManager; @@ -19,6 +20,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.legacy.domain.Select; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; @@ -70,6 +72,7 @@ public SearchResponse getResponseWithHits( boolean ordered = select.isOrderdSelect(); if (!ordered) { request.addSort(DOC_FIELD_NAME, ASC); + request.addSort(METADATA_FIELD_ID, SortOrder.ASC); } // Set PIT request.setPointInTime(new PointInTimeBuilder(pit.getPitId())); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java index 7b6228a3d2..70e7118ad5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.join; import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -39,6 +40,7 @@ public static SearchResponse scrollOneTimeWithHits( boolean ordered = originalSelect.isOrderdSelect(); if (!ordered) { scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + scrollRequest.addSort(METADATA_FIELD_ID, SortOrder.ASC); } SearchResponse responseWithHits = scrollRequest.get(); // on ordered select - not using SCAN , elastic returns hits on first scroll diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java index 9877b17a8f..0e9d09d3e7 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.query; import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -110,6 +111,7 @@ public void checkAndSetScroll() { boolean ordered = select.isOrderdSelect(); if (!ordered) { request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); + request.addSort(METADATA_FIELD_ID, SortOrder.ASC); } // Request also requires PointInTime, but we should create pit while execution. } else { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java index 9ddbde2d29..a879a21ee8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java @@ -1,5 +1,7 @@ package org.opensearch.sql.legacy.query.planner.physical.node.pointInTime; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; + import org.opensearch.common.unit.TimeValue; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.sort.FieldSortBuilder; @@ -42,6 +44,7 @@ protected void loadFirstBatch() { request .getRequestBuilder() .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) + .addSort(METADATA_FIELD_ID, SortOrder.ASC) .setSize(pageSize) .setTimeout(TimeValue.timeValueSeconds(timeout)) .setPointInTime(new PointInTimeBuilder(pitId)) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java index 5019e9cde8..9a8deba46a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy.query.planner.physical.node.scroll; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; + import org.opensearch.action.search.ClearScrollResponse; import org.opensearch.common.unit.TimeValue; import org.opensearch.search.sort.FieldSortBuilder; @@ -40,6 +42,7 @@ protected void loadFirstBatch() { request .getRequestBuilder() .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) + .addSort(METADATA_FIELD_ID, SortOrder.ASC) .setSize(pageSize) .setScroll(TimeValue.timeValueSeconds(timeout)) .get(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index fff252f3b4..18ad1809c1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -8,6 +8,7 @@ import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; +import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; import java.io.IOException; import java.util.Collections; @@ -189,6 +190,9 @@ public OpenSearchResponse searchWithPIT(Function<SearchRequest, SearchResponse> // Set sort field for search_after if (this.sourceBuilder.sorts() == null) { this.sourceBuilder.sort(DOC_FIELD_NAME, ASC); + // Workaround to preserve sort location more exactly, + // see https://github.com/opensearch-project/sql/pull/3061 + this.sourceBuilder.sort(METADATA_FIELD_ID, ASC); } SearchRequest searchRequest = new SearchRequest().source(this.sourceBuilder); this.searchResponse = searchAction.apply(searchRequest); From 7666a44b7e07bd673bdd35d94a33f87cd98e3905 Mon Sep 17 00:00:00 2001 From: amityadav005 <amityadav005@ymail.com> Date: Thu, 17 Oct 2024 01:21:35 +0530 Subject: [PATCH 63/96] Updating JobExecutionResponseReader interface to add RequestContext (#3062) * Updating JobExecutionResponseReader interface to add RequestContext Signed-off-by: AMIT YADAV <amitydv@amazon.com> * Updating JobExecutionResponseReader interface to add RequestContext Signed-off-by: AMIT YADAV <amitydv@amazon.com> --------- Signed-off-by: AMIT YADAV <amitydv@amazon.com> Co-authored-by: AMIT YADAV <amitydv@amazon.com> --- .../spark/dispatcher/BatchQueryHandler.java | 4 +-- .../sql/spark/dispatcher/IndexDMLHandler.java | 2 +- .../dispatcher/InteractiveQueryHandler.java | 2 +- .../response/JobExecutionResponseReader.java | 14 +++++--- .../asyncquery/AsyncQueryCoreIntegTest.java | 19 ++++++++-- .../dispatcher/SparkQueryDispatcherTest.java | 35 +++++++++++++++---- .../OpenSearchJobExecutionResponseReader.java | 13 +++++-- .../AsyncQueryGetResultSpecTest.java | 18 +++++++--- ...nSearchJobExecutionResponseReaderTest.java | 28 +++++++++++---- 9 files changed, 105 insertions(+), 30 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 5a775aa243..4396b45898 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -47,8 +47,8 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { // either empty json when the result is not available or data with status // Fetch from Result Index - return jobExecutionResponseReader.getResultWithJobId( - asyncQueryJobMetadata.getJobId(), asyncQueryJobMetadata.getResultIndex()); + return jobExecutionResponseReader.getResultFromResultIndex( + asyncQueryJobMetadata, asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java index fe848593a7..71b20b4311 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java @@ -169,7 +169,7 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( - queryId, asyncQueryJobMetadata.getResultIndex()); + queryId, asyncQueryJobMetadata.getResultIndex(), asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java index 75912f3a7c..1eaad1ca9d 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java @@ -56,7 +56,7 @@ protected JSONObject getResponseFromResultIndex( AsyncQueryRequestContext asyncQueryRequestContext) { String queryId = asyncQueryJobMetadata.getQueryId(); return jobExecutionResponseReader.getResultWithQueryId( - queryId, asyncQueryJobMetadata.getResultIndex()); + queryId, asyncQueryJobMetadata.getResultIndex(), asyncQueryRequestContext); } @Override diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java b/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java index e3184b7326..237ce9c7f6 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java @@ -6,17 +6,22 @@ package org.opensearch.sql.spark.response; import org.json.JSONObject; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; /** Interface for reading job execution result */ public interface JobExecutionResponseReader { /** * Retrieves the job execution result based on the job ID. * - * @param jobId The job ID. - * @param resultLocation The location identifier where the result is stored (optional). + * @param asyncQueryJobMetadata metadata will have jobId and resultLocation and other required + * params. + * @param asyncQueryRequestContext request context passed to AsyncQueryExecutorService * @return A JSONObject containing the result data. */ - JSONObject getResultWithJobId(String jobId, String resultLocation); + JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext); /** * Retrieves the job execution result based on the query ID. @@ -25,5 +30,6 @@ public interface JobExecutionResponseReader { * @param resultLocation The location identifier where the result is stored (optional). * @return A JSONObject containing the result data. */ - JSONObject getResultWithQueryId(String queryId, String resultLocation); + JSONObject getResultWithQueryId( + String queryId, String resultLocation, AsyncQueryRequestContext asyncQueryRequestContext); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 57ad4ecf42..d4a6b544c4 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -452,7 +452,8 @@ public void getResultOfInteractiveQuery() { .sessionId(SESSION_ID) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) + when(jobExecutionResponseReader.getResultWithQueryId( + QUERY_ID, RESULT_INDEX, asyncQueryRequestContext)) .thenReturn(result); AsyncQueryExecutionResponse response = @@ -471,7 +472,8 @@ public void getResultOfIndexDMLQuery() { .jobId(DROP_INDEX_JOB_ID) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithQueryId(QUERY_ID, RESULT_INDEX)) + when(jobExecutionResponseReader.getResultWithQueryId( + QUERY_ID, RESULT_INDEX, asyncQueryRequestContext)) .thenReturn(result); AsyncQueryExecutionResponse response = @@ -491,7 +493,18 @@ public void getResultOfRefreshQuery() { .jobType(JobType.BATCH) .resultIndex(RESULT_INDEX)); JSONObject result = getValidExecutionResponse(); - when(jobExecutionResponseReader.getResultWithJobId(JOB_ID, RESULT_INDEX)).thenReturn(result); + when(jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder() + .applicationId(APPLICATION_ID) + .queryId(QUERY_ID) + .jobId(JOB_ID) + .datasourceName(DATASOURCE_NAME) + .resultIndex(RESULT_INDEX) + .jobType(JobType.BATCH) + .metadata(ImmutableMap.of()) + .build(), + asyncQueryRequestContext)) + .thenReturn(result); AsyncQueryExecutionResponse response = asyncQueryExecutorService.getAsyncQueryResults(QUERY_ID, asyncQueryRequestContext); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 1a38b6977f..343dc3040a 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -765,7 +765,17 @@ void testGetQueryResponse() { when(emrServerlessClient.getJobRunResult(EMRS_APPLICATION_ID, EMR_JOB_ID)) .thenReturn(new GetJobRunResult().withJobRun(new JobRun().withState(JobRunState.PENDING))); // simulate result index is not created yet - when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)) + when(jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder() + .jobId(EMR_JOB_ID) + .queryId(QUERY_ID) + .applicationId(EMRS_APPLICATION_ID) + .jobId(EMR_JOB_ID) + .jobType(JobType.INTERACTIVE) + .datasourceName(MY_GLUE) + .metadata(ImmutableMap.of()) + .build(), + asyncQueryRequestContext)) .thenReturn(new JSONObject()); JSONObject result = @@ -782,7 +792,7 @@ void testGetQueryResponseWithSession() { doReturn(StatementState.WAITING).when(statement).getStatementState(); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); JSONObject result = sparkQueryDispatcher.getQueryResponse( @@ -798,7 +808,7 @@ void testGetQueryResponseWithInvalidSession() { doReturn(Optional.empty()).when(sessionManager).getSession(MOCK_SESSION_ID, MY_GLUE); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); IllegalArgumentException exception = Assertions.assertThrows( @@ -818,7 +828,7 @@ void testGetQueryResponseWithStatementNotExist() { doReturn(Optional.empty()).when(session).get(any(), eq(asyncQueryRequestContext)); doReturn(new JSONObject()) .when(jobExecutionResponseReader) - .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any()); + .getResultWithQueryId(eq(MOCK_STATEMENT_ID), any(), eq(asyncQueryRequestContext)); IllegalArgumentException exception = Assertions.assertThrows( @@ -840,12 +850,25 @@ void testGetQueryResponseWithSuccess() { resultMap.put(STATUS_FIELD, "SUCCESS"); resultMap.put(ERROR_FIELD, ""); queryResult.put(DATA_FIELD, resultMap); - when(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)).thenReturn(queryResult); + AsyncQueryJobMetadata asyncQueryJobMetadata = + AsyncQueryJobMetadata.builder() + .queryId(QUERY_ID) + .applicationId(EMRS_APPLICATION_ID) + .jobId(EMR_JOB_ID) + .jobType(JobType.INTERACTIVE) + .datasourceName(MY_GLUE) + .metadata(ImmutableMap.of()) + .jobId(EMR_JOB_ID) + .build(); + when(jobExecutionResponseReader.getResultFromResultIndex( + asyncQueryJobMetadata, asyncQueryRequestContext)) + .thenReturn(queryResult); JSONObject result = sparkQueryDispatcher.getQueryResponse(asyncQueryJobMetadata(), asyncQueryRequestContext); - verify(jobExecutionResponseReader, times(1)).getResultWithJobId(EMR_JOB_ID, null); + verify(jobExecutionResponseReader, times(1)) + .getResultFromResultIndex(asyncQueryJobMetadata, asyncQueryRequestContext); assertEquals( new HashSet<>(Arrays.asList(DATA_FIELD, STATUS_FIELD, ERROR_FIELD)), result.keySet()); JSONObject dataJson = new JSONObject(); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java b/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java index 10113ece8d..c969a3a6dc 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java @@ -21,6 +21,8 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; /** JobExecutionResponseReader implementation for reading response from OpenSearch index. */ public class OpenSearchJobExecutionResponseReader implements JobExecutionResponseReader { @@ -32,12 +34,17 @@ public OpenSearchJobExecutionResponseReader(Client client) { } @Override - public JSONObject getResultWithJobId(String jobId, String resultLocation) { - return searchInSparkIndex(QueryBuilders.termQuery(JOB_ID_FIELD, jobId), resultLocation); + public JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { + return searchInSparkIndex( + QueryBuilders.termQuery(JOB_ID_FIELD, asyncQueryJobMetadata.getJobId()), + asyncQueryJobMetadata.getResultIndex()); } @Override - public JSONObject getResultWithQueryId(String queryId, String resultLocation) { + public JSONObject getResultWithQueryId( + String queryId, String resultLocation, AsyncQueryRequestContext asyncQueryRequestContext) { return searchInSparkIndex(QueryBuilders.termQuery("queryId", queryId), resultLocation); } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java index 7ccbad969d..ef98e955f6 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java @@ -24,6 +24,7 @@ import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.asyncquery.model.MockFlintSparkJob; import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext; @@ -428,12 +429,21 @@ private class AssertionHelper { */ new JobExecutionResponseReader() { @Override - public JSONObject getResultWithJobId(String jobId, String resultIndex) { - return interaction.interact(new InteractionStep(emrClient, jobId, resultIndex)); + public JSONObject getResultFromResultIndex( + AsyncQueryJobMetadata asyncQueryJobMetadata, + AsyncQueryRequestContext asyncQueryRequestContext) { + return interaction.interact( + new InteractionStep( + emrClient, + asyncQueryJobMetadata.getJobId(), + asyncQueryJobMetadata.getResultIndex())); } @Override - public JSONObject getResultWithQueryId(String queryId, String resultIndex) { + public JSONObject getResultWithQueryId( + String queryId, + String resultIndex, + AsyncQueryRequestContext asyncQueryRequestContext) { return interaction.interact(new InteractionStep(emrClient, queryId, resultIndex)); } }); @@ -501,7 +511,7 @@ private InteractionStep(LocalEMRSClient emrClient, String queryId, String result /** Simulate PPL plugin search query_execution_result */ JSONObject pluginSearchQueryResult() { return new OpenSearchJobExecutionResponseReader(client) - .getResultWithQueryId(queryId, resultIndex); + .getResultWithQueryId(queryId, resultIndex, null); } /** Simulate EMR-S bulk writes query_execution_result with refresh = wait_for */ diff --git a/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java b/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java index 66230464e5..4de3a56dd9 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java @@ -29,6 +29,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; @ExtendWith(MockitoExtension.class) public class OpenSearchJobExecutionResponseReaderTest { @@ -50,7 +51,11 @@ public void testGetResultFromOpensearchIndex() { new SearchHit[] {searchHit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsMap()).thenReturn(Map.of("stepId", EMR_JOB_ID)); - assertFalse(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null).isEmpty()); + assertFalse( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null) + .isEmpty()); } @Test @@ -64,7 +69,11 @@ public void testGetResultFromCustomIndex() { new SearchHit[] {searchHit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsMap()).thenReturn(Map.of("stepId", EMR_JOB_ID)); - assertFalse(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, "foo").isEmpty()); + assertFalse( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).resultIndex("foo").build(), null) + .isEmpty()); } @Test @@ -76,7 +85,9 @@ public void testInvalidSearchResponse() { RuntimeException exception = assertThrows( RuntimeException.class, - () -> jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)); + () -> + jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null)); Assertions.assertEquals( "Fetching result from " @@ -92,13 +103,18 @@ public void testSearchFailure() { assertThrows( RuntimeException.class, - () -> jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, null)); + () -> + jobExecutionResponseReader.getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).build(), null)); } @Test public void testIndexNotFoundException() { when(client.search(any())).thenThrow(IndexNotFoundException.class); - - assertTrue(jobExecutionResponseReader.getResultWithJobId(EMR_JOB_ID, "foo").isEmpty()); + assertTrue( + jobExecutionResponseReader + .getResultFromResultIndex( + AsyncQueryJobMetadata.builder().jobId(EMR_JOB_ID).resultIndex("foo").build(), null) + .isEmpty()); } } From a87061d9a21a59e71365aae18523e47701b24b5e Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Fri, 18 Oct 2024 14:02:20 -0700 Subject: [PATCH 64/96] Optimize scheduler interval conf (#3049) * Optimize scheduler interval conf Signed-off-by: Louis Chu <clingzhi@amazon.com> * Update OpenSearchAsyncQuerySchedulerConfigComposer.java Signed-off-by: Louis Chu <clingzhi@amazon.com> * Update OpenSearchAsyncQuerySchedulerConfigComposer.java Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com> --- .../OpenSearchAsyncQuerySchedulerConfigComposer.java | 9 +++++++-- .../OpenSearchAsyncQuerySchedulerConfigComposerTest.java | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java index f791b050a1..28fd4b1b58 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL; import lombok.RequiredArgsConstructor; +import org.opensearch.core.common.Strings; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; @@ -30,7 +31,11 @@ public void compose( settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL); sparkSubmitParameters.setConfigItem( FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, String.valueOf(externalSchedulerEnabled)); - sparkSubmitParameters.setConfigItem( - FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); + if (!Strings.isNullOrEmpty(externalSchedulerInterval)) { + externalSchedulerInterval = + "\"" + externalSchedulerInterval + "\""; // Wrap the value with double quotes + sparkSubmitParameters.setConfigItem( + FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); + } } } diff --git a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java index 1556d4db3f..19ab091e25 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java @@ -1,5 +1,6 @@ package org.opensearch.sql.spark.config; +import static org.junit.Assert.assertNull; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -40,7 +41,7 @@ public void testCompose() { verify(sparkSubmitParameters) .setConfigItem("spark.flint.job.externalScheduler.enabled", "true"); verify(sparkSubmitParameters) - .setConfigItem("spark.flint.job.externalScheduler.interval", "10 minutes"); + .setConfigItem("spark.flint.job.externalScheduler.interval", "\"10 minutes\""); } @Test @@ -63,6 +64,6 @@ public void testComposeWithMissingInterval() { composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); - verify(sparkSubmitParameters).setConfigItem("spark.flint.job.externalScheduler.interval", ""); + assertNull(sparkSubmitParameters.getConfigItem("spark.flint.job.externalScheduler.interval")); } } From 0e3e57c2d6a83f63e00c6cb6abf94ce1d6963bed Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" <dblock@amazon.com> Date: Mon, 21 Oct 2024 13:32:03 -0400 Subject: [PATCH 65/96] Fix grammar in log warnings. (#2928) Signed-off-by: dblock <dblock@amazon.com> --- .../test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java | 2 +- .../opensearch/executor/protector/ResourceMonitorPlan.java | 4 ++-- .../sql/opensearch/monitor/OpenSearchMemoryHealthy.java | 2 +- .../sql/opensearch/executor/ResourceMonitorPlanTest.java | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java index eed2369590..2799ab1016 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ResourceMonitorIT.java @@ -34,7 +34,7 @@ public void queryExceedResourceLimitShouldFail() throws IOException { assertEquals(500, exception.getResponse().getStatusLine().getStatusCode()); assertThat( exception.getMessage(), - Matchers.containsString("resource is not enough to run the" + " query, quit.")); + Matchers.containsString("insufficient resources to run the query, quit.")); // update plugins.ppl.query.memory_limit to default value 85% updateClusterSettings( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java index e3bc48ba72..150a749358 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/ResourceMonitorPlan.java @@ -44,7 +44,7 @@ public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { @Override public void open() { if (!this.monitor.isHealthy()) { - throw new IllegalStateException("resource is not enough to run the query, quit."); + throw new IllegalStateException("insufficient resources to run the query, quit."); } delegate.open(); } @@ -68,7 +68,7 @@ public boolean hasNext() { public ExprValue next() { boolean shouldCheck = (++nextCallCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); if (shouldCheck && !this.monitor.isHealthy()) { - throw new IllegalStateException("resource is not enough to load next row, quit."); + throw new IllegalStateException("insufficient resources to load next row, quit."); } return delegate.next(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java index 4b7b6c5dcb..bc038cb42f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/monitor/OpenSearchMemoryHealthy.java @@ -36,7 +36,7 @@ public boolean isMemoryHealthy(long limitBytes) { } else { log.warn("Memory usage:{} exceed limit:{}", memoryUsage, limitBytes); if (randomFail.shouldFail()) { - log.warn("Fast failure the current request"); + log.warn("Fast failing the current request"); throw new MemoryUsageExceedFastFailureException(); } else { throw new MemoryUsageExceedException(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java index 26bcdf6d89..82062bee76 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ResourceMonitorPlanTest.java @@ -47,7 +47,7 @@ void openExceedResourceLimit() { IllegalStateException exception = assertThrows(IllegalStateException.class, () -> monitorPlan.open()); - assertEquals("resource is not enough to run the query, quit.", exception.getMessage()); + assertEquals("insufficient resources to run the query, quit.", exception.getMessage()); } @Test @@ -79,7 +79,7 @@ void nextExceedResourceLimit() { IllegalStateException exception = assertThrows(IllegalStateException.class, () -> monitorPlan.next()); - assertEquals("resource is not enough to load next row, quit.", exception.getMessage()); + assertEquals("insufficient resources to load next row, quit.", exception.getMessage()); } @Test From f4def3111020bf5147bffce666e9efbb5738820e Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Mon, 21 Oct 2024 10:39:30 -0700 Subject: [PATCH 66/96] Refactor test workflow to separate bwc tests (#3070) * Refactor test workflow to separate bwc tests Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Try to fix PWD perms Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * More fixes Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix broken cursor test Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Undo indent Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --------- Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .../workflows/sql-test-and-build-workflow.yml | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 6aa674b73f..0d0c96a367 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -53,11 +53,6 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew --continue build" - - name: Run backward compatibility tests - run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./scripts/bwctest.sh" - - name: Create Artifact Path run: | mkdir -p opensearch-sql-builds @@ -159,3 +154,48 @@ jobs: plugin/build/reports/** doctest/build/testclusters/docTestCluster-0/logs/* integ-test/build/testclusters/*/logs/* + + bwc-tests: + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + strategy: + matrix: + java: [21] + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: --user root + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Run backward compatibility tests + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./scripts/bwctest.sh" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* From 07101343cf91249c1f2a01f6fc9f96f0ff777ebe Mon Sep 17 00:00:00 2001 From: Sean Kao <seankao@amazon.com> Date: Mon, 21 Oct 2024 15:43:36 -0700 Subject: [PATCH 67/96] bump commons-io to 2.14.0 (#3083) Signed-off-by: Sean Kao <seankao@amazon.com> --- async-query/build.gradle | 2 +- datasources/build.gradle | 2 +- spark/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/async-query/build.gradle b/async-query/build.gradle index 53fdcbe292..fba74aa216 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation group: 'org.json', name: 'json', version: '20231013' api group: 'com.amazonaws', name: 'aws-java-sdk-emr', version: "${aws_java_sdk_version}" api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' testImplementation(platform("org.junit:junit-bom:5.9.3")) diff --git a/datasources/build.gradle b/datasources/build.gradle index 9456141e80..1d1127ad0d 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' // FIXME. upgrade aws-encryption-sdk-java once the bouncycastle dependency update to 1.78. implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' diff --git a/spark/build.gradle b/spark/build.gradle index d9d5c96413..103c017791 100644 --- a/spark/build.gradle +++ b/spark/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation group: 'org.json', name: 'json', version: '20231013' api group: 'com.amazonaws', name: 'aws-java-sdk-emr', version: "${aws_java_sdk_version}" api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}" - implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.14.0' testImplementation(platform("org.junit:junit-bom:5.9.3")) From 366d165680a0674ee630cc7bad1522166641331b Mon Sep 17 00:00:00 2001 From: Louis Chu <clingzhi@amazon.com> Date: Wed, 23 Oct 2024 06:31:33 -0700 Subject: [PATCH 68/96] Remove scheduler index from SystemIndexDescriptor (#3092) --- plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 560c5edadd..766edc42c0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -342,9 +342,6 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett systemIndexDescriptors.add( new SystemIndexDescriptor( SPARK_REQUEST_BUFFER_INDEX_NAME + "*", "SQL Spark Request Buffer index pattern")); - systemIndexDescriptors.add( - new SystemIndexDescriptor( - OpenSearchAsyncQueryScheduler.SCHEDULER_INDEX_NAME, "SQL Scheduler job index")); return systemIndexDescriptors; } } From 5852ce30d4ee551ff3db22a2b2e042b536f143ad Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Wed, 23 Oct 2024 12:35:56 -0700 Subject: [PATCH 69/96] Improve error handling for some more edge cases (#3080) * Add failing tests Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix the first test Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Revise the tests Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix wildcard tests Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Add license header Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Fix rerunning SQL parsing Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --------- Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .../sql/legacy/MalformedQueryIT.java | 82 +++++++++++++++++++ .../legacy/query/OpenSearchActionFactory.java | 10 ++- .../matchtoterm/TermFieldRewriter.java | 17 ++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java new file mode 100644 index 0000000000..84b60fdabd --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy; + +import java.io.IOException; +import java.util.Locale; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.json.JSONObject; +import org.junit.Assert; +import org.opensearch.client.ResponseException; + +/** Tests for clean handling of various types of invalid queries */ +public class MalformedQueryIT extends SQLIntegTestCase { + @Override + protected void init() throws Exception { + loadIndex(Index.BANK); + loadIndex(Index.BANK_TWO); + } + + public void testJoinWithInvalidCondition() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected Join query with malformed 'ON' to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.firstname, b.age FROM %s AS a INNER JOIN %s AS b %%" + + " a.account_number=b.account_number", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + + Assert.assertEquals("SqlParseException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(400, errMsg.getInt("status")); + } + + public void testWrappedWildcardInSubquery() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected wildcard subquery to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.first_name FROM %s AS a WHERE a.age IN (SELECT age FROM" + + " `opensearch-sql_test_index_*` WHERE age > 30)", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + System.err.println("Full response: " + errMsg); + + Assert.assertEquals("IndexNotFoundException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(404, errMsg.getInt("status")); + } + + public void testUnwrappedWildcardInSubquery() throws IOException, ParseException { + ResponseException result = + assertThrows( + "Expected wildcard subquery to raise error, but didn't", + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "SELECT a.first_name FROM %s AS a WHERE a.age IN (SELECT age FROM * WHERE" + + " age > 30)", + TestsConstants.TEST_INDEX_BANK, + TestsConstants.TEST_INDEX_BANK_TWO))); + var errMsg = new JSONObject(EntityUtils.toString(result.getResponse().getEntity())); + System.err.println("Full response: " + errMsg); + + Assert.assertEquals("IndexNotFoundException", errMsg.getJSONObject("error").getString("type")); + Assert.assertEquals(404, errMsg.getInt("status")); + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java index b9a7c9f218..a5bfeebfbd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/OpenSearchActionFactory.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.domain.IndexStatement.StatementType; import static org.opensearch.sql.legacy.utils.Util.toSqlExpr; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; @@ -86,7 +87,14 @@ public static QueryAction create(Client client, QueryActionRequest request) switch (getFirstWord(sql)) { case "SELECT": - SQLQueryExpr sqlExpr = (SQLQueryExpr) toSqlExpr(sql); + SQLExpr rawExpr = toSqlExpr(sql); + if (!(rawExpr instanceof SQLQueryExpr)) { + throw new SqlParseException( + "Expected a query expression, but found a " + + rawExpr.getClass().getSimpleName() + + ". The query is not runnable."); + } + SQLQueryExpr sqlExpr = (SQLQueryExpr) rawExpr; RewriteRuleExecutor<SQLQueryExpr> ruleExecutor = RewriteRuleExecutor.builder() diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java index f9744ab841..922e3b930e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Stream; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; import org.opensearch.sql.legacy.esdomain.mapping.IndexMappings; @@ -122,7 +123,23 @@ public boolean visit(SQLIdentifierExpr expr) { String fullFieldName = arr[1]; String index = curScope().getAliases().get(alias); + if (index == null) { + throw new IndexNotFoundException( + String.format( + "The requested table '%s' does not correspond to any known index. Only indices or" + + " table aliases are allowed.", + alias.replaceFirst("_\\d+$", ""))); + } + FieldMappings fieldMappings = curScope().getMapper().mapping(index); + if (fieldMappings == null) { + throw new IndexNotFoundException( + String.format( + "The index '%s' could not be found. Note that wildcard indices are not permitted" + + " in SQL.", + index)); + } + if (fieldMappings.has(fullFieldName)) { source = fieldMappings.mapping(fullFieldName); } else { From e109417bb3ce770fb54cfd7b231b8219a22ed46f Mon Sep 17 00:00:00 2001 From: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:21:37 -0700 Subject: [PATCH 70/96] Array values are preserved (#1300) (#3095) Signed-off-by: Norman Jordan <norman.jordan@improving.com> --- .../sql/common/setting/Settings.java | 3 + docs/user/beyond/partiql.rst | 10 +- docs/user/ppl/general/datatypes.rst | 10 +- .../sql/legacy/ObjectFieldSelectIT.java | 16 +- .../org/opensearch/sql/ppl/StandaloneIT.java | 1 + .../java/org/opensearch/sql/sql/NestedIT.java | 2 +- .../sql/sql/StandalonePaginationIT.java | 1 + .../value/OpenSearchExprValueFactory.java | 9 +- .../request/OpenSearchQueryRequest.java | 4 +- .../request/OpenSearchScrollRequest.java | 4 +- .../setting/OpenSearchSettings.java | 18 +- .../opensearch/storage/OpenSearchIndex.java | 7 +- .../storage/scan/OpenSearchIndexScan.java | 2 - .../storage/script/core/ExpressionScript.java | 2 +- .../client/OpenSearchNodeClientTest.java | 1 - .../client/OpenSearchRestClientTest.java | 1 - .../value/OpenSearchExprValueFactoryTest.java | 213 ++++++++++++++++-- .../request/OpenSearchQueryRequestTest.java | 3 +- .../request/OpenSearchRequestBuilderTest.java | 1 + .../response/OpenSearchResponseTest.java | 16 +- .../storage/OpenSearchIndexTest.java | 11 + .../OpenSearchIndexScanPaginationTest.java | 4 +- .../storage/scan/OpenSearchIndexScanTest.java | 4 +- 23 files changed, 282 insertions(+), 61 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 0037032d22..a9fa693a22 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -28,6 +28,9 @@ public enum Key { /** PPL Settings. */ PPL_ENABLED("plugins.ppl.enabled"), + /** Query Settings. */ + FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"), + /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), diff --git a/docs/user/beyond/partiql.rst b/docs/user/beyond/partiql.rst index 6a93a55c94..d8e4b0722b 100644 --- a/docs/user/beyond/partiql.rst +++ b/docs/user/beyond/partiql.rst @@ -202,11 +202,11 @@ Selecting top level for object fields, object fields of array value and nested f os> SELECT city, accounts, projects FROM people; fetched rows / total rows = 1/1 - +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ - | city | accounts | projects | - |-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------| - | {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | - +-----------------------------------------------------+-----------+----------------------------------------------------------------------------------------------------------------+ + +-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+ + | city | accounts | projects | + |-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | [{'id': 1},{'id': 2}] | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] | + +-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+ Example 2: Selecting Deeper Levels ---------------------------------- diff --git a/docs/user/ppl/general/datatypes.rst b/docs/user/ppl/general/datatypes.rst index a205626dbd..ba8281b6a9 100644 --- a/docs/user/ppl/general/datatypes.rst +++ b/docs/user/ppl/general/datatypes.rst @@ -385,8 +385,8 @@ Select deeper level for object fields of array value which returns the first ele os> source = people | fields accounts, accounts.id; fetched rows / total rows = 1/1 - +-----------+-------------+ - | accounts | accounts.id | - |-----------+-------------| - | {'id': 1} | 1 | - +-----------+-------------+ \ No newline at end of file + +-----------------------+-------------+ + | accounts | accounts.id | + |-----------------------+-------------| + | [{'id': 1},{'id': 2}] | 1 | + +-----------------------+-------------+ \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java index 3a2f48d497..aadd79469d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java @@ -14,6 +14,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; /** @@ -79,9 +80,20 @@ public void testSelectNestedFieldItself() { @Test public void testSelectObjectFieldOfArrayValuesItself() { JSONObject response = new JSONObject(query("SELECT accounts FROM %s")); + verifyDataRows(response, rows(new JSONArray("[{\"id\":1},{\"id\":2}]"))); + } - // Only the first element of the list of is returned. - verifyDataRows(response, rows(new JSONObject("{\"id\": 1}"))); + @Test + public void testSelectObjectFieldOfArrayValuesItselfNoFieldTypeTolerance() throws Exception { + updateClusterSettings( + new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "false")); + try { + JSONObject response = new JSONObject(query("SELECT accounts FROM %s")); + verifyDataRows(response, rows(new JSONObject("{\"id\":1}"))); + } finally { + updateClusterSettings( + new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "true")); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 66f85b0754..d484f3c4d0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -153,6 +153,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder<Key, Object>() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 4ae683c229..18d93dbb2a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -427,7 +427,7 @@ public void test_nested_in_where_as_predicate_expression_with_multiple_condition + " nested(message.dayOfWeek) >= 4"; JSONObject result = executeJdbcRequest(query); assertEquals(2, result.getInt("total")); - verifyDataRows(result, rows("c", "ab", 4), rows("zz", "aa", 6)); + verifyDataRows(result, rows("c", "ab", 4), rows("zz", new JSONArray(List.of("aa", "bb")), 6)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index 698e185abb..f6951f4a2c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -167,6 +167,7 @@ private Settings defaultSettings() { .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) + .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 417aaddaee..d9e21436b7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -74,6 +74,9 @@ public class OpenSearchExprValueFactory { /** The Mapping of Field and ExprType. */ private final Map<String, OpenSearchDataType> typeMapping; + /** Whether to support nested value types (such as arrays) */ + private final boolean fieldTypeTolerance; + /** * Extend existing mapping by new data without overwrite. Called from aggregation only {@see * AggregationQueryBuilder#buildTypeMapping}. @@ -140,8 +143,10 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) { .build(); /** Constructor of OpenSearchExprValueFactory. */ - public OpenSearchExprValueFactory(Map<String, OpenSearchDataType> typeMapping) { + public OpenSearchExprValueFactory( + Map<String, OpenSearchDataType> typeMapping, boolean fieldTypeTolerance) { this.typeMapping = OpenSearchDataType.traverseAndFlatten(typeMapping); + this.fieldTypeTolerance = fieldTypeTolerance; } /** @@ -164,7 +169,7 @@ public ExprValue construct(String jsonString, boolean supportArrays) { new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)), TOP_PATH, Optional.of(STRUCT), - supportArrays); + fieldTypeTolerance || supportArrays); } catch (JsonProcessingException e) { throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 18ad1809c1..3461660795 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -151,7 +151,9 @@ public OpenSearchQueryRequest(StreamInput in, OpenSearchStorageEngine engine) th } OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); - exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + exprValueFactory = + new OpenSearchExprValueFactory( + index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance()); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index c9490f0767..d793b53fca 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -178,6 +178,8 @@ public OpenSearchScrollRequest(StreamInput in, OpenSearchStorageEngine engine) includes = in.readStringList(); indexName = new IndexName(in); OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString()); - exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes()); + exprValueFactory = + new OpenSearchExprValueFactory( + index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 1083dbd836..612771eea4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -229,6 +229,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting<?> FIELD_TYPE_TOLERANCE_SETTING = + Setting.boolSetting( + Key.FIELD_TYPE_TOLERANCE.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** Construct OpenSearchSetting. The OpenSearchSetting must be singleton. */ @SuppressWarnings("unchecked") public OpenSearchSettings(ClusterSettings clusterSettings) { @@ -372,13 +379,19 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { clusterSettings, Key.SESSION_INACTIVITY_TIMEOUT_MILLIS, SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING, - new Updater((Key.SESSION_INACTIVITY_TIMEOUT_MILLIS))); + new Updater(Key.SESSION_INACTIVITY_TIMEOUT_MILLIS)); register( settingBuilder, clusterSettings, Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL, STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING, - new Updater((Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL))); + new Updater(Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL)); + register( + settingBuilder, + clusterSettings, + Key.FIELD_TYPE_TOLERANCE, + FIELD_TYPE_TOLERANCE_SETTING, + new Updater(Key.FIELD_TYPE_TOLERANCE)); defaultSettings = settingBuilder.build(); } @@ -455,6 +468,7 @@ public static List<Setting<?>> pluginSettings() { .add(DATASOURCES_LIMIT_SETTING) .add(SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING) .add(STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING) + .add(FIELD_TYPE_TOLERANCE_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index a6fe83c8c4..b8822cd1e8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -177,7 +177,12 @@ private OpenSearchExprValueFactory createExprValueFactory() { Map<String, OpenSearchDataType> allFields = new HashMap<>(); getReservedFieldTypes().forEach((k, v) -> allFields.put(k, OpenSearchDataType.of(v))); allFields.putAll(getFieldOpenSearchTypes()); - return new OpenSearchExprValueFactory(allFields); + return new OpenSearchExprValueFactory( + allFields, settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)); + } + + public boolean isFieldTypeTolerance() { + return settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE); } @VisibleForTesting diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java index b17773cb03..74cbd1f167 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java @@ -47,8 +47,6 @@ public class OpenSearchIndexScan extends TableScanOperator implements Serializab /** Search response for current batch. */ private Iterator<ExprValue> iterator; - private Settings pluginSettings; - /** Creates index scan based on a provided OpenSearchRequestBuilder. */ public OpenSearchIndexScan( OpenSearchClient client, int maxResponseSize, OpenSearchRequest request) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java index 3a9ff02ba0..460a9b4567 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java @@ -102,7 +102,7 @@ private OpenSearchExprValueFactory buildValueFactory(Set<ReferenceExpression> fi Map<String, OpenSearchDataType> typeEnv = fields.stream() .collect(toMap(ReferenceExpression::getAttr, e -> OpenSearchDataType.of(e.type()))); - return new OpenSearchExprValueFactory(typeEnv); + return new OpenSearchExprValueFactory(typeEnv, false); } private Environment<Expression, ExprValue> buildValueEnv( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index ba0fb85422..12a906b25e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -297,7 +297,6 @@ void search() { new SearchHits( new SearchHit[] {searchHit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); - when(searchHit.getInnerHits()).thenReturn(null); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 82d9e74422..eb2355a36b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -286,7 +286,6 @@ void search() throws IOException { new SearchHits( new SearchHit[] {searchHit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); - when(searchHit.getInnerHits()).thenReturn(null); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 6b4d825ab1..5fd40ef6c4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -118,7 +118,10 @@ class OpenSearchExprValueFactoryTest { .build(); private final OpenSearchExprValueFactory exprValueFactory = - new OpenSearchExprValueFactory(MAPPING); + new OpenSearchExprValueFactory(MAPPING, true); + + private final OpenSearchExprValueFactory exprValueFactoryNoArrays = + new OpenSearchExprValueFactory(MAPPING, false); @Test public void constructNullValue() { @@ -464,6 +467,13 @@ public void constructArrayOfStrings() { constructFromObject("arrayV", List.of("zz", "au"))); } + @Test + public void constructArrayOfStringsWithArrays() { + assertEquals( + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))), + constructFromObjectWithArraySupport("arrayV", List.of("zz", "au"))); + } + @Test public void constructNestedArraysOfStrings() { assertEquals( @@ -473,15 +483,23 @@ public void constructNestedArraysOfStrings() { } @Test - public void constructNestedArraysOfStringsReturnsFirstIndex() { + public void constructNestedArraysOfStringsReturnsAll() { assertEquals( - stringValue("zz"), tupleValue("{\"stringV\":[[\"zz\", \"au\"],[\"ss\"]]}").get("stringV")); + new ExprCollectionValue( + List.of( + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))), + new ExprCollectionValue(List.of(stringValue("ss"))))), + tupleValue("{\"stringV\":[[\"zz\", \"au\"],[\"ss\"]]}").get("stringV")); } @Test - public void constructMultiNestedArraysOfStringsReturnsFirstIndex() { + public void constructMultiNestedArraysOfStringsReturnsAll() { assertEquals( - stringValue("z"), + new ExprCollectionValue( + List.of( + stringValue("z"), + new ExprCollectionValue(List.of(stringValue("s"))), + new ExprCollectionValue(List.of(stringValue("zz"), stringValue("au"))))), tupleValue("{\"stringV\":[\"z\",[\"s\"],[\"zz\", \"au\"]]}").get("stringV")); } @@ -577,6 +595,20 @@ public void constructNestedArrayNode() { tupleValueWithArraySupport("{\"nestedV\":[1969,2011]}").get("nestedV")); } + @Test + public void constructNestedArrayNodeNotSupported() { + assertEquals( + Map.of("stringV", stringValue("foo")), + tupleValueWithoutArraySupport("[{\"stringV\":\"foo\"}]")); + } + + @Test + public void constructNestedArrayNodeNotSupportedNoFieldTolerance() { + assertEquals( + Map.of("stringV", stringValue("foo")), + tupleValueWithoutArraySupportNoFieldTolerance("{\"stringV\":\"foo\"}")); + } + @Test public void constructNestedObjectNode() { assertEquals( @@ -600,9 +632,24 @@ public void constructArrayOfGeoPoints() { } @Test - public void constructArrayOfGeoPointsReturnsFirstIndex() { + public void constructArrayOfGeoPointsNoArrays() { assertEquals( new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + tupleValueWithoutArraySupport( + "{\"geoV\":[" + + "{\"lat\":42.60355556,\"lon\":-97.25263889}," + + "{\"lat\":-33.6123556,\"lon\":66.287449}" + + "]}") + .get("geoV")); + } + + @Test + public void constructArrayOfGeoPointsReturnsAll() { + assertEquals( + new ExprCollectionValue( + List.of( + new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + new OpenSearchExprGeoPointValue(-33.6123556, 66.287449))), tupleValue( "{\"geoV\":[" + "{\"lat\":42.60355556,\"lon\":-97.25263889}," @@ -612,46 +659,60 @@ public void constructArrayOfGeoPointsReturnsFirstIndex() { } @Test - public void constructArrayOfIPsReturnsFirstIndex() { + public void constructArrayOfIPsReturnsAll() { assertEquals( - new OpenSearchExprIpValue("192.168.0.1"), + new ExprCollectionValue( + List.of( + new OpenSearchExprIpValue("192.168.0.1"), + new OpenSearchExprIpValue("192.168.0.2"))), tupleValue("{\"ipV\":[\"192.168.0.1\",\"192.168.0.2\"]}").get("ipV")); } @Test - public void constructBinaryArrayReturnsFirstIndex() { + public void constructBinaryArrayReturnsAll() { assertEquals( - new OpenSearchExprBinaryValue("U29tZSBiaWsdfsdfgYmxvYg=="), + new ExprCollectionValue( + List.of( + new OpenSearchExprBinaryValue("U29tZSBiaWsdfsdfgYmxvYg=="), + new OpenSearchExprBinaryValue("U987yuhjjiy8jhk9vY+98jjdf"))), tupleValue("{\"binaryV\":[\"U29tZSBiaWsdfsdfgYmxvYg==\",\"U987yuhjjiy8jhk9vY+98jjdf\"]}") .get("binaryV")); } @Test - public void constructArrayOfCustomEpochMillisReturnsFirstIndex() { + public void constructArrayOfCustomEpochMillisReturnsAll() { assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), + new ExprCollectionValue( + List.of( + new ExprTimestampValue("2015-01-01 12:10:30"), + new ExprTimestampValue("1999-11-09 01:09:44"))), tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01 12:10:30\",\"1999-11-09 01:09:44\"]}") .get("customAndEpochMillisV")); } @Test - public void constructArrayOfDateStringsReturnsFirstIndex() { + public void constructArrayOfDateStringsReturnsAll() { assertEquals( - new ExprDateValue("1984-04-12"), + new ExprCollectionValue( + List.of(new ExprDateValue("1984-04-12"), new ExprDateValue("2033-05-03"))), tupleValue("{\"dateStringV\":[\"1984-04-12\",\"2033-05-03\"]}").get("dateStringV")); } @Test - public void constructArrayOfTimeStringsReturnsFirstIndex() { + public void constructArrayOfTimeStringsReturnsAll() { assertEquals( - new ExprTimeValue("12:10:30"), + new ExprCollectionValue( + List.of(new ExprTimeValue("12:10:30"), new ExprTimeValue("18:33:55"))), tupleValue("{\"timeStringV\":[\"12:10:30.000Z\",\"18:33:55.000Z\"]}").get("timeStringV")); } @Test public void constructArrayOfEpochMillis() { assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + new ExprCollectionValue( + List.of( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + new ExprTimestampValue(Instant.ofEpochMilli(1454251113333L)))), tupleValue("{\"dateOrEpochMillisV\":[\"1420070400001\",\"1454251113333\"]}") .get("dateOrEpochMillisV")); } @@ -763,12 +824,75 @@ public void constructBinary() { } /** - * Return the first element if is OpenSearch Array. + * Return the all elements if is OpenSearch Array. * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. */ @Test - public void constructFromOpenSearchArrayReturnFirstElement() { - assertEquals(integerValue(1), tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + public void constructFromOpenSearchArrayReturnAll() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValue("{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return the all elements if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithArraySupport() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValueWithArraySupport( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return only the first element if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithoutArraySupport() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); assertEquals( new ExprTupleValue( new LinkedHashMap<String, ExprValue>() { @@ -777,7 +901,39 @@ public void constructFromOpenSearchArrayReturnFirstElement() { put("state", stringValue("WA")); } }), - tupleValue("{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + tupleValueWithoutArraySupport( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") + .get("structV")); + } + + /** + * Return only the first element if is OpenSearch Array. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + */ + @Test + public void constructFromOpenSearchArrayReturnAllWithoutArraySupportNoFieldTolerance() { + assertEquals( + new ExprCollectionValue(List.of(integerValue(1), integerValue(2), integerValue(3))), + tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals( + new ExprCollectionValue( + List.of( + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(1)); + put("state", stringValue("WA")); + } + }), + new ExprTupleValue( + new LinkedHashMap<String, ExprValue>() { + { + put("id", integerValue(2)); + put("state", stringValue("CA")); + } + }))), + tupleValueWithoutArraySupportNoFieldTolerance( + "{\"structV\":[{\"id\":1,\"state\":\"WA\"},{\"id\":2,\"state\":\"CA\"}]}}") .get("structV")); } @@ -798,7 +954,7 @@ public void noTypeFoundForMapping() { @Test public void constructUnsupportedTypeThrowException() { OpenSearchExprValueFactory exprValueFactory = - new OpenSearchExprValueFactory(Map.of("type", new TestType())); + new OpenSearchExprValueFactory(Map.of("type", new TestType()), true); IllegalStateException exception = assertThrows( IllegalStateException.class, () -> exprValueFactory.construct("{\"type\":1}", false)); @@ -815,7 +971,8 @@ public void constructUnsupportedTypeThrowException() { // it is accepted without overwriting existing data. public void factoryMappingsAreExtendableWithoutOverWrite() throws NoSuchFieldException, IllegalAccessException { - var factory = new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER))); + var factory = + new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER)), true); factory.extendTypeMapping( Map.of( "value", OpenSearchDataType.of(DOUBLE), @@ -843,6 +1000,16 @@ public Map<String, ExprValue> tupleValueWithArraySupport(String jsonString) { return construct.tupleValue(); } + public Map<String, ExprValue> tupleValueWithoutArraySupport(String jsonString) { + final ExprValue construct = exprValueFactoryNoArrays.construct(jsonString, false); + return construct.tupleValue(); + } + + public Map<String, ExprValue> tupleValueWithoutArraySupportNoFieldTolerance(String jsonString) { + final ExprValue construct = exprValueFactoryNoArrays.construct(jsonString, true); + return construct.tupleValue(); + } + private ExprValue constructFromObject(String fieldName, Object value) { return exprValueFactory.construct(fieldName, value, false); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index 89b51207b5..0847e520cc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchRequest; @@ -72,7 +71,7 @@ public class OpenSearchQueryRequestTest { @Mock private OpenSearchStorageEngine engine; @Mock private PointInTimeBuilder pointInTimeBuilder; - @InjectMocks private OpenSearchQueryRequest serializationRequest; + private OpenSearchQueryRequest serializationRequest; private SearchSourceBuilder sourceBuilderForSerializer; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index bf87840b60..a2430a671d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -86,6 +86,7 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(false); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 6f4605bc2f..217145a052 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -14,10 +14,12 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -163,7 +165,8 @@ void iterator_metafields() { "_sort", new ExprLongValue(123456L), "_score", new ExprFloatValue(3.75F), "_maxscore", new ExprFloatValue(3.75F))); - List includes = List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore"); + List<String> includes = + List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore"); int i = 0; for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { if (i == 0) { @@ -248,20 +251,15 @@ void iterator_metafields_scoreNaN() { @Test void iterator_with_inner_hits() { + Map<String, SearchHits> innerHits = new HashMap<>(); + innerHits.put("a", mock(SearchHits.class)); + when(searchHit1.getInnerHits()).thenReturn(innerHits); when(searchResponse.getHits()) .thenReturn( new SearchHits( new SearchHit[] {searchHit1}, new TotalHits(2L, TotalHits.Relation.EQUAL_TO), 1.0F)); - when(searchHit1.getInnerHits()) - .thenReturn( - Map.of( - "innerHit", - new SearchHits( - new SearchHit[] {searchHit1}, - new TotalHits(2L, TotalHits.Relation.EQUAL_TO), - 1.0F))); when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue1); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index ef6b86c42a..3f8a07f495 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -10,6 +10,7 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; @@ -82,6 +83,7 @@ void setUp() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Test @@ -270,4 +272,13 @@ void implementOtherLogicalOperators() { include), index.implement(plan)); } + + @Test + void isFieldTypeTolerance() { + when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)) + .thenReturn(true) + .thenReturn(false); + assertTrue(index.isFieldTypeTolerance()); + assertFalse(index.isFieldTypeTolerance()); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index e6a17aceaf..6f923cf5c4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -59,6 +59,7 @@ void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Mock private OpenSearchClient client; @@ -67,7 +68,8 @@ void setup() { new OpenSearchExprValueFactory( Map.of( "name", OpenSearchDataType.of(STRING), - "department", OpenSearchDataType.of(STRING))); + "department", OpenSearchDataType.of(STRING)), + true); @Test void query_empty_result() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index e680c6b3a6..da30442bae 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -78,13 +78,15 @@ class OpenSearchIndexScanTest { private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory( Map.of( - "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING))); + "name", OpenSearchDataType.of(STRING), "department", OpenSearchDataType.of(STRING)), + true); @BeforeEach void setup() { lenient() .when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)) .thenReturn(true); + lenient().when(settings.getSettingValue(Settings.Key.FIELD_TYPE_TOLERANCE)).thenReturn(true); } @Test From f00244eccafc50226950afa46dfe44defdfd9142 Mon Sep 17 00:00:00 2001 From: Chen Dai <daichen@amazon.com> Date: Fri, 25 Oct 2024 15:45:57 -0700 Subject: [PATCH 71/96] Add release notes for v2.18.0 (#3125) * Add release notes for 2.18 Signed-off-by: Chen Dai <daichen@amazon.com> * Remove all backport labels Signed-off-by: Chen Dai <daichen@amazon.com> --------- Signed-off-by: Chen Dai <daichen@amazon.com> --- .../opensearch-sql.release-notes-2.18.0.0.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.18.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.18.0.0.md b/release-notes/opensearch-sql.release-notes-2.18.0.0.md new file mode 100644 index 0000000000..1acd0c7d21 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.18.0.0.md @@ -0,0 +1,20 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.18.0 + +### Features + +* Backport #2981 to 2.x ([#3111](https://github.com/opensearch-project/sql/pull/3111)) + +### Bug Fixes + +* Improve error handling for some more edge cases ([#3112](https://github.com/opensearch-project/sql/pull/3112)) +* Resolve Alias Issues in Legacy SQL with Filters ([#3109](https://github.com/opensearch-project/sql/pull/3109)) +* Bug Fixes for minor issues with SQL PIT refactor ([#3108](https://github.com/opensearch-project/sql/pull/3108)) +* Correct regular expression range ([#3107](https://github.com/opensearch-project/sql/pull/3107)) +* SQL pagination should work with the `pretty` parameter ([#3106](https://github.com/opensearch-project/sql/pull/3106)) +* Improve error handling for malformed query cursors ([#3084](https://github.com/opensearch-project/sql/pull/3084)) +* Remove scheduler index from SystemIndexDescriptor ([#3097](https://github.com/opensearch-project/sql/pull/3097)) + +### Maintenance + +* bump commons-io to 2.14.0 ([#3091](https://github.com/opensearch-project/sql/pull/3091)) +* Fix tests on 2.18 ([#3113](https://github.com/opensearch-project/sql/pull/3113)) From 3b8661201efe893cd80da5f8dbd3b21333a8eda9 Mon Sep 17 00:00:00 2001 From: Chase <62891993+engechas@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:34:49 -0700 Subject: [PATCH 72/96] Add validation method for Flint extension queries and wire it into the dispatcher (#3096) * Add validation method for Flint extension queries and wire it into the dispatcher Signed-off-by: Chase Engelbrecht <engechas@amazon.com> * Add unit test Signed-off-by: Chase Engelbrecht <engechas@amazon.com> * Run spotless Signed-off-by: Chase Engelbrecht <engechas@amazon.com> * Fix NPE in test Signed-off-by: Chase Engelbrecht <engechas@amazon.com> * Add java doc Signed-off-by: Chase Engelbrecht <engechas@amazon.com> --------- Signed-off-by: Chase Engelbrecht <engechas@amazon.com> --- .../sql/spark/dispatcher/SparkQueryDispatcher.java | 1 + .../sql/spark/validator/SQLQueryValidator.java | 8 ++++++++ .../sql/spark/dispatcher/SparkQueryDispatcherTest.java | 5 +++++ .../sql/spark/validator/SQLQueryValidatorTest.java | 10 ++++++++++ 4 files changed, 24 insertions(+) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index ff8c8d1fe8..5f88ea9ca0 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -51,6 +51,7 @@ public DispatchQueryResponse dispatch( String query = dispatchQueryRequest.getQuery(); if (SQLQueryUtils.isFlintExtensionQuery(query)) { + sqlQueryValidator.validateFlintExtensionQuery(query, dataSourceMetadata.getConnector()); return handleFlintExtensionQuery( dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java index f387cbad25..5d934411bf 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java @@ -36,4 +36,12 @@ public void validate(String sqlQuery, DataSourceType datasourceType) { throw e; } } + + /** + * Validates a query from the Flint extension grammar. The method is currently a no-op. + * + * @param sqlQuery The Flint extension query to be validated + * @param dataSourceType The type of the datasource the query is being run on + */ + public void validateFlintExtensionQuery(String sqlQuery, DataSourceType dataSourceType) {} } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 343dc3040a..405fdf511d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -641,6 +641,11 @@ void testDispatchVacuumIndexQuery() { @Test void testDispatchRecoverIndexQuery() { + DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); + when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( + MY_GLUE, asyncQueryRequestContext)) + .thenReturn(dataSourceMetadata); + String query = "RECOVER INDEX JOB `flint_spark_catalog_default_test_skipping_index`"; Assertions.assertThrows( IllegalArgumentException.class, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 695a083809..fbc00109da 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -5,11 +5,13 @@ package org.opensearch.sql.spark.validator; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.util.Arrays; +import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Getter; import org.antlr.v4.runtime.CommonTokenStream; @@ -561,6 +563,14 @@ void testSecurityLakeQueries() { v.ng(TestElement.INTEGRATION_WITH_HIVE_UDFS_UDAFS_UDTFS); } + @Test + void testValidateFlintExtensionQuery() { + assertDoesNotThrow( + () -> + sqlQueryValidator.validateFlintExtensionQuery( + UUID.randomUUID().toString(), DataSourceType.SECURITY_LAKE)); + } + @AllArgsConstructor private static class VerifyValidator { private final SQLQueryValidator validator; From 5716cab002085342241023ef11b3a72f5f9e0151 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 30 Oct 2024 15:59:16 -0700 Subject: [PATCH 73/96] Call updateState when query is cancelled (#3139) * Call updateState when query is cancelled Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix code style Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../AsyncQueryExecutorServiceImpl.java | 7 +++++- .../AsyncQueryJobMetadataStorageService.java | 6 +++++ .../AsyncQueryExecutorServiceImplTest.java | 25 ++++++++++--------- ...chAsyncQueryJobMetadataStorageService.java | 9 +++++++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java index 0e9e128896..0639768354 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java @@ -19,6 +19,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; import org.opensearch.sql.spark.dispatcher.SparkQueryDispatcher; @@ -116,7 +117,11 @@ public String cancelQuery(String queryId, AsyncQueryRequestContext asyncQueryReq Optional<AsyncQueryJobMetadata> asyncQueryJobMetadata = asyncQueryJobMetadataStorageService.getJobMetadata(queryId); if (asyncQueryJobMetadata.isPresent()) { - return sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get(), asyncQueryRequestContext); + String result = + sparkQueryDispatcher.cancelJob(asyncQueryJobMetadata.get(), asyncQueryRequestContext); + asyncQueryJobMetadataStorageService.updateState( + asyncQueryJobMetadata.get(), QueryState.CANCELLED, asyncQueryRequestContext); + return result; } throw new AsyncQueryNotFoundException(String.format("QueryId: %s not found", queryId)); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java index b4e94c984d..86e925f58f 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java @@ -10,6 +10,7 @@ import java.util.Optional; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; public interface AsyncQueryJobMetadataStorageService { @@ -17,5 +18,10 @@ void storeJobMetadata( AsyncQueryJobMetadata asyncQueryJobMetadata, AsyncQueryRequestContext asyncQueryRequestContext); + void updateState( + AsyncQueryJobMetadata asyncQueryJobMetadata, + QueryState newState, + AsyncQueryRequestContext asyncQueryRequestContext); + Optional<AsyncQueryJobMetadata> getJobMetadata(String jobId); } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index 73850db83d..3177c335d9 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.spark.asyncquery; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -33,6 +34,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; import org.opensearch.sql.spark.config.SparkSubmitParameterModifier; @@ -109,7 +111,7 @@ void testCreateAsyncQuery() { .getSparkExecutionEngineConfig(asyncQueryRequestContext); verify(sparkQueryDispatcher, times(1)) .dispatch(expectedDispatchQueryRequest, asyncQueryRequestContext); - Assertions.assertEquals(QUERY_ID, createAsyncQueryResponse.getQueryId()); + assertEquals(QUERY_ID, createAsyncQueryResponse.getQueryId()); } @Test @@ -153,8 +155,7 @@ void testGetAsyncQueryResultsWithJobNotFoundException() { AsyncQueryNotFoundException.class, () -> jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext)); - Assertions.assertEquals( - "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); + assertEquals("QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); verifyNoInteractions(sparkQueryDispatcher); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -174,7 +175,7 @@ void testGetAsyncQueryResultsWithInProgressJob() { Assertions.assertNull(asyncQueryExecutionResponse.getResults()); Assertions.assertNull(asyncQueryExecutionResponse.getSchema()); - Assertions.assertEquals("PENDING", asyncQueryExecutionResponse.getStatus()); + assertEquals("PENDING", asyncQueryExecutionResponse.getStatus()); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -191,11 +192,10 @@ void testGetAsyncQueryResultsWithSuccessJob() throws IOException { AsyncQueryExecutionResponse asyncQueryExecutionResponse = jobExecutorService.getAsyncQueryResults(EMR_JOB_ID, asyncQueryRequestContext); - Assertions.assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); - Assertions.assertEquals(1, asyncQueryExecutionResponse.getSchema().getColumns().size()); - Assertions.assertEquals( - "1", asyncQueryExecutionResponse.getSchema().getColumns().get(0).getName()); - Assertions.assertEquals( + assertEquals("SUCCESS", asyncQueryExecutionResponse.getStatus()); + assertEquals(1, asyncQueryExecutionResponse.getSchema().getColumns().size()); + assertEquals("1", asyncQueryExecutionResponse.getSchema().getColumns().get(0).getName()); + assertEquals( 1, ((HashMap<String, String>) asyncQueryExecutionResponse.getResults().get(0).value()) .get("1")); @@ -212,8 +212,7 @@ void testCancelJobWithJobNotFound() { AsyncQueryNotFoundException.class, () -> jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext)); - Assertions.assertEquals( - "QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); + assertEquals("QueryId: " + EMR_JOB_ID + " not found", asyncQueryNotFoundException.getMessage()); verifyNoInteractions(sparkQueryDispatcher); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } @@ -227,7 +226,9 @@ void testCancelJob() { String jobId = jobExecutorService.cancelQuery(EMR_JOB_ID, asyncQueryRequestContext); - Assertions.assertEquals(EMR_JOB_ID, jobId); + assertEquals(EMR_JOB_ID, jobId); + verify(asyncQueryJobMetadataStorageService) + .updateState(any(), eq(QueryState.CANCELLED), eq(asyncQueryRequestContext)); verifyNoInteractions(sparkExecutionEngineConfigSupplier); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java index 4847c8e00f..eb377a5cff 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java @@ -12,6 +12,7 @@ import org.opensearch.sql.spark.asyncquery.exceptions.AsyncQueryNotFoundException; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; +import org.opensearch.sql.spark.asyncquery.model.QueryState; import org.opensearch.sql.spark.execution.statestore.OpenSearchStateStoreUtil; import org.opensearch.sql.spark.execution.statestore.StateStore; import org.opensearch.sql.spark.execution.xcontent.AsyncQueryJobMetadataXContentSerializer; @@ -39,6 +40,14 @@ public void storeJobMetadata( OpenSearchStateStoreUtil.getIndexName(asyncQueryJobMetadata.getDatasourceName())); } + @Override + public void updateState( + AsyncQueryJobMetadata asyncQueryJobMetadata, + QueryState newState, + AsyncQueryRequestContext asyncQueryRequestContext) { + // NoOp since AsyncQueryJobMetadata record does not store state now + } + private String mapIdToDocumentId(String id) { return "qid" + id; } From 9bb4c08acdc5e0412bc5d35c1dc6ee6d219d7245 Mon Sep 17 00:00:00 2001 From: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Date: Thu, 31 Oct 2024 08:28:36 -0700 Subject: [PATCH 74/96] Added documentation for the plugins.query.field_type_tolerance setting (#1300) (#3118) Signed-off-by: Norman Jordan <norman.jordan@improving.com> --- docs/user/admin/settings.rst | 58 +++++++++++++++++++++++++ docs/user/limitations/limitations.rst | 29 +++++++++++++ doctest/test_data/multi_value_long.json | 5 +++ 3 files changed, 92 insertions(+) create mode 100644 doctest/test_data/multi_value_long.json diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 71718d1726..cbcb4f329d 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -824,3 +824,61 @@ To Re-enable Data Sources::: } } +plugins.query.field_type_tolerance +================================== + +Description +----------- + +This setting controls whether preserve arrays. If this setting is set to false, then an array is reduced +to the first non array value of any level of nesting. + +1. The default value is true (preserve arrays) +2. This setting is node scope +3. This setting can be updated dynamically + +Querying a field containing array values will return the full array values:: + + os> SELECT accounts FROM people; + fetched rows / total rows = 1/1 + +-----------------------+ + | accounts | + +-----------------------+ + | [{'id': 1},{'id': 2}] | + +-----------------------+ + +Disable field type tolerance:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.field_type_tolerance" : false + } + }' + +When field type tolerance is disabled, arrays are collapsed to the first non array value:: + + os> SELECT accounts FROM people; + fetched rows / total rows = 1/1 + +-----------+ + | accounts | + +-----------+ + | {'id': 1} | + +-----------+ + +Reenable field type tolerance:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.field_type_tolerance" : true + } + }' + +Limitations: +------------ +OpenSearch does not natively support the ARRAY data type but does allow multi-value fields implicitly. The +SQL/PPL plugin adheres strictly to the data type semantics defined in index mappings. When parsing OpenSearch +responses, it expects data to match the declared type and does not account for data in array format. If the +plugins.query.field_type_tolerance setting is enabled, the SQL/PPL plugin will handle array datasets by returning +scalar data types, allowing basic queries (e.g., SELECT * FROM tbl WHERE condition). However, using multi-value +fields in expressions or functions will result in exceptions. If this setting is disabled or absent, only the +first element of an array is returned, preserving the default behavior. \ No newline at end of file diff --git a/docs/user/limitations/limitations.rst b/docs/user/limitations/limitations.rst index 8ce75a0e25..22ad3c2a17 100644 --- a/docs/user/limitations/limitations.rst +++ b/docs/user/limitations/limitations.rst @@ -101,3 +101,32 @@ The response in JDBC format with cursor id:: } The query with `aggregation` and `join` does not support pagination for now. + +Limitations on Using Multi-valued Fields +======================================== + +OpenSearch does not natively support the ARRAY data type but does allow multi-value fields implicitly. The +SQL/PPL plugin adheres strictly to the data type semantics defined in index mappings. When parsing OpenSearch +responses, it expects data to match the declared type and does not account for data in array format. If the +plugins.query.field_type_tolerance setting is enabled, the SQL/PPL plugin will handle array datasets by returning +scalar data types, allowing basic queries (e.g., SELECT * FROM tbl WHERE condition). However, using multi-value +fields in expressions or functions will result in exceptions. If this setting is disabled or absent, only the +first element of an array is returned, preserving the default behavior. + +For example, the following query tries to calculate the absolute value of a field that contains arrays of +longs:: + + POST _plugins/_sql/ + { + "query": "SELECT id, ABS(long_array) FROM multi_value_long" + } +The response in JSON format is:: + + { + "error": { + "reason": "Invalid SQL query", + "details": "invalid to get longValue from value of type ARRAY", + "type": "ExpressionEvaluationException" + }, + "status": 400 + } diff --git a/doctest/test_data/multi_value_long.json b/doctest/test_data/multi_value_long.json new file mode 100644 index 0000000000..3c139630f6 --- /dev/null +++ b/doctest/test_data/multi_value_long.json @@ -0,0 +1,5 @@ +{"id": 1, "long_array": [1, 2]} +{"id": 2, "long_array": [3, 4]} +{"id": 3, "long_array": [1, 5]} +{"id": 4, "long_array": [1, 2]} +{"id": 5, "long_array": [2, 3]} \ No newline at end of file From 4ff1fe3cb48a17ff17b0cf134131283fc0282205 Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Fri, 1 Nov 2024 08:17:48 +0800 Subject: [PATCH 75/96] Fix a regression issue of parsing datetime string with custom time format in Span (#3079) --- .../planner/physical/collector/Rounding.java | 6 +-- .../physical/collector/RoundingTest.java | 54 +++++++++++++++++++ .../sql/ppl/DateTimeImplementationIT.java | 37 +++++++++++++ .../value/OpenSearchExprValueFactory.java | 2 +- 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 82c8af52cd..7645213c67 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -46,13 +46,13 @@ public static Rounding<?> createRounding(SpanExpression span) { if (DOUBLE.isCompatible(type)) { return new DoubleRounding(interval); } - if (type.equals(TIMESTAMP)) { + if (type.equals(TIMESTAMP) || type.typeName().equalsIgnoreCase(TIMESTAMP.typeName())) { return new TimestampRounding(interval, span.getUnit().getName()); } - if (type.equals(DATE)) { + if (type.equals(DATE) || type.typeName().equalsIgnoreCase(DATE.typeName())) { return new DateRounding(interval, span.getUnit().getName()); } - if (type.equals(TIME)) { + if (type.equals(TIME) || type.typeName().equalsIgnoreCase(TIME.typeName())) { return new TimeRounding(interval, span.getUnit().getName()); } return new UnknownRounding(); diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java index 3a2601a874..4f6d51c901 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java @@ -5,14 +5,18 @@ package org.opensearch.sql.planner.physical.collector; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.span.SpanExpression; @@ -26,6 +30,35 @@ void time_rounding_illegal_span() { ExpressionEvaluationException.class, () -> rounding.round(new ExprTimeValue("23:30:00"))); } + @Test + void datetime_rounding_span() { + SpanExpression dateSpan = DSL.span(DSL.ref("date", DATE), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = DSL.span(DSL.ref("time", TIME), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = DSL.span(DSL.ref("timestamp", TIMESTAMP), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + } + + @Test + void datetime_rounding_non_core_type_span() { + SpanExpression dateSpan = + DSL.span(DSL.ref("date", new MockDateExprType()), DSL.literal(1), "d"); + Rounding rounding = Rounding.createRounding(dateSpan); + assertInstanceOf(Rounding.DateRounding.class, rounding); + SpanExpression timeSpan = + DSL.span(DSL.ref("time", new MockTimeExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timeSpan); + assertInstanceOf(Rounding.TimeRounding.class, rounding); + SpanExpression timestampSpan = + DSL.span(DSL.ref("timestamp", new MockTimestampExprType()), DSL.literal(1), "h"); + rounding = Rounding.createRounding(timestampSpan); + assertInstanceOf(Rounding.TimestampRounding.class, rounding); + } + @Test void round_unknown_type() { SpanExpression span = DSL.span(DSL.ref("unknown", STRING), DSL.literal(1), ""); @@ -41,4 +74,25 @@ void resolve() { () -> Rounding.DateTimeUnit.resolve(illegalUnit), "Unable to resolve unit " + illegalUnit); } + + static class MockDateExprType implements ExprType { + @Override + public String typeName() { + return "DATE"; + } + } + + static class MockTimeExprType implements ExprType { + @Override + public String typeName() { + return "TIME"; + } + } + + static class MockTimestampExprType implements ExprType { + @Override + public String typeName() { + return "TIMESTAMP"; + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java index f9dc7d8027..e777a4f454 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeImplementationIT.java @@ -6,8 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySome; @@ -20,6 +22,7 @@ public class DateTimeImplementationIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.DATE); + loadIndex(Index.DATE_FORMATS); } @Test @@ -176,4 +179,38 @@ public void nullDateTimeInvalidDateValueMonth() throws IOException { verifySchema(result, schema("f", null, "timestamp")); verifySome(result.getJSONArray("datarows"), rows(new Object[] {null})); } + + @Test + public void testSpanDatetimeWithCustomFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "date")); + verifyDataRows(result, rows(2, "1984-04-12")); + } + + @Test + public void testSpanDatetimeWithEpochMillisFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(epoch_millis, 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } + + @Test + public void testSpanDatetimeWithDisjunctiveDifferentFormats() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval a = 1 | stats count() as cnt by span(yyyy-MM-dd_OR_epoch_millis," + + " 1d) as span", + TEST_INDEX_DATE_FORMATS)); + verifySchema(result, schema("cnt", null, "integer"), schema("span", null, "timestamp")); + verifyDataRows(result, rows(2, "1984-04-12 00:00:00")); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index d9e21436b7..1ce504a4c5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -300,7 +300,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) } } else { // custom format - return parseDateTimeString(value.stringValue(), dt); + return parseDateTimeString(value.objectValue().toString(), dt); } } if (value.isString()) { From cfe38d7d6678dfbd5597f1098a1a47e29f84a708 Mon Sep 17 00:00:00 2001 From: Mike Swierczek <441523+Michael-S@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:03:54 -0500 Subject: [PATCH 76/96] Fix: CSV and Raw output, escape quotes (#3063) Fixes #3050 Signed-off-by: Mike Swierczek <441523+Michael-S@users.noreply.github.com> --- .../protocol/response/format/FlatResponseBase.java | 6 +++++- .../response/format/CsvResponseFormatterTest.java | 7 +++++-- .../response/format/RawResponseFormatterTest.java | 14 +++++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java index 98e79a4048..be2517bf6c 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java @@ -84,6 +84,10 @@ protected List<List<String>> formatData(List<List<String>> lines) { protected String quoteIfRequired(String separator, String cell) { final String quote = "\""; - return cell.contains(separator) ? quote + cell.replaceAll("\"", "\"\"") + quote : cell; + if (cell != null && (cell.contains(separator) || cell.contains(quote))) { + return quote + cell.replaceAll(quote, quote + quote) + quote; + } else { + return cell; + } } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 554d5ae8bb..ef2f2e8da8 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -108,8 +108,11 @@ void quoteIfRequired() { QueryResult response = new QueryResult( schema, - Arrays.asList(tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")))); - String expected = "\"na,me\",\",,age\"%n\"John,Smith\",\"30,,,\""; + Arrays.asList( + tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")), + tupleValue(ImmutableMap.of("na,me", "\"Janice Jones", ",,age", "26\"")))); + String expected = + "\"na,me\",\",,age\"%n\"John,Smith\",\"30,,,\"%n\"\"\"Janice Jones\",\"26\"\"\""; assertEquals(format(expected), formatter.format(response)); } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index fd057437a0..ebdadcd50b 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -120,10 +120,18 @@ void quoteIfRequired() { QueryResult response = new QueryResult( schema, - Arrays.asList(tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); - String expected = "\"na|me\"|\"||age\"%n" + "\"John|Smith\"|\"30|||\""; + Arrays.asList( + tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")), + tupleValue(ImmutableMap.of("na|me", "Ja\"ne J\"ones", "||age", "\"40\"")))); + String expected = + "\"na|me\"|\"||age\"%n" + + "\"John|Smith\"|\"30|||\"%n" + + "\"Ja\"\"ne J\"\"ones\"|\"\"\"40\"\"\""; assertEquals(format(expected), getRawFormatter().format(response)); - String expectedPretty = "\"na|me\" |\"||age\"%n" + "\"John|Smith\"|\"30|||\""; + String expectedPretty = + "\"na|me\" |\"||age\" %n" + + "\"John|Smith\" |\"30|||\" %n" + + "\"Ja\"\"ne J\"\"ones\"|\"\"\"40\"\"\""; assertEquals(format(expectedPretty), getRawFormatterPretty().format(response)); } From 95a16436a43ae474ca06aa1011081e3f3fc62ee9 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 14 Nov 2024 10:01:44 -0800 Subject: [PATCH 77/96] Fix coverage issue for #3063 (#3155) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../sql/protocol/response/format/FlatResponseBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java index be2517bf6c..ee6e2051d2 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java @@ -84,7 +84,7 @@ protected List<List<String>> formatData(List<List<String>> lines) { protected String quoteIfRequired(String separator, String cell) { final String quote = "\""; - if (cell != null && (cell.contains(separator) || cell.contains(quote))) { + if (cell.contains(separator) || cell.contains(quote)) { return quote + cell.replaceAll(quote, quote + quote) + quote; } else { return cell; From 5b3cdd8ce132a4d9e6aa72559d8762021a802357 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Thu, 14 Nov 2024 11:58:45 -0800 Subject: [PATCH 78/96] Call LeaseManager for BatchQuery (#3153) * Call LeaseManager for BatchQuery Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Reformat code Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix unit test for coverage Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Reformat Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../spark/dispatcher/BatchQueryHandler.java | 11 +++++ .../spark/dispatcher/RefreshQueryHandler.java | 6 ++- .../asyncquery/AsyncQueryCoreIntegTest.java | 40 +++++++++++++++-- .../execution/statestore/StateStore.java | 3 +- .../leasemanager/DefaultLeaseManager.java | 3 +- .../leasemanager/DefaultLeaseManagerTest.java | 43 +++++++++++++------ 6 files changed, 87 insertions(+), 19 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java index 4396b45898..dbd6411998 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java @@ -25,6 +25,7 @@ import org.opensearch.sql.spark.dispatcher.model.DispatchQueryResponse; import org.opensearch.sql.spark.dispatcher.model.JobType; import org.opensearch.sql.spark.leasemanager.LeaseManager; +import org.opensearch.sql.spark.leasemanager.model.LeaseRequest; import org.opensearch.sql.spark.metrics.MetricsService; import org.opensearch.sql.spark.parameter.SparkSubmitParametersBuilderProvider; import org.opensearch.sql.spark.response.JobExecutionResponseReader; @@ -75,6 +76,14 @@ public String cancelJob( return asyncQueryJobMetadata.getQueryId(); } + /** + * This method allows RefreshQueryHandler to override the job type when calling + * leaseManager.borrow. + */ + protected void borrow(String datasource) { + leaseManager.borrow(new LeaseRequest(JobType.BATCH, datasource)); + } + @Override public DispatchQueryResponse submit( DispatchQueryRequest dispatchQueryRequest, DispatchQueryContext context) { @@ -82,6 +91,8 @@ public DispatchQueryResponse submit( Map<String, String> tags = context.getTags(); DataSourceMetadata dataSourceMetadata = context.getDataSourceMetadata(); + this.borrow(dispatchQueryRequest.getDatasource()); + tags.put(JOB_TYPE_TAG_KEY, JobType.BATCH.getText()); StartJobRequest startJobRequest = new StartJobRequest( diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java index 52cd863081..659166e8a6 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java @@ -71,10 +71,14 @@ public String cancelJob( return asyncQueryJobMetadata.getQueryId(); } + @Override + protected void borrow(String datasource) { + leaseManager.borrow(new LeaseRequest(JobType.REFRESH, datasource)); + } + @Override public DispatchQueryResponse submit( DispatchQueryRequest dispatchQueryRequest, DispatchQueryContext context) { - leaseManager.borrow(new LeaseRequest(JobType.REFRESH, dispatchQueryRequest.getDatasource())); DispatchQueryResponse resp = super.submit(dispatchQueryRequest, context); DataSourceMetadata dataSourceMetadata = context.getDataSourceMetadata(); diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index d4a6b544c4..5ef8343dcc 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -77,6 +77,7 @@ import org.opensearch.sql.spark.flint.IndexDMLResultStorageService; import org.opensearch.sql.spark.flint.operation.FlintIndexOpFactory; import org.opensearch.sql.spark.leasemanager.LeaseManager; +import org.opensearch.sql.spark.leasemanager.model.LeaseRequest; import org.opensearch.sql.spark.metrics.MetricsService; import org.opensearch.sql.spark.parameter.SparkParameterComposerCollection; import org.opensearch.sql.spark.parameter.SparkSubmitParametersBuilderProvider; @@ -137,6 +138,7 @@ public class AsyncQueryCoreIntegTest { @Captor ArgumentCaptor<FlintIndexOptions> flintIndexOptionsArgumentCaptor; @Captor ArgumentCaptor<StartJobRunRequest> startJobRunRequestArgumentCaptor; @Captor ArgumentCaptor<CreateSessionRequest> createSessionRequestArgumentCaptor; + @Captor ArgumentCaptor<LeaseRequest> leaseRequestArgumentCaptor; AsyncQueryExecutorService asyncQueryExecutorService; @@ -267,7 +269,8 @@ public void createVacuumIndexQuery() { assertEquals(SESSION_ID, response.getSessionId()); verifyGetQueryIdCalled(); verifyGetSessionIdCalled(); - verify(leaseManager).borrow(any()); + verify(leaseManager).borrow(leaseRequestArgumentCaptor.capture()); + assertEquals(JobType.INTERACTIVE, leaseRequestArgumentCaptor.getValue().getJobType()); verifyStartJobRunCalled(); verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.INTERACTIVE); } @@ -356,11 +359,38 @@ public void createStreamingQuery() { assertEquals(QUERY_ID, response.getQueryId()); assertNull(response.getSessionId()); verifyGetQueryIdCalled(); - verify(leaseManager).borrow(any()); + verify(leaseManager).borrow(leaseRequestArgumentCaptor.capture()); + assertEquals(JobType.STREAMING, leaseRequestArgumentCaptor.getValue().getJobType()); verifyStartJobRunCalled(); verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.STREAMING); } + @Test + public void createBatchQuery() { + givenSparkExecutionEngineConfigIsSupplied(); + givenValidDataSourceMetadataExist(); + when(queryIdProvider.getQueryId(any(), eq(asyncQueryRequestContext))).thenReturn(QUERY_ID); + when(awsemrServerless.startJobRun(any())) + .thenReturn(new StartJobRunResult().withApplicationId(APPLICATION_ID).withJobRunId(JOB_ID)); + + CreateAsyncQueryResponse response = + asyncQueryExecutorService.createAsyncQuery( + new CreateAsyncQueryRequest( + "CREATE INDEX index_name ON table_name(l_orderkey, l_quantity)" + + " WITH (auto_refresh = false)", + DATASOURCE_NAME, + LangType.SQL), + asyncQueryRequestContext); + + assertEquals(QUERY_ID, response.getQueryId()); + assertNull(response.getSessionId()); + verifyGetQueryIdCalled(); + verify(leaseManager).borrow(leaseRequestArgumentCaptor.capture()); + assertEquals(JobType.BATCH, leaseRequestArgumentCaptor.getValue().getJobType()); + verifyStartJobRunCalled(); + verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.BATCH); + } + private void verifyStartJobRunCalled() { verify(awsemrServerless).startJobRun(startJobRunRequestArgumentCaptor.capture()); StartJobRunRequest startJobRunRequest = startJobRunRequestArgumentCaptor.getValue(); @@ -413,7 +443,8 @@ public void createRefreshQuery() { assertEquals(QUERY_ID, response.getQueryId()); assertNull(response.getSessionId()); verifyGetQueryIdCalled(); - verify(leaseManager).borrow(any()); + verify(leaseManager).borrow(leaseRequestArgumentCaptor.capture()); + assertEquals(JobType.REFRESH, leaseRequestArgumentCaptor.getValue().getJobType()); verifyStartJobRunCalled(); verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.REFRESH); } @@ -439,7 +470,8 @@ public void createInteractiveQuery() { assertEquals(SESSION_ID, response.getSessionId()); verifyGetQueryIdCalled(); verifyGetSessionIdCalled(); - verify(leaseManager).borrow(any()); + verify(leaseManager).borrow(leaseRequestArgumentCaptor.capture()); + assertEquals(JobType.INTERACTIVE, leaseRequestArgumentCaptor.getValue().getJobType()); verifyStartJobRunCalled(); verifyStoreJobMetadataCalled(JOB_ID, QueryState.WAITING, JobType.INTERACTIVE); } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java index 8d57198277..18dce7b7b2 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java @@ -237,7 +237,8 @@ private void createIndex(String indexName) { } } - private long count(String indexName, QueryBuilder query) { + @VisibleForTesting + public long count(String indexName, QueryBuilder query) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(query); searchSourceBuilder.size(0); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java b/async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java index 375fa7b11e..db8ca1ad2b 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java @@ -92,7 +92,8 @@ public String description() { @Override public boolean test(LeaseRequest leaseRequest) { - if (leaseRequest.getJobType() == JobType.INTERACTIVE) { + if (leaseRequest.getJobType() != JobType.REFRESH + && leaseRequest.getJobType() != JobType.STREAMING) { return true; } return activeRefreshJobCount(stateStore, ALL_DATASOURCE).get() < refreshJobLimit(); diff --git a/async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java index 558f7f7b3a..a7ea6aa22f 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java @@ -5,7 +5,9 @@ package org.opensearch.sql.spark.leasemanager; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -23,19 +25,36 @@ class DefaultLeaseManagerTest { @Mock private StateStore stateStore; @Test - public void concurrentSessionRuleOnlyApplyToInteractiveQuery() { - assertTrue( - new DefaultLeaseManager.ConcurrentSessionRule(settings, stateStore) - .test(new LeaseRequest(JobType.BATCH, "mys3"))); - assertTrue( - new DefaultLeaseManager.ConcurrentSessionRule(settings, stateStore) - .test(new LeaseRequest(JobType.STREAMING, "mys3"))); + public void leaseManagerRejectsJobs() { + when(stateStore.count(any(), any())).thenReturn(3L); + when(settings.getSettingValue(any())).thenReturn(3); + DefaultLeaseManager defaultLeaseManager = new DefaultLeaseManager(settings, stateStore); + + defaultLeaseManager.borrow(getLeaseRequest(JobType.BATCH)); + assertThrows( + ConcurrencyLimitExceededException.class, + () -> defaultLeaseManager.borrow(getLeaseRequest(JobType.INTERACTIVE))); + assertThrows( + ConcurrencyLimitExceededException.class, + () -> defaultLeaseManager.borrow(getLeaseRequest(JobType.STREAMING))); + assertThrows( + ConcurrencyLimitExceededException.class, + () -> defaultLeaseManager.borrow(getLeaseRequest(JobType.REFRESH))); } @Test - public void concurrentRefreshRuleOnlyNotAppliedToInteractiveQuery() { - assertTrue( - new DefaultLeaseManager.ConcurrentRefreshJobRule(settings, stateStore) - .test(new LeaseRequest(JobType.INTERACTIVE, "mys3"))); + public void leaseManagerAcceptsJobs() { + when(stateStore.count(any(), any())).thenReturn(2L); + when(settings.getSettingValue(any())).thenReturn(3); + DefaultLeaseManager defaultLeaseManager = new DefaultLeaseManager(settings, stateStore); + + defaultLeaseManager.borrow(getLeaseRequest(JobType.BATCH)); + defaultLeaseManager.borrow(getLeaseRequest(JobType.INTERACTIVE)); + defaultLeaseManager.borrow(getLeaseRequest(JobType.STREAMING)); + defaultLeaseManager.borrow(getLeaseRequest(JobType.REFRESH)); + } + + private LeaseRequest getLeaseRequest(JobType jobType) { + return new LeaseRequest(jobType, "mys3"); } } From 380dd34b3c43f31b9d8eb1eed1e2541fedc80c31 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 20 Nov 2024 10:30:32 -0800 Subject: [PATCH 79/96] Make GrammarElement public (#3161) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../java/org/opensearch/sql/spark/validator/GrammarElement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java index 217640bada..0dc0e8944b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java @@ -8,7 +8,7 @@ import lombok.AllArgsConstructor; @AllArgsConstructor -enum GrammarElement { +public enum GrammarElement { ALTER_NAMESPACE("ALTER (DATABASE|TABLE|NAMESPACE)"), ALTER_VIEW("ALTER VIEW"), CREATE_NAMESPACE("CREATE (DATABASE|TABLE|NAMESPACE)"), From 8b2d01e2adb073ee5df923c4c3e787e6318b54d2 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Wed, 20 Nov 2024 16:25:01 -0800 Subject: [PATCH 80/96] Update grammar validation settings (#3165) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../org/opensearch/sql/spark/validator/GrammarElement.java | 2 ++ .../sql/spark/validator/S3GlueGrammarElementValidator.java | 2 ++ .../sql/spark/validator/SQLQueryValidationVisitor.java | 6 ++++++ .../sql/spark/validator/SQLQueryValidatorTest.java | 6 +++--- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java index 0dc0e8944b..ab89348f33 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java @@ -74,7 +74,9 @@ public enum GrammarElement { // Functions MAP_FUNCTIONS("Map functions"), + BITWISE_FUNCTIONS("Bitwise functions"), CSV_FUNCTIONS("CSV functions"), + GENERATOR_FUNCTIONS("Generator functions"), MISC_FUNCTIONS("Misc functions"), // UDF diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java index 9ed1fd9e9e..668ab26d68 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -6,6 +6,7 @@ package org.opensearch.sql.spark.validator; import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.BITWISE_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; @@ -71,6 +72,7 @@ public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidat SET, SHOW_FUNCTIONS, SHOW_VIEWS, + BITWISE_FUNCTIONS, MISC_FUNCTIONS, UDF) .build(); diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index d50503418e..2312c0de7a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -561,12 +561,18 @@ private void validateFunctionAllowed(String function) { case MAP: validateAllowed(GrammarElement.MAP_FUNCTIONS); break; + case BITWISE: + validateAllowed(GrammarElement.BITWISE_FUNCTIONS); + break; case CSV: validateAllowed(GrammarElement.CSV_FUNCTIONS); break; case MISC: validateAllowed(GrammarElement.MISC_FUNCTIONS); break; + case GENERATOR: + validateAllowed(GrammarElement.GENERATOR_FUNCTIONS); + break; case UDF: validateAllowed(GrammarElement.UDF); break; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index fbc00109da..520fc96ba8 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -309,7 +309,7 @@ void testDenyAllValidator() { v.ok(TestElement.JSON_FUNCTIONS); v.ok(TestElement.MATHEMATICAL_FUNCTIONS); v.ok(TestElement.STRING_FUNCTIONS); - v.ok(TestElement.BITWISE_FUNCTIONS); + v.ng(TestElement.BITWISE_FUNCTIONS); v.ok(TestElement.CONVERSION_FUNCTIONS); v.ok(TestElement.CONDITIONAL_FUNCTIONS); v.ok(TestElement.PREDICATE_FUNCTIONS); @@ -321,7 +321,7 @@ void testDenyAllValidator() { v.ok(TestElement.WINDOW_FUNCTIONS); // Generator Functions - v.ok(TestElement.GENERATOR_FUNCTIONS); + v.ng(TestElement.GENERATOR_FUNCTIONS); // UDFs v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS); @@ -426,7 +426,7 @@ void testS3glueQueries() { v.ok(TestElement.JSON_FUNCTIONS); v.ok(TestElement.MATHEMATICAL_FUNCTIONS); v.ok(TestElement.STRING_FUNCTIONS); - v.ok(TestElement.BITWISE_FUNCTIONS); + v.ng(TestElement.BITWISE_FUNCTIONS); v.ok(TestElement.CONVERSION_FUNCTIONS); v.ok(TestElement.CONDITIONAL_FUNCTIONS); v.ok(TestElement.PREDICATE_FUNCTIONS); From 40b5b7949e230eef3a531a6fcea202d754cc5633 Mon Sep 17 00:00:00 2001 From: Andy Kwok <andy.kwok@improving.com> Date: Tue, 26 Nov 2024 14:57:55 -0800 Subject: [PATCH 81/96] Doc fixs (#3163) Signed-off-by: Andy Kwok <andy.kwok@improving.com> --- docs/dev/intro-architecture.md | 2 +- docs/user/general/datatypes.rst | 2 +- docs/user/general/identifiers.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/intro-architecture.md b/docs/dev/intro-architecture.md index 88b7065864..34fdeec958 100644 --- a/docs/dev/intro-architecture.md +++ b/docs/dev/intro-architecture.md @@ -12,7 +12,7 @@ In the high level, the OD-SQL Engine could be divided into four major sub-module * *Parser*: Currently, there are two Lex&Parser coexists. The Druid Lex&Parser is the original one from NLPChina. The input AST of Core Engine is from the Druid Lex&Parser. The [ANTLR](https://github.com/opensearch-project/sql/blob/main/legacy/src/main/antlr/OpenSearchLegacySqlLexer.g4) Lex&Parser is added by us to customized the verification and exception handling. * *Analyzer*: The analyzer module take the output from ANTLR Lex&Parser then perform syntax and semantic analyze. -* *Core Engine*: The QueryAction take the output from Druid Lex&Parser and translate to the OpenSearch DSL if possible. This is an NLPChina original module. The QueryPlanner Builder is added by us to support the JOIN and Post-processing logic. The QueryPlanner will take the take the output from Druid Lex&Parser and build the PhysicalPlan +* *Core Engine*: The QueryAction take the output from Druid Lex&Parser and translate to the OpenSearch DSL if possible. This is an NLPChina original module. The QueryPlanner Builder is added by us to support the JOIN and Post-processing logic. The QueryPlanner will take the output from Druid Lex&Parser and build the PhysicalPlan * *Execution*: The execution module execute QueryAction or QueryPlanner and return the response to the client. Different from the Frontend, Analyzer and Core Engine which running on the Transport Thread and can’t do any blocking operation. The Execution module running on the client threadpool and can perform the blocking operation. There are also others modules include in the OD-SQL engine. diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index f18b5b716a..3e115b249e 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -115,7 +115,7 @@ A data type can be converted to another, implicitly or explicitly or impossibly, The general rules and design tenets for data type conversion include: -1. Implicit conversion is defined by type precedence which is represented by the type hierarchy tree. See `Data Type Conversion in SQL/PPL </docs/dev/TypeConversion.md>`_ for more details. +1. Implicit conversion is defined by type precedence which is represented by the type hierarchy tree. See `Data Type Conversion in SQL/PPL </docs/dev/query-type-conversion.md>`_ for more details. 2. Explicit conversion defines the complete set of conversion allowed. If no explicit conversion defined, implicit conversion should be impossible too. 3. On the other hand, if implicit conversion can occur between 2 types, then explicit conversion should be allowed too. 4. Conversion within a data type family is considered as conversion between different data representation and should be supported as much as possible. diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index 562bf38526..033525f99f 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -241,7 +241,7 @@ tableName = ``logs.12.13.1``. 3. ``my_prometheus.http_requests_total`` -datasourceName = ```my_prometheus`` [Is in the list of datasources configured]. +datasourceName = ``my_prometheus`` [Is in the list of datasources configured]. schemaName = ``default`` [No supported schema found, so default to `default`]. From 871d9f2a927d6ca05e31e865988a9036ace85cee Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Tue, 26 Nov 2024 16:28:38 -0800 Subject: [PATCH 82/96] Add grammar validation for PPL (#3167) * Add grammar validation for PPL Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Remove specific PPL grammar validator Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- async-query-core/build.gradle | 4 +- .../src/main/antlr/OpenSearchPPLLexer.g4 | 496 +++++++ .../src/main/antlr/OpenSearchPPLParser.g4 | 1174 +++++++++++++++++ .../dispatcher/SparkQueryDispatcher.java | 7 +- .../sql/spark/validator/GrammarElement.java | 85 +- .../spark/validator/PPLGrammarElement.java | 30 + .../validator/PPLQueryValidationVisitor.java | 87 ++ .../spark/validator/PPLQueryValidator.java | 50 + .../S3GlueGrammarElementValidator.java | 83 -- .../S3GlueSQLGrammarElementValidator.java | 83 ++ .../spark/validator/SQLGrammarElement.java | 91 ++ .../validator/SQLQueryValidationVisitor.java | 172 +-- .../SecurityLakeGrammarElementValidator.java | 123 -- ...ecurityLakeSQLGrammarElementValidator.java | 123 ++ .../asyncquery/AsyncQueryCoreIntegTest.java | 12 +- .../dispatcher/SparkQueryDispatcherTest.java | 19 +- .../GrammarElementValidatorProviderTest.java | 15 +- .../validator/PPLQueryValidatorTest.java | 202 +++ .../validator/SQLQueryValidatorTest.java | 4 +- .../config/AsyncExecutorServiceModule.java | 27 +- .../AsyncQueryExecutorServiceSpec.java | 12 +- 21 files changed, 2494 insertions(+), 405 deletions(-) create mode 100644 async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 create mode 100644 async-query-core/src/main/antlr/OpenSearchPPLParser.g4 create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLGrammarElement.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidationVisitor.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidator.java delete mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueSQLGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLGrammarElement.java delete mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java create mode 100644 async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeSQLGrammarElementValidator.java create mode 100644 async-query-core/src/test/java/org/opensearch/sql/spark/validator/PPLQueryValidatorTest.java diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 330b418681..37bf6748c9 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -21,12 +21,14 @@ tasks.register('downloadG4Files', Exec) { executable 'curl' - def opensearchSparkBranch = "0.5" + def opensearchSparkBranch = "0.6" def apacheSparkVersionTag = "v3.5.1" args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4" args '-o', 'src/main/antlr/SparkSqlBase.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4" args '-o', 'src/main/antlr/SqlBaseParser.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4" args '-o', 'src/main/antlr/SqlBaseLexer.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4" + args '-o', 'src/main/antlr/OpenSearchPPLParser.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4" + args '-o', 'src/main/antlr/OpenSearchPPLLexer.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4" } generateGrammarSource { diff --git a/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 new file mode 100644 index 0000000000..cb323f7942 --- /dev/null +++ b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 @@ -0,0 +1,496 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +lexer grammar OpenSearchPPLLexer; + +channels { WHITESPACE, ERRORCHANNEL } + + +// COMMAND KEYWORDS +SEARCH: 'SEARCH'; +DESCRIBE: 'DESCRIBE'; +SHOW: 'SHOW'; +FROM: 'FROM'; +WHERE: 'WHERE'; +FIELDS: 'FIELDS'; +RENAME: 'RENAME'; +STATS: 'STATS'; +EVENTSTATS: 'EVENTSTATS'; +DEDUP: 'DEDUP'; +SORT: 'SORT'; +EVAL: 'EVAL'; +HEAD: 'HEAD'; +TOP: 'TOP'; +RARE: 'RARE'; +PARSE: 'PARSE'; +METHOD: 'METHOD'; +REGEX: 'REGEX'; +PUNCT: 'PUNCT'; +GROK: 'GROK'; +PATTERN: 'PATTERN'; +PATTERNS: 'PATTERNS'; +NEW_FIELD: 'NEW_FIELD'; +KMEANS: 'KMEANS'; +AD: 'AD'; +ML: 'ML'; +FILLNULL: 'FILLNULL'; +EXPAND: 'EXPAND'; +FLATTEN: 'FLATTEN'; +TRENDLINE: 'TRENDLINE'; + +//Native JOIN KEYWORDS +JOIN: 'JOIN'; +ON: 'ON'; +INNER: 'INNER'; +OUTER: 'OUTER'; +FULL: 'FULL'; +SEMI: 'SEMI'; +ANTI: 'ANTI'; +CROSS: 'CROSS'; +LEFT_HINT: 'HINT.LEFT'; +RIGHT_HINT: 'HINT.RIGHT'; + +//CORRELATION KEYWORDS +CORRELATE: 'CORRELATE'; +SELF: 'SELF'; +EXACT: 'EXACT'; +APPROXIMATE: 'APPROXIMATE'; +SCOPE: 'SCOPE'; +MAPPING: 'MAPPING'; + +//EXPLAIN KEYWORDS +EXPLAIN: 'EXPLAIN'; +FORMATTED: 'FORMATTED'; +COST: 'COST'; +CODEGEN: 'CODEGEN'; +EXTENDED: 'EXTENDED'; +SIMPLE: 'SIMPLE'; + +// COMMAND ASSIST KEYWORDS +AS: 'AS'; +BY: 'BY'; +SOURCE: 'SOURCE'; +INDEX: 'INDEX'; +D: 'D'; +DESC: 'DESC'; +DATASOURCES: 'DATASOURCES'; +USING: 'USING'; +WITH: 'WITH'; + +// FIELD KEYWORDS +AUTO: 'AUTO'; +STR: 'STR'; +IP: 'IP'; +NUM: 'NUM'; + + +// FIELDSUMMARY keywords +FIELDSUMMARY: 'FIELDSUMMARY'; +INCLUDEFIELDS: 'INCLUDEFIELDS'; +NULLS: 'NULLS'; + +//TRENDLINE KEYWORDS +SMA: 'SMA'; +WMA: 'WMA'; + +// ARGUMENT KEYWORDS +KEEPEMPTY: 'KEEPEMPTY'; +CONSECUTIVE: 'CONSECUTIVE'; +DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; +PARTITIONS: 'PARTITIONS'; +ALLNUM: 'ALLNUM'; +DELIM: 'DELIM'; +CENTROIDS: 'CENTROIDS'; +ITERATIONS: 'ITERATIONS'; +DISTANCE_TYPE: 'DISTANCE_TYPE'; +NUMBER_OF_TREES: 'NUMBER_OF_TREES'; +SHINGLE_SIZE: 'SHINGLE_SIZE'; +SAMPLE_SIZE: 'SAMPLE_SIZE'; +OUTPUT_AFTER: 'OUTPUT_AFTER'; +TIME_DECAY: 'TIME_DECAY'; +ANOMALY_RATE: 'ANOMALY_RATE'; +CATEGORY_FIELD: 'CATEGORY_FIELD'; +TIME_FIELD: 'TIME_FIELD'; +TIME_ZONE: 'TIME_ZONE'; +TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; +ANOMALY_SCORE_THRESHOLD: 'ANOMALY_SCORE_THRESHOLD'; +APPEND: 'APPEND'; + +// COMPARISON FUNCTION KEYWORDS +CASE: 'CASE'; +ELSE: 'ELSE'; +IN: 'IN'; +EXISTS: 'EXISTS'; + +// LOGICAL KEYWORDS +NOT: 'NOT'; +OR: 'OR'; +AND: 'AND'; +XOR: 'XOR'; +TRUE: 'TRUE'; +FALSE: 'FALSE'; +REGEXP: 'REGEXP'; + +// DATETIME, INTERVAL AND UNIT KEYWORDS +CONVERT_TZ: 'CONVERT_TZ'; +DATETIME: 'DATETIME'; +DAY: 'DAY'; +DAY_HOUR: 'DAY_HOUR'; +DAY_MICROSECOND: 'DAY_MICROSECOND'; +DAY_MINUTE: 'DAY_MINUTE'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; +DAY_SECOND: 'DAY_SECOND'; +HOUR: 'HOUR'; +HOUR_MICROSECOND: 'HOUR_MICROSECOND'; +HOUR_MINUTE: 'HOUR_MINUTE'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; +HOUR_SECOND: 'HOUR_SECOND'; +INTERVAL: 'INTERVAL'; +MICROSECOND: 'MICROSECOND'; +MILLISECOND: 'MILLISECOND'; +MINUTE: 'MINUTE'; +MINUTE_MICROSECOND: 'MINUTE_MICROSECOND'; +MINUTE_OF_DAY: 'MINUTE_OF_DAY'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MINUTE_SECOND: 'MINUTE_SECOND'; +MONTH: 'MONTH'; +MONTH_OF_YEAR: 'MONTH_OF_YEAR'; +QUARTER: 'QUARTER'; +SECOND: 'SECOND'; +SECOND_MICROSECOND: 'SECOND_MICROSECOND'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; +WEEK: 'WEEK'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +YEAR: 'YEAR'; +YEAR_MONTH: 'YEAR_MONTH'; + +// DATASET TYPES +DATAMODEL: 'DATAMODEL'; +LOOKUP: 'LOOKUP'; +SAVEDSEARCH: 'SAVEDSEARCH'; + +// CONVERTED DATA TYPES +INT: 'INT'; +INTEGER: 'INTEGER'; +DOUBLE: 'DOUBLE'; +LONG: 'LONG'; +FLOAT: 'FLOAT'; +STRING: 'STRING'; +BOOLEAN: 'BOOLEAN'; + +// SPECIAL CHARACTERS AND OPERATORS +PIPE: '|'; +COMMA: ','; +DOT: '.'; +EQUAL: '='; +GREATER: '>'; +LESS: '<'; +NOT_GREATER: '<' '='; +NOT_LESS: '>' '='; +NOT_EQUAL: '!' '='; +PLUS: '+'; +MINUS: '-'; +STAR: '*'; +DIVIDE: '/'; +MODULE: '%'; +EXCLAMATION_SYMBOL: '!'; +COLON: ':'; +LT_PRTHS: '('; +RT_PRTHS: ')'; +LT_SQR_PRTHS: '['; +RT_SQR_PRTHS: ']'; +SINGLE_QUOTE: '\''; +DOUBLE_QUOTE: '"'; +BACKTICK: '`'; +ARROW: '->'; + +// Operators. Bit + +BIT_NOT_OP: '~'; +BIT_AND_OP: '&'; +BIT_XOR_OP: '^'; + +// AGGREGATIONS +AVG: 'AVG'; +COUNT: 'COUNT'; +DISTINCT_COUNT: 'DISTINCT_COUNT'; +ESTDC: 'ESTDC'; +ESTDC_ERROR: 'ESTDC_ERROR'; +MAX: 'MAX'; +MEAN: 'MEAN'; +MEDIAN: 'MEDIAN'; +MIN: 'MIN'; +MODE: 'MODE'; +RANGE: 'RANGE'; +STDEV: 'STDEV'; +STDEVP: 'STDEVP'; +SUM: 'SUM'; +SUMSQ: 'SUMSQ'; +VAR_SAMP: 'VAR_SAMP'; +VAR_POP: 'VAR_POP'; +STDDEV_SAMP: 'STDDEV_SAMP'; +STDDEV_POP: 'STDDEV_POP'; +PERCENTILE: 'PERCENTILE'; +PERCENTILE_APPROX: 'PERCENTILE_APPROX'; +TAKE: 'TAKE'; +FIRST: 'FIRST'; +LAST: 'LAST'; +LIST: 'LIST'; +VALUES: 'VALUES'; +EARLIEST: 'EARLIEST'; +EARLIEST_TIME: 'EARLIEST_TIME'; +LATEST: 'LATEST'; +LATEST_TIME: 'LATEST_TIME'; +PER_DAY: 'PER_DAY'; +PER_HOUR: 'PER_HOUR'; +PER_MINUTE: 'PER_MINUTE'; +PER_SECOND: 'PER_SECOND'; +RATE: 'RATE'; +SPARKLINE: 'SPARKLINE'; +C: 'C'; +DC: 'DC'; + +// BASIC FUNCTIONS +ABS: 'ABS'; +CBRT: 'CBRT'; +CEIL: 'CEIL'; +CEILING: 'CEILING'; +CONV: 'CONV'; +CRC32: 'CRC32'; +E: 'E'; +EXP: 'EXP'; +FLOOR: 'FLOOR'; +LN: 'LN'; +LOG: 'LOG'; +LOG10: 'LOG10'; +LOG2: 'LOG2'; +MOD: 'MOD'; +PI: 'PI'; +POSITION: 'POSITION'; +POW: 'POW'; +POWER: 'POWER'; +RAND: 'RAND'; +ROUND: 'ROUND'; +SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; +SQRT: 'SQRT'; +TRUNCATE: 'TRUNCATE'; + +// TRIGONOMETRIC FUNCTIONS +ACOS: 'ACOS'; +ASIN: 'ASIN'; +ATAN: 'ATAN'; +ATAN2: 'ATAN2'; +COS: 'COS'; +COT: 'COT'; +DEGREES: 'DEGREES'; +RADIANS: 'RADIANS'; +SIN: 'SIN'; +TAN: 'TAN'; + +// CRYPTOGRAPHIC FUNCTIONS +MD5: 'MD5'; +SHA1: 'SHA1'; +SHA2: 'SHA2'; + +// DATE AND TIME FUNCTIONS +ADDDATE: 'ADDDATE'; +ADDTIME: 'ADDTIME'; +CURDATE: 'CURDATE'; +CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_TIME: 'CURRENT_TIME'; +CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_TIMEZONE: 'CURRENT_TIMEZONE'; +CURTIME: 'CURTIME'; +DATE: 'DATE'; +DATEDIFF: 'DATEDIFF'; +DATE_ADD: 'DATE_ADD'; +DATE_FORMAT: 'DATE_FORMAT'; +DATE_SUB: 'DATE_SUB'; +DAYNAME: 'DAYNAME'; +DAYOFMONTH: 'DAYOFMONTH'; +DAYOFWEEK: 'DAYOFWEEK'; +DAYOFYEAR: 'DAYOFYEAR'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; +DURATION: 'DURATION'; +EXTRACT: 'EXTRACT'; +FROM_DAYS: 'FROM_DAYS'; +FROM_UNIXTIME: 'FROM_UNIXTIME'; +GET_FORMAT: 'GET_FORMAT'; +LAST_DAY: 'LAST_DAY'; +LOCALTIME: 'LOCALTIME'; +LOCALTIMESTAMP: 'LOCALTIMESTAMP'; +MAKEDATE: 'MAKEDATE'; +MAKE_DATE: 'MAKE_DATE'; +MAKETIME: 'MAKETIME'; +MONTHNAME: 'MONTHNAME'; +NOW: 'NOW'; +PERIOD_ADD: 'PERIOD_ADD'; +PERIOD_DIFF: 'PERIOD_DIFF'; +SEC_TO_TIME: 'SEC_TO_TIME'; +STR_TO_DATE: 'STR_TO_DATE'; +SUBDATE: 'SUBDATE'; +SUBTIME: 'SUBTIME'; +SYSDATE: 'SYSDATE'; +TIME: 'TIME'; +TIMEDIFF: 'TIMEDIFF'; +TIMESTAMP: 'TIMESTAMP'; +TIMESTAMPADD: 'TIMESTAMPADD'; +TIMESTAMPDIFF: 'TIMESTAMPDIFF'; +TIME_FORMAT: 'TIME_FORMAT'; +TIME_TO_SEC: 'TIME_TO_SEC'; +TO_DAYS: 'TO_DAYS'; +TO_SECONDS: 'TO_SECONDS'; +UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; +UTC_DATE: 'UTC_DATE'; +UTC_TIME: 'UTC_TIME'; +UTC_TIMESTAMP: 'UTC_TIMESTAMP'; +WEEKDAY: 'WEEKDAY'; +YEARWEEK: 'YEARWEEK'; + +// TEXT FUNCTIONS +SUBSTR: 'SUBSTR'; +SUBSTRING: 'SUBSTRING'; +LTRIM: 'LTRIM'; +RTRIM: 'RTRIM'; +TRIM: 'TRIM'; +TO: 'TO'; +LOWER: 'LOWER'; +UPPER: 'UPPER'; +CONCAT: 'CONCAT'; +CONCAT_WS: 'CONCAT_WS'; +LENGTH: 'LENGTH'; +STRCMP: 'STRCMP'; +RIGHT: 'RIGHT'; +LEFT: 'LEFT'; +ASCII: 'ASCII'; +LOCATE: 'LOCATE'; +REPLACE: 'REPLACE'; +REVERSE: 'REVERSE'; +CAST: 'CAST'; +ISEMPTY: 'ISEMPTY'; +ISBLANK: 'ISBLANK'; + +// JSON TEXT FUNCTIONS +JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; +JSON_ARRAY: 'JSON_ARRAY'; +JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; +TO_JSON_STRING: 'TO_JSON_STRING'; +JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_KEYS: 'JSON_KEYS'; +JSON_VALID: 'JSON_VALID'; +//JSON_APPEND: 'JSON_APPEND'; +//JSON_DELETE: 'JSON_DELETE'; +//JSON_EXTEND: 'JSON_EXTEND'; +//JSON_SET: 'JSON_SET'; +//JSON_ARRAY_ALL_MATCH: 'JSON_ARRAY_ALL_MATCH'; +//JSON_ARRAY_ANY_MATCH: 'JSON_ARRAY_ANY_MATCH'; +//JSON_ARRAY_FILTER: 'JSON_ARRAY_FILTER'; +//JSON_ARRAY_MAP: 'JSON_ARRAY_MAP'; +//JSON_ARRAY_REDUCE: 'JSON_ARRAY_REDUCE'; + +// COLLECTION FUNCTIONS +ARRAY: 'ARRAY'; +ARRAY_LENGTH: 'ARRAY_LENGTH'; + +// LAMBDA FUNCTIONS +//EXISTS: 'EXISTS'; +FORALL: 'FORALL'; +FILTER: 'FILTER'; +TRANSFORM: 'TRANSFORM'; +REDUCE: 'REDUCE'; + +// BOOL FUNCTIONS +LIKE: 'LIKE'; +ISNULL: 'ISNULL'; +ISNOTNULL: 'ISNOTNULL'; +ISPRESENT: 'ISPRESENT'; +BETWEEN: 'BETWEEN'; +CIDRMATCH: 'CIDRMATCH'; + +// FLOWCONTROL FUNCTIONS +IFNULL: 'IFNULL'; +NULLIF: 'NULLIF'; +IF: 'IF'; +TYPEOF: 'TYPEOF'; + +//OTHER CONDITIONAL EXPRESSIONS +COALESCE: 'COALESCE'; + +// RELEVANCE FUNCTIONS AND PARAMETERS +MATCH: 'MATCH'; +MATCH_PHRASE: 'MATCH_PHRASE'; +MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX'; +MATCH_BOOL_PREFIX: 'MATCH_BOOL_PREFIX'; +SIMPLE_QUERY_STRING: 'SIMPLE_QUERY_STRING'; +MULTI_MATCH: 'MULTI_MATCH'; +QUERY_STRING: 'QUERY_STRING'; + +ALLOW_LEADING_WILDCARD: 'ALLOW_LEADING_WILDCARD'; +ANALYZE_WILDCARD: 'ANALYZE_WILDCARD'; +ANALYZER: 'ANALYZER'; +AUTO_GENERATE_SYNONYMS_PHRASE_QUERY:'AUTO_GENERATE_SYNONYMS_PHRASE_QUERY'; +BOOST: 'BOOST'; +CUTOFF_FREQUENCY: 'CUTOFF_FREQUENCY'; +DEFAULT_FIELD: 'DEFAULT_FIELD'; +DEFAULT_OPERATOR: 'DEFAULT_OPERATOR'; +ENABLE_POSITION_INCREMENTS: 'ENABLE_POSITION_INCREMENTS'; +ESCAPE: 'ESCAPE'; +FLAGS: 'FLAGS'; +FUZZY_MAX_EXPANSIONS: 'FUZZY_MAX_EXPANSIONS'; +FUZZY_PREFIX_LENGTH: 'FUZZY_PREFIX_LENGTH'; +FUZZY_TRANSPOSITIONS: 'FUZZY_TRANSPOSITIONS'; +FUZZY_REWRITE: 'FUZZY_REWRITE'; +FUZZINESS: 'FUZZINESS'; +LENIENT: 'LENIENT'; +LOW_FREQ_OPERATOR: 'LOW_FREQ_OPERATOR'; +MAX_DETERMINIZED_STATES: 'MAX_DETERMINIZED_STATES'; +MAX_EXPANSIONS: 'MAX_EXPANSIONS'; +MINIMUM_SHOULD_MATCH: 'MINIMUM_SHOULD_MATCH'; +OPERATOR: 'OPERATOR'; +PHRASE_SLOP: 'PHRASE_SLOP'; +PREFIX_LENGTH: 'PREFIX_LENGTH'; +QUOTE_ANALYZER: 'QUOTE_ANALYZER'; +QUOTE_FIELD_SUFFIX: 'QUOTE_FIELD_SUFFIX'; +REWRITE: 'REWRITE'; +SLOP: 'SLOP'; +TIE_BREAKER: 'TIE_BREAKER'; +TYPE: 'TYPE'; +ZERO_TERMS_QUERY: 'ZERO_TERMS_QUERY'; + +// SPAN KEYWORDS +SPAN: 'SPAN'; +MS: 'MS'; +S: 'S'; +M: 'M'; +H: 'H'; +W: 'W'; +Q: 'Q'; +Y: 'Y'; + + +// LITERALS AND VALUES +//STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; +ID: ID_LITERAL; +CLUSTER: CLUSTER_PREFIX_LITERAL; +INTEGER_LITERAL: DEC_DIGIT+; +DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; + +fragment DATE_SUFFIX: ([\-.][*0-9]+)+; +fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; +fragment CLUSTER_PREFIX_LITERAL: [*A-Z]+?[*A-Z_\-0-9]* COLON; +ID_DATE_SUFFIX: CLUSTER_PREFIX_LITERAL? ID_LITERAL DATE_SUFFIX; +DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; +SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; +BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; +fragment DEC_DIGIT: [0-9]; + +LINE_COMMENT: '//' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN); +BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); + +ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); diff --git a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 new file mode 100644 index 0000000000..133cf64be5 --- /dev/null +++ b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 @@ -0,0 +1,1174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +parser grammar OpenSearchPPLParser; + + +options { tokenVocab = OpenSearchPPLLexer; } +root + : pplStatement? EOF + ; + +// statement +pplStatement + : dmlStatement + ; + +dmlStatement + : (explainCommand PIPE)? queryStatement + ; + +queryStatement + : pplCommands (PIPE commands)* + ; + +subSearch + : searchCommand (PIPE commands)* + ; + +// commands +pplCommands + : searchCommand + | describeCommand + ; + +commands + : whereCommand + | correlateCommand + | joinCommand + | fieldsCommand + | statsCommand + | dedupCommand + | sortCommand + | headCommand + | topCommand + | rareCommand + | evalCommand + | grokCommand + | parseCommand + | patternsCommand + | lookupCommand + | renameCommand + | fillnullCommand + | fieldsummaryCommand + | flattenCommand + | expandCommand + | trendlineCommand + ; + +commandName + : SEARCH + | DESCRIBE + | SHOW + | AD + | ML + | KMEANS + | WHERE + | CORRELATE + | JOIN + | FIELDS + | STATS + | EVENTSTATS + | DEDUP + | EXPLAIN + | SORT + | HEAD + | TOP + | RARE + | EVAL + | GROK + | PARSE + | PATTERNS + | LOOKUP + | RENAME + | EXPAND + | FILLNULL + | FIELDSUMMARY + | FLATTEN + | TRENDLINE + ; + +searchCommand + : (SEARCH)? fromClause # searchFrom + | (SEARCH)? fromClause logicalExpression # searchFromFilter + | (SEARCH)? logicalExpression fromClause # searchFilterFrom + ; + +fieldsummaryCommand + : FIELDSUMMARY (fieldsummaryParameter)* + ; + +fieldsummaryParameter + : INCLUDEFIELDS EQUAL fieldList # fieldsummaryIncludeFields + | NULLS EQUAL booleanLiteral # fieldsummaryNulls + ; + +describeCommand + : DESCRIBE tableSourceClause + ; + +explainCommand + : EXPLAIN explainMode + ; + +explainMode + : FORMATTED + | COST + | CODEGEN + | EXTENDED + | SIMPLE + ; + +showDataSourcesCommand + : SHOW DATASOURCES + ; + +whereCommand + : WHERE logicalExpression + ; + +correlateCommand + : CORRELATE correlationType FIELDS LT_PRTHS fieldList RT_PRTHS (scopeClause)? mappingList + ; + +correlationType + : SELF + | EXACT + | APPROXIMATE + ; + +scopeClause + : SCOPE LT_PRTHS fieldExpression COMMA value = literalValue (unit = timespanUnit)? RT_PRTHS + ; + +mappingList + : MAPPING LT_PRTHS ( mappingClause (COMMA mappingClause)* ) RT_PRTHS + ; + +mappingClause + : left = qualifiedName comparisonOperator right = qualifiedName # mappingCompareExpr + ; + +fieldsCommand + : FIELDS (PLUS | MINUS)? fieldList + ; + +renameCommand + : RENAME renameClasue (COMMA renameClasue)* + ; + +statsCommand + : (STATS | EVENTSTATS) (PARTITIONS EQUAL partitions = integerLiteral)? (ALLNUM EQUAL allnum = booleanLiteral)? (DELIM EQUAL delim = stringLiteral)? statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral)? + ; + +dedupCommand + : DEDUP (number = integerLiteral)? fieldList (KEEPEMPTY EQUAL keepempty = booleanLiteral)? (CONSECUTIVE EQUAL consecutive = booleanLiteral)? + ; + +sortCommand + : SORT sortbyClause + ; + +evalCommand + : EVAL evalClause (COMMA evalClause)* + ; + +headCommand + : HEAD (number = integerLiteral)? (FROM from = integerLiteral)? + ; + +topCommand + : TOP (number = integerLiteral)? fieldList (byClause)? + ; + +rareCommand + : RARE fieldList (byClause)? + ; + +grokCommand + : GROK (source_field = expression) (pattern = stringLiteral) + ; + +parseCommand + : PARSE (source_field = expression) (pattern = stringLiteral) + ; + +patternsCommand + : PATTERNS (patternsParameter)* (source_field = expression) + ; + +patternsParameter + : (NEW_FIELD EQUAL new_field = stringLiteral) + | (PATTERN EQUAL pattern = stringLiteral) + ; + +patternsMethod + : PUNCT + | REGEX + ; + +// lookup +lookupCommand + : LOOKUP tableSource lookupMappingList ((APPEND | REPLACE) outputCandidateList)? + ; + +lookupMappingList + : lookupPair (COMMA lookupPair)* + ; + +outputCandidateList + : lookupPair (COMMA lookupPair)* + ; + + // The lookup pair will generate a K-V pair. + // The format is Key -> Alias(outputFieldName, inputField), Value -> outputField. For example: + // 1. When lookupPair is "name AS cName", the key will be Alias(cName, Field(name)), the value will be Field(cName) + // 2. When lookupPair is "dept", the key is Alias(dept, Field(dept)), value is Field(dept) +lookupPair + : inputField = fieldExpression (AS outputField = fieldExpression)? + ; + +fillnullCommand + : FILLNULL (fillNullWithTheSameValue + | fillNullWithFieldVariousValues) + ; + + fillNullWithTheSameValue + : WITH nullReplacement IN nullableField (COMMA nullableField)* + ; + + fillNullWithFieldVariousValues + : USING nullableField EQUAL nullReplacement (COMMA nullableField EQUAL nullReplacement)* + ; + + + nullableField + : fieldExpression + ; + + nullReplacement + : expression + ; + +expandCommand + : EXPAND fieldExpression (AS alias = qualifiedName)? + ; + +flattenCommand + : FLATTEN fieldExpression (AS alias = identifierSeq)? + ; + +trendlineCommand + : TRENDLINE (SORT sortField)? trendlineClause (trendlineClause)* + ; + +trendlineClause + : trendlineType LT_PRTHS numberOfDataPoints = INTEGER_LITERAL COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? + ; + +trendlineType + : SMA + | WMA + ; + +kmeansCommand + : KMEANS (kmeansParameter)* + ; + +kmeansParameter + : (CENTROIDS EQUAL centroids = integerLiteral) + | (ITERATIONS EQUAL iterations = integerLiteral) + | (DISTANCE_TYPE EQUAL distance_type = stringLiteral) + ; + +adCommand + : AD (adParameter)* + ; + +adParameter + : (NUMBER_OF_TREES EQUAL number_of_trees = integerLiteral) + | (SHINGLE_SIZE EQUAL shingle_size = integerLiteral) + | (SAMPLE_SIZE EQUAL sample_size = integerLiteral) + | (OUTPUT_AFTER EQUAL output_after = integerLiteral) + | (TIME_DECAY EQUAL time_decay = decimalLiteral) + | (ANOMALY_RATE EQUAL anomaly_rate = decimalLiteral) + | (CATEGORY_FIELD EQUAL category_field = stringLiteral) + | (TIME_FIELD EQUAL time_field = stringLiteral) + | (DATE_FORMAT EQUAL date_format = stringLiteral) + | (TIME_ZONE EQUAL time_zone = stringLiteral) + | (TRAINING_DATA_SIZE EQUAL training_data_size = integerLiteral) + | (ANOMALY_SCORE_THRESHOLD EQUAL anomaly_score_threshold = decimalLiteral) + ; + +mlCommand + : ML (mlArg)* + ; + +mlArg + : (argName = ident EQUAL argValue = literalValue) + ; + +// clauses +fromClause + : SOURCE EQUAL tableOrSubqueryClause + | INDEX EQUAL tableOrSubqueryClause + ; + +tableOrSubqueryClause + : LT_SQR_PRTHS subSearch RT_SQR_PRTHS (AS alias = qualifiedName)? + | tableSourceClause + ; + +// One tableSourceClause will generate one Relation node with/without one alias +// even if the relation contains more than one table sources. +// These table sources in one relation will be readed one by one in OpenSearch. +// But it may have different behaivours in different execution backends. +// For example, a Spark UnresovledRelation node only accepts one data source. +tableSourceClause + : tableSource (COMMA tableSource)* (AS alias = qualifiedName)? + ; + +// join +joinCommand + : (joinType) JOIN sideAlias joinHintList? joinCriteria? right = tableOrSubqueryClause + ; + +joinType + : INNER? + | CROSS + | LEFT OUTER? + | RIGHT OUTER? + | FULL OUTER? + | LEFT? SEMI + | LEFT? ANTI + ; + +sideAlias + : (LEFT EQUAL leftAlias = ident)? COMMA? (RIGHT EQUAL rightAlias = ident)? + ; + +joinCriteria + : ON logicalExpression + ; + +joinHintList + : hintPair (COMMA? hintPair)* + ; + +hintPair + : leftHintKey = LEFT_HINT DOT ID EQUAL leftHintValue = ident #leftHint + | rightHintKey = RIGHT_HINT DOT ID EQUAL rightHintValue = ident #rightHint + ; + +renameClasue + : orignalField = wcFieldExpression AS renamedField = wcFieldExpression + ; + +byClause + : BY fieldList + ; + +statsByClause + : BY fieldList + | BY bySpanClause + | BY bySpanClause COMMA fieldList + ; + +bySpanClause + : spanClause (AS alias = qualifiedName)? + ; + +spanClause + : SPAN LT_PRTHS fieldExpression COMMA value = literalValue (unit = timespanUnit)? RT_PRTHS + ; + +sortbyClause + : sortField (COMMA sortField)* + ; + +evalClause + : fieldExpression EQUAL expression + ; + +// aggregation terms +statsAggTerm + : statsFunction (AS alias = wcFieldExpression)? + ; + +// aggregation functions +statsFunction + : statsFunctionName LT_PRTHS valueExpression RT_PRTHS # statsFunctionCall + | COUNT LT_PRTHS RT_PRTHS # countAllFunctionCall + | (DISTINCT_COUNT | DC) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall + | percentileFunctionName = (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS valueExpression COMMA percent = integerLiteral RT_PRTHS # percentileFunctionCall + ; + +statsFunctionName + : AVG + | COUNT + | SUM + | MIN + | MAX + | STDDEV_SAMP + | STDDEV_POP + ; + +// expressions +expression + : logicalExpression + | valueExpression + ; + +logicalExpression + : NOT logicalExpression # logicalNot + | LT_PRTHS logicalExpression RT_PRTHS # parentheticLogicalExpr + | comparisonExpression # comparsion + | left = logicalExpression (AND)? right = logicalExpression # logicalAnd + | left = logicalExpression OR right = logicalExpression # logicalOr + | left = logicalExpression XOR right = logicalExpression # logicalXor + | booleanExpression # booleanExpr + ; + +comparisonExpression + : left = valueExpression comparisonOperator right = valueExpression # compareExpr + | valueExpression NOT? IN valueList # inExpr + | expr1 = functionArg NOT? BETWEEN expr2 = functionArg AND expr3 = functionArg # between + ; + +valueExpressionList + : valueExpression + | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS + ; + +valueExpression + : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic + | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic + | primaryExpression # valueExpressionDefault + | positionFunction # positionFunctionCall + | caseFunction # caseExpr + | timestampFunction # timestampFunctionCall + | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr + | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr + | ident ARROW expression # lambda + | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW expression # lambda + ; + +primaryExpression + : evalFunctionCall + | fieldExpression + | literalValue + ; + +positionFunction + : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS + ; + +booleanExpression + : booleanFunctionCall # booleanFunctionCallExpr + | isEmptyExpression # isEmptyExpr + | valueExpressionList NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr + | EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr + | cidrMatchFunctionCall # cidrFunctionCallExpr + ; + + isEmptyExpression + : (ISEMPTY | ISBLANK) LT_PRTHS functionArg RT_PRTHS + ; + + caseFunction + : CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS + ; + +relevanceExpression + : singleFieldRelevanceFunction + | multiFieldRelevanceFunction + ; + +// Field is a single column +singleFieldRelevanceFunction + : singleFieldRelevanceFunctionName LT_PRTHS field = relevanceField COMMA query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS + ; + +// Field is a list of columns +multiFieldRelevanceFunction + : multiFieldRelevanceFunctionName LT_PRTHS LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS + ; + +// tables +tableSource + : tableQualifiedName + | ID_DATE_SUFFIX + ; + +tableFunction + : qualifiedName LT_PRTHS functionArgs RT_PRTHS + ; + +// fields +fieldList + : fieldExpression (COMMA fieldExpression)* + ; + +wcFieldList + : wcFieldExpression (COMMA wcFieldExpression)* + ; + +sortField + : (PLUS | MINUS)? sortFieldExpression + ; + +sortFieldExpression + : fieldExpression + | AUTO LT_PRTHS fieldExpression RT_PRTHS + | STR LT_PRTHS fieldExpression RT_PRTHS + | IP LT_PRTHS fieldExpression RT_PRTHS + | NUM LT_PRTHS fieldExpression RT_PRTHS + ; + +fieldExpression + : qualifiedName + ; + +wcFieldExpression + : wcQualifiedName + ; + +// functions +evalFunctionCall + : evalFunctionName LT_PRTHS functionArgs RT_PRTHS + ; + +// cast function +dataTypeFunctionCall + : CAST LT_PRTHS expression AS convertedDataType RT_PRTHS + ; + +// boolean functions +booleanFunctionCall + : conditionFunctionBase LT_PRTHS functionArgs RT_PRTHS + ; + +cidrMatchFunctionCall + : CIDRMATCH LT_PRTHS ipAddress = functionArg COMMA cidrBlock = functionArg RT_PRTHS + ; + +convertedDataType + : typeName = DATE + | typeName = TIME + | typeName = TIMESTAMP + | typeName = INT + | typeName = INTEGER + | typeName = DOUBLE + | typeName = LONG + | typeName = FLOAT + | typeName = STRING + | typeName = BOOLEAN + ; + +evalFunctionName + : mathematicalFunctionName + | dateTimeFunctionName + | textFunctionName + | conditionFunctionBase + | systemFunctionName + | positionFunctionName + | coalesceFunctionName + | cryptographicFunctionName + | jsonFunctionName + | collectionFunctionName + | lambdaFunctionName + ; + +functionArgs + : (functionArg (COMMA functionArg)*)? + ; + +functionArg + : (ident EQUAL)? valueExpression + ; + +relevanceArg + : relevanceArgName EQUAL relevanceArgValue + ; + +relevanceArgName + : ALLOW_LEADING_WILDCARD + | ANALYZER + | ANALYZE_WILDCARD + | AUTO_GENERATE_SYNONYMS_PHRASE_QUERY + | BOOST + | CUTOFF_FREQUENCY + | DEFAULT_FIELD + | DEFAULT_OPERATOR + | ENABLE_POSITION_INCREMENTS + | ESCAPE + | FIELDS + | FLAGS + | FUZZINESS + | FUZZY_MAX_EXPANSIONS + | FUZZY_PREFIX_LENGTH + | FUZZY_REWRITE + | FUZZY_TRANSPOSITIONS + | LENIENT + | LOW_FREQ_OPERATOR + | MAX_DETERMINIZED_STATES + | MAX_EXPANSIONS + | MINIMUM_SHOULD_MATCH + | OPERATOR + | PHRASE_SLOP + | PREFIX_LENGTH + | QUOTE_ANALYZER + | QUOTE_FIELD_SUFFIX + | REWRITE + | SLOP + | TIE_BREAKER + | TIME_ZONE + | TYPE + | ZERO_TERMS_QUERY + ; + +relevanceFieldAndWeight + : field = relevanceField + | field = relevanceField weight = relevanceFieldWeight + | field = relevanceField BIT_XOR_OP weight = relevanceFieldWeight + ; + +relevanceFieldWeight + : integerLiteral + | decimalLiteral + ; + +relevanceField + : qualifiedName + | stringLiteral + ; + +relevanceQuery + : relevanceArgValue + ; + +relevanceArgValue + : qualifiedName + | literalValue + ; + +mathematicalFunctionName + : ABS + | CBRT + | CEIL + | CEILING + | CONV + | CRC32 + | E + | EXP + | FLOOR + | LN + | LOG + | LOG10 + | LOG2 + | MOD + | PI + | POW + | POWER + | RAND + | ROUND + | SIGN + | SIGNUM + | SQRT + | TRUNCATE + | trigonometricFunctionName + ; + +trigonometricFunctionName + : ACOS + | ASIN + | ATAN + | ATAN2 + | COS + | COT + | DEGREES + | RADIANS + | SIN + | TAN + ; + +cryptographicFunctionName + : MD5 + | SHA1 + | SHA2 + ; + +dateTimeFunctionName + : ADDDATE + | ADDTIME + | CONVERT_TZ + | CURDATE + | CURRENT_DATE + | CURRENT_TIME + | CURRENT_TIMESTAMP + | CURRENT_TIMEZONE + | CURTIME + | DATE + | DATEDIFF + | DATETIME + | DATE_ADD + | DATE_FORMAT + | DATE_SUB + | DAY + | DAYNAME + | DAYOFMONTH + | DAYOFWEEK + | DAYOFYEAR + | DAY_OF_MONTH + | DAY_OF_WEEK + | DAY_OF_YEAR + | FROM_DAYS + | FROM_UNIXTIME + | HOUR + | HOUR_OF_DAY + | LAST_DAY + | LOCALTIME + | LOCALTIMESTAMP + | MAKEDATE + | MAKE_DATE + | MAKETIME + | MICROSECOND + | MINUTE + | MINUTE_OF_DAY + | MINUTE_OF_HOUR + | MONTH + | MONTHNAME + | MONTH_OF_YEAR + | NOW + | PERIOD_ADD + | PERIOD_DIFF + | QUARTER + | SECOND + | SECOND_OF_MINUTE + | SEC_TO_TIME + | STR_TO_DATE + | SUBDATE + | SUBTIME + | SYSDATE + | TIME + | TIMEDIFF + | TIMESTAMP + | TIME_FORMAT + | TIME_TO_SEC + | TO_DAYS + | TO_SECONDS + | UNIX_TIMESTAMP + | UTC_DATE + | UTC_TIME + | UTC_TIMESTAMP + | WEEK + | WEEKDAY + | WEEK_OF_YEAR + | YEAR + | YEARWEEK + ; + +getFormatFunction + : GET_FORMAT LT_PRTHS getFormatType COMMA functionArg RT_PRTHS + ; + +getFormatType + : DATE + | DATETIME + | TIME + | TIMESTAMP + ; + +extractFunction + : EXTRACT LT_PRTHS datetimePart FROM functionArg RT_PRTHS + ; + +simpleDateTimePart + : MICROSECOND + | SECOND + | MINUTE + | HOUR + | DAY + | WEEK + | MONTH + | QUARTER + | YEAR + ; + +complexDateTimePart + : SECOND_MICROSECOND + | MINUTE_MICROSECOND + | MINUTE_SECOND + | HOUR_MICROSECOND + | HOUR_SECOND + | HOUR_MINUTE + | DAY_MICROSECOND + | DAY_SECOND + | DAY_MINUTE + | DAY_HOUR + | YEAR_MONTH + ; + +datetimePart + : simpleDateTimePart + | complexDateTimePart + ; + +timestampFunction + : timestampFunctionName LT_PRTHS simpleDateTimePart COMMA firstArg = functionArg COMMA secondArg = functionArg RT_PRTHS + ; + +timestampFunctionName + : TIMESTAMPADD + | TIMESTAMPDIFF + ; + +// condition function return boolean value +conditionFunctionBase + : LIKE + | IF + | ISNULL + | ISNOTNULL + | IFNULL + | NULLIF + | ISPRESENT + | JSON_VALID + ; + +systemFunctionName + : TYPEOF + ; + +textFunctionName + : SUBSTR + | SUBSTRING + | TRIM + | LTRIM + | RTRIM + | LOWER + | UPPER + | CONCAT + | CONCAT_WS + | LENGTH + | STRCMP + | RIGHT + | LEFT + | ASCII + | LOCATE + | REPLACE + | REVERSE + | ISEMPTY + | ISBLANK + ; + +jsonFunctionName + : JSON + | JSON_OBJECT + | JSON_ARRAY + | JSON_ARRAY_LENGTH + | TO_JSON_STRING + | JSON_EXTRACT + | JSON_KEYS + | JSON_VALID +// | JSON_APPEND +// | JSON_DELETE +// | JSON_EXTEND +// | JSON_SET +// | JSON_ARRAY_ALL_MATCH +// | JSON_ARRAY_ANY_MATCH +// | JSON_ARRAY_FILTER +// | JSON_ARRAY_MAP +// | JSON_ARRAY_REDUCE + ; + +collectionFunctionName + : ARRAY + | ARRAY_LENGTH + ; + +lambdaFunctionName + : FORALL + | EXISTS + | FILTER + | TRANSFORM + | REDUCE + ; + +positionFunctionName + : POSITION + ; + +coalesceFunctionName + : COALESCE + ; + +// operators + comparisonOperator + : EQUAL + | NOT_EQUAL + | LESS + | NOT_LESS + | GREATER + | NOT_GREATER + | REGEXP + ; + +singleFieldRelevanceFunctionName + : MATCH + | MATCH_PHRASE + | MATCH_BOOL_PREFIX + | MATCH_PHRASE_PREFIX + ; + +multiFieldRelevanceFunctionName + : SIMPLE_QUERY_STRING + | MULTI_MATCH + | QUERY_STRING + ; + +// literals and values +literalValue + : stringLiteral + | integerLiteral + | decimalLiteral + | booleanLiteral + | datetimeLiteral //#datetime + | intervalLiteral + ; + +intervalLiteral + : INTERVAL valueExpression intervalUnit + ; + +stringLiteral + : DQUOTA_STRING + | SQUOTA_STRING + ; + +integerLiteral + : (PLUS | MINUS)? INTEGER_LITERAL + ; + +decimalLiteral + : (PLUS | MINUS)? DECIMAL_LITERAL + ; + +booleanLiteral + : TRUE + | FALSE + ; + +// Date and Time Literal, follow ANSI 92 +datetimeLiteral + : dateLiteral + | timeLiteral + | timestampLiteral + ; + +dateLiteral + : DATE date = stringLiteral + ; + +timeLiteral + : TIME time = stringLiteral + ; + +timestampLiteral + : TIMESTAMP timestamp = stringLiteral + ; + +intervalUnit + : MICROSECOND + | SECOND + | MINUTE + | HOUR + | DAY + | WEEK + | MONTH + | QUARTER + | YEAR + | SECOND_MICROSECOND + | MINUTE_MICROSECOND + | MINUTE_SECOND + | HOUR_MICROSECOND + | HOUR_SECOND + | HOUR_MINUTE + | DAY_MICROSECOND + | DAY_SECOND + | DAY_MINUTE + | DAY_HOUR + | YEAR_MONTH + ; + +timespanUnit + : MS + | S + | M + | H + | D + | W + | Q + | Y + | MILLISECOND + | SECOND + | MINUTE + | HOUR + | DAY + | WEEK + | MONTH + | QUARTER + | YEAR + ; + +valueList + : LT_PRTHS literalValue (COMMA literalValue)* RT_PRTHS + ; + +qualifiedName + : ident (DOT ident)* # identsAsQualifiedName + ; + +identifierSeq + : qualifiedName (COMMA qualifiedName)* # identsAsQualifiedNameSeq + | LT_PRTHS qualifiedName (COMMA qualifiedName)* RT_PRTHS # identsAsQualifiedNameSeq + ; + +tableQualifiedName + : tableIdent (DOT ident)* # identsAsTableQualifiedName + ; + +wcQualifiedName + : wildcard (DOT wildcard)* # identsAsWildcardQualifiedName + ; + +ident + : (DOT)? ID + | BACKTICK ident BACKTICK + | BQUOTA_STRING + | keywordsCanBeId + ; + +tableIdent + : (CLUSTER)? ident + ; + +wildcard + : ident (MODULE ident)* (MODULE)? + | SINGLE_QUOTE wildcard SINGLE_QUOTE + | DOUBLE_QUOTE wildcard DOUBLE_QUOTE + | BACKTICK wildcard BACKTICK + ; + +keywordsCanBeId + : D // OD SQL and ODBC special + | timespanUnit + | SPAN + | evalFunctionName + | relevanceArgName + | intervalUnit + | dateTimeFunctionName + | textFunctionName + | jsonFunctionName + | mathematicalFunctionName + | positionFunctionName + | cryptographicFunctionName + | singleFieldRelevanceFunctionName + | multiFieldRelevanceFunctionName + | commandName + | comparisonOperator + | explainMode + | correlationType + // commands assist keywords + | IN + | SOURCE + | INDEX + | DESC + | DATASOURCES + | AUTO + | STR + | IP + | NUM + | FROM + | PATTERN + | NEW_FIELD + | SCOPE + | MAPPING + | WITH + | USING + | CAST + | GET_FORMAT + | EXTRACT + | INTERVAL + | PLUS + | MINUS + | INCLUDEFIELDS + | NULLS + // ARGUMENT KEYWORDS + | KEEPEMPTY + | CONSECUTIVE + | DEDUP_SPLITVALUES + | PARTITIONS + | ALLNUM + | DELIM + | CENTROIDS + | ITERATIONS + | DISTANCE_TYPE + | NUMBER_OF_TREES + | SHINGLE_SIZE + | SAMPLE_SIZE + | OUTPUT_AFTER + | TIME_DECAY + | ANOMALY_RATE + | CATEGORY_FIELD + | TIME_FIELD + | TIME_ZONE + | TRAINING_DATA_SIZE + | ANOMALY_SCORE_THRESHOLD + // AGGREGATIONS + | statsFunctionName + | DISTINCT_COUNT + | PERCENTILE + | PERCENTILE_APPROX + | ESTDC + | ESTDC_ERROR + | MEAN + | MEDIAN + | MODE + | RANGE + | STDEV + | STDEVP + | SUMSQ + | VAR_SAMP + | VAR_POP + | TAKE + | FIRST + | LAST + | LIST + | VALUES + | EARLIEST + | EARLIEST_TIME + | LATEST + | LATEST_TIME + | PER_DAY + | PER_HOUR + | PER_MINUTE + | PER_SECOND + | RATE + | SPARKLINE + | C + | DC + // JOIN TYPE + | OUTER + | INNER + | CROSS + | LEFT + | RIGHT + | FULL + | SEMI + | ANTI + | BETWEEN + | CIDRMATCH + | trendlineType + ; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java index 5f88ea9ca0..a390924e85 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java @@ -23,6 +23,7 @@ import org.opensearch.sql.spark.execution.session.SessionManager; import org.opensearch.sql.spark.rest.model.LangType; import org.opensearch.sql.spark.utils.SQLQueryUtils; +import org.opensearch.sql.spark.validator.PPLQueryValidator; import org.opensearch.sql.spark.validator.SQLQueryValidator; /** This class takes care of understanding query and dispatching job query to emr serverless. */ @@ -39,6 +40,7 @@ public class SparkQueryDispatcher { private final QueryHandlerFactory queryHandlerFactory; private final QueryIdProvider queryIdProvider; private final SQLQueryValidator sqlQueryValidator; + private final PPLQueryValidator pplQueryValidator; public DispatchQueryResponse dispatch( DispatchQueryRequest dispatchQueryRequest, @@ -47,9 +49,8 @@ public DispatchQueryResponse dispatch( this.dataSourceService.verifyDataSourceAccessAndGetRawMetadata( dispatchQueryRequest.getDatasource(), asyncQueryRequestContext); + String query = dispatchQueryRequest.getQuery(); if (LangType.SQL.equals(dispatchQueryRequest.getLangType())) { - String query = dispatchQueryRequest.getQuery(); - if (SQLQueryUtils.isFlintExtensionQuery(query)) { sqlQueryValidator.validateFlintExtensionQuery(query, dataSourceMetadata.getConnector()); return handleFlintExtensionQuery( @@ -57,6 +58,8 @@ public DispatchQueryResponse dispatch( } sqlQueryValidator.validate(query, dataSourceMetadata.getConnector()); + } else if (LangType.PPL.equals(dispatchQueryRequest.getLangType())) { + pplQueryValidator.validate(query, dataSourceMetadata.getConnector()); } return handleDefaultQuery(dispatchQueryRequest, asyncQueryRequestContext, dataSourceMetadata); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java index ab89348f33..be2394393c 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElement.java @@ -5,87 +5,4 @@ package org.opensearch.sql.spark.validator; -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public enum GrammarElement { - ALTER_NAMESPACE("ALTER (DATABASE|TABLE|NAMESPACE)"), - ALTER_VIEW("ALTER VIEW"), - CREATE_NAMESPACE("CREATE (DATABASE|TABLE|NAMESPACE)"), - CREATE_FUNCTION("CREATE FUNCTION"), - CREATE_VIEW("CREATE VIEW"), - DROP_NAMESPACE("DROP (DATABASE|TABLE|NAMESPACE)"), - DROP_FUNCTION("DROP FUNCTION"), - DROP_VIEW("DROP VIEW"), - DROP_TABLE("DROP TABLE"), - REPAIR_TABLE("REPAIR TABLE"), - TRUNCATE_TABLE("TRUNCATE TABLE"), - // DML Statements - INSERT("INSERT"), - LOAD("LOAD"), - - // Data Retrieval Statements - EXPLAIN("EXPLAIN"), - WITH("WITH"), - CLUSTER_BY("CLUSTER BY"), - DISTRIBUTE_BY("DISTRIBUTE BY"), - // GROUP_BY("GROUP BY"), - // HAVING("HAVING"), - HINTS("HINTS"), - INLINE_TABLE("Inline Table(VALUES)"), - FILE("File"), - INNER_JOIN("INNER JOIN"), - CROSS_JOIN("CROSS JOIN"), - LEFT_OUTER_JOIN("LEFT OUTER JOIN"), - LEFT_SEMI_JOIN("LEFT SEMI JOIN"), - RIGHT_OUTER_JOIN("RIGHT OUTER JOIN"), - FULL_OUTER_JOIN("FULL OUTER JOIN"), - LEFT_ANTI_JOIN("LEFT ANTI JOIN"), - TABLESAMPLE("TABLESAMPLE"), - TABLE_VALUED_FUNCTION("Table-valued function"), - LATERAL_VIEW("LATERAL VIEW"), - LATERAL_SUBQUERY("LATERAL SUBQUERY"), - TRANSFORM("TRANSFORM"), - - // Auxiliary Statements - MANAGE_RESOURCE("Resource management statements"), - ANALYZE_TABLE("ANALYZE TABLE(S)"), - CACHE_TABLE("CACHE TABLE"), - CLEAR_CACHE("CLEAR CACHE"), - DESCRIBE_NAMESPACE("DESCRIBE (NAMESPACE|DATABASE|SCHEMA)"), - DESCRIBE_FUNCTION("DESCRIBE FUNCTION"), - DESCRIBE_QUERY("DESCRIBE QUERY"), - DESCRIBE_TABLE("DESCRIBE TABLE"), - REFRESH_RESOURCE("REFRESH"), - REFRESH_TABLE("REFRESH TABLE"), - REFRESH_FUNCTION("REFRESH FUNCTION"), - RESET("RESET"), - SET("SET"), - SHOW_COLUMNS("SHOW COLUMNS"), - SHOW_CREATE_TABLE("SHOW CREATE TABLE"), - SHOW_NAMESPACES("SHOW (DATABASES|SCHEMAS)"), - SHOW_FUNCTIONS("SHOW FUNCTIONS"), - SHOW_PARTITIONS("SHOW PARTITIONS"), - SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED"), - SHOW_TABLES("SHOW TABLES"), - SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES"), - SHOW_VIEWS("SHOW VIEWS"), - UNCACHE_TABLE("UNCACHE TABLE"), - - // Functions - MAP_FUNCTIONS("Map functions"), - BITWISE_FUNCTIONS("Bitwise functions"), - CSV_FUNCTIONS("CSV functions"), - GENERATOR_FUNCTIONS("Generator functions"), - MISC_FUNCTIONS("Misc functions"), - - // UDF - UDF("User Defined functions"); - - String description; - - @Override - public String toString() { - return description; - } -} +public interface GrammarElement {} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLGrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLGrammarElement.java new file mode 100644 index 0000000000..9e426f39fb --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLGrammarElement.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public enum PPLGrammarElement implements GrammarElement { + PATTERNS_COMMAND("patterns command"), + JOIN_COMMAND("join command"), + LOOKUP_COMMAND("lookup command"), + SUBQUERY_COMMAND("subquery command"), + FLATTEN_COMMAND("flatten command"), + FILLNULL_COMMAND("fillnull command"), + EXPAND_COMMAND("expand command"), + DESCRIBE_COMMAND("describe command"), + IPADDRESS_FUNCTIONS("IP address functions"), + JSON_FUNCTIONS("JSON functions"), + LAMBDA_FUNCTIONS("Lambda functions"); + + String description; + + @Override + public String toString() { + return description; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidationVisitor.java new file mode 100644 index 0000000000..d829dd17a5 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidationVisitor.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.opensearch.sql.spark.antlr.parser.OpenSearchPPLParser.*; + +@AllArgsConstructor +public class PPLQueryValidationVisitor + extends org.opensearch.sql.spark.antlr.parser.OpenSearchPPLParserBaseVisitor<Void> { + private final GrammarElementValidator grammarElementValidator; + + @Override + public Void visitPatternsCommand(PatternsCommandContext ctx) { + validateAllowed(PPLGrammarElement.PATTERNS_COMMAND); + return super.visitPatternsCommand(ctx); + } + + @Override + public Void visitJoinCommand(JoinCommandContext ctx) { + validateAllowed(PPLGrammarElement.JOIN_COMMAND); + return super.visitJoinCommand(ctx); + } + + @Override + public Void visitLookupCommand(LookupCommandContext ctx) { + validateAllowed(PPLGrammarElement.LOOKUP_COMMAND); + return super.visitLookupCommand(ctx); + } + + @Override + public Void visitSubSearch(SubSearchContext ctx) { + validateAllowed(PPLGrammarElement.SUBQUERY_COMMAND); + return super.visitSubSearch(ctx); + } + + @Override + public Void visitFlattenCommand(FlattenCommandContext ctx) { + validateAllowed(PPLGrammarElement.FLATTEN_COMMAND); + return super.visitFlattenCommand(ctx); + } + + @Override + public Void visitFillnullCommand(FillnullCommandContext ctx) { + validateAllowed(PPLGrammarElement.FILLNULL_COMMAND); + return super.visitFillnullCommand(ctx); + } + + @Override + public Void visitExpandCommand(ExpandCommandContext ctx) { + validateAllowed(PPLGrammarElement.EXPAND_COMMAND); + return super.visitExpandCommand(ctx); + } + + @Override + public Void visitDescribeCommand(DescribeCommandContext ctx) { + validateAllowed(PPLGrammarElement.DESCRIBE_COMMAND); + return super.visitDescribeCommand(ctx); + } + + @Override + public Void visitCidrMatchFunctionCall(CidrMatchFunctionCallContext ctx) { + validateAllowed(PPLGrammarElement.IPADDRESS_FUNCTIONS); + return super.visitCidrMatchFunctionCall(ctx); + } + + @Override + public Void visitJsonFunctionName(JsonFunctionNameContext ctx) { + validateAllowed(PPLGrammarElement.JSON_FUNCTIONS); + return super.visitJsonFunctionName(ctx); + } + + @Override + public Void visitLambdaFunctionName(LambdaFunctionNameContext ctx) { + validateAllowed(PPLGrammarElement.LAMBDA_FUNCTIONS); + return super.visitLambdaFunctionName(ctx); + } + + private void validateAllowed(PPLGrammarElement element) { + if (!grammarElementValidator.isValid(element)) { + throw new IllegalArgumentException(element + " is not allowed."); + } + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidator.java new file mode 100644 index 0000000000..e630ffc45f --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidator.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; +import org.antlr.v4.runtime.CommonTokenStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.antlr.parser.OpenSearchPPLLexer; +import org.opensearch.sql.spark.antlr.parser.OpenSearchPPLParser; + +@AllArgsConstructor +public class PPLQueryValidator { + private static final Logger log = LogManager.getLogger(SQLQueryValidator.class); + + private final GrammarElementValidatorProvider grammarElementValidatorProvider; + + /** + * It will look up validator associated with the DataSourceType, and throw + * IllegalArgumentException if invalid grammar element is found. + * + * @param pplQuery The query to be validated + * @param datasourceType + */ + public void validate(String pplQuery, DataSourceType datasourceType) { + GrammarElementValidator grammarElementValidator = + grammarElementValidatorProvider.getValidatorForDatasource(datasourceType); + PPLQueryValidationVisitor visitor = new PPLQueryValidationVisitor(grammarElementValidator); + try { + visitor.visit(getPplParser(pplQuery).root()); + } catch (IllegalArgumentException e) { + log.error("Query validation failed. DataSourceType=" + datasourceType, e); + throw e; + } + } + + public static OpenSearchPPLParser getPplParser(String pplQuery) { + OpenSearchPPLParser sqlBaseParser = + new OpenSearchPPLParser( + new CommonTokenStream(new OpenSearchPPLLexer(new CaseInsensitiveCharStream(pplQuery)))); + sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener()); + return sqlBaseParser; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java deleted file mode 100644 index 668ab26d68..0000000000 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.validator; - -import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.BITWISE_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; -import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; -import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.FILE; -import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; -import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; -import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; -import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; -import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; -import static org.opensearch.sql.spark.validator.GrammarElement.RESET; -import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.SET; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; -import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; -import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; -import static org.opensearch.sql.spark.validator.GrammarElement.UDF; - -import com.google.common.collect.ImmutableSet; -import java.util.Set; - -public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidator { - private static final Set<GrammarElement> S3GLUE_DENY_LIST = - ImmutableSet.<GrammarElement>builder() - .add( - ALTER_VIEW, - CREATE_FUNCTION, - CREATE_VIEW, - DROP_FUNCTION, - DROP_VIEW, - INSERT, - LOAD, - CLUSTER_BY, - DISTRIBUTE_BY, - HINTS, - INLINE_TABLE, - FILE, - CROSS_JOIN, - LEFT_SEMI_JOIN, - RIGHT_OUTER_JOIN, - FULL_OUTER_JOIN, - LEFT_ANTI_JOIN, - TABLESAMPLE, - TABLE_VALUED_FUNCTION, - TRANSFORM, - MANAGE_RESOURCE, - DESCRIBE_FUNCTION, - REFRESH_RESOURCE, - REFRESH_FUNCTION, - RESET, - SET, - SHOW_FUNCTIONS, - SHOW_VIEWS, - BITWISE_FUNCTIONS, - MISC_FUNCTIONS, - UDF) - .build(); - - public S3GlueGrammarElementValidator() { - super(S3GLUE_DENY_LIST); - } -} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueSQLGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueSQLGrammarElementValidator.java new file mode 100644 index 0000000000..870fb9412d --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueSQLGrammarElementValidator.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.SQLGrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.BITWISE_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CROSS_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.FILE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.FULL_OUTER_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LEFT_SEMI_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.RESET; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.RIGHT_OUTER_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SET; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.UDF; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class S3GlueSQLGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set<GrammarElement> S3GLUE_DENY_LIST = + ImmutableSet.<GrammarElement>builder() + .add( + ALTER_VIEW, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_VIEW, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + DESCRIBE_FUNCTION, + REFRESH_RESOURCE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_FUNCTIONS, + SHOW_VIEWS, + BITWISE_FUNCTIONS, + MISC_FUNCTIONS, + UDF) + .build(); + + public S3GlueSQLGrammarElementValidator() { + super(S3GLUE_DENY_LIST); + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLGrammarElement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLGrammarElement.java new file mode 100644 index 0000000000..ef3e1f2c8c --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLGrammarElement.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public enum SQLGrammarElement implements GrammarElement { + ALTER_NAMESPACE("ALTER (DATABASE|TABLE|NAMESPACE)"), + ALTER_VIEW("ALTER VIEW"), + CREATE_NAMESPACE("CREATE (DATABASE|TABLE|NAMESPACE)"), + CREATE_FUNCTION("CREATE FUNCTION"), + CREATE_VIEW("CREATE VIEW"), + DROP_NAMESPACE("DROP (DATABASE|TABLE|NAMESPACE)"), + DROP_FUNCTION("DROP FUNCTION"), + DROP_VIEW("DROP VIEW"), + DROP_TABLE("DROP TABLE"), + REPAIR_TABLE("REPAIR TABLE"), + TRUNCATE_TABLE("TRUNCATE TABLE"), + // DML Statements + INSERT("INSERT"), + LOAD("LOAD"), + + // Data Retrieval Statements + EXPLAIN("EXPLAIN"), + WITH("WITH"), + CLUSTER_BY("CLUSTER BY"), + DISTRIBUTE_BY("DISTRIBUTE BY"), + // GROUP_BY("GROUP BY"), + // HAVING("HAVING"), + HINTS("HINTS"), + INLINE_TABLE("Inline Table(VALUES)"), + FILE("File"), + INNER_JOIN("INNER JOIN"), + CROSS_JOIN("CROSS JOIN"), + LEFT_OUTER_JOIN("LEFT OUTER JOIN"), + LEFT_SEMI_JOIN("LEFT SEMI JOIN"), + RIGHT_OUTER_JOIN("RIGHT OUTER JOIN"), + FULL_OUTER_JOIN("FULL OUTER JOIN"), + LEFT_ANTI_JOIN("LEFT ANTI JOIN"), + TABLESAMPLE("TABLESAMPLE"), + TABLE_VALUED_FUNCTION("Table-valued function"), + LATERAL_VIEW("LATERAL VIEW"), + LATERAL_SUBQUERY("LATERAL SUBQUERY"), + TRANSFORM("TRANSFORM"), + + // Auxiliary Statements + MANAGE_RESOURCE("Resource management statements"), + ANALYZE_TABLE("ANALYZE TABLE(S)"), + CACHE_TABLE("CACHE TABLE"), + CLEAR_CACHE("CLEAR CACHE"), + DESCRIBE_NAMESPACE("DESCRIBE (NAMESPACE|DATABASE|SCHEMA)"), + DESCRIBE_FUNCTION("DESCRIBE FUNCTION"), + DESCRIBE_QUERY("DESCRIBE QUERY"), + DESCRIBE_TABLE("DESCRIBE TABLE"), + REFRESH_RESOURCE("REFRESH"), + REFRESH_TABLE("REFRESH TABLE"), + REFRESH_FUNCTION("REFRESH FUNCTION"), + RESET("RESET"), + SET("SET"), + SHOW_COLUMNS("SHOW COLUMNS"), + SHOW_CREATE_TABLE("SHOW CREATE TABLE"), + SHOW_NAMESPACES("SHOW (DATABASES|SCHEMAS)"), + SHOW_FUNCTIONS("SHOW FUNCTIONS"), + SHOW_PARTITIONS("SHOW PARTITIONS"), + SHOW_TABLE_EXTENDED("SHOW TABLE EXTENDED"), + SHOW_TABLES("SHOW TABLES"), + SHOW_TBLPROPERTIES("SHOW TBLPROPERTIES"), + SHOW_VIEWS("SHOW VIEWS"), + UNCACHE_TABLE("UNCACHE TABLE"), + + // Functions + MAP_FUNCTIONS("Map functions"), + BITWISE_FUNCTIONS("Bitwise functions"), + CSV_FUNCTIONS("CSV functions"), + GENERATOR_FUNCTIONS("Generator functions"), + MISC_FUNCTIONS("Misc functions"), + + // UDF + UDF("User Defined functions"); + + String description; + + @Override + public String toString() { + return description; + } +} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index 2312c0de7a..f54ac49b4e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -87,100 +87,100 @@ public class SQLQueryValidationVisitor extends SqlBaseParserBaseVisitor<Void> { @Override public Void visitCreateFunction(SqlBaseParser.CreateFunctionContext ctx) { - validateAllowed(GrammarElement.CREATE_FUNCTION); + validateAllowed(SQLGrammarElement.CREATE_FUNCTION); return super.visitCreateFunction(ctx); } @Override public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitSetNamespaceProperties(ctx); } @Override public Void visitAddTableColumns(AddTableColumnsContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitAddTableColumns(ctx); } @Override public Void visitAddTablePartition(AddTablePartitionContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitAddTablePartition(ctx); } @Override public Void visitRenameTableColumn(RenameTableColumnContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitRenameTableColumn(ctx); } @Override public Void visitDropTableColumns(DropTableColumnsContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitDropTableColumns(ctx); } @Override public Void visitAlterTableAlterColumn(AlterTableAlterColumnContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitAlterTableAlterColumn(ctx); } @Override public Void visitHiveReplaceColumns(HiveReplaceColumnsContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitHiveReplaceColumns(ctx); } @Override public Void visitSetTableSerDe(SetTableSerDeContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitSetTableSerDe(ctx); } @Override public Void visitRenameTablePartition(RenameTablePartitionContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitRenameTablePartition(ctx); } @Override public Void visitDropTablePartitions(DropTablePartitionsContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitDropTablePartitions(ctx); } @Override public Void visitSetTableLocation(SetTableLocationContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitSetTableLocation(ctx); } @Override public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitRecoverPartitions(ctx); } @Override public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); return super.visitSetNamespaceLocation(ctx); } @Override public Void visitAlterViewQuery(AlterViewQueryContext ctx) { - validateAllowed(GrammarElement.ALTER_VIEW); + validateAllowed(SQLGrammarElement.ALTER_VIEW); return super.visitAlterViewQuery(ctx); } @Override public Void visitRenameTable(RenameTableContext ctx) { if (ctx.VIEW() != null) { - validateAllowed(GrammarElement.ALTER_VIEW); + validateAllowed(SQLGrammarElement.ALTER_VIEW); } else { - validateAllowed(GrammarElement.ALTER_NAMESPACE); + validateAllowed(SQLGrammarElement.ALTER_NAMESPACE); } return super.visitRenameTable(ctx); @@ -188,109 +188,109 @@ public Void visitRenameTable(RenameTableContext ctx) { @Override public Void visitCreateNamespace(CreateNamespaceContext ctx) { - validateAllowed(GrammarElement.CREATE_NAMESPACE); + validateAllowed(SQLGrammarElement.CREATE_NAMESPACE); return super.visitCreateNamespace(ctx); } @Override public Void visitCreateTable(CreateTableContext ctx) { - validateAllowed(GrammarElement.CREATE_NAMESPACE); + validateAllowed(SQLGrammarElement.CREATE_NAMESPACE); return super.visitCreateTable(ctx); } @Override public Void visitCreateTableLike(CreateTableLikeContext ctx) { - validateAllowed(GrammarElement.CREATE_NAMESPACE); + validateAllowed(SQLGrammarElement.CREATE_NAMESPACE); return super.visitCreateTableLike(ctx); } @Override public Void visitReplaceTable(ReplaceTableContext ctx) { - validateAllowed(GrammarElement.CREATE_NAMESPACE); + validateAllowed(SQLGrammarElement.CREATE_NAMESPACE); return super.visitReplaceTable(ctx); } @Override public Void visitDropNamespace(DropNamespaceContext ctx) { - validateAllowed(GrammarElement.DROP_NAMESPACE); + validateAllowed(SQLGrammarElement.DROP_NAMESPACE); return super.visitDropNamespace(ctx); } @Override public Void visitDropTable(DropTableContext ctx) { - validateAllowed(GrammarElement.DROP_NAMESPACE); + validateAllowed(SQLGrammarElement.DROP_NAMESPACE); return super.visitDropTable(ctx); } @Override public Void visitCreateView(CreateViewContext ctx) { - validateAllowed(GrammarElement.CREATE_VIEW); + validateAllowed(SQLGrammarElement.CREATE_VIEW); return super.visitCreateView(ctx); } @Override public Void visitDropView(DropViewContext ctx) { - validateAllowed(GrammarElement.DROP_VIEW); + validateAllowed(SQLGrammarElement.DROP_VIEW); return super.visitDropView(ctx); } @Override public Void visitDropFunction(DropFunctionContext ctx) { - validateAllowed(GrammarElement.DROP_FUNCTION); + validateAllowed(SQLGrammarElement.DROP_FUNCTION); return super.visitDropFunction(ctx); } @Override public Void visitRepairTable(RepairTableContext ctx) { - validateAllowed(GrammarElement.REPAIR_TABLE); + validateAllowed(SQLGrammarElement.REPAIR_TABLE); return super.visitRepairTable(ctx); } @Override public Void visitTruncateTable(TruncateTableContext ctx) { - validateAllowed(GrammarElement.TRUNCATE_TABLE); + validateAllowed(SQLGrammarElement.TRUNCATE_TABLE); return super.visitTruncateTable(ctx); } @Override public Void visitInsertOverwriteTable(InsertOverwriteTableContext ctx) { - validateAllowed(GrammarElement.INSERT); + validateAllowed(SQLGrammarElement.INSERT); return super.visitInsertOverwriteTable(ctx); } @Override public Void visitInsertIntoReplaceWhere(InsertIntoReplaceWhereContext ctx) { - validateAllowed(GrammarElement.INSERT); + validateAllowed(SQLGrammarElement.INSERT); return super.visitInsertIntoReplaceWhere(ctx); } @Override public Void visitInsertIntoTable(InsertIntoTableContext ctx) { - validateAllowed(GrammarElement.INSERT); + validateAllowed(SQLGrammarElement.INSERT); return super.visitInsertIntoTable(ctx); } @Override public Void visitInsertOverwriteDir(InsertOverwriteDirContext ctx) { - validateAllowed(GrammarElement.INSERT); + validateAllowed(SQLGrammarElement.INSERT); return super.visitInsertOverwriteDir(ctx); } @Override public Void visitInsertOverwriteHiveDir(InsertOverwriteHiveDirContext ctx) { - validateAllowed(GrammarElement.INSERT); + validateAllowed(SQLGrammarElement.INSERT); return super.visitInsertOverwriteHiveDir(ctx); } @Override public Void visitLoadData(LoadDataContext ctx) { - validateAllowed(GrammarElement.LOAD); + validateAllowed(SQLGrammarElement.LOAD); return super.visitLoadData(ctx); } @Override public Void visitExplain(ExplainContext ctx) { - validateAllowed(GrammarElement.EXPLAIN); + validateAllowed(SQLGrammarElement.EXPLAIN); return super.visitExplain(ctx); } @@ -298,7 +298,7 @@ public Void visitExplain(ExplainContext ctx) { public Void visitTableName(TableNameContext ctx) { String reference = ctx.identifierReference().getText(); if (isFileReference(reference)) { - validateAllowed(GrammarElement.FILE); + validateAllowed(SQLGrammarElement.FILE); } return super.visitTableName(ctx); } @@ -311,74 +311,74 @@ private boolean isFileReference(String reference) { @Override public Void visitCtes(CtesContext ctx) { - validateAllowed(GrammarElement.WITH); + validateAllowed(SQLGrammarElement.WITH); return super.visitCtes(ctx); } @Override public Void visitQueryOrganization(QueryOrganizationContext ctx) { if (ctx.CLUSTER() != null) { - validateAllowed(GrammarElement.CLUSTER_BY); + validateAllowed(SQLGrammarElement.CLUSTER_BY); } else if (ctx.DISTRIBUTE() != null) { - validateAllowed(GrammarElement.DISTRIBUTE_BY); + validateAllowed(SQLGrammarElement.DISTRIBUTE_BY); } return super.visitQueryOrganization(ctx); } @Override public Void visitHint(HintContext ctx) { - validateAllowed(GrammarElement.HINTS); + validateAllowed(SQLGrammarElement.HINTS); return super.visitHint(ctx); } @Override public Void visitInlineTable(InlineTableContext ctx) { - validateAllowed(GrammarElement.INLINE_TABLE); + validateAllowed(SQLGrammarElement.INLINE_TABLE); return super.visitInlineTable(ctx); } @Override public Void visitJoinType(JoinTypeContext ctx) { if (ctx.CROSS() != null) { - validateAllowed(GrammarElement.CROSS_JOIN); + validateAllowed(SQLGrammarElement.CROSS_JOIN); } else if (ctx.LEFT() != null && ctx.SEMI() != null) { - validateAllowed(GrammarElement.LEFT_SEMI_JOIN); + validateAllowed(SQLGrammarElement.LEFT_SEMI_JOIN); } else if (ctx.ANTI() != null) { - validateAllowed(GrammarElement.LEFT_ANTI_JOIN); + validateAllowed(SQLGrammarElement.LEFT_ANTI_JOIN); } else if (ctx.LEFT() != null) { - validateAllowed(GrammarElement.LEFT_OUTER_JOIN); + validateAllowed(SQLGrammarElement.LEFT_OUTER_JOIN); } else if (ctx.RIGHT() != null) { - validateAllowed(GrammarElement.RIGHT_OUTER_JOIN); + validateAllowed(SQLGrammarElement.RIGHT_OUTER_JOIN); } else if (ctx.FULL() != null) { - validateAllowed(GrammarElement.FULL_OUTER_JOIN); + validateAllowed(SQLGrammarElement.FULL_OUTER_JOIN); } else { - validateAllowed(GrammarElement.INNER_JOIN); + validateAllowed(SQLGrammarElement.INNER_JOIN); } return super.visitJoinType(ctx); } @Override public Void visitSample(SampleContext ctx) { - validateAllowed(GrammarElement.TABLESAMPLE); + validateAllowed(SQLGrammarElement.TABLESAMPLE); return super.visitSample(ctx); } @Override public Void visitTableValuedFunction(TableValuedFunctionContext ctx) { - validateAllowed(GrammarElement.TABLE_VALUED_FUNCTION); + validateAllowed(SQLGrammarElement.TABLE_VALUED_FUNCTION); return super.visitTableValuedFunction(ctx); } @Override public Void visitLateralView(LateralViewContext ctx) { - validateAllowed(GrammarElement.LATERAL_VIEW); + validateAllowed(SQLGrammarElement.LATERAL_VIEW); return super.visitLateralView(ctx); } @Override public Void visitRelation(RelationContext ctx) { if (ctx.LATERAL() != null) { - validateAllowed(GrammarElement.LATERAL_SUBQUERY); + validateAllowed(SQLGrammarElement.LATERAL_SUBQUERY); } return super.visitRelation(ctx); } @@ -386,7 +386,7 @@ public Void visitRelation(RelationContext ctx) { @Override public Void visitJoinRelation(JoinRelationContext ctx) { if (ctx.LATERAL() != null) { - validateAllowed(GrammarElement.LATERAL_SUBQUERY); + validateAllowed(SQLGrammarElement.LATERAL_SUBQUERY); } return super.visitJoinRelation(ctx); } @@ -394,158 +394,158 @@ public Void visitJoinRelation(JoinRelationContext ctx) { @Override public Void visitTransformClause(TransformClauseContext ctx) { if (ctx.TRANSFORM() != null) { - validateAllowed(GrammarElement.TRANSFORM); + validateAllowed(SQLGrammarElement.TRANSFORM); } return super.visitTransformClause(ctx); } @Override public Void visitManageResource(ManageResourceContext ctx) { - validateAllowed(GrammarElement.MANAGE_RESOURCE); + validateAllowed(SQLGrammarElement.MANAGE_RESOURCE); return super.visitManageResource(ctx); } @Override public Void visitAnalyze(AnalyzeContext ctx) { - validateAllowed(GrammarElement.ANALYZE_TABLE); + validateAllowed(SQLGrammarElement.ANALYZE_TABLE); return super.visitAnalyze(ctx); } @Override public Void visitAnalyzeTables(AnalyzeTablesContext ctx) { - validateAllowed(GrammarElement.ANALYZE_TABLE); + validateAllowed(SQLGrammarElement.ANALYZE_TABLE); return super.visitAnalyzeTables(ctx); } @Override public Void visitCacheTable(CacheTableContext ctx) { - validateAllowed(GrammarElement.CACHE_TABLE); + validateAllowed(SQLGrammarElement.CACHE_TABLE); return super.visitCacheTable(ctx); } @Override public Void visitClearCache(ClearCacheContext ctx) { - validateAllowed(GrammarElement.CLEAR_CACHE); + validateAllowed(SQLGrammarElement.CLEAR_CACHE); return super.visitClearCache(ctx); } @Override public Void visitDescribeNamespace(DescribeNamespaceContext ctx) { - validateAllowed(GrammarElement.DESCRIBE_NAMESPACE); + validateAllowed(SQLGrammarElement.DESCRIBE_NAMESPACE); return super.visitDescribeNamespace(ctx); } @Override public Void visitDescribeFunction(DescribeFunctionContext ctx) { - validateAllowed(GrammarElement.DESCRIBE_FUNCTION); + validateAllowed(SQLGrammarElement.DESCRIBE_FUNCTION); return super.visitDescribeFunction(ctx); } @Override public Void visitDescribeRelation(DescribeRelationContext ctx) { - validateAllowed(GrammarElement.DESCRIBE_TABLE); + validateAllowed(SQLGrammarElement.DESCRIBE_TABLE); return super.visitDescribeRelation(ctx); } @Override public Void visitDescribeQuery(DescribeQueryContext ctx) { - validateAllowed(GrammarElement.DESCRIBE_QUERY); + validateAllowed(SQLGrammarElement.DESCRIBE_QUERY); return super.visitDescribeQuery(ctx); } @Override public Void visitRefreshResource(RefreshResourceContext ctx) { - validateAllowed(GrammarElement.REFRESH_RESOURCE); + validateAllowed(SQLGrammarElement.REFRESH_RESOURCE); return super.visitRefreshResource(ctx); } @Override public Void visitRefreshTable(RefreshTableContext ctx) { - validateAllowed(GrammarElement.REFRESH_TABLE); + validateAllowed(SQLGrammarElement.REFRESH_TABLE); return super.visitRefreshTable(ctx); } @Override public Void visitRefreshFunction(RefreshFunctionContext ctx) { - validateAllowed(GrammarElement.REFRESH_FUNCTION); + validateAllowed(SQLGrammarElement.REFRESH_FUNCTION); return super.visitRefreshFunction(ctx); } @Override public Void visitResetConfiguration(ResetConfigurationContext ctx) { - validateAllowed(GrammarElement.RESET); + validateAllowed(SQLGrammarElement.RESET); return super.visitResetConfiguration(ctx); } @Override public Void visitResetQuotedConfiguration(ResetQuotedConfigurationContext ctx) { - validateAllowed(GrammarElement.RESET); + validateAllowed(SQLGrammarElement.RESET); return super.visitResetQuotedConfiguration(ctx); } @Override public Void visitSetConfiguration(SetConfigurationContext ctx) { - validateAllowed(GrammarElement.SET); + validateAllowed(SQLGrammarElement.SET); return super.visitSetConfiguration(ctx); } @Override public Void visitShowColumns(ShowColumnsContext ctx) { - validateAllowed(GrammarElement.SHOW_COLUMNS); + validateAllowed(SQLGrammarElement.SHOW_COLUMNS); return super.visitShowColumns(ctx); } @Override public Void visitShowCreateTable(ShowCreateTableContext ctx) { - validateAllowed(GrammarElement.SHOW_CREATE_TABLE); + validateAllowed(SQLGrammarElement.SHOW_CREATE_TABLE); return super.visitShowCreateTable(ctx); } @Override public Void visitShowNamespaces(ShowNamespacesContext ctx) { - validateAllowed(GrammarElement.SHOW_NAMESPACES); + validateAllowed(SQLGrammarElement.SHOW_NAMESPACES); return super.visitShowNamespaces(ctx); } @Override public Void visitShowFunctions(ShowFunctionsContext ctx) { - validateAllowed(GrammarElement.SHOW_FUNCTIONS); + validateAllowed(SQLGrammarElement.SHOW_FUNCTIONS); return super.visitShowFunctions(ctx); } @Override public Void visitShowPartitions(ShowPartitionsContext ctx) { - validateAllowed(GrammarElement.SHOW_PARTITIONS); + validateAllowed(SQLGrammarElement.SHOW_PARTITIONS); return super.visitShowPartitions(ctx); } @Override public Void visitShowTableExtended(ShowTableExtendedContext ctx) { - validateAllowed(GrammarElement.SHOW_TABLE_EXTENDED); + validateAllowed(SQLGrammarElement.SHOW_TABLE_EXTENDED); return super.visitShowTableExtended(ctx); } @Override public Void visitShowTables(ShowTablesContext ctx) { - validateAllowed(GrammarElement.SHOW_TABLES); + validateAllowed(SQLGrammarElement.SHOW_TABLES); return super.visitShowTables(ctx); } @Override public Void visitShowTblProperties(ShowTblPropertiesContext ctx) { - validateAllowed(GrammarElement.SHOW_TBLPROPERTIES); + validateAllowed(SQLGrammarElement.SHOW_TBLPROPERTIES); return super.visitShowTblProperties(ctx); } @Override public Void visitShowViews(ShowViewsContext ctx) { - validateAllowed(GrammarElement.SHOW_VIEWS); + validateAllowed(SQLGrammarElement.SHOW_VIEWS); return super.visitShowViews(ctx); } @Override public Void visitUncacheTable(UncacheTableContext ctx) { - validateAllowed(GrammarElement.UNCACHE_TABLE); + validateAllowed(SQLGrammarElement.UNCACHE_TABLE); return super.visitUncacheTable(ctx); } @@ -559,27 +559,27 @@ private void validateFunctionAllowed(String function) { FunctionType type = FunctionType.fromFunctionName(function.toLowerCase()); switch (type) { case MAP: - validateAllowed(GrammarElement.MAP_FUNCTIONS); + validateAllowed(SQLGrammarElement.MAP_FUNCTIONS); break; case BITWISE: - validateAllowed(GrammarElement.BITWISE_FUNCTIONS); + validateAllowed(SQLGrammarElement.BITWISE_FUNCTIONS); break; case CSV: - validateAllowed(GrammarElement.CSV_FUNCTIONS); + validateAllowed(SQLGrammarElement.CSV_FUNCTIONS); break; case MISC: - validateAllowed(GrammarElement.MISC_FUNCTIONS); + validateAllowed(SQLGrammarElement.MISC_FUNCTIONS); break; case GENERATOR: - validateAllowed(GrammarElement.GENERATOR_FUNCTIONS); + validateAllowed(SQLGrammarElement.GENERATOR_FUNCTIONS); break; case UDF: - validateAllowed(GrammarElement.UDF); + validateAllowed(SQLGrammarElement.UDF); break; } } - private void validateAllowed(GrammarElement element) { + private void validateAllowed(SQLGrammarElement element) { if (!grammarElementValidator.isValid(element)) { throw new IllegalArgumentException(element + " is not allowed."); } diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java deleted file mode 100644 index 7dd2b0ee89..0000000000 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spark.validator; - -import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_NAMESPACE; -import static org.opensearch.sql.spark.validator.GrammarElement.ALTER_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.ANALYZE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.CACHE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.CLEAR_CACHE; -import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; -import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; -import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; -import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_QUERY; -import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; -import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; -import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; -import static org.opensearch.sql.spark.validator.GrammarElement.FILE; -import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; -import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; -import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; -import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; -import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; -import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.RESET; -import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; -import static org.opensearch.sql.spark.validator.GrammarElement.SET; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_NAMESPACES; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_PARTITIONS; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLES; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TABLE_EXTENDED; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_TBLPROPERTIES; -import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; -import static org.opensearch.sql.spark.validator.GrammarElement.TABLESAMPLE; -import static org.opensearch.sql.spark.validator.GrammarElement.TABLE_VALUED_FUNCTION; -import static org.opensearch.sql.spark.validator.GrammarElement.TRANSFORM; -import static org.opensearch.sql.spark.validator.GrammarElement.TRUNCATE_TABLE; -import static org.opensearch.sql.spark.validator.GrammarElement.UDF; -import static org.opensearch.sql.spark.validator.GrammarElement.UNCACHE_TABLE; - -import com.google.common.collect.ImmutableSet; -import java.util.Set; - -public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementValidator { - private static final Set<GrammarElement> SECURITY_LAKE_DENY_LIST = - ImmutableSet.<GrammarElement>builder() - .add( - ALTER_NAMESPACE, - ALTER_VIEW, - CREATE_NAMESPACE, - CREATE_FUNCTION, - CREATE_VIEW, - DROP_FUNCTION, - DROP_NAMESPACE, - DROP_VIEW, - REPAIR_TABLE, - TRUNCATE_TABLE, - INSERT, - LOAD, - CLUSTER_BY, - DISTRIBUTE_BY, - HINTS, - INLINE_TABLE, - FILE, - CROSS_JOIN, - LEFT_SEMI_JOIN, - RIGHT_OUTER_JOIN, - FULL_OUTER_JOIN, - LEFT_ANTI_JOIN, - TABLESAMPLE, - TABLE_VALUED_FUNCTION, - TRANSFORM, - MANAGE_RESOURCE, - ANALYZE_TABLE, - CACHE_TABLE, - CLEAR_CACHE, - DESCRIBE_NAMESPACE, - DESCRIBE_FUNCTION, - DESCRIBE_QUERY, - DESCRIBE_TABLE, - REFRESH_RESOURCE, - REFRESH_TABLE, - REFRESH_FUNCTION, - RESET, - SET, - SHOW_COLUMNS, - SHOW_CREATE_TABLE, - SHOW_NAMESPACES, - SHOW_FUNCTIONS, - SHOW_PARTITIONS, - SHOW_TABLE_EXTENDED, - SHOW_TABLES, - SHOW_TBLPROPERTIES, - SHOW_VIEWS, - UNCACHE_TABLE, - CSV_FUNCTIONS, - MISC_FUNCTIONS, - UDF) - .build(); - - public SecurityLakeGrammarElementValidator() { - super(SECURITY_LAKE_DENY_LIST); - } -} diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeSQLGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeSQLGrammarElementValidator.java new file mode 100644 index 0000000000..89af6f31a4 --- /dev/null +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeSQLGrammarElementValidator.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.opensearch.sql.spark.validator.SQLGrammarElement.ALTER_NAMESPACE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.ALTER_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.ANALYZE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CACHE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CLEAR_CACHE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CLUSTER_BY; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CREATE_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CREATE_NAMESPACE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CROSS_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.CSV_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DESCRIBE_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DESCRIBE_NAMESPACE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DESCRIBE_QUERY; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DESCRIBE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DISTRIBUTE_BY; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DROP_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DROP_NAMESPACE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.DROP_VIEW; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.FILE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.FULL_OUTER_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.HINTS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.INLINE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LEFT_SEMI_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.LOAD; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.MANAGE_RESOURCE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.MISC_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REFRESH_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REFRESH_RESOURCE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REFRESH_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.REPAIR_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.RESET; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.RIGHT_OUTER_JOIN; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SET; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_COLUMNS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_CREATE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_FUNCTIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_NAMESPACES; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_PARTITIONS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_TABLES; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_TABLE_EXTENDED; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_TBLPROPERTIES; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.SHOW_VIEWS; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TABLESAMPLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TABLE_VALUED_FUNCTION; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TRANSFORM; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.TRUNCATE_TABLE; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.UDF; +import static org.opensearch.sql.spark.validator.SQLGrammarElement.UNCACHE_TABLE; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +public class SecurityLakeSQLGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set<GrammarElement> SECURITY_LAKE_DENY_LIST = + ImmutableSet.<GrammarElement>builder() + .add( + ALTER_NAMESPACE, + ALTER_VIEW, + CREATE_NAMESPACE, + CREATE_FUNCTION, + CREATE_VIEW, + DROP_FUNCTION, + DROP_NAMESPACE, + DROP_VIEW, + REPAIR_TABLE, + TRUNCATE_TABLE, + INSERT, + LOAD, + CLUSTER_BY, + DISTRIBUTE_BY, + HINTS, + INLINE_TABLE, + FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, + TABLESAMPLE, + TABLE_VALUED_FUNCTION, + TRANSFORM, + MANAGE_RESOURCE, + ANALYZE_TABLE, + CACHE_TABLE, + CLEAR_CACHE, + DESCRIBE_NAMESPACE, + DESCRIBE_FUNCTION, + DESCRIBE_QUERY, + DESCRIBE_TABLE, + REFRESH_RESOURCE, + REFRESH_TABLE, + REFRESH_FUNCTION, + RESET, + SET, + SHOW_COLUMNS, + SHOW_CREATE_TABLE, + SHOW_NAMESPACES, + SHOW_FUNCTIONS, + SHOW_PARTITIONS, + SHOW_TABLE_EXTENDED, + SHOW_TABLES, + SHOW_TBLPROPERTIES, + SHOW_VIEWS, + UNCACHE_TABLE, + CSV_FUNCTIONS, + MISC_FUNCTIONS, + UDF) + .build(); + + public SecurityLakeSQLGrammarElementValidator() { + super(SECURITY_LAKE_DENY_LIST); + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java index 5ef8343dcc..382b560727 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java @@ -88,7 +88,8 @@ import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; -import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.PPLQueryValidator; +import org.opensearch.sql.spark.validator.S3GlueSQLGrammarElementValidator; import org.opensearch.sql.spark.validator.SQLQueryValidator; /** @@ -184,15 +185,20 @@ public void setUp() { SQLQueryValidator sqlQueryValidator = new SQLQueryValidator( new GrammarElementValidatorProvider( - ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueSQLGrammarElementValidator()), new DefaultGrammarElementValidator())); + PPLQueryValidator pplQueryValidator = + new PPLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(), new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider, - sqlQueryValidator); + sqlQueryValidator, + pplQueryValidator); asyncQueryExecutorService = new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 405fdf511d..61d72773d9 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -91,7 +91,8 @@ import org.opensearch.sql.spark.scheduler.AsyncQueryScheduler; import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; -import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.PPLQueryValidator; +import org.opensearch.sql.spark.validator.S3GlueSQLGrammarElementValidator; import org.opensearch.sql.spark.validator.SQLQueryValidator; @ExtendWith(MockitoExtension.class) @@ -120,9 +121,14 @@ public class SparkQueryDispatcherTest { private final SQLQueryValidator sqlQueryValidator = new SQLQueryValidator( new GrammarElementValidatorProvider( - ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueSQLGrammarElementValidator()), new DefaultGrammarElementValidator())); + private final PPLQueryValidator pplQueryValidator = + new PPLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(), new DefaultGrammarElementValidator())); + private DataSourceSparkParameterComposer dataSourceSparkParameterComposer = (datasourceMetadata, sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, "basic"); @@ -175,7 +181,8 @@ void setUp() { sessionManager, queryHandlerFactory, queryIdProvider, - sqlQueryValidator); + sqlQueryValidator, + pplQueryValidator); } @Test @@ -584,7 +591,8 @@ void testDispatchAlterToManualRefreshIndexQuery() { sessionManager, queryHandlerFactory, queryIdProvider, - sqlQueryValidator); + sqlQueryValidator, + pplQueryValidator); String query = "ALTER INDEX elb_and_requestUri ON my_glue.default.http_logs WITH" + " (auto_refresh = false)"; @@ -614,7 +622,8 @@ void testDispatchDropIndexQuery() { sessionManager, queryHandlerFactory, queryIdProvider, - sqlQueryValidator); + sqlQueryValidator, + pplQueryValidator); String query = "DROP INDEX elb_and_requestUri ON my_glue.default.http_logs"; DataSourceMetadata dataSourceMetadata = constructMyGlueDataSourceMetadata(); when(dataSourceService.verifyDataSourceAccessAndGetRawMetadata( diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java index 7d4b255356..3b24c3c8ab 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java @@ -12,25 +12,26 @@ import org.opensearch.sql.datasource.model.DataSourceType; class GrammarElementValidatorProviderTest { - S3GlueGrammarElementValidator s3GlueGrammarElementValidator = new S3GlueGrammarElementValidator(); - SecurityLakeGrammarElementValidator securityLakeGrammarElementValidator = - new SecurityLakeGrammarElementValidator(); + S3GlueSQLGrammarElementValidator s3GlueSQLGrammarElementValidator = + new S3GlueSQLGrammarElementValidator(); + SecurityLakeSQLGrammarElementValidator securityLakeSQLGrammarElementValidator = + new SecurityLakeSQLGrammarElementValidator(); DefaultGrammarElementValidator defaultGrammarElementValidator = new DefaultGrammarElementValidator(); GrammarElementValidatorProvider grammarElementValidatorProvider = new GrammarElementValidatorProvider( ImmutableMap.of( - DataSourceType.S3GLUE, s3GlueGrammarElementValidator, - DataSourceType.SECURITY_LAKE, securityLakeGrammarElementValidator), + DataSourceType.S3GLUE, s3GlueSQLGrammarElementValidator, + DataSourceType.SECURITY_LAKE, securityLakeSQLGrammarElementValidator), defaultGrammarElementValidator); @Test public void test() { assertEquals( - s3GlueGrammarElementValidator, + s3GlueSQLGrammarElementValidator, grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.S3GLUE)); assertEquals( - securityLakeGrammarElementValidator, + securityLakeSQLGrammarElementValidator, grammarElementValidatorProvider.getValidatorForDatasource(DataSourceType.SECURITY_LAKE)); assertEquals( defaultGrammarElementValidator, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/PPLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/PPLQueryValidatorTest.java new file mode 100644 index 0000000000..8d02bb3c72 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/PPLQueryValidatorTest.java @@ -0,0 +1,202 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.validator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.DESCRIBE_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.EXPAND_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.FILLNULL_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.FLATTEN_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.IPADDRESS_FUNCTIONS; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.JOIN_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.JSON_FUNCTIONS; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.LAMBDA_FUNCTIONS; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.LOOKUP_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.PATTERNS_COMMAND; +import static org.opensearch.sql.spark.validator.PPLGrammarElement.SUBQUERY_COMMAND; + +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.antlr.v4.runtime.CommonTokenStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SingleStatementContext; + +@ExtendWith(MockitoExtension.class) +public class PPLQueryValidatorTest { + @Mock GrammarElementValidatorProvider mockedProvider; + + @InjectMocks PPLQueryValidator pplQueryValidator; + + private static final String SOURCE_PREFIX = "source = t | "; + + private enum TestElement { + FIELDS("fields field1, field1"), + WHERE("where field1=\"success\""), + STATS("stats count(), count(`field1`), min(`field1`), max(`field1`)"), + PARSE("parse `field1` \".*/(?<field2>[^/]+$)\""), + PATTERNS("patterns new_field='no_numbers' pattern='[0-9]' message"), + SORT("sort -field1Alias"), + EVAL("eval field2 = `field` * 2"), + RENAME("rename field2 as field1"), + HEAD("head 10"), + GROK("grok email '.+@%{HOSTNAME:host)'"), + TOP("top 2 Field1 by Field2"), + DEDUP("dedup field1"), + JOIN("join on c_custkey = o_custkey orders"), + LOOKUP("lookup account_list mkt_id AS mkt_code REPLACE amount, account_name AS name"), + SUBQUERY("where a > [ source = inner | stats min(c) ]"), + RARE("rare Field1 by Field2"), + TRENDLINE("trendline sma(2, field1) as Field1Alias"), + EVENTSTATS("eventstats sum(field1) by field2"), + FLATTEN("flatten field1"), + FIELD_SUMMARY("fieldsummary includefields=field1 nulls=true"), + FILLNULL("fillnull with 0 in field1"), + EXPAND("expand employee"), + DESCRIBE(false, "describe schema.table"), + STRING_FUNCTIONS("eval cl1Len = LENGTH(col1)"), + DATETIME_FUNCTIONS("eval newDate = ADDDATE(DATE('2020-08-26'), 1)"), + CONDITION_FUNCTIONS("eval field2 = isnull(col1)"), + MATH_FUNCTIONS("eval field2 = ACOS(col1)"), + EXPRESSIONS("where age > (25 + 5)"), + IPADDRESS_FUNCTIONS("where cidrmatch(ip, '192.168.0.1/24')"), + JSON_FUNCTIONS("where cidrmatch(ip, '192.168.0.1/24')"), + LAMBDA_FUNCTIONS("eval array = json_array(1, -1, 2), result = filter(array, x -> x > 0)"), + CRYPTO_FUNCTIONS("eval field1 = MD5('hello')"); + + @Getter private final String[] queries; + + TestElement(String... queries) { + this.queries = addPrefix(queries); + } + + // For describe + TestElement(boolean addPrefix, String... queries) { + this.queries = addPrefix ? addPrefix(queries) : queries; + } + + private String[] addPrefix(String... queries) { + return Arrays.stream(queries).map(query -> SOURCE_PREFIX + query).toArray(String[]::new); + } + } + + @Test + void testAllowAllByDefault() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new DefaultGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(pplQueryValidator, DataSourceType.SPARK); + Arrays.stream(PPLQueryValidatorTest.TestElement.values()).forEach(v::ok); + } + + private static class TestPPLGrammarElementValidator extends DenyListGrammarElementValidator { + private static final Set<GrammarElement> DENY_LIST = + ImmutableSet.<GrammarElement>builder() + .add( + PATTERNS_COMMAND, + JOIN_COMMAND, + LOOKUP_COMMAND, + SUBQUERY_COMMAND, + FLATTEN_COMMAND, + FILLNULL_COMMAND, + EXPAND_COMMAND, + DESCRIBE_COMMAND, + IPADDRESS_FUNCTIONS, + JSON_FUNCTIONS, + LAMBDA_FUNCTIONS) + .build(); + + public TestPPLGrammarElementValidator() { + super(DENY_LIST); + } + } + + @Test + void testCwlValidator() { + when(mockedProvider.getValidatorForDatasource(any())) + .thenReturn(new TestPPLGrammarElementValidator()); + VerifyValidator v = new VerifyValidator(pplQueryValidator, DataSourceType.SPARK); + + v.ok(TestElement.FIELDS); + v.ok(TestElement.WHERE); + v.ok(TestElement.STATS); + v.ok(TestElement.PARSE); + v.ng(TestElement.PATTERNS); + v.ok(TestElement.SORT); + v.ok(TestElement.EVAL); + v.ok(TestElement.RENAME); + v.ok(TestElement.HEAD); + v.ok(TestElement.GROK); + v.ok(TestElement.TOP); + v.ok(TestElement.DEDUP); + v.ng(TestElement.JOIN); + v.ng(TestElement.LOOKUP); + v.ng(TestElement.SUBQUERY); + v.ok(TestElement.RARE); + v.ok(TestElement.TRENDLINE); + v.ok(TestElement.EVENTSTATS); + v.ng(TestElement.FLATTEN); + v.ok(TestElement.FIELD_SUMMARY); + v.ng(TestElement.FILLNULL); + v.ng(TestElement.EXPAND); + v.ng(TestElement.DESCRIBE); + v.ok(TestElement.STRING_FUNCTIONS); + v.ok(TestElement.DATETIME_FUNCTIONS); + v.ok(TestElement.CONDITION_FUNCTIONS); + v.ok(TestElement.MATH_FUNCTIONS); + v.ok(TestElement.EXPRESSIONS); + v.ng(TestElement.IPADDRESS_FUNCTIONS); + v.ng(TestElement.JSON_FUNCTIONS); + v.ng(TestElement.LAMBDA_FUNCTIONS); + v.ok(TestElement.CRYPTO_FUNCTIONS); + } + + @AllArgsConstructor + private static class VerifyValidator { + private final PPLQueryValidator validator; + private final DataSourceType dataSourceType; + + public void ok(PPLQueryValidatorTest.TestElement query) { + runValidate(query.getQueries()); + } + + public void ng(PPLQueryValidatorTest.TestElement element) { + Arrays.stream(element.queries) + .forEach( + query -> + assertThrows( + IllegalArgumentException.class, + () -> runValidate(query), + "The query should throw: query=`" + query.toString() + "`")); + } + + void runValidate(String[] queries) { + Arrays.stream(queries).forEach(query -> validator.validate(query, dataSourceType)); + } + + void runValidate(String query) { + validator.validate(query, dataSourceType); + } + + SingleStatementContext getParser(String query) { + org.opensearch.sql.spark.antlr.parser.SqlBaseParser sqlBaseParser = + new org.opensearch.sql.spark.antlr.parser.SqlBaseParser( + new CommonTokenStream( + new org.opensearch.sql.spark.antlr.parser.SqlBaseLexer( + new CaseInsensitiveCharStream(query)))); + return sqlBaseParser.singleStatement(); + } + } +} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 520fc96ba8..3e4eef52fd 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -332,7 +332,7 @@ void testDenyAllValidator() { @Test void testS3glueQueries() { when(mockedProvider.getValidatorForDatasource(any())) - .thenReturn(new S3GlueGrammarElementValidator()); + .thenReturn(new S3GlueSQLGrammarElementValidator()); VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.S3GLUE); // DDL Statements @@ -449,7 +449,7 @@ void testS3glueQueries() { @Test void testSecurityLakeQueries() { when(mockedProvider.getValidatorForDatasource(any())) - .thenReturn(new SecurityLakeGrammarElementValidator()); + .thenReturn(new SecurityLakeSQLGrammarElementValidator()); VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SECURITY_LAKE); // DDL Statements diff --git a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java index db070182a3..d21f8c7665 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java @@ -67,9 +67,10 @@ import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; -import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.PPLQueryValidator; +import org.opensearch.sql.spark.validator.S3GlueSQLGrammarElementValidator; import org.opensearch.sql.spark.validator.SQLQueryValidator; -import org.opensearch.sql.spark.validator.SecurityLakeGrammarElementValidator; +import org.opensearch.sql.spark.validator.SecurityLakeSQLGrammarElementValidator; @RequiredArgsConstructor public class AsyncExecutorServiceModule extends AbstractModule { @@ -108,9 +109,15 @@ public SparkQueryDispatcher sparkQueryDispatcher( SessionManager sessionManager, QueryHandlerFactory queryHandlerFactory, QueryIdProvider queryIdProvider, - SQLQueryValidator sqlQueryValidator) { + SQLQueryValidator sqlQueryValidator, + PPLQueryValidator pplQueryValidator) { return new SparkQueryDispatcher( - dataSourceService, sessionManager, queryHandlerFactory, queryIdProvider, sqlQueryValidator); + dataSourceService, + sessionManager, + queryHandlerFactory, + queryIdProvider, + sqlQueryValidator, + pplQueryValidator); } @Provides @@ -187,13 +194,21 @@ public SQLQueryValidator sqlQueryValidator() { new GrammarElementValidatorProvider( ImmutableMap.of( DataSourceType.S3GLUE, - new S3GlueGrammarElementValidator(), + new S3GlueSQLGrammarElementValidator(), DataSourceType.SECURITY_LAKE, - new SecurityLakeGrammarElementValidator()), + new SecurityLakeSQLGrammarElementValidator()), new DefaultGrammarElementValidator()); return new SQLQueryValidator(validatorProvider); } + @Provides + public PPLQueryValidator pplQueryValidator() { + GrammarElementValidatorProvider validatorProvider = + new GrammarElementValidatorProvider( + ImmutableMap.of(), new DefaultGrammarElementValidator()); + return new PPLQueryValidator(validatorProvider); + } + @Provides public IndexDMLResultStorageService indexDMLResultStorageService( DataSourceService dataSourceService, StateStore stateStore) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index 72ed17f5aa..53b465aa6d 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -104,7 +104,8 @@ import org.opensearch.sql.spark.scheduler.OpenSearchAsyncQueryScheduler; import org.opensearch.sql.spark.validator.DefaultGrammarElementValidator; import org.opensearch.sql.spark.validator.GrammarElementValidatorProvider; -import org.opensearch.sql.spark.validator.S3GlueGrammarElementValidator; +import org.opensearch.sql.spark.validator.PPLQueryValidator; +import org.opensearch.sql.spark.validator.S3GlueSQLGrammarElementValidator; import org.opensearch.sql.spark.validator.SQLQueryValidator; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.test.OpenSearchIntegTestCase; @@ -315,8 +316,12 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( SQLQueryValidator sqlQueryValidator = new SQLQueryValidator( new GrammarElementValidatorProvider( - ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueGrammarElementValidator()), + ImmutableMap.of(DataSourceType.S3GLUE, new S3GlueSQLGrammarElementValidator()), new DefaultGrammarElementValidator())); + PPLQueryValidator pplQueryValidator = + new PPLQueryValidator( + new GrammarElementValidatorProvider( + ImmutableMap.of(), new DefaultGrammarElementValidator())); SparkQueryDispatcher sparkQueryDispatcher = new SparkQueryDispatcher( this.dataSourceService, @@ -328,7 +333,8 @@ protected AsyncQueryExecutorService createAsyncQueryExecutorService( sessionIdProvider), queryHandlerFactory, new DatasourceEmbeddedQueryIdProvider(), - sqlQueryValidator); + sqlQueryValidator, + pplQueryValidator); return new AsyncQueryExecutorServiceImpl( asyncQueryJobMetadataStorageService, sparkQueryDispatcher, From eb88e3cbd5c5902b3c48f3151c62d0c9b3ff0076 Mon Sep 17 00:00:00 2001 From: Andy Kwok <andy.kwok@improving.com> Date: Mon, 2 Dec 2024 09:22:51 -0800 Subject: [PATCH 83/96] Fix: Date field format parsing for legacy query engine (#3160) * Test cases Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Minimise code changes Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Format Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Update integration test Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Update unit test Signed-off-by: Andy Kwok <andy.kwok@improving.com> --------- Signed-off-by: Andy Kwok <andy.kwok@improving.com> --- .../org/opensearch/sql/legacy/CursorIT.java | 4 ++- .../executor/format/DateFieldFormatter.java | 32 +++++++++++++------ .../format/DateFieldFormatterTest.java | 20 +++++++++++- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index e4ba593844..565c40b121 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -280,7 +280,9 @@ public void testRegressionOnDateFormatChange() throws IOException { Arrays.asList( "2015-01-01 00:00:00.000", "2015-01-01 12:10:30.000", - "1585882955", // by existing design, this is not formatted in MySQL standard format + // Conversion will be applied when dateTime is stored on unix timestamp, + // https://github.com/opensearch-project/sql/pull/3160 + "2020-04-03 03:02:35.000", "2020-04-08 06:10:30.000"); assertThat(actualDateList, equalTo(expectedDateList)); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index dc239abd84..feba13d139 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; +import org.opensearch.sql.legacy.utils.StringUtils; /** Formatter to transform date fields into a consistent format for consumption by clients. */ public class DateFieldFormatter { @@ -83,7 +84,6 @@ public void applyJDBCDateFormat(Map<String, Object> rowSource) { Date date = parseDateString(formats, columnOriginalDate.toString()); if (date != null) { rowSource.put(columnName, DateFormat.getFormattedDate(date, FORMAT_JDBC)); - break; } else { LOG.warn("Could not parse date value; returning original value"); } @@ -152,15 +152,27 @@ private Date parseDateString(List<String> formats, String columnOriginalDate) { switch (columnFormat) { case "date_optional_time": case "strict_date_optional_time": - parsedDate = - DateUtils.parseDate( - columnOriginalDate, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, - FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, - FORMAT_DOT_DATE_AND_TIME, - FORMAT_DOT_DATE); + // It's possible to have date stored in second / millisecond form without explicit + // format hint. + // Parse it on a best-effort basis. + if (StringUtils.isNumeric(columnOriginalDate)) { + long timestamp = Long.parseLong(columnOriginalDate); + if (timestamp > Integer.MAX_VALUE) { + parsedDate = new Date(timestamp); + } else { + parsedDate = new Date(timestamp * 1000); + } + } else { + parsedDate = + DateUtils.parseDate( + columnOriginalDate, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_LOGS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_FLIGHTS_EXCEPTION_NO_TIME, + FORMAT_DOT_OPENSEARCH_DASHBOARDS_SAMPLE_DATA_ECOMMERCE_EXCEPTION, + FORMAT_DOT_DATE_AND_TIME, + FORMAT_DOT_DATE); + } break; case "epoch_millis": parsedDate = new Date(Long.parseLong(columnOriginalDate)); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java index 1c2d1bae62..7d43ea0383 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatterTest.java @@ -575,7 +575,7 @@ public void testIncorrectFormat() { String dateFormat = "date_optional_time"; String originalDateValue = "1581724085"; // Invalid format for date value; should return original value - String expectedDateValue = "1581724085"; + String expectedDateValue = "2020-02-14 23:48:05.000"; verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } @@ -609,6 +609,24 @@ public void testStrictDateOptionalTimeOrEpochMillsShouldPass() { verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); } + @Test + public void testDateInTimestampFormInSecondWithoutHint() { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1732057981"; + String expectedDateValue = "2024-11-19 23:13:01.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + + @Test + public void testDateInTimestampFormInMilliSecondWithoutHint() { + String columnName = "date_field"; + String dateFormat = "date_optional_time"; + String originalDateValue = "1732057981000"; + String expectedDateValue = "2024-11-19 23:13:01.000"; + verifyFormatting(columnName, dateFormat, originalDateValue, expectedDateValue); + } + private void verifyFormatting( String columnName, String dateFormatProperty, From 9d9730bc5556cc2dea9ec01c181998f34df3dc4e Mon Sep 17 00:00:00 2001 From: Andy Kwok <andy.kwok@improving.com> Date: Mon, 2 Dec 2024 11:00:51 -0800 Subject: [PATCH 84/96] Fix: CI Github Action (#3177) * Fix Linux jobs Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Bump Github action to v4 Signed-off-by: Andy Kwok <andy.kwok@improving.com> * Remove unused ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: option Signed-off-by: Andy Kwok <andy.kwok@improving.com> --------- Signed-off-by: Andy Kwok <andy.kwok@improving.com> --- .../workflows/integ-tests-with-security.yml | 16 +++++----- .github/workflows/sql-pitest.yml | 13 ++++---- .../workflows/sql-test-and-build-workflow.yml | 31 ++++++++++--------- .github/workflows/sql-test-workflow.yml | 12 +++---- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 9d214a45cb..e7c52f4231 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -21,21 +21,21 @@ jobs: fail-fast: false matrix: java: [21] - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - uses: actions/checkout@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} @@ -66,10 +66,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index d9034cce9f..fed98e4926 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -22,21 +22,22 @@ jobs: matrix: java: - 21 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: - - uses: actions/checkout@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 0d0c96a367..a13d5ccf6d 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,21 +29,21 @@ jobs: fail-fast: false matrix: java: [21] - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - uses: actions/checkout@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} @@ -61,7 +61,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report if: ${{ always() }} - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: sql-engine @@ -104,10 +104,10 @@ jobs: runs-on: ${{ matrix.entry.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.entry.java }} @@ -123,7 +123,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: sql-engine @@ -163,15 +163,16 @@ jobs: java: [21] container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: --user root - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - uses: actions/checkout@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index f2f22bd44f..9cbec80037 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -22,21 +22,21 @@ jobs: matrix: java: - 21 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - uses: actions/checkout@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} From 6712526d967a47e7685e84b5295b3c42197ac3a0 Mon Sep 17 00:00:00 2001 From: Taylor Curran <taylor.curran@improving.com> Date: Tue, 3 Dec 2024 09:14:26 -0800 Subject: [PATCH 85/96] Add CIDR function to PPL (#3036) (#3110) Signed-off-by: currantw <taylor.curran@improving.com> --- core/build.gradle | 1 + .../sql/exception/QueryEngineException.java | 4 + .../sql/exception/SemanticCheckException.java | 5 + .../org/opensearch/sql/expression/DSL.java | 4 + ...Function.java => AggregatorFunctions.java} | 2 +- ...meFunction.java => DateTimeFunctions.java} | 299 +++++++++--------- .../function/BuiltinFunctionName.java | 3 + .../function/BuiltinFunctionRepository.java | 34 +- .../function/DefaultFunctionResolver.java | 2 +- .../sql/expression/ip/IPFunctions.java | 105 ++++++ ...Function.java => ArithmeticFunctions.java} | 2 +- ...nction.java => MathematicalFunctions.java} | 2 +- ...stOperator.java => TypeCastOperators.java} | 3 +- ...tor.java => BinaryPredicateOperators.java} | 4 +- ...ator.java => UnaryPredicateOperators.java} | 39 ++- .../{TextFunction.java => TextFunctions.java} | 24 +- .../sql/planner/physical/FilterOperator.java | 4 +- .../opensearch/sql/analysis/AnalyzerTest.java | 2 +- .../PercentileApproxAggregatorTest.java | 15 +- .../datetime/DateAddAndAddDateTest.java | 2 +- .../datetime/DateSubAndSubDateTest.java | 2 +- .../expression/datetime/ToSecondsTest.java | 2 +- .../function/DefaultFunctionResolverTest.java | 2 +- .../sql/expression/ip/IPFunctionTest.java | 120 +++++++ .../BinaryPredicateOperatorTest.java | 2 +- .../predicate/UnaryPredicateOperatorTest.java | 4 +- docs/category.json | 9 +- docs/user/dql/metadata.rst | 3 +- docs/user/ppl/functions/condition.rst | 4 +- docs/user/ppl/functions/ip.rst | 38 +++ docs/user/ppl/index.rst | 2 + doctest/test_data/weblogs.json | 6 + doctest/test_docs.py | 5 +- .../org/opensearch/sql/legacy/JdbcTestIT.java | 4 +- .../org/opensearch/sql/ppl/IPFunctionIT.java | 58 ++++ .../weblogs_index_mapping.json | 5 +- integ-test/src/test/resources/weblogs.json | 6 +- .../value/OpenSearchExprValueFactory.java | 4 +- .../data/value/OpenSearchExprIpValueTest.java | 19 +- .../value/OpenSearchExprValueFactoryTest.java | 23 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + 42 files changed, 623 insertions(+), 254 deletions(-) rename core/src/main/java/org/opensearch/sql/expression/aggregation/{AggregatorFunction.java => AggregatorFunctions.java} (99%) rename core/src/main/java/org/opensearch/sql/expression/datetime/{DateTimeFunction.java => DateTimeFunctions.java} (86%) create mode 100644 core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java rename core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/{ArithmeticFunction.java => ArithmeticFunctions.java} (99%) rename core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/{MathematicalFunction.java => MathematicalFunctions.java} (99%) rename core/src/main/java/org/opensearch/sql/expression/operator/convert/{TypeCastOperator.java => TypeCastOperators.java} (99%) rename core/src/main/java/org/opensearch/sql/expression/operator/predicate/{BinaryPredicateOperator.java => BinaryPredicateOperators.java} (98%) rename core/src/main/java/org/opensearch/sql/expression/operator/predicate/{UnaryPredicateOperator.java => UnaryPredicateOperators.java} (83%) rename core/src/main/java/org/opensearch/sql/expression/text/{TextFunction.java => TextFunctions.java} (95%) create mode 100644 core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java create mode 100644 docs/user/ppl/functions/ip.rst create mode 100644 doctest/test_data/weblogs.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java diff --git a/core/build.gradle b/core/build.gradle index f36777030c..c596251342 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -57,6 +57,7 @@ dependencies { api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api project(':common') + implementation "com.github.seancfoley:ipaddress:5.4.2" testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' diff --git a/core/src/main/java/org/opensearch/sql/exception/QueryEngineException.java b/core/src/main/java/org/opensearch/sql/exception/QueryEngineException.java index b3d13bef71..122d4963fa 100644 --- a/core/src/main/java/org/opensearch/sql/exception/QueryEngineException.java +++ b/core/src/main/java/org/opensearch/sql/exception/QueryEngineException.java @@ -11,4 +11,8 @@ public class QueryEngineException extends RuntimeException { public QueryEngineException(String message) { super(message); } + + public QueryEngineException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/core/src/main/java/org/opensearch/sql/exception/SemanticCheckException.java b/core/src/main/java/org/opensearch/sql/exception/SemanticCheckException.java index 6e0c184af8..c43dfdffc8 100644 --- a/core/src/main/java/org/opensearch/sql/exception/SemanticCheckException.java +++ b/core/src/main/java/org/opensearch/sql/exception/SemanticCheckException.java @@ -7,7 +7,12 @@ /** Semantic Check Exception. */ public class SemanticCheckException extends QueryEngineException { + public SemanticCheckException(String message) { super(message); } + + public SemanticCheckException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 9975afac7f..54bd35e70f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -563,6 +563,10 @@ public static FunctionExpression regexp(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.REGEXP, expressions); } + public static FunctionExpression cidrmatch(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.CIDRMATCH, expressions); + } + public static FunctionExpression concat(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.CONCAT, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunctions.java similarity index 99% rename from core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java rename to core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunctions.java index 631eb2e613..698fb20408 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunctions.java @@ -40,7 +40,7 @@ * <em>count</em> accepts values of all types. */ @UtilityClass -public class AggregatorFunction { +public class AggregatorFunctions { /** * Register Aggregation Function. * diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java similarity index 86% rename from core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java rename to core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java index a42a599ad8..411bd27993 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java @@ -101,7 +101,7 @@ */ @UtilityClass @SuppressWarnings("unchecked") -public class DateTimeFunction { +public class DateTimeFunctions { // The number of seconds per day public static final long SECONDS_PER_DAY = 86400; @@ -357,8 +357,8 @@ private DefaultFunctionResolver adddate() { BuiltinFunctionName.ADDDATE.getName(), (SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>[]) (Stream.concat( - get_date_add_date_sub_signatures(DateTimeFunction::exprAddDateInterval), - get_adddate_subdate_signatures(DateTimeFunction::exprAddDateDays)) + get_date_add_date_sub_signatures(DateTimeFunctions::exprAddDateInterval), + get_adddate_subdate_signatures(DateTimeFunctions::exprAddDateDays)) .toArray(SerializableFunction<?, ?>[]::new))); } @@ -375,41 +375,41 @@ private DefaultFunctionResolver addtime() { return define( BuiltinFunctionName.ADDTIME.getName(), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), TIME, TIME, TIME), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIME, TIME, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), TIME, TIME, DATE), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIME, TIME, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIME, TIME, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, DATE, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, DATE, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, DATE, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, TIMESTAMP, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, TIMESTAMP, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprAddTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprAddTime), TIMESTAMP, TIMESTAMP, TIMESTAMP)); @@ -425,13 +425,13 @@ private DefaultFunctionResolver convert_tz() { return define( BuiltinFunctionName.CONVERT_TZ.getName(), impl( - nullMissingHandling(DateTimeFunction::exprConvertTZ), + nullMissingHandling(DateTimeFunctions::exprConvertTZ), TIMESTAMP, TIMESTAMP, STRING, STRING), impl( - nullMissingHandling(DateTimeFunction::exprConvertTZ), + nullMissingHandling(DateTimeFunctions::exprConvertTZ), TIMESTAMP, STRING, STRING, @@ -445,9 +445,9 @@ private DefaultFunctionResolver convert_tz() { private DefaultFunctionResolver date() { return define( BuiltinFunctionName.DATE.getName(), - impl(nullMissingHandling(DateTimeFunction::exprDate), DATE, STRING), - impl(nullMissingHandling(DateTimeFunction::exprDate), DATE, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDate), DATE, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprDate), DATE, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprDate), DATE, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDate), DATE, TIMESTAMP)); } /** @@ -458,35 +458,35 @@ private DefaultFunctionResolver datediff() { return define( BuiltinFunctionName.DATEDIFF.getName(), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), LONG, DATE, DATE), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, DATE, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), LONG, DATE, TIME), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, DATE, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), LONG, TIME, DATE), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIME, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), LONG, TIME, TIME), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIME, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIMESTAMP, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, DATE, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIMESTAMP, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIMESTAMP, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + nullMissingHandlingWithProperties(DateTimeFunctions::exprDateDiff), LONG, TIME, TIMESTAMP)); @@ -501,15 +501,15 @@ private DefaultFunctionResolver datediff() { private FunctionResolver datetime() { return define( BuiltinFunctionName.DATETIME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprDateTime), TIMESTAMP, STRING, STRING), - impl(nullMissingHandling(DateTimeFunction::exprDateTimeNoTimezone), TIMESTAMP, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprDateTime), TIMESTAMP, STRING, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprDateTimeNoTimezone), TIMESTAMP, STRING)); } private DefaultFunctionResolver date_add() { return define( BuiltinFunctionName.DATE_ADD.getName(), (SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>[]) - get_date_add_date_sub_signatures(DateTimeFunction::exprAddDateInterval) + get_date_add_date_sub_signatures(DateTimeFunctions::exprAddDateInterval) .toArray(SerializableFunction<?, ?>[]::new)); } @@ -517,7 +517,7 @@ private DefaultFunctionResolver date_sub() { return define( BuiltinFunctionName.DATE_SUB.getName(), (SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>[]) - get_date_add_date_sub_signatures(DateTimeFunction::exprSubDateInterval) + get_date_add_date_sub_signatures(DateTimeFunctions::exprSubDateInterval) .toArray(SerializableFunction<?, ?>[]::new)); } @@ -525,9 +525,9 @@ private DefaultFunctionResolver date_sub() { private DefaultFunctionResolver day() { return define( BuiltinFunctionName.DAY.getName(), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, STRING)); } /** @@ -537,9 +537,9 @@ private DefaultFunctionResolver day() { private DefaultFunctionResolver dayName() { return define( BuiltinFunctionName.DAYNAME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprDayName), STRING, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDayName), STRING, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprDayName), STRING, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprDayName), STRING, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDayName), STRING, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprDayName), STRING, STRING)); } /** DAYOFMONTH(STRING/DATE/TIMESTAMP). return the day of the month (1-31). */ @@ -549,12 +549,12 @@ private DefaultFunctionResolver dayOfMonth(BuiltinFunctionName name) { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.dayOfMonthToday(functionProperties.getQueryStartClock())), + DateTimeFunctions.dayOfMonthToday(functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfMonth), INTEGER, TIMESTAMP)); } /** @@ -567,12 +567,12 @@ private DefaultFunctionResolver dayOfWeek(FunctionName name) { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.dayOfWeekToday(functionProperties.getQueryStartClock())), + DateTimeFunctions.dayOfWeekToday(functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprDayOfWeek), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprDayOfWeek), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfWeek), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfWeek), INTEGER, STRING)); } /** DAYOFYEAR(STRING/DATE/TIMESTAMP). return the day of the year for date (1-366). */ @@ -582,111 +582,114 @@ private DefaultFunctionResolver dayOfYear(BuiltinFunctionName dayOfYear) { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.dayOfYearToday(functionProperties.getQueryStartClock())), + DateTimeFunctions.dayOfYearToday(functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprDayOfYear), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprDayOfYear), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfYear), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprDayOfYear), INTEGER, STRING)); } private DefaultFunctionResolver extract() { return define( BuiltinFunctionName.EXTRACT.getName(), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprExtractForTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprExtractForTime), LONG, STRING, TIME), - impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, DATE), - impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprExtract), LONG, STRING, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprExtract), LONG, STRING, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprExtract), LONG, STRING, STRING)); } /** FROM_DAYS(LONG). return the date value given the day number N. */ private DefaultFunctionResolver from_days() { return define( BuiltinFunctionName.FROM_DAYS.getName(), - impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG)); + impl(nullMissingHandling(DateTimeFunctions::exprFromDays), DATE, LONG)); } private FunctionResolver from_unixtime() { return define( BuiltinFunctionName.FROM_UNIXTIME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), TIMESTAMP, DOUBLE), + impl(nullMissingHandling(DateTimeFunctions::exprFromUnixTime), TIMESTAMP, DOUBLE), impl( - nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, DOUBLE, STRING)); + nullMissingHandling(DateTimeFunctions::exprFromUnixTimeFormat), + STRING, + DOUBLE, + STRING)); } private DefaultFunctionResolver get_format() { return define( BuiltinFunctionName.GET_FORMAT.getName(), - impl(nullMissingHandling(DateTimeFunction::exprGetFormat), STRING, STRING, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprGetFormat), STRING, STRING, STRING)); } /** HOUR(STRING/TIME/DATE/TIMESTAMP). return the hour value for time. */ private DefaultFunctionResolver hour(BuiltinFunctionName name) { return define( name.getName(), - impl(nullMissingHandling(DateTimeFunction::exprHour), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprHour), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprHour), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprHour), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprHour), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprHour), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprHour), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprHour), INTEGER, TIMESTAMP)); } private DefaultFunctionResolver last_day() { return define( BuiltinFunctionName.LAST_DAY.getName(), - impl(nullMissingHandling(DateTimeFunction::exprLastDay), DATE, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprLastDay), DATE, STRING), implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.exprLastDayToday(functionProperties.getQueryStartClock())), + DateTimeFunctions.exprLastDayToday(functionProperties.getQueryStartClock())), DATE, TIME), - impl(nullMissingHandling(DateTimeFunction::exprLastDay), DATE, DATE), - impl(nullMissingHandling(DateTimeFunction::exprLastDay), DATE, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprLastDay), DATE, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprLastDay), DATE, TIMESTAMP)); } private FunctionResolver makedate() { return define( BuiltinFunctionName.MAKEDATE.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMakeDate), DATE, DOUBLE, DOUBLE)); + impl(nullMissingHandling(DateTimeFunctions::exprMakeDate), DATE, DOUBLE, DOUBLE)); } private FunctionResolver maketime() { return define( BuiltinFunctionName.MAKETIME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMakeTime), TIME, DOUBLE, DOUBLE, DOUBLE)); + impl(nullMissingHandling(DateTimeFunctions::exprMakeTime), TIME, DOUBLE, DOUBLE, DOUBLE)); } /** MICROSECOND(STRING/TIME/TIMESTAMP). return the microsecond value for time. */ private DefaultFunctionResolver microsecond() { return define( BuiltinFunctionName.MICROSECOND.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMicrosecond), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprMicrosecond), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprMicrosecond), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprMicrosecond), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprMicrosecond), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprMicrosecond), INTEGER, TIMESTAMP)); } /** MINUTE(STRING/TIME/TIMESTAMP). return the minute value for time. */ private DefaultFunctionResolver minute(BuiltinFunctionName name) { return define( name.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprMinute), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprMinute), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprMinute), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprMinute), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprMinute), INTEGER, TIMESTAMP)); } /** MINUTE(STRING/TIME/TIMESTAMP). return the minute value for time. */ private DefaultFunctionResolver minute_of_day() { return define( BuiltinFunctionName.MINUTE_OF_DAY.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprMinuteOfDay), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprMinuteOfDay), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprMinuteOfDay), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprMinuteOfDay), INTEGER, TIMESTAMP)); } /** MONTH(STRING/DATE/TIMESTAMP). return the month for date (1-12). */ @@ -696,21 +699,21 @@ private DefaultFunctionResolver month(BuiltinFunctionName month) { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.monthOfYearToday(functionProperties.getQueryStartClock())), + DateTimeFunctions.monthOfYearToday(functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprMonth), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprMonth), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprMonth), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprMonth), INTEGER, STRING)); } /** MONTHNAME(STRING/DATE/TIMESTAMP). return the full name of the month for date. */ private DefaultFunctionResolver monthName() { return define( BuiltinFunctionName.MONTHNAME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprMonthName), STRING, DATE), - impl(nullMissingHandling(DateTimeFunction::exprMonthName), STRING, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprMonthName), STRING, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprMonthName), STRING, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprMonthName), STRING, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprMonthName), STRING, STRING)); } /** @@ -720,7 +723,7 @@ private DefaultFunctionResolver monthName() { private DefaultFunctionResolver period_add() { return define( BuiltinFunctionName.PERIOD_ADD.getName(), - impl(nullMissingHandling(DateTimeFunction::exprPeriodAdd), INTEGER, INTEGER, INTEGER)); + impl(nullMissingHandling(DateTimeFunctions::exprPeriodAdd), INTEGER, INTEGER, INTEGER)); } /** @@ -731,35 +734,35 @@ private DefaultFunctionResolver period_add() { private DefaultFunctionResolver period_diff() { return define( BuiltinFunctionName.PERIOD_DIFF.getName(), - impl(nullMissingHandling(DateTimeFunction::exprPeriodDiff), INTEGER, INTEGER, INTEGER)); + impl(nullMissingHandling(DateTimeFunctions::exprPeriodDiff), INTEGER, INTEGER, INTEGER)); } /** QUARTER(STRING/DATE/TIMESTAMP). return the month for date (1-4). */ private DefaultFunctionResolver quarter() { return define( BuiltinFunctionName.QUARTER.getName(), - impl(nullMissingHandling(DateTimeFunction::exprQuarter), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprQuarter), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprQuarter), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprQuarter), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprQuarter), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprQuarter), INTEGER, STRING)); } private DefaultFunctionResolver sec_to_time() { return define( BuiltinFunctionName.SEC_TO_TIME.getName(), - impl((nullMissingHandling(DateTimeFunction::exprSecToTime)), TIME, INTEGER), - impl((nullMissingHandling(DateTimeFunction::exprSecToTime)), TIME, LONG), - impl((nullMissingHandling(DateTimeFunction::exprSecToTimeWithNanos)), TIME, DOUBLE), - impl((nullMissingHandling(DateTimeFunction::exprSecToTimeWithNanos)), TIME, FLOAT)); + impl((nullMissingHandling(DateTimeFunctions::exprSecToTime)), TIME, INTEGER), + impl((nullMissingHandling(DateTimeFunctions::exprSecToTime)), TIME, LONG), + impl((nullMissingHandling(DateTimeFunctions::exprSecToTimeWithNanos)), TIME, DOUBLE), + impl((nullMissingHandling(DateTimeFunctions::exprSecToTimeWithNanos)), TIME, FLOAT)); } /** SECOND(STRING/TIME/TIMESTAMP). return the second value for time. */ private DefaultFunctionResolver second(BuiltinFunctionName name) { return define( name.getName(), - impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, STRING), - impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprSecond), INTEGER, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprSecond), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprSecond), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprSecond), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprSecond), INTEGER, TIMESTAMP)); } private DefaultFunctionResolver subdate() { @@ -767,8 +770,8 @@ private DefaultFunctionResolver subdate() { BuiltinFunctionName.SUBDATE.getName(), (SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>[]) (Stream.concat( - get_date_add_date_sub_signatures(DateTimeFunction::exprSubDateInterval), - get_adddate_subdate_signatures(DateTimeFunction::exprSubDateDays)) + get_date_add_date_sub_signatures(DateTimeFunctions::exprSubDateInterval), + get_adddate_subdate_signatures(DateTimeFunctions::exprSubDateDays)) .toArray(SerializableFunction<?, ?>[]::new))); } @@ -785,41 +788,41 @@ private DefaultFunctionResolver subtime() { return define( BuiltinFunctionName.SUBTIME.getName(), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), TIME, TIME, TIME), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIME, TIME, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), TIME, TIME, DATE), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIME, TIME, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIME, TIME, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, TIMESTAMP, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, TIMESTAMP, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, DATE, TIME), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, DATE, DATE), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, DATE, TIMESTAMP), implWithProperties( - nullMissingHandlingWithProperties(DateTimeFunction::exprSubTime), + nullMissingHandlingWithProperties(DateTimeFunctions::exprSubTime), TIMESTAMP, TIMESTAMP, TIMESTAMP)); @@ -835,7 +838,7 @@ private DefaultFunctionResolver str_to_date() { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg, format) -> - DateTimeFunction.exprStrToDate(functionProperties, arg, format)), + DateTimeFunctions.exprStrToDate(functionProperties, arg, format)), TIMESTAMP, STRING, STRING)); @@ -848,10 +851,10 @@ private DefaultFunctionResolver str_to_date() { private DefaultFunctionResolver time() { return define( BuiltinFunctionName.TIME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprTime), TIME, STRING), - impl(nullMissingHandling(DateTimeFunction::exprTime), TIME, DATE), - impl(nullMissingHandling(DateTimeFunction::exprTime), TIME, TIME), - impl(nullMissingHandling(DateTimeFunction::exprTime), TIME, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprTime), TIME, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprTime), TIME, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprTime), TIME, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprTime), TIME, TIMESTAMP)); } /** @@ -867,16 +870,16 @@ private DefaultFunctionResolver time() { private DefaultFunctionResolver timediff() { return define( BuiltinFunctionName.TIMEDIFF.getName(), - impl(nullMissingHandling(DateTimeFunction::exprTimeDiff), TIME, TIME, TIME)); + impl(nullMissingHandling(DateTimeFunctions::exprTimeDiff), TIME, TIME, TIME)); } /** TIME_TO_SEC(STRING/TIME/TIMESTAMP). return the time argument, converted to seconds. */ private DefaultFunctionResolver time_to_sec() { return define( BuiltinFunctionName.TIME_TO_SEC.getName(), - impl(nullMissingHandling(DateTimeFunction::exprTimeToSec), LONG, STRING), - impl(nullMissingHandling(DateTimeFunction::exprTimeToSec), LONG, TIME), - impl(nullMissingHandling(DateTimeFunction::exprTimeToSec), LONG, TIMESTAMP)); + impl(nullMissingHandling(DateTimeFunctions::exprTimeToSec), LONG, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprTimeToSec), LONG, TIME), + impl(nullMissingHandling(DateTimeFunctions::exprTimeToSec), LONG, TIMESTAMP)); } /** @@ -914,7 +917,7 @@ private DefaultFunctionResolver timestampadd() { return define( BuiltinFunctionName.TIMESTAMPADD.getName(), impl( - nullMissingHandling(DateTimeFunction::exprTimestampAdd), + nullMissingHandling(DateTimeFunctions::exprTimestampAdd), TIMESTAMP, STRING, INTEGER, @@ -943,7 +946,7 @@ private DefaultFunctionResolver timestampdiff() { return define( BuiltinFunctionName.TIMESTAMPDIFF.getName(), impl( - nullMissingHandling(DateTimeFunction::exprTimestampDiff), + nullMissingHandling(DateTimeFunctions::exprTimestampDiff), TIMESTAMP, STRING, TIMESTAMP, @@ -962,9 +965,9 @@ private DefaultFunctionResolver timestampdiff() { private DefaultFunctionResolver to_days() { return define( BuiltinFunctionName.TO_DAYS.getName(), - impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, STRING), - impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATE)); + impl(nullMissingHandling(DateTimeFunctions::exprToDays), LONG, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprToDays), LONG, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprToDays), LONG, DATE)); } /** @@ -975,8 +978,8 @@ private DefaultFunctionResolver to_days() { private DefaultFunctionResolver to_seconds() { return define( BuiltinFunctionName.TO_SECONDS.getName(), - impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprToSecondsForIntType), LONG, LONG)); + impl(nullMissingHandling(DateTimeFunctions::exprToSeconds), LONG, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprToSecondsForIntType), LONG, LONG)); } private FunctionResolver unix_timestamp() { @@ -984,11 +987,11 @@ private FunctionResolver unix_timestamp() { BuiltinFunctionName.UNIX_TIMESTAMP.getName(), implWithProperties( functionProperties -> - DateTimeFunction.unixTimeStamp(functionProperties.getQueryStartClock()), + DateTimeFunctions.unixTimeStamp(functionProperties.getQueryStartClock()), LONG), - impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE), - impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DOUBLE)); + impl(nullMissingHandling(DateTimeFunctions::unixTimeStampOf), DOUBLE, DATE), + impl(nullMissingHandling(DateTimeFunctions::unixTimeStampOf), DOUBLE, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::unixTimeStampOf), DOUBLE, DOUBLE)); } /** UTC_DATE(). return the current UTC Date in format yyyy-MM-dd */ @@ -1019,24 +1022,24 @@ private DefaultFunctionResolver week(BuiltinFunctionName week) { implWithProperties( nullMissingHandlingWithProperties( (functionProperties, arg) -> - DateTimeFunction.weekOfYearToday( + DateTimeFunctions.weekOfYearToday( DEFAULT_WEEK_OF_YEAR_MODE, functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprWeekWithoutMode), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprWeekWithoutMode), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprWeekWithoutMode), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprWeekWithoutMode), INTEGER, STRING), implWithProperties( nullMissingHandlingWithProperties( (functionProperties, time, modeArg) -> - DateTimeFunction.weekOfYearToday( + DateTimeFunctions.weekOfYearToday( modeArg, functionProperties.getQueryStartClock())), INTEGER, TIME, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, DATE, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, TIMESTAMP, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprWeek), INTEGER, STRING, INTEGER)); + impl(nullMissingHandling(DateTimeFunctions::exprWeek), INTEGER, DATE, INTEGER), + impl(nullMissingHandling(DateTimeFunctions::exprWeek), INTEGER, TIMESTAMP, INTEGER), + impl(nullMissingHandling(DateTimeFunctions::exprWeek), INTEGER, STRING, INTEGER)); } private DefaultFunctionResolver weekday() { @@ -1050,18 +1053,18 @@ private DefaultFunctionResolver weekday() { - 1)), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprWeekday), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprWeekday), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprWeekday), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprWeekday), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprWeekday), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprWeekday), INTEGER, STRING)); } /** YEAR(STRING/DATE/TIMESTAMP). return the year for date (1000-9999). */ private DefaultFunctionResolver year() { return define( BuiltinFunctionName.YEAR.getName(), - impl(nullMissingHandling(DateTimeFunction::exprYear), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprYear), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprYear), INTEGER, STRING)); + impl(nullMissingHandling(DateTimeFunctions::exprYear), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprYear), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprYear), INTEGER, STRING)); } /** YEARWEEK(DATE[,mode]). return the week number for date. */ @@ -1075,9 +1078,9 @@ private DefaultFunctionResolver yearweek() { DEFAULT_WEEK_OF_YEAR_MODE, functionProperties.getQueryStartClock())), INTEGER, TIME), - impl(nullMissingHandling(DateTimeFunction::exprYearweekWithoutMode), INTEGER, DATE), - impl(nullMissingHandling(DateTimeFunction::exprYearweekWithoutMode), INTEGER, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprYearweekWithoutMode), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunctions::exprYearweekWithoutMode), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunctions::exprYearweekWithoutMode), INTEGER, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunctions::exprYearweekWithoutMode), INTEGER, STRING), implWithProperties( nullMissingHandlingWithProperties( (functionProperties, time, modeArg) -> @@ -1085,9 +1088,9 @@ private DefaultFunctionResolver yearweek() { INTEGER, TIME, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprYearweek), INTEGER, DATE, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprYearweek), INTEGER, TIMESTAMP, INTEGER), - impl(nullMissingHandling(DateTimeFunction::exprYearweek), INTEGER, STRING, INTEGER)); + impl(nullMissingHandling(DateTimeFunctions::exprYearweek), INTEGER, DATE, INTEGER), + impl(nullMissingHandling(DateTimeFunctions::exprYearweek), INTEGER, TIMESTAMP, INTEGER), + impl(nullMissingHandling(DateTimeFunctions::exprYearweek), INTEGER, STRING, INTEGER)); } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index fd5ea14a2e..a67308c96a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -132,6 +132,9 @@ public enum BuiltinFunctionName { /** Text Functions. */ TOSTRING(FunctionName.of("tostring")), + /** IP Functions. */ + CIDRMATCH(FunctionName.of("cidrmatch")), + /** Arithmetic Operators. */ ADD(FunctionName.of("+")), ADDFUNCTION(FunctionName.of("add")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 2e16d5f01f..79ea58b860 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -24,16 +24,17 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.aggregation.AggregatorFunction; -import org.opensearch.sql.expression.datetime.DateTimeFunction; +import org.opensearch.sql.expression.aggregation.AggregatorFunctions; +import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.datetime.IntervalClause; -import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunction; -import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunction; -import org.opensearch.sql.expression.operator.convert.TypeCastOperator; -import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator; -import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator; +import org.opensearch.sql.expression.ip.IPFunctions; +import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions; +import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions; +import org.opensearch.sql.expression.operator.convert.TypeCastOperators; +import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperators; +import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperators; import org.opensearch.sql.expression.system.SystemFunctions; -import org.opensearch.sql.expression.text.TextFunction; +import org.opensearch.sql.expression.text.TextFunctions; import org.opensearch.sql.expression.window.WindowFunctions; import org.opensearch.sql.storage.StorageEngine; @@ -69,18 +70,19 @@ public static synchronized BuiltinFunctionRepository getInstance() { instance = new BuiltinFunctionRepository(new HashMap<>()); // Register all built-in functions - ArithmeticFunction.register(instance); - BinaryPredicateOperator.register(instance); - MathematicalFunction.register(instance); - UnaryPredicateOperator.register(instance); - AggregatorFunction.register(instance); - DateTimeFunction.register(instance); + ArithmeticFunctions.register(instance); + BinaryPredicateOperators.register(instance); + MathematicalFunctions.register(instance); + UnaryPredicateOperators.register(instance); + AggregatorFunctions.register(instance); + DateTimeFunctions.register(instance); IntervalClause.register(instance); WindowFunctions.register(instance); - TextFunction.register(instance); - TypeCastOperator.register(instance); + TextFunctions.register(instance); + TypeCastOperators.register(instance); SystemFunctions.register(instance); OpenSearchFunctions.register(instance); + IPFunctions.register(instance); } return instance; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/DefaultFunctionResolver.java b/core/src/main/java/org/opensearch/sql/expression/function/DefaultFunctionResolver.java index 5d0f31594b..e1d0052723 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/DefaultFunctionResolver.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/DefaultFunctionResolver.java @@ -61,7 +61,7 @@ public Pair<FunctionSignature, FunctionBuilder> resolve(FunctionSignature unreso && !FunctionSignature.isVarArgFunction(bestMatchEntry.getValue().getParamTypeList())) { throw new ExpressionEvaluationException( String.format( - "%s function expected %s, but get %s", + "%s function expected %s, but got %s", functionName, formatFunctions(functionBundle.keySet()), unresolvedSignature.formatTypes())); diff --git a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java new file mode 100644 index 0000000000..b3e7fad211 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.ip; + +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.expression.function.FunctionDSL.define; +import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; + +import inet.ipaddr.AddressStringException; +import inet.ipaddr.IPAddressString; +import inet.ipaddr.IPAddressStringParameters; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + +/** Utility class that defines and registers IP functions. */ +@UtilityClass +public class IPFunctions { + + public void register(BuiltinFunctionRepository repository) { + repository.register(cidrmatch()); + } + + private DefaultFunctionResolver cidrmatch() { + + // TODO #3145: Add support for IP address data type. + return define( + BuiltinFunctionName.CIDRMATCH.getName(), + impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, STRING, STRING)); + } + + /** + * Returns whether the given IP address is within the specified inclusive CIDR IP address range. + * Supports both IPv4 and IPv6 addresses. + * + * @param addressExprValue IP address as a string (e.g. "198.51.100.14" or + * "2001:0db8::ff00:42:8329"). + * @param rangeExprValue IP address range in CIDR notation as a string (e.g. "198.51.100.0/24" or + * "2001:0db8::/32") + * @return true if the address is in the range; otherwise false. + * @throws SemanticCheckException if the address or range is not valid, or if they do not use the + * same version (IPv4 or IPv6). + */ + private ExprValue exprCidrMatch(ExprValue addressExprValue, ExprValue rangeExprValue) { + + // TODO #3145: Update to support IP address data type. + String addressString = addressExprValue.stringValue(); + String rangeString = rangeExprValue.stringValue(); + + final IPAddressStringParameters validationOptions = + new IPAddressStringParameters.Builder() + .allowEmpty(false) + .setEmptyAsLoopback(false) + .allow_inet_aton(false) + .allowSingleSegment(false) + .toParams(); + + // Get and validate IP address. + IPAddressString address = + new IPAddressString(addressExprValue.stringValue(), validationOptions); + + try { + address.validate(); + } catch (AddressStringException e) { + String msg = + String.format( + "IP address '%s' is not valid. Error details: %s", addressString, e.getMessage()); + throw new SemanticCheckException(msg, e); + } + + // Get and validate CIDR IP address range. + IPAddressString range = new IPAddressString(rangeExprValue.stringValue(), validationOptions); + + try { + range.validate(); + } catch (AddressStringException e) { + String msg = + String.format( + "CIDR IP address range '%s' is not valid. Error details: %s", + rangeString, e.getMessage()); + throw new SemanticCheckException(msg, e); + } + + // Address and range must use the same IP version (IPv4 or IPv6). + if (address.isIPv4() ^ range.isIPv4()) { + String msg = + String.format( + "IP address '%s' and CIDR IP address range '%s' are not compatible. Both must be" + + " either IPv4 or IPv6.", + addressString, rangeString); + throw new SemanticCheckException(msg); + } + + return ExprValueUtils.booleanValue(range.contains(address)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunction.java b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctions.java similarity index 99% rename from core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunction.java rename to core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctions.java index 82b91e1d34..164de6d74c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctions.java @@ -37,7 +37,7 @@ * module, Accepts two numbers and produces a number. */ @UtilityClass -public class ArithmeticFunction { +public class ArithmeticFunctions { /** * Register Arithmetic Function. * diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctions.java similarity index 99% rename from core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java rename to core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctions.java index 22f4b76573..102834f60d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctions.java @@ -46,7 +46,7 @@ import org.opensearch.sql.expression.function.SerializableFunction; @UtilityClass -public class MathematicalFunction { +public class MathematicalFunctions { /** * Register Mathematical Functions. * diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java similarity index 99% rename from core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java rename to core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index db4b29f3b9..55e223d94c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -42,7 +42,8 @@ import org.opensearch.sql.expression.function.FunctionDSL; @UtilityClass -public class TypeCastOperator { +public class TypeCastOperators { + /** Register Type Cast Operator. */ public static void register(BuiltinFunctionRepository repository) { repository.register(castToString()); diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java similarity index 98% rename from core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperator.java rename to core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java index bf6b3c22f5..96ff7785b7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java @@ -36,7 +36,7 @@ * equalTo, Compare the left expression and right expression and produces a Boolean. */ @UtilityClass -public class BinaryPredicateOperator { +public class BinaryPredicateOperators { /** * Register Binary Predicate Function. * @@ -401,7 +401,7 @@ private static DefaultFunctionResolver notLike() { BuiltinFunctionName.NOT_LIKE.getName(), impl( nullMissingHandling( - (v1, v2) -> UnaryPredicateOperator.not(OperatorUtils.matches(v1, v2))), + (v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches(v1, v2))), BOOLEAN, STRING, STRING)); diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperators.java similarity index 83% rename from core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperator.java rename to core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperators.java index ad9d9ac934..07bb5b2299 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperators.java @@ -30,7 +30,8 @@ * The definition of unary predicate function not, Accepts one Boolean value and produces a Boolean. */ @UtilityClass -public class UnaryPredicateOperator { +public class UnaryPredicateOperators { + /** Register Unary Predicate Function. */ public static void register(BuiltinFunctionRepository repository) { repository.register(not()); @@ -45,7 +46,7 @@ public static void register(BuiltinFunctionRepository repository) { private static DefaultFunctionResolver not() { return FunctionDSL.define( BuiltinFunctionName.NOT.getName(), - FunctionDSL.impl(UnaryPredicateOperator::not, BOOLEAN, BOOLEAN)); + FunctionDSL.impl(UnaryPredicateOperators::not, BOOLEAN, BOOLEAN)); } /** @@ -108,11 +109,10 @@ private static DefaultFunctionResolver ifFunction() { org.apache.commons.lang3.tuple.Pair<FunctionSignature, FunctionBuilder>>> functionsOne = typeList.stream() - .map(v -> impl((UnaryPredicateOperator::exprIf), v, BOOLEAN, v, v)) + .map(v -> impl((UnaryPredicateOperators::exprIf), v, BOOLEAN, v, v)) .collect(Collectors.toList()); - DefaultFunctionResolver functionResolver = FunctionDSL.define(functionName, functionsOne); - return functionResolver; + return FunctionDSL.define(functionName, functionsOne); } private static DefaultFunctionResolver ifNull() { @@ -125,31 +125,28 @@ private static DefaultFunctionResolver ifNull() { org.apache.commons.lang3.tuple.Pair<FunctionSignature, FunctionBuilder>>> functionsOne = typeList.stream() - .map(v -> impl((UnaryPredicateOperator::exprIfNull), v, v, v)) + .map(v -> impl((UnaryPredicateOperators::exprIfNull), v, v, v)) .collect(Collectors.toList()); - DefaultFunctionResolver functionResolver = FunctionDSL.define(functionName, functionsOne); - return functionResolver; + return FunctionDSL.define(functionName, functionsOne); } private static DefaultFunctionResolver nullIf() { FunctionName functionName = BuiltinFunctionName.NULLIF.getName(); List<ExprCoreType> typeList = ExprCoreType.coreTypes(); - DefaultFunctionResolver functionResolver = - FunctionDSL.define( - functionName, - typeList.stream() - .map(v -> impl((UnaryPredicateOperator::exprNullIf), v, v, v)) - .collect(Collectors.toList())); - return functionResolver; + return FunctionDSL.define( + functionName, + typeList.stream() + .map(v -> impl((UnaryPredicateOperators::exprNullIf), v, v, v)) + .collect(Collectors.toList())); } /** * v2 if v1 is null. * - * @param v1 varable 1 - * @param v2 varable 2 + * @param v1 variable 1 + * @param v2 variable 2 * @return v2 if v1 is null */ public static ExprValue exprIfNull(ExprValue v1, ExprValue v2) { @@ -157,11 +154,11 @@ public static ExprValue exprIfNull(ExprValue v1, ExprValue v2) { } /** - * return null if v1 equls to v2. + * return null if v1 equals to v2. * - * @param v1 varable 1 - * @param v2 varable 2 - * @return null if v1 equls to v2 + * @param v1 variable 1 + * @param v2 variable 2 + * @return null if v1 equals to v2 */ public static ExprValue exprNullIf(ExprValue v1, ExprValue v2) { return v1.equals(v2) ? LITERAL_NULL : v1; diff --git a/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java b/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java similarity index 95% rename from core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java rename to core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java index d670843551..8a5302070c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java @@ -38,7 +38,7 @@ * implementation should rely on ExprValue. */ @UtilityClass -public class TextFunction { +public class TextFunctions { private static String EMPTY_STRING = ""; /** @@ -76,9 +76,9 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver substringSubstr(FunctionName functionName) { return define( functionName, - impl(nullMissingHandling(TextFunction::exprSubstrStart), STRING, STRING, INTEGER), + impl(nullMissingHandling(TextFunctions::exprSubstrStart), STRING, STRING, INTEGER), impl( - nullMissingHandling(TextFunction::exprSubstrStartLength), + nullMissingHandling(TextFunctions::exprSubstrStartLength), STRING, STRING, INTEGER, @@ -267,7 +267,7 @@ private DefaultFunctionResolver strcmp() { private DefaultFunctionResolver right() { return define( BuiltinFunctionName.RIGHT.getName(), - impl(nullMissingHandling(TextFunction::exprRight), STRING, STRING, INTEGER)); + impl(nullMissingHandling(TextFunctions::exprRight), STRING, STRING, INTEGER)); } /** @@ -279,7 +279,7 @@ private DefaultFunctionResolver right() { private DefaultFunctionResolver left() { return define( BuiltinFunctionName.LEFT.getName(), - impl(nullMissingHandling(TextFunction::exprLeft), STRING, STRING, INTEGER)); + impl(nullMissingHandling(TextFunctions::exprLeft), STRING, STRING, INTEGER)); } /** @@ -292,7 +292,7 @@ private DefaultFunctionResolver left() { private DefaultFunctionResolver ascii() { return define( BuiltinFunctionName.ASCII.getName(), - impl(nullMissingHandling(TextFunction::exprAscii), INTEGER, STRING)); + impl(nullMissingHandling(TextFunctions::exprAscii), INTEGER, STRING)); } /** @@ -310,14 +310,15 @@ private DefaultFunctionResolver locate() { BuiltinFunctionName.LOCATE.getName(), impl( nullMissingHandling( - (SerializableBiFunction<ExprValue, ExprValue, ExprValue>) TextFunction::exprLocate), + (SerializableBiFunction<ExprValue, ExprValue, ExprValue>) + TextFunctions::exprLocate), INTEGER, STRING, STRING), impl( nullMissingHandling( (SerializableTriFunction<ExprValue, ExprValue, ExprValue, ExprValue>) - TextFunction::exprLocate), + TextFunctions::exprLocate), INTEGER, STRING, STRING, @@ -337,7 +338,8 @@ private DefaultFunctionResolver position() { BuiltinFunctionName.POSITION.getName(), impl( nullMissingHandling( - (SerializableBiFunction<ExprValue, ExprValue, ExprValue>) TextFunction::exprLocate), + (SerializableBiFunction<ExprValue, ExprValue, ExprValue>) + TextFunctions::exprLocate), INTEGER, STRING, STRING)); @@ -353,7 +355,7 @@ private DefaultFunctionResolver position() { private DefaultFunctionResolver replace() { return define( BuiltinFunctionName.REPLACE.getName(), - impl(nullMissingHandling(TextFunction::exprReplace), STRING, STRING, STRING, STRING)); + impl(nullMissingHandling(TextFunctions::exprReplace), STRING, STRING, STRING, STRING)); } /** @@ -365,7 +367,7 @@ private DefaultFunctionResolver replace() { private DefaultFunctionResolver reverse() { return define( BuiltinFunctionName.REVERSE.getName(), - impl(nullMissingHandling(TextFunction::exprReverse), STRING, STRING)); + impl(nullMissingHandling(TextFunctions::exprReverse), STRING, STRING)); } private static ExprValue exprSubstrStart(ExprValue exprValue, ExprValue start) { diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java index ec61d53163..192ea5cb4f 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java @@ -13,13 +13,13 @@ import lombok.ToString; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator; +import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperators; import org.opensearch.sql.storage.bindingtuple.BindingTuple; /** * The Filter operator represents WHERE clause and uses the conditions to evaluate the input {@link * BindingTuple}. The Filter operator only returns the results that evaluated to true. The NULL and - * MISSING are handled by the logic defined in {@link BinaryPredicateOperator}. + * MISSING are handled by the logic defined in {@link BinaryPredicateOperators}. */ @EqualsAndHashCode(callSuper = false) @ToString diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 8d935b11d2..2412bd9474 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -158,7 +158,7 @@ public void filter_relation_with_invalid_qualifiedName_ExpressionEvaluationExcep "= function expected {[BYTE,BYTE],[SHORT,SHORT],[INTEGER,INTEGER],[LONG,LONG]," + "[FLOAT,FLOAT],[DOUBLE,DOUBLE],[STRING,STRING],[BOOLEAN,BOOLEAN],[DATE,DATE]," + "[TIME,TIME],[TIMESTAMP,TIMESTAMP],[INTERVAL,INTERVAL]," - + "[STRUCT,STRUCT],[ARRAY,ARRAY]}, but get [STRING,INTEGER]", + + "[STRUCT,STRUCT],[ARRAY,ARRAY]}, but got [STRING,INTEGER]", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java index ac617e7b32..33fc325204 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java @@ -13,11 +13,18 @@ package org.opensearch.sql.expression.aggregation; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.model.ExprValueUtils.*; -import static org.opensearch.sql.data.type.ExprCoreType.*; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import java.util.ArrayList; import java.util.List; @@ -195,7 +202,7 @@ public void test_percentile_with_invalid_size() { "percentile_approx function expected" + " {[INTEGER,DOUBLE],[INTEGER,DOUBLE,DOUBLE],[LONG,DOUBLE],[LONG,DOUBLE,DOUBLE]," + "[FLOAT,DOUBLE],[FLOAT,DOUBLE,DOUBLE],[DOUBLE,DOUBLE],[DOUBLE,DOUBLE,DOUBLE]}," - + " but get [DOUBLE,STRING]", + + " but got [DOUBLE,STRING]", exception2.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateAddAndAddDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateAddAndAddDateTest.java index 519e97bdc6..b4ab3a8567 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateAddAndAddDateTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateAddAndAddDateTest.java @@ -156,7 +156,7 @@ public void adddate_has_second_signature_but_not_date_add() { () -> date_add(LocalDateTime.of(1961, 4, 12, 9, 7), 100500)); assertEquals( "date_add function expected {[DATE,INTERVAL],[TIMESTAMP,INTERVAL]," - + "[TIME,INTERVAL]}, but get [TIMESTAMP,INTEGER]", + + "[TIME,INTERVAL]}, but got [TIMESTAMP,INTEGER]", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java index 123ecda0bd..897f49cfee 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java @@ -139,7 +139,7 @@ public void subdate_has_second_signature_but_not_date_sub() { ExpressionEvaluationException.class, () -> date_sub(LocalDateTime.of(1961, 4, 12, 9, 7), 100500)); assertEquals( - "date_sub function expected {[DATE,INTERVAL],[TIMESTAMP,INTERVAL],[TIME,INTERVAL]}, but get" + "date_sub function expected {[DATE,INTERVAL],[TIMESTAMP,INTERVAL],[TIME,INTERVAL]}, but got" + " [TIMESTAMP,INTEGER]", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index 910fe42a52..e983eb28f6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.LONG; -import static org.opensearch.sql.expression.datetime.DateTimeFunction.SECONDS_PER_DAY; +import static org.opensearch.sql.expression.datetime.DateTimeFunctions.SECONDS_PER_DAY; import java.time.Duration; import java.time.LocalDate; diff --git a/core/src/test/java/org/opensearch/sql/expression/function/DefaultFunctionResolverTest.java b/core/src/test/java/org/opensearch/sql/expression/function/DefaultFunctionResolverTest.java index ad9e8a6661..0c0439a764 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/DefaultFunctionResolverTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/DefaultFunctionResolverTest.java @@ -72,7 +72,7 @@ void resolve_function_not_match() { assertThrows( ExpressionEvaluationException.class, () -> resolver.resolve(functionSignature)); assertEquals( - "add function expected {[INTEGER,INTEGER]}, but get [BOOLEAN,BOOLEAN]", + "add function expected {[INTEGER,INTEGER]}, but got [BOOLEAN,BOOLEAN]", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java new file mode 100644 index 0000000000..b50bf9fd1f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.ip; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; + +@ExtendWith(MockitoExtension.class) +public class IPFunctionTest { + + // IP range and address constants for testing. + private static final ExprValue IPv4Range = ExprValueUtils.stringValue("198.51.100.0/24"); + private static final ExprValue IPv6Range = ExprValueUtils.stringValue("2001:0db8::/32"); + + // TODO #3145: Add tests for IP address data type. + private static final ExprValue IPv4AddressBelow = ExprValueUtils.stringValue("198.51.99.1"); + private static final ExprValue IPv4AddressWithin = ExprValueUtils.stringValue("198.51.100.1"); + private static final ExprValue IPv4AddressAbove = ExprValueUtils.stringValue("198.51.101.2"); + + private static final ExprValue IPv6AddressBelow = + ExprValueUtils.stringValue("2001:0db7::ff00:42:8329"); + private static final ExprValue IPv6AddressWithin = + ExprValueUtils.stringValue("2001:0db8::ff00:42:8329"); + private static final ExprValue IPv6AddressAbove = + ExprValueUtils.stringValue("2001:0db9::ff00:42:8329"); + + // Mock value environment for testing. + @Mock private Environment<Expression, ExprValue> env; + + @Test + public void cidrmatch_invalid_address() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> execute(ExprValueUtils.stringValue("INVALID"), IPv4Range)); + assertTrue( + exception.getMessage().matches("IP address 'INVALID' is not valid. Error details: .*")); + } + + @Test + public void cidrmatch_invalid_range() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> execute(IPv4AddressWithin, ExprValueUtils.stringValue("INVALID"))); + assertTrue( + exception + .getMessage() + .matches("CIDR IP address range 'INVALID' is not valid. Error details: .*")); + } + + @Test + public void cidrmatch_different_versions() { + SemanticCheckException exception; + + exception = + assertThrows(SemanticCheckException.class, () -> execute(IPv4AddressWithin, IPv6Range)); + assertEquals( + "IP address '198.51.100.1' and CIDR IP address range '2001:0db8::/32' are not compatible." + + " Both must be either IPv4 or IPv6.", + exception.getMessage()); + + exception = + assertThrows(SemanticCheckException.class, () -> execute(IPv6AddressWithin, IPv4Range)); + assertEquals( + "IP address '2001:0db8::ff00:42:8329' and CIDR IP address range '198.51.100.0/24' are not" + + " compatible. Both must be either IPv4 or IPv6.", + exception.getMessage()); + } + + @Test + public void cidrmatch_valid_ipv4() { + assertEquals(LITERAL_FALSE, execute(IPv4AddressBelow, IPv4Range)); + assertEquals(LITERAL_TRUE, execute(IPv4AddressWithin, IPv4Range)); + assertEquals(LITERAL_FALSE, execute(IPv4AddressAbove, IPv4Range)); + } + + @Test + public void cidrmatch_valid_ipv6() { + assertEquals(LITERAL_FALSE, execute(IPv6AddressBelow, IPv6Range)); + assertEquals(LITERAL_TRUE, execute(IPv6AddressWithin, IPv6Range)); + assertEquals(LITERAL_FALSE, execute(IPv6AddressAbove, IPv6Range)); + } + + /** + * Builds and evaluates a CIDR function expression with the given field and range expression + * values, and returns the resulting value. + */ + private ExprValue execute(ExprValue field, ExprValue range) { + + final String fieldName = "ip_address"; + FunctionExpression exp = DSL.cidrmatch(DSL.ref(fieldName, STRING), DSL.literal(range)); + + // Mock the value environment to return the specified field + // expression as the value for the "ip_address" field. + when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(field); + + return exp.valueOf(env); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 55dfbd35c2..19cbb4674e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -584,7 +584,7 @@ void testRegexpString(StringPatternPair stringPatternPair) { assertEquals(stringPatternPair.regExpTest(), expression.valueOf(valueEnv()).integerValue()); } - /** Todo. remove this test cases after script serilization implemented. */ + /** Todo. remove this test cases after script serialization implemented. */ @Test public void serializationTest() throws Exception { Expression expression = DSL.equal(DSL.literal("v1"), DSL.literal("v2")); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java index f7a1a7008a..7de4f456c9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java @@ -221,12 +221,12 @@ public void test_if_predicate(Expression v1, Expression v2, Expression v3, Expre @ParameterizedTest @MethodSource("exprIfNullArguments") public void test_exprIfNull_predicate(ExprValue v1, ExprValue v2, ExprValue expected) { - assertEquals(expected.value(), UnaryPredicateOperator.exprIfNull(v1, v2).value()); + assertEquals(expected.value(), UnaryPredicateOperators.exprIfNull(v1, v2).value()); } @ParameterizedTest @MethodSource("exprNullIfArguments") public void test_exprNullIf_predicate(ExprValue v1, ExprValue v2, ExprValue expected) { - assertEquals(expected.value(), UnaryPredicateOperator.exprNullIf(v1, v2).value()); + assertEquals(expected.value(), UnaryPredicateOperators.exprNullIf(v1, v2).value()); } } diff --git a/docs/category.json b/docs/category.json index e90c674a2e..ca3d345e8b 100644 --- a/docs/category.json +++ b/docs/category.json @@ -28,12 +28,13 @@ "user/ppl/cmd/where.rst", "user/ppl/general/identifiers.rst", "user/ppl/general/datatypes.rst", - "user/ppl/functions/math.rst", - "user/ppl/functions/datetime.rst", - "user/ppl/functions/string.rst", "user/ppl/functions/condition.rst", + "user/ppl/functions/datetime.rst", + "user/ppl/functions/expressions.rst", + "user/ppl/functions/ip.rst", + "user/ppl/functions/math.rst", "user/ppl/functions/relevance.rst", - "user/ppl/functions/expressions.rst" + "user/ppl/functions/string.rst" ], "sql_cli": [ "user/dql/expressions.rst", diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index fa233020a3..aba4eb0c75 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 9/9 + fetched rows / total rows = 10/10 +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -47,6 +47,7 @@ SQL query:: | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | weblogs | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index 96c3e64e72..9ce130072e 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -101,7 +101,7 @@ NULLIF Description >>>>>>>>>>> -Usage: nullif(field1, field2) return null if two parameters are same, otherwiser return field1. +Usage: nullif(field1, field2) return null if two parameters are same, otherwise return field1. Argument type: all the supported data type, (NOTE : if two parameters has different type, if two parameters has different type, you will fail semantic check) @@ -152,7 +152,7 @@ IF Description >>>>>>>>>>> -Usage: if(condition, expr1, expr2) return expr1 if condition is true, otherwiser return expr2. +Usage: if(condition, expr1, expr2) return expr1 if condition is true, otherwise return expr2. Argument type: all the supported data type, (NOTE : if expr1 and expr2 are different type, you will fail semantic check diff --git a/docs/user/ppl/functions/ip.rst b/docs/user/ppl/functions/ip.rst new file mode 100644 index 0000000000..3387974af5 --- /dev/null +++ b/docs/user/ppl/functions/ip.rst @@ -0,0 +1,38 @@ +==================== +IP Address Functions +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +CIDRMATCH +--------- + +Description +>>>>>>>>>>> + +Usage: `cidrmatch(ip, cidr)` checks if `ip` is within the specified `cidr` range. + +Argument type: STRING, STRING + +Return type: BOOLEAN + +Example: + + os> source=weblogs | where cidrmatch(host, '199.120.110.0/24') | fields host + fetched rows / total rows = 1/1 + +----------------+ + | host | + |----------------| + | 199.120.110.21 | + +----------------+ + +Note: + - `ip` can be an IPv4 or an IPv6 address + - `cidr` can be an IPv4 or an IPv6 block + - `ip` and `cidr` must be either both IPv4 or both IPv6 + - `ip` and `cidr` must both be valid and non-empty/non-null + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 1fa981b1b7..9525874c59 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -102,6 +102,8 @@ The query start with search command and then flowing a set of command delimited - `System Functions <functions/system.rst>`_ + - `IP Address Functions <functions/ip.rst>`_ + * **Optimization** - `Optimization <../../user/optimization/optimization.rst>`_ diff --git a/doctest/test_data/weblogs.json b/doctest/test_data/weblogs.json new file mode 100644 index 0000000000..4228e9c4d2 --- /dev/null +++ b/doctest/test_data/weblogs.json @@ -0,0 +1,6 @@ +{"index":{}} +{"host": "199.72.81.55", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} +{"index":{}} +{"host": "199.120.110.21", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} +{"index":{}} +{"host": "205.212.115.106", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 881078a9bd..1d46766c6d 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -29,7 +29,7 @@ WILDCARD = "wildcard" NESTED = "nested" DATASOURCES = ".ql-datasources" - +WEBLOGS = "weblogs" class DocTestConnection(OpenSearchConnection): @@ -122,6 +122,7 @@ def set_up_test_indices(test): load_file("wildcard.json", index_name=WILDCARD) load_file("nested_objects.json", index_name=NESTED) load_file("datasources.json", index_name=DATASOURCES) + load_file("weblogs.json", index_name=WEBLOGS) def load_file(filename, index_name): @@ -150,7 +151,7 @@ def set_up(test): def tear_down(test): # drop leftover tables after each test - test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED], ignore_unavailable=True) + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS], ignore_unavailable=True) docsuite = partial(doctest.DocFileSuite, diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 74acad4f52..005119a9bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -155,7 +155,9 @@ public void dateFunctionNameCaseInsensitiveTest() { public void ipTypeShouldPassJdbcFormatter() { assertThat( executeQuery( - "SELECT host AS hostIP FROM " + TestsConstants.TEST_INDEX_WEBLOG + " ORDER BY hostIP", + "SELECT host_ip AS hostIP FROM " + + TestsConstants.TEST_INDEX_WEBLOG + + " ORDER BY hostIP", "jdbc"), containsString("\"type\": \"ip\"")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java new file mode 100644 index 0000000000..adb044d0d2 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOG; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class IPFunctionIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.WEBLOG); + } + + @Test + public void test_cidrmatch() throws IOException { + + // TODO #3145: Add tests for IP address data type. + JSONObject result; + + // No matches + result = + executeQuery( + String.format( + "source=%s | where cidrmatch(host_string, '199.120.111.0/24') | fields host_string", + TEST_INDEX_WEBLOG)); + verifySchema(result, schema("host_string", null, "string")); + verifyDataRows(result); + + // One match + result = + executeQuery( + String.format( + "source=%s | where cidrmatch(host_string, '199.120.110.0/24') | fields host_string", + TEST_INDEX_WEBLOG)); + verifySchema(result, schema("host_string", null, "string")); + verifyDataRows(result, rows("199.120.110.21")); + + // Multiple matches + result = + executeQuery( + String.format( + "source=%s | where cidrmatch(host_string, '199.0.0.0/8') | fields host_string", + TEST_INDEX_WEBLOG)); + verifySchema(result, schema("host_string", null, "string")); + verifyDataRows(result, rows("199.72.81.55"), rows("199.120.110.21")); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json index 05b9784313..bff3e20bb9 100644 --- a/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json @@ -1,9 +1,12 @@ { "mappings": { "properties": { - "host": { + "host_ip": { "type": "ip" }, + "host_string": { + "type": "keyword" + }, "method": { "type": "text" }, diff --git a/integ-test/src/test/resources/weblogs.json b/integ-test/src/test/resources/weblogs.json index 4228e9c4d2..d2e9a968f8 100644 --- a/integ-test/src/test/resources/weblogs.json +++ b/integ-test/src/test/resources/weblogs.json @@ -1,6 +1,6 @@ {"index":{}} -{"host": "199.72.81.55", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} +{"host_ip": "199.72.81.55", "host_string": "199.72.81.55", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} {"index":{}} -{"host": "199.120.110.21", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} +{"host_ip": "199.120.110.21", "host_string": "199.120.110.21", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} {"index":{}} -{"host": "205.212.115.106", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} +{"host_ip": "205.212.115.106", "host_string": "205.212.115.106", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 1ce504a4c5..41d6667ded 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -87,9 +87,7 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) { for (var field : typeMapping.keySet()) { // Prevent overwriting, because aggregation engine may be not aware // of all niceties of all types. - if (!this.typeMapping.containsKey(field)) { - this.typeMapping.put(field, typeMapping.get(field)); - } + this.typeMapping.putIfAbsent(field, typeMapping.get(field)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java index 38a4ad3199..5ee175f304 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java @@ -14,27 +14,28 @@ public class OpenSearchExprIpValueTest { - private OpenSearchExprIpValue ipValue = new OpenSearchExprIpValue("192.168.0.1"); + private final String ipString = "192.168.0.1"; + private final OpenSearchExprIpValue ipValue = new OpenSearchExprIpValue(ipString); @Test - void value() { - assertEquals("192.168.0.1", ipValue.value()); + void testValue() { + assertEquals(ipString, ipValue.value()); } @Test - void type() { + void testType() { assertEquals(OpenSearchIpType.of(), ipValue.type()); } @Test - void compare() { - assertEquals(0, ipValue.compareTo(new OpenSearchExprIpValue("192.168.0.1"))); - assertEquals(ipValue, new OpenSearchExprIpValue("192.168.0.1")); + void testCompare() { + assertEquals(0, ipValue.compareTo(new OpenSearchExprIpValue(ipString))); + assertEquals(ipValue, new OpenSearchExprIpValue(ipString)); } @Test - void equal() { - assertTrue(ipValue.equal(new OpenSearchExprIpValue("192.168.0.1"))); + void testEqual() { + assertTrue(ipValue.equal(new OpenSearchExprIpValue(ipString))); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 5fd40ef6c4..d82926077e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -62,6 +62,8 @@ class OpenSearchExprValueFactoryTest { + static final String fieldIp = "ipV"; + private static final Map<String, OpenSearchDataType> MAPPING = new ImmutableMap.Builder<String, OpenSearchDataType>() .put("byteV", OpenSearchDataType.of(BYTE)) @@ -112,14 +114,13 @@ class OpenSearchExprValueFactoryTest { "textKeywordV", OpenSearchTextType.of( Map.of("words", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))) - .put("ipV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip)) + .put(fieldIp, OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip)) .put("geoV", OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint)) .put("binaryV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary)) .build(); - + private static final double TOLERANCE = 1E-5; private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory(MAPPING, true); - private final OpenSearchExprValueFactory exprValueFactoryNoArrays = new OpenSearchExprValueFactory(MAPPING, false); @@ -660,12 +661,13 @@ public void constructArrayOfGeoPointsReturnsAll() { @Test public void constructArrayOfIPsReturnsAll() { + final String ip1 = "192.168.0.1"; + final String ip2 = "192.168.0.2"; + assertEquals( new ExprCollectionValue( - List.of( - new OpenSearchExprIpValue("192.168.0.1"), - new OpenSearchExprIpValue("192.168.0.2"))), - tupleValue("{\"ipV\":[\"192.168.0.1\",\"192.168.0.2\"]}").get("ipV")); + List.of(new OpenSearchExprIpValue(ip1), new OpenSearchExprIpValue(ip2))), + tupleValue(String.format("{\"%s\":[\"%s\",\"%s\"]}", fieldIp, ip1, ip2)).get(fieldIp)); } @Test @@ -741,13 +743,12 @@ public void constructStruct() { @Test public void constructIP() { + final String valueIp = "192.168.0.1"; assertEquals( - new OpenSearchExprIpValue("192.168.0.1"), - tupleValue("{\"ipV\":\"192.168.0.1\"}").get("ipV")); + new OpenSearchExprIpValue(valueIp), + tupleValue(String.format("{\"%s\":\"%s\"}", fieldIp, valueIp)).get(fieldIp)); } - private static final double TOLERANCE = 1E-5; - @Test public void constructGeoPoint() { final double lat = 42.60355556; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9f707c13cd..21cee12675 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -322,6 +322,7 @@ CAST: 'CAST'; LIKE: 'LIKE'; ISNULL: 'ISNULL'; ISNOTNULL: 'ISNOTNULL'; +CIDRMATCH: 'CIDRMATCH'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 4dc223b028..54ec23dcb9 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -629,6 +629,7 @@ conditionFunctionName : LIKE | ISNULL | ISNOTNULL + | CIDRMATCH ; // flow control function return non-boolean value @@ -829,6 +830,7 @@ keywordsCanBeId | textFunctionName | mathematicalFunctionName | positionFunctionName + | conditionFunctionName // commands | SEARCH | DESCRIBE From b6846ceba06df1548a6fd24cf380baf3ae9fce06 Mon Sep 17 00:00:00 2001 From: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:59:33 -0800 Subject: [PATCH 86/96] Add FILLNULL command in PPL (#3032) (#3075) Signed-off-by: Norman Jordan <norman.jordan@improving.com> Signed-off-by: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Co-authored-by: Andrew Carbonetto <andrew.carbonetto@improving.com> --- .../org/opensearch/sql/analysis/Analyzer.java | 24 ++ .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/dsl/AstDSL.java | 21 ++ .../org/opensearch/sql/ast/tree/FillNull.java | 89 ++++++++ .../opensearch/sql/analysis/AnalyzerTest.java | 44 ++++ docs/category.json | 1 + docs/user/ppl/cmd/fillnull.rst | 62 +++++ .../org/opensearch/sql/ppl/ExplainIT.java | 11 + .../opensearch/sql/ppl/FillNullCommandIT.java | 214 ++++++++++++++++++ .../ppl/explain_fillnull_push.json | 28 +++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 19 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 29 +++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 28 +++ .../sql/ppl/antlr/PPLSyntaxParserTest.java | 12 + .../sql/ppl/parser/AstBuilderTest.java | 32 +++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 13 ++ 17 files changed, 635 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java create mode 100644 docs/user/ppl/cmd/fillnull.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index d5e8b93b13..71db736f78 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -46,6 +46,7 @@ import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; @@ -558,6 +559,29 @@ public LogicalPlan visitAD(AD node, AnalysisContext context) { return new LogicalAD(child, options); } + /** Build {@link LogicalEval} for fillnull command. */ + @Override + public LogicalPlan visitFillNull(final FillNull node, final AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + + ImmutableList.Builder<Pair<ReferenceExpression, Expression>> expressionsBuilder = + new Builder<>(); + for (FillNull.NullableFieldFill fieldFill : node.getNullableFieldFills()) { + Expression fieldExpr = + expressionAnalyzer.analyze(fieldFill.getNullableFieldReference(), context); + ReferenceExpression ref = + DSL.ref(fieldFill.getNullableFieldReference().getField().toString(), fieldExpr.type()); + FunctionExpression ifNullFunction = + DSL.ifnull(ref, expressionAnalyzer.analyze(fieldFill.getReplaceNullWithMe(), context)); + expressionsBuilder.add(new ImmutablePair<>(ref, ifNullFunction)); + TypeEnvironment typeEnvironment = context.peek(); + // define the new reference in type env. + typeEnvironment.define(ref); + } + + return new LogicalEval(child, expressionsBuilder.build()); + } + /** Build {@link LogicalML} for ml command. */ @Override public LogicalPlan visitML(ML node, AnalysisContext context) { diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 973b10310b..a0520dc70e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -45,6 +45,7 @@ import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; @@ -312,4 +313,8 @@ public T visitFetchCursor(FetchCursor cursor, C context) { public T visitCloseCursor(CloseCursor closeCursor, C context) { return visitChildren(closeCursor, context); } + + public T visitFillNull(FillNull fillNull, C context) { + return visitChildren(fillNull, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 4f3056b0f7..8135731ff6 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -5,10 +5,12 @@ package org.opensearch.sql.ast.dsl; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.Alias; @@ -46,6 +48,7 @@ import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Limit; @@ -471,4 +474,22 @@ public static Parse parse( java.util.Map<String, Literal> arguments) { return new Parse(parseMethod, sourceField, pattern, arguments, input); } + + public static FillNull fillNull(UnresolvedExpression replaceNullWithMe, Field... fields) { + return new FillNull( + FillNull.ContainNullableFieldFill.ofSameValue( + replaceNullWithMe, ImmutableList.copyOf(fields))); + } + + public static FillNull fillNull( + List<ImmutablePair<Field, UnresolvedExpression>> fieldAndReplacements) { + ImmutableList.Builder<FillNull.NullableFieldFill> replacementsBuilder = ImmutableList.builder(); + for (ImmutablePair<Field, UnresolvedExpression> fieldAndReplacement : fieldAndReplacements) { + replacementsBuilder.add( + new FillNull.NullableFieldFill( + fieldAndReplacement.getLeft(), fieldAndReplacement.getRight())); + } + return new FillNull( + FillNull.ContainNullableFieldFill.ofVariousValue(replacementsBuilder.build())); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java b/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java new file mode 100644 index 0000000000..e1e56229b4 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.List; +import java.util.Objects; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** AST node represent FillNull operation. */ +@RequiredArgsConstructor +@AllArgsConstructor +public class FillNull extends UnresolvedPlan { + + @Getter + @RequiredArgsConstructor + public static class NullableFieldFill { + @NonNull private final Field nullableFieldReference; + @NonNull private final UnresolvedExpression replaceNullWithMe; + } + + public interface ContainNullableFieldFill { + List<NullableFieldFill> getNullFieldFill(); + + static ContainNullableFieldFill ofVariousValue(List<NullableFieldFill> replacements) { + return new VariousValueNullFill(replacements); + } + + static ContainNullableFieldFill ofSameValue( + UnresolvedExpression replaceNullWithMe, List<Field> nullableFieldReferences) { + return new SameValueNullFill(replaceNullWithMe, nullableFieldReferences); + } + } + + private static class SameValueNullFill implements ContainNullableFieldFill { + @Getter(onMethod_ = @Override) + private final List<NullableFieldFill> nullFieldFill; + + public SameValueNullFill( + UnresolvedExpression replaceNullWithMe, List<Field> nullableFieldReferences) { + Objects.requireNonNull(replaceNullWithMe, "Null replacement is required"); + this.nullFieldFill = + Objects.requireNonNull(nullableFieldReferences, "Nullable field reference is required") + .stream() + .map(nullableReference -> new NullableFieldFill(nullableReference, replaceNullWithMe)) + .toList(); + } + } + + @RequiredArgsConstructor + private static class VariousValueNullFill implements ContainNullableFieldFill { + @NonNull + @Getter(onMethod_ = @Override) + private final List<NullableFieldFill> nullFieldFill; + } + + private UnresolvedPlan child; + + @NonNull private final ContainNullableFieldFill containNullableFieldFill; + + public List<NullableFieldFill> getNullableFieldFills() { + return containNullableFieldFill.getNullFieldFill(); + } + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List<? extends Node> getChild() { + return child == null ? List.of() : List.of(child); + } + + @Override + public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) { + return nodeVisitor.visitFillNull(this, context); + } +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 2412bd9474..4f06ce9d23 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -73,6 +73,7 @@ import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.ParseMethod; @@ -81,6 +82,7 @@ import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.CloseCursor; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Paginate; @@ -1437,6 +1439,48 @@ public void kmeanns_relation() { new Kmeans(AstDSL.relation("schema"), argumentMap)); } + @Test + public void fillnull_same_value() { + assertAnalyzeEqual( + LogicalPlanDSL.eval( + LogicalPlanDSL.relation("schema", table), + ImmutablePair.of( + DSL.ref("integer_value", INTEGER), + DSL.ifnull(DSL.ref("integer_value", INTEGER), DSL.literal(0))), + ImmutablePair.of( + DSL.ref("int_null_value", INTEGER), + DSL.ifnull(DSL.ref("int_null_value", INTEGER), DSL.literal(0)))), + new FillNull( + AstDSL.relation("schema"), + FillNull.ContainNullableFieldFill.ofSameValue( + AstDSL.intLiteral(0), + ImmutableList.<Field>builder() + .add(AstDSL.field("integer_value")) + .add(AstDSL.field("int_null_value")) + .build()))); + } + + @Test + public void fillnull_various_values() { + assertAnalyzeEqual( + LogicalPlanDSL.eval( + LogicalPlanDSL.relation("schema", table), + ImmutablePair.of( + DSL.ref("integer_value", INTEGER), + DSL.ifnull(DSL.ref("integer_value", INTEGER), DSL.literal(0))), + ImmutablePair.of( + DSL.ref("int_null_value", INTEGER), + DSL.ifnull(DSL.ref("int_null_value", INTEGER), DSL.literal(1)))), + new FillNull( + AstDSL.relation("schema"), + FillNull.ContainNullableFieldFill.ofVariousValue( + ImmutableList.of( + new FillNull.NullableFieldFill( + AstDSL.field("integer_value"), AstDSL.intLiteral(0)), + new FillNull.NullableFieldFill( + AstDSL.field("int_null_value"), AstDSL.intLiteral(1)))))); + } + @Test public void ad_batchRCF_relation() { Map<String, Literal> argumentMap = diff --git a/docs/category.json b/docs/category.json index ca3d345e8b..aacfc43478 100644 --- a/docs/category.json +++ b/docs/category.json @@ -14,6 +14,7 @@ "user/ppl/cmd/information_schema.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", + "user/ppl/cmd/fillnull.rst", "user/ppl/cmd/grok.rst", "user/ppl/cmd/head.rst", "user/ppl/cmd/parse.rst", diff --git a/docs/user/ppl/cmd/fillnull.rst b/docs/user/ppl/cmd/fillnull.rst new file mode 100644 index 0000000000..4a9e38d353 --- /dev/null +++ b/docs/user/ppl/cmd/fillnull.rst @@ -0,0 +1,62 @@ +============= +fillnull +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +Using ``fillnull`` command to fill null with provided value in one or more fields in the search result. + + +Syntax +============ +`fillnull [with <null-replacement> in <nullable-field>["," <nullable-field>]] | [using <source-field> = <null-replacement> [","<source-field> = <null-replacement>]]` + +* null-replacement: mandatory. The value used to replace `null`s. +* nullable-field: mandatory. Field reference. The `null` values in the field referred to by the property will be replaced with the values from the null-replacement. + +Example 1: fillnull one field +====================================================================== + +The example show fillnull one field. + +PPL query:: + + os> source=accounts | fields email, employer | fillnull with '<not found>' in email ; + fetched rows / total rows = 4/4 + +-----------------------+----------+ + | email | employer | + |-----------------------+----------| + | amberduke@pyrami.com | Pyrami | + | hattiebond@netagy.com | Netagy | + | <not found> | Quility | + | daleadams@boink.com | null | + +-----------------------+----------+ + +Example 2: fillnull applied to multiple fields +======================================================================== + +The example show fillnull applied to multiple fields. + +PPL query:: + + os> source=accounts | fields email, employer | fillnull using email = '<not found>', employer = '<no employer>' ; + fetched rows / total rows = 4/4 + +-----------------------+---------------+ + | email | employer | + |-----------------------+---------------| + | amberduke@pyrami.com | Pyrami | + | hattiebond@netagy.com | Netagy | + | <not found> | Quility | + | daleadams@boink.com | <no employer> | + +-----------------------+---------------+ + +Limitation +========== +The ``fillnull`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index c6b21e1605..b9c7f89ba0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -89,6 +89,17 @@ public void testLimitPushDownExplain() throws Exception { + "| fields ageMinus")); } + @Test + public void testFillNullPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_fillnull_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + " | fillnull with -1 in age,balance | fields age, balance")); + } + String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java new file mode 100644 index 0000000000..d88d31c997 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java @@ -0,0 +1,214 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class FillNullCommandIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.CALCS); + } + + @Test + public void testFillNullSameValueOneField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, num0 | fillnull with -1 in num0", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", -1), + rows("nine", 10), + rows("ten", -1), + rows("eleven", -1), + rows("twelve", -1), + rows(null, -1), + rows("fourteen", -1), + rows("fifteen", -1), + rows("sixteen", -1), + rows(null, -1)); + } + + @Test + public void testFillNullSameValueTwoFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields num0, num2 | fillnull with -1 in num0,num2", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows(12.3, 17.86), + rows(-12.3, 16.73), + rows(15.7, -1), + rows(-15.7, 8.51), + rows(3.5, 6.46), + rows(-3.5, 8.98), + rows(0, 11.69), + rows(-1, 17.25), + rows(10, -1), + rows(-1, 11.5), + rows(-1, 6.8), + rows(-1, 3.79), + rows(-1, -1), + rows(-1, 13.04), + rows(-1, -1), + rows(-1, 10.98), + rows(-1, 7.87)); + } + + @Test + public void testFillNullVariousValuesOneField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, num0 | fillnull using num0 = -1", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", -1), + rows("nine", 10), + rows("ten", -1), + rows("eleven", -1), + rows("twelve", -1), + rows(null, -1), + rows("fourteen", -1), + rows("fifteen", -1), + rows("sixteen", -1), + rows(null, -1)); + } + + @Test + public void testFillNullVariousValuesTwoFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields num0, num2 | fillnull using num0 = -1, num2 = -2", + TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows(12.3, 17.86), + rows(-12.3, 16.73), + rows(15.7, -2), + rows(-15.7, 8.51), + rows(3.5, 6.46), + rows(-3.5, 8.98), + rows(0, 11.69), + rows(-1, 17.25), + rows(10, -2), + rows(-1, 11.5), + rows(-1, 6.8), + rows(-1, 3.79), + rows(-1, -2), + rows(-1, 13.04), + rows(-1, -2), + rows(-1, 10.98), + rows(-1, 7.87)); + } + + @Test + public void testFillNullWithOtherField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fillnull using num0 = num1 | fields str2, num0", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", 11.38), + rows("nine", 10), + rows("ten", 12.4), + rows("eleven", 10.32), + rows("twelve", 2.47), + rows(null, 12.05), + rows("fourteen", 10.37), + rows("fifteen", 7.1), + rows("sixteen", 16.81), + rows(null, 7.12)); + } + + @Test + public void testFillNullWithFunctionOnOtherField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fillnull with ceil(num1) in num0 | fields str2, num0", + TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", 12), + rows("nine", 10), + rows("ten", 13), + rows("eleven", 11), + rows("twelve", 3), + rows(null, 13), + rows("fourteen", 11), + rows("fifteen", 8), + rows("sixteen", 17), + rows(null, 8)); + } + + @Test + public void testFillNullWithFunctionMultipleCommands() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fillnull with num1 in num0 | fields str2, num0 | fillnull with" + + " 'unknown' in str2", + TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows("unknown", -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows("unknown", 0), + rows("eight", 11.38), + rows("nine", 10), + rows("ten", 12.4), + rows("eleven", 10.32), + rows("twelve", 2.47), + rows("unknown", 12.05), + rows("fourteen", 10.37), + rows("fifteen", 7.1), + rows("sixteen", 16.81), + rows("unknown", 7.12)); + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json new file mode 100644 index 0000000000..7e5e1c1c20 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json @@ -0,0 +1,28 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[age, balance]" + }, + "children": [ + { + "name": "EvalOperator", + "description": { + "expressions": { + "balance": "ifnull(balance, -1)", + "age": "ifnull(age, -1)" + } + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } + ] + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 21cee12675..3ba8da74f4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -35,6 +35,7 @@ NEW_FIELD: 'NEW_FIELD'; KMEANS: 'KMEANS'; AD: 'AD'; ML: 'ML'; +FILLNULL: 'FILLNULL'; // COMMAND ASSIST KEYWORDS AS: 'AS'; @@ -44,6 +45,8 @@ INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; DATASOURCES: 'DATASOURCES'; +USING: 'USING'; +WITH: 'WITH'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 54ec23dcb9..89a32abe23 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -49,6 +49,7 @@ commands | kmeansCommand | adCommand | mlCommand + | fillnullCommand ; searchCommand @@ -127,6 +128,23 @@ patternsMethod | REGEX ; +fillnullCommand + : FILLNULL (fillNullWithTheSameValue + | fillNullWithFieldVariousValues) + ; + +fillNullWithTheSameValue + : WITH nullReplacement = valueExpression IN nullableFieldList = fieldList + ; + +fillNullWithFieldVariousValues + : USING nullReplacementExpression (COMMA nullReplacementExpression)* + ; + +nullReplacementExpression + : nullableField = fieldExpression EQUAL nullReplacement = valueExpression + ; + kmeansCommand : KMEANS (kmeansParameter)* ; @@ -843,6 +861,7 @@ keywordsCanBeId | DEDUP | SORT | EVAL + | FILLNULL | HEAD | TOP | RARE diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 78fe28b49e..2fccb8e635 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -51,6 +51,7 @@ import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; @@ -392,6 +393,34 @@ public UnresolvedPlan visitMlCommand(OpenSearchPPLParser.MlCommandContext ctx) { return new ML(builder.build()); } + /** fillnull command. */ + @Override + public UnresolvedPlan visitFillNullWithTheSameValue( + OpenSearchPPLParser.FillNullWithTheSameValueContext ctx) { + return new FillNull( + FillNull.ContainNullableFieldFill.ofSameValue( + internalVisitExpression(ctx.nullReplacement), + ctx.nullableFieldList.fieldExpression().stream() + .map(f -> (Field) internalVisitExpression(f)) + .toList())); + } + + /** fillnull command. */ + @Override + public UnresolvedPlan visitFillNullWithFieldVariousValues( + OpenSearchPPLParser.FillNullWithFieldVariousValuesContext ctx) { + ImmutableList.Builder<FillNull.NullableFieldFill> replacementsBuilder = ImmutableList.builder(); + for (int i = 0; i < ctx.nullReplacementExpression().size(); i++) { + replacementsBuilder.add( + new FillNull.NullableFieldFill( + (Field) internalVisitExpression(ctx.nullReplacementExpression(i).nullableField), + internalVisitExpression(ctx.nullReplacementExpression(i).nullReplacement))); + } + + return new FillNull( + FillNull.ContainNullableFieldFill.ofVariousValue(replacementsBuilder.build())); + } + /** Get original text in query. */ private String getTextInQuery(ParserRuleContext ctx) { Token start = ctx.getStart(); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index d28e5d122b..a1ca0fd69a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -34,6 +34,7 @@ import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Project; @@ -234,6 +235,33 @@ private String visitExpression(UnresolvedExpression expression) { return expressionAnalyzer.analyze(expression, null); } + @Override + public String visitFillNull(FillNull node, String context) { + String child = node.getChild().get(0).accept(this, context); + List<FillNull.NullableFieldFill> fieldFills = node.getNullableFieldFills(); + final UnresolvedExpression firstReplacement = fieldFills.getFirst().getReplaceNullWithMe(); + if (fieldFills.stream().allMatch(n -> firstReplacement == n.getReplaceNullWithMe())) { + return StringUtils.format( + "%s | fillnull with %s in %s", + child, + firstReplacement, + node.getNullableFieldFills().stream() + .map(n -> visitExpression(n.getNullableFieldReference())) + .collect(Collectors.joining(", "))); + } else { + return StringUtils.format( + "%s | fillnull using %s", + child, + node.getNullableFieldFills().stream() + .map( + n -> + StringUtils.format( + "%s = %s", + visitExpression(n.getNullableFieldReference()), n.getReplaceNullWithMe())) + .collect(Collectors.joining(", "))); + } + } + private String groupBy(String groupBy) { return Strings.isNullOrEmpty(groupBy) ? "" : StringUtils.format("by %s", groupBy); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 943953d416..5601bda485 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -417,4 +417,16 @@ public void testCanParseTimestampdiffFunction() { new PPLSyntaxParser() .parse("SOURCE=test | eval k = TIMESTAMPDIFF(WEEK,'2003-01-02','2003-01-02')")); } + + @Test + public void testCanParseFillNullSameValue() { + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a")); + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a, b")); + } + + @Test + public void testCanParseFillNullVariousValues() { + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0")); + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0, b = 1")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index ced266ed78..488cb7dc14 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -42,6 +42,7 @@ import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import org.junit.Ignore; @@ -50,10 +51,12 @@ import org.junit.rules.ExpectedException; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.ParseMethod; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; +import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; @@ -660,6 +663,35 @@ public void testMLCommand() { .build())); } + @Test + public void testFillNullCommandSameValue() { + assertEqual( + "source=t | fillnull with 0 in a, b, c", + new FillNull( + relation("t"), + FillNull.ContainNullableFieldFill.ofSameValue( + intLiteral(0), + ImmutableList.<Field>builder() + .add(field("a")) + .add(field("b")) + .add(field("c")) + .build()))); + } + + @Test + public void testFillNullCommandVariousValues() { + assertEqual( + "source=t | fillnull using a = 1, b = 2, c = 3", + new FillNull( + relation("t"), + FillNull.ContainNullableFieldFill.ofVariousValue( + ImmutableList.<FillNull.NullableFieldFill>builder() + .add(new FillNull.NullableFieldFill(field("a"), intLiteral(1))) + .add(new FillNull.NullableFieldFill(field("b"), intLiteral(2))) + .add(new FillNull.NullableFieldFill(field("c"), intLiteral(3))) + .build()))); + } + @Test public void testDescribeCommand() { assertEqual("describe t", relation(mappingTable("t"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index cd51ea07df..b5b4c97f13 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -105,6 +105,19 @@ public void testEvalCommand() { assertEquals("source=t | eval r=abs(f)", anonymize("source=t | eval r=abs(f)")); } + @Test + public void testFillNullSameValue() { + assertEquals( + "source=t | fillnull with 0 in f1, f2", anonymize("source=t | fillnull with 0 in f1, f2")); + } + + @Test + public void testFillNullVariousValues() { + assertEquals( + "source=t | fillnull using f1 = 0, f2 = -1", + anonymize("source=t | fillnull using f1 = 0, f2 = -1")); + } + @Test public void testRareCommandWithGroupBy() { assertEquals("source=t | rare 10 a by b", anonymize("source=t | rare a by b")); From e9af11f1e298f3ea009407d28c6652afd62fab89 Mon Sep 17 00:00:00 2001 From: Taylor Curran <taylor.curran@improving.com> Date: Thu, 5 Dec 2024 16:28:20 -0800 Subject: [PATCH 87/96] Remove unused imports (#3192) Signed-off-by: currantw <taylor.curran@improving.com> --- .../sql/legacy/unittest/query/DefaultQueryActionTest.java | 2 -- .../sql/opensearch/client/OpenSearchNodeClientTest.java | 1 - .../sql/opensearch/request/OpenSearchQueryRequestTest.java | 1 - .../sql/opensearch/storage/scan/OpenSearchIndexScanTest.java | 1 - 4 files changed, 5 deletions(-) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java index ec6ab00f97..50f094bf72 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/query/DefaultQueryActionTest.java @@ -6,8 +6,6 @@ package org.opensearch.sql.legacy.unittest.query; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.*; import java.util.*; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 12a906b25e..73c4f0e7f8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -14,7 +14,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index 0847e520cc..52c208da15 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -6,7 +6,6 @@ package org.opensearch.sql.opensearch.request; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.opensearch.sql.opensearch.request.OpenSearchRequest.DEFAULT_QUERY_TIMEOUT; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index da30442bae..5381c4a7a7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -11,7 +11,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; import static org.opensearch.search.sort.SortOrder.ASC; From bc2fd586010c0aa9c6e263b7b443b2e17948f047 Mon Sep 17 00:00:00 2001 From: Taylor Curran <taylor.curran@improving.com> Date: Fri, 6 Dec 2024 09:58:13 -0800 Subject: [PATCH 88/96] Add `final` to constant fields (#3191) Signed-off-by: currantw <taylor.curran@improving.com> --- .../sql/spark/dispatcher/model/JobType.java | 2 +- .../spark/execution/session/SessionState.java | 2 +- .../SparkParameterComposerCollection.java | 6 +-- .../parameter/SparkSubmitParameters.java | 2 +- .../sql/spark/utils/SQLQueryUtils.java | 3 +- ...DatasourceEmbeddedQueryIdProviderTest.java | 2 +- .../dispatcher/SparkQueryDispatcherTest.java | 4 +- .../execution/statestore/StateModelTest.java | 2 +- .../SparkParameterComposerCollectionTest.java | 6 +-- .../GrammarElementValidatorProviderTest.java | 8 ++-- .../cluster/ClusterManagerEventListener.java | 14 +++---- .../execution/statestore/StateStore.java | 6 +-- .../job/ScheduledAsyncQueryJobRunner.java | 2 +- ...AsyncQueryExecutorServiceImplSpecTest.java | 2 +- .../AsyncQueryExecutorServiceSpec.java | 2 +- .../AsyncQueryGetResultSpecTest.java | 2 +- ...yncQueryJobMetadataStorageServiceTest.java | 3 +- .../asyncquery/model/MockFlintSparkJob.java | 7 ++-- .../session/InteractiveSessionTest.java | 7 ++-- .../execution/statement/StatementTest.java | 9 +++-- ...IndexStateModelXContentSerializerTest.java | 2 +- .../interceptors/AwsSigningInterceptor.java | 4 +- .../BasicAuthenticationInterceptor.java | 2 +- .../opensearch/sql/common/grok/BasicTest.java | 2 +- ...ataSourceSchemaIdentifierNameResolver.java | 4 +- .../sql/analysis/symbol/SymbolTable.java | 4 +- .../org/opensearch/sql/ast/tree/Filter.java | 2 +- .../org/opensearch/sql/ast/tree/Project.java | 2 +- .../datasource/model/DataSourceMetadata.java | 24 ++++++------ .../datasource/model/DataSourceStatus.java | 2 +- .../sql/datasource/model/DataSourceType.java | 12 +++--- .../streaming/DefaultMetadataLog.java | 2 +- .../predicate/BinaryPredicateOperators.java | 6 +-- .../sql/expression/text/TextFunctions.java | 2 +- .../sql/planner/logical/LogicalNested.java | 2 +- .../sql/planner/physical/NestedOperator.java | 2 +- .../storage/bindingtuple/BindingTuple.java | 2 +- .../opensearch/sql/utils/ExpressionUtils.java | 2 +- .../sql/analysis/AnalyzerTestBase.java | 16 ++++---- .../sql/analysis/TypeEnvironmentTest.java | 2 +- .../org/opensearch/sql/config/TestConfig.java | 2 +- .../sql/data/model/ExprValueUtilsTest.java | 25 ++++++------ .../sql/executor/QueryManagerTest.java | 2 +- .../IntervalTriggerExecutionTest.java | 4 +- .../execution/StreamingQueryPlanTest.java | 2 +- .../sql/expression/ExpressionTestBase.java | 2 +- .../aggregation/AggregationTest.java | 8 ++-- .../function/FunctionSignatureTest.java | 4 +- .../function/WideningTypeRuleTest.java | 2 +- .../BinaryPredicateOperatorTest.java | 2 +- .../planner/physical/PhysicalPlanTest.java | 2 +- .../physical/PhysicalPlanTestBase.java | 2 +- .../datasources/exceptions/ErrorMessage.java | 2 +- .../TransportCreateDataSourceAction.java | 4 +- .../TransportDeleteDataSourceAction.java | 2 +- .../TransportGetDataSourceAction.java | 2 +- .../TransportPatchDataSourceAction.java | 2 +- .../TransportUpdateDataSourceAction.java | 2 +- .../sql/correctness/tests/TestReportTest.java | 2 +- .../sql/correctness/testset/TestQuerySet.java | 2 +- .../sql/legacy/GetEndpointQueryIT.java | 2 +- .../sql/legacy/JoinAliasWriterRuleIT.java | 2 +- .../opensearch/sql/legacy/MultiQueryIT.java | 4 +- .../org/opensearch/sql/legacy/SubqueryIT.java | 2 +- .../sql/ppl/DateTimeComparisonIT.java | 6 +-- .../org/opensearch/sql/ppl/PPLPluginIT.java | 2 +- .../sql/sql/DateTimeComparisonIT.java | 6 +-- .../org/opensearch/sql/sql/ExpressionIT.java | 2 +- .../opensearch/sql/sql/QueryValidationIT.java | 2 +- .../antlr/semantic/scope/SymbolTable.java | 2 +- .../types/function/AggregateFunction.java | 2 +- .../sql/legacy/domain/Condition.java | 14 +++---- .../opensearch/sql/legacy/domain/Having.java | 2 +- .../sql/legacy/domain/IndexStatement.java | 4 +- .../sql/legacy/domain/JoinSelect.java | 4 +- .../opensearch/sql/legacy/domain/KVValue.java | 2 +- .../opensearch/sql/legacy/domain/Order.java | 2 +- .../opensearch/sql/legacy/domain/Paramer.java | 2 +- .../opensearch/sql/legacy/domain/Query.java | 2 +- .../opensearch/sql/legacy/domain/Select.java | 8 ++-- .../opensearch/sql/legacy/domain/Where.java | 2 +- .../legacy/domain/bucketpath/BucketPath.java | 2 +- .../sql/legacy/domain/hints/Hint.java | 4 +- .../executor/GetIndexRequestRestListener.java | 2 +- .../executor/cursor/CursorCloseExecutor.java | 2 +- .../executor/cursor/CursorResultExecutor.java | 4 +- .../sql/legacy/executor/format/DataRows.java | 8 ++-- .../executor/format/DateFieldFormatter.java | 2 +- .../legacy/executor/format/DateFormat.java | 38 +++++++++---------- .../executor/format/DeleteResultSet.java | 4 +- .../executor/format/DescribeResultSet.java | 4 +- .../legacy/executor/format/ErrorMessage.java | 10 ++--- .../sql/legacy/executor/format/Protocol.java | 2 +- .../sql/legacy/executor/format/Schema.java | 6 +-- .../executor/format/SelectResultSet.java | 6 +-- .../legacy/executor/format/ShowResultSet.java | 4 +- .../executor/join/ElasticJoinExecutor.java | 8 ++-- .../join/HashJoinComparisonStructure.java | 4 +- .../join/HashJoinElasticExecutor.java | 6 +-- .../join/NestedLoopsElasticExecutor.java | 4 +- .../executor/multi/ComperableHitResult.java | 6 +-- .../legacy/executor/multi/MinusExecutor.java | 6 +-- .../MinusOneFieldAndOptimizationResult.java | 4 +- .../legacy/executor/multi/UnionExecutor.java | 2 +- .../sql/legacy/metrics/BasicCounter.java | 2 +- .../sql/legacy/metrics/GaugeMetric.java | 2 +- .../opensearch/sql/legacy/metrics/Metric.java | 2 +- .../sql/legacy/metrics/MetricName.java | 4 +- .../sql/legacy/metrics/Metrics.java | 5 ++- .../sql/legacy/metrics/NumericMetric.java | 2 +- .../sql/legacy/parser/CaseWhenParser.java | 6 +-- .../legacy/parser/ElasticSqlExprParser.java | 2 +- .../sql/legacy/parser/FieldMaker.java | 2 +- .../sql/legacy/parser/SqlParser.java | 2 +- .../sql/legacy/parser/SubQueryExpression.java | 2 +- .../sql/legacy/parser/WhereParser.java | 4 +- .../legacy/pit/PointInTimeHandlerImpl.java | 2 +- .../legacy/query/AggregationQueryAction.java | 2 +- .../sql/legacy/query/QueryAction.java | 4 +- ...SqlElasticDeleteByQueryRequestBuilder.java | 2 +- .../query/SqlOpenSearchRequestBuilder.java | 2 +- .../query/join/BackOffRetryStrategy.java | 4 +- .../legacy/query/join/JoinRequestBuilder.java | 4 +- .../query/join/OpenSearchJoinQueryAction.java | 2 +- .../sql/legacy/query/maker/AggMaker.java | 2 +- .../legacy/query/multi/MultiQueryAction.java | 2 +- .../query/multi/MultiQueryRequestBuilder.java | 8 ++-- .../legacy/query/multi/MultiQuerySelect.java | 6 +-- .../converter/SQLAggregationParser.java | 2 +- .../core/BindingTupleQueryPlanner.java | 4 +- .../query/planner/core/ExecuteParams.java | 2 +- .../query/planner/logical/node/Group.java | 4 +- .../physical/estimation/Estimation.java | 2 +- .../physical/node/join/CombinedRow.java | 4 +- .../physical/node/join/JoinAlgorithm.java | 2 +- .../physical/node/join/ListHashTable.java | 2 +- .../legacy/query/planner/resource/Stats.java | 6 +-- .../resource/blocksize/AdaptiveBlockSize.java | 2 +- .../planner/resource/blocksize/BlockSize.java | 2 +- .../request/PreparedStatementRequest.java | 6 +-- .../legacy/rewriter/join/JoinRewriteRule.java | 4 +- .../matchtoterm/TermFieldRewriter.java | 4 +- .../nestedfield/NestedFieldRewriter.java | 2 +- .../legacy/rewriter/nestedfield/Scope.java | 4 +- .../rewriter/ordinal/OrdinalRewriterRule.java | 8 ++-- .../subquery/SubQueryRewriteRule.java | 2 +- .../rewriter/SubqueryAliasRewriter.java | 4 +- .../spatial/BoundingBoxFilterParams.java | 4 +- .../sql/legacy/spatial/CellFilterParams.java | 6 +-- .../legacy/spatial/DistanceFilterParams.java | 4 +- .../opensearch/sql/legacy/spatial/Point.java | 4 +- .../legacy/spatial/PolygonFilterParams.java | 2 +- .../spatial/RangeDistanceFilterParams.java | 2 +- .../legacy/spatial/SpatialParamsFactory.java | 2 +- .../sql/legacy/utils/SQLFunctions.java | 2 +- .../sql/legacy/antlr/SyntaxAnalysisTest.java | 4 +- .../semantic/SemanticAnalyzerConfigTest.java | 2 +- .../semantic/SemanticAnalyzerTestBase.java | 4 +- .../semantic/scope/TypeSupplierTest.java | 2 +- .../visitor/AntlrSqlParseTreeVisitorTest.java | 4 +- .../executor/AsyncRestExecutorTest.java | 2 +- .../pit/PointInTimeHandlerImplTest.java | 2 +- .../unittest/ErrorMessageFactoryTest.java | 4 +- .../legacy/unittest/SqlRequestParamTest.java | 2 +- .../expression/core/BinaryExpressionTest.java | 2 +- .../expression/core/UnaryExpressionTest.java | 2 +- .../expression/model/ExprValueUtilsTest.java | 2 +- .../unittest/parser/BucketPathTest.java | 2 +- .../legacy/unittest/parser/SqlParserTest.java | 2 +- .../unittest/parser/SubQueryParserTest.java | 2 +- .../planner/QueryPlannerBatchTest.java | 18 +++++---- .../planner/QueryPlannerMonitorTest.java | 2 +- .../converter/SQLAggregationParserTest.java | 2 +- .../SQLExprToExpressionConverterTest.java | 2 +- .../identifier/UnquoteIdentifierRuleTest.java | 4 +- .../ordinal/OrdinalRewriterRuleTest.java | 6 +-- .../parent/SQLExprParentSetterRuleTest.java | 2 +- .../subquery/ExistsSubQueryRewriterTest.java | 2 +- .../subquery/InSubqueryRewriterTest.java | 2 +- .../subquery/SubQueryRewriteRuleTest.java | 2 +- .../rewriter/term/TermFieldRewriterTest.java | 2 +- .../unittest/utils/SQLFunctionsTest.java | 2 +- .../util/MultipleIndexClusterUtils.java | 4 +- .../response/error/ErrorMessage.java | 2 +- .../scan/OpenSearchIndexScanQueryBuilder.java | 2 +- .../storage/script/sort/SortQueryBuilder.java | 4 +- .../OpenSearchExprGeoPointValueTest.java | 3 +- .../planner/physical/MLOperatorTest.java | 2 +- .../response/OpenSearchResponseTest.java | 6 +-- .../error/ErrorMessageFactoryTest.java | 4 +- .../TransportPPLQueryRequestTest.java | 2 +- .../sql/ppl/parser/AstExpressionBuilder.java | 2 +- .../opensearch/sql/ppl/PPLServiceTest.java | 4 +- .../sql/ppl/antlr/PPLSyntaxParserTest.java | 2 +- .../sql/ppl/domain/PPLQueryRequestTest.java | 2 +- .../sql/ppl/parser/AstBuilderTest.java | 2 +- .../ppl/parser/AstStatementBuilderTest.java | 2 +- .../protocol/response/QueryResultTest.java | 2 +- .../sql/spark/client/EmrClientImpl.java | 2 +- .../opensearch/sql/sql/SQLServiceTest.java | 4 +- 200 files changed, 393 insertions(+), 383 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java index af1f69d74b..26b391933b 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java @@ -11,7 +11,7 @@ public enum JobType { REFRESH("refresh"), BATCH("batch"); - private String text; + private final String text; JobType(String text) { this.text = text; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java index bd5d14c603..fc15308e05 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java @@ -20,7 +20,7 @@ public enum SessionState { DEAD("dead"), FAIL("fail"); - public static List<SessionState> END_STATE = ImmutableList.of(DEAD, FAIL); + public static final List<SessionState> END_STATE = ImmutableList.of(DEAD, FAIL); private final String sessionState; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java index a6a88738bf..c4d9509bd2 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java @@ -21,9 +21,9 @@ * be registered during initialization such as in Guice Module. */ public class SparkParameterComposerCollection { - private Collection<GeneralSparkParameterComposer> generalComposers = new ArrayList<>(); - private Map<DataSourceType, Collection<DataSourceSparkParameterComposer>> datasourceComposers = - new HashMap<>(); + private final Collection<GeneralSparkParameterComposer> generalComposers = new ArrayList<>(); + private final Map<DataSourceType, Collection<DataSourceSparkParameterComposer>> + datasourceComposers = new HashMap<>(); /** * Register composers for specific DataSourceType. The registered composer is called only if the diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java index 84fd49b712..a496ad7503 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkSubmitParameters.java @@ -17,7 +17,7 @@ public class SparkSubmitParameters { public static final String EQUALS = "="; @Setter private String className = DEFAULT_CLASS_NAME; - private Map<String, String> config = new LinkedHashMap<>(); + private final Map<String, String> config = new LinkedHashMap<>(); /** Extra parameters to append finally */ @Setter private String extraParameters; diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java index 3ba9c23ed7..bd7060b776 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java @@ -90,7 +90,8 @@ public static SqlBaseParser getBaseParser(String sqlQuery) { public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor<Void> { - @Getter private List<FullyQualifiedTableName> fullyQualifiedTableNames = new LinkedList<>(); + @Getter + private final List<FullyQualifiedTableName> fullyQualifiedTableNames = new LinkedList<>(); public SparkSqlTableNameVisitor() {} diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java index 7f1c92dff3..2cd50d755d 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProviderTest.java @@ -19,7 +19,7 @@ class DatasourceEmbeddedQueryIdProviderTest { @Mock AsyncQueryRequestContext asyncQueryRequestContext; - DatasourceEmbeddedQueryIdProvider datasourceEmbeddedQueryIdProvider = + final DatasourceEmbeddedQueryIdProvider datasourceEmbeddedQueryIdProvider = new DatasourceEmbeddedQueryIdProvider(); @Test diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 61d72773d9..b5ea349045 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -129,7 +129,7 @@ public class SparkQueryDispatcherTest { new GrammarElementValidatorProvider( ImmutableMap.of(), new DefaultGrammarElementValidator())); - private DataSourceSparkParameterComposer dataSourceSparkParameterComposer = + private final DataSourceSparkParameterComposer dataSourceSparkParameterComposer = (datasourceMetadata, sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_AUTH_KEY, "basic"); sparkSubmitParameters.setConfigItem(FLINT_INDEX_STORE_HOST_KEY, "HOST"); @@ -139,7 +139,7 @@ public class SparkQueryDispatcherTest { KEY_FROM_DATASOURCE_COMPOSER, VALUE_FROM_DATASOURCE_COMPOSER); }; - private GeneralSparkParameterComposer generalSparkParameterComposer = + private final GeneralSparkParameterComposer generalSparkParameterComposer = (sparkSubmitParameters, dispatchQueryRequest, context) -> { sparkSubmitParameters.setConfigItem(KEY_FROM_COMPOSER, VALUE_FROM_COMPOSER); }; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java index 15d1ec2ecc..fdbbbc17e0 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java @@ -25,7 +25,7 @@ public String getId() { } } - ConcreteStateModel model = + final ConcreteStateModel model = ConcreteStateModel.builder().metadata(ImmutableMap.of(METADATA_KEY, METADATA_VALUE)).build(); @Test diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollectionTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollectionTest.java index c0c97caa58..8cd1de8b27 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollectionTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollectionTest.java @@ -30,9 +30,9 @@ class SparkParameterComposerCollectionTest { @Mock DispatchQueryRequest dispatchQueryRequest; @Mock AsyncQueryRequestContext asyncQueryRequestContext; - DataSourceType type1 = new DataSourceType("TYPE1"); - DataSourceType type2 = new DataSourceType("TYPE2"); - DataSourceType type3 = new DataSourceType("TYPE3"); + final DataSourceType type1 = new DataSourceType("TYPE1"); + final DataSourceType type2 = new DataSourceType("TYPE2"); + final DataSourceType type3 = new DataSourceType("TYPE3"); SparkParameterComposerCollection collection; diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java index 3b24c3c8ab..a42c9f7cd5 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java @@ -12,13 +12,13 @@ import org.opensearch.sql.datasource.model.DataSourceType; class GrammarElementValidatorProviderTest { - S3GlueSQLGrammarElementValidator s3GlueSQLGrammarElementValidator = + final S3GlueSQLGrammarElementValidator s3GlueSQLGrammarElementValidator = new S3GlueSQLGrammarElementValidator(); - SecurityLakeSQLGrammarElementValidator securityLakeSQLGrammarElementValidator = + final SecurityLakeSQLGrammarElementValidator securityLakeSQLGrammarElementValidator = new SecurityLakeSQLGrammarElementValidator(); - DefaultGrammarElementValidator defaultGrammarElementValidator = + final DefaultGrammarElementValidator defaultGrammarElementValidator = new DefaultGrammarElementValidator(); - GrammarElementValidatorProvider grammarElementValidatorProvider = + final GrammarElementValidatorProvider grammarElementValidatorProvider = new GrammarElementValidatorProvider( ImmutableMap.of( DataSourceType.S3GLUE, s3GlueSQLGrammarElementValidator, diff --git a/async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java index 6c660f073c..52c829318a 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java @@ -30,13 +30,13 @@ public class ClusterManagerEventListener implements LocalNodeClusterManagerListe private Cancellable flintIndexRetentionCron; private Cancellable flintStreamingJobHouseKeeperCron; - private ClusterService clusterService; - private ThreadPool threadPool; - private Client client; - private Clock clock; - private DataSourceService dataSourceService; - private FlintIndexMetadataService flintIndexMetadataService; - private FlintIndexOpFactory flintIndexOpFactory; + private final ClusterService clusterService; + private final ThreadPool threadPool; + private final Client client; + private final Clock clock; + private final DataSourceService dataSourceService; + private final FlintIndexMetadataService flintIndexMetadataService; + private final FlintIndexOpFactory flintIndexOpFactory; private Duration sessionTtlDuration; private Duration resultTtlDuration; private TimeValue streamingJobHouseKeepingInterval; diff --git a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java index 18dce7b7b2..552c646cbe 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java @@ -67,9 +67,9 @@ */ @RequiredArgsConstructor public class StateStore { - public static String SETTINGS_FILE_NAME = "query_execution_request_settings.yml"; - public static String MAPPING_FILE_NAME = "query_execution_request_mapping.yml"; - public static String ALL_DATASOURCE = "*"; + public static final String SETTINGS_FILE_NAME = "query_execution_request_settings.yml"; + public static final String MAPPING_FILE_NAME = "query_execution_request_mapping.yml"; + public static final String ALL_DATASOURCE = "*"; private static final Logger LOG = LogManager.getLogger(); diff --git a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java index 3652acf295..a2abb8e944 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java @@ -40,7 +40,7 @@ public class ScheduledAsyncQueryJobRunner implements ScheduledJobRunner { AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME; private static final Logger LOGGER = LogManager.getLogger(ScheduledAsyncQueryJobRunner.class); - private static ScheduledAsyncQueryJobRunner INSTANCE = new ScheduledAsyncQueryJobRunner(); + private static final ScheduledAsyncQueryJobRunner INSTANCE = new ScheduledAsyncQueryJobRunner(); public static ScheduledAsyncQueryJobRunner getJobRunnerInstance() { return INSTANCE; diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java index 175f9ac914..e6459c752e 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java @@ -44,7 +44,7 @@ import org.opensearch.sql.spark.utils.IDUtils; public class AsyncQueryExecutorServiceImplSpecTest extends AsyncQueryExecutorServiceSpec { - AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); + final AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); @Disabled("batch query is unsupported") public void withoutSessionCreateAsyncQueryThenGetResultThenCancel() { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java index 53b465aa6d..9511359f86 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java @@ -133,7 +133,7 @@ public class AsyncQueryExecutorServiceSpec extends OpenSearchIntegTestCase { protected StatementStorageService statementStorageService; protected AsyncQueryScheduler asyncQueryScheduler; protected AsyncQueryRequestContext asyncQueryRequestContext; - protected SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); + protected final SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); @Override protected Collection<Class<? extends Plugin>> nodePlugins() { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java index ef98e955f6..5f6d7190da 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java @@ -41,7 +41,7 @@ import org.opensearch.sql.spark.transport.model.AsyncQueryResult; public class AsyncQueryGetResultSpecTest extends AsyncQueryExecutorServiceSpec { - AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); + final AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); /** Mock Flint index and index state */ private final FlintDatasetMock mockIndex = diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java index c84d68421d..d373f451a6 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java @@ -28,7 +28,8 @@ public class OpenSearchAsyncQueryJobMetadataStorageServiceTest extends OpenSearc private static final String MOCK_RESULT_INDEX = "resultIndex"; private static final String MOCK_QUERY_ID = "00fdo6u94n7abo0q"; private OpenSearchAsyncQueryJobMetadataStorageService openSearchJobMetadataStorageService; - private AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); + private final AsyncQueryRequestContext asyncQueryRequestContext = + new NullAsyncQueryRequestContext(); @Before public void setup() { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java index 0dc8f02820..e61d550d68 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java @@ -16,9 +16,10 @@ public class MockFlintSparkJob { private FlintIndexStateModel stateModel; - private FlintIndexStateModelService flintIndexStateModelService; - private String datasource; - private AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); + private final FlintIndexStateModelService flintIndexStateModelService; + private final String datasource; + private final AsyncQueryRequestContext asyncQueryRequestContext = + new NullAsyncQueryRequestContext(); public MockFlintSparkJob( FlintIndexStateModelService flintIndexStateModelService, String latestId, String datasource) { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java index d0bfed94c0..30a71607b0 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java @@ -44,10 +44,11 @@ public class InteractiveSessionTest extends OpenSearchIntegTestCase { private StartJobRequest startJobRequest; private SessionStorageService sessionStorageService; private StatementStorageService statementStorageService; - private SessionConfigSupplier sessionConfigSupplier = () -> 600000L; + private final SessionConfigSupplier sessionConfigSupplier = () -> 600000L; private SessionManager sessionManager; - private AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); - private SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); + private final AsyncQueryRequestContext asyncQueryRequestContext = + new NullAsyncQueryRequestContext(); + private final SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); @Before public void setup() { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java index fe3d5f3177..49a4d69222 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java @@ -47,12 +47,13 @@ public class StatementTest extends OpenSearchIntegTestCase { private StatementStorageService statementStorageService; private SessionStorageService sessionStorageService; - private TestEMRServerlessClient emrsClient = new TestEMRServerlessClient(); - private SessionConfigSupplier sessionConfigSupplier = () -> 600000L; - private SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); + private final TestEMRServerlessClient emrsClient = new TestEMRServerlessClient(); + private final SessionConfigSupplier sessionConfigSupplier = () -> 600000L; + private final SessionIdProvider sessionIdProvider = new DatasourceEmbeddedSessionIdProvider(); private SessionManager sessionManager; - private AsyncQueryRequestContext asyncQueryRequestContext = new NullAsyncQueryRequestContext(); + private final AsyncQueryRequestContext asyncQueryRequestContext = + new NullAsyncQueryRequestContext(); @Before public void setup() { diff --git a/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java index 0d6d5f3119..1840c52457 100644 --- a/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java @@ -23,7 +23,7 @@ @ExtendWith(MockitoExtension.class) class FlintIndexStateModelXContentSerializerTest { - private FlintIndexStateModelXContentSerializer serializer = + private final FlintIndexStateModelXContentSerializer serializer = new FlintIndexStateModelXContentSerializer(); @Test diff --git a/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java b/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java index 16196544b5..1ab1bb8976 100644 --- a/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java @@ -24,9 +24,9 @@ public class AwsSigningInterceptor implements Interceptor { - private OkHttpAwsV4Signer okHttpAwsV4Signer; + private final OkHttpAwsV4Signer okHttpAwsV4Signer; - private AWSCredentialsProvider awsCredentialsProvider; + private final AWSCredentialsProvider awsCredentialsProvider; private static final Logger LOG = LogManager.getLogger(); diff --git a/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java b/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java index 15e9a0fc12..0ade25520f 100644 --- a/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java @@ -16,7 +16,7 @@ public class BasicAuthenticationInterceptor implements Interceptor { - private String credentials; + private final String credentials; public BasicAuthenticationInterceptor(@NonNull String username, @NonNull String password) { this.credentials = Credentials.basic(username, password); diff --git a/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java b/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java index c724b58f3e..748495bff6 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java @@ -33,7 +33,7 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class BasicTest { - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); private GrokCompiler compiler; diff --git a/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java index 01145dc7df..31719d2fe3 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java +++ b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java @@ -18,8 +18,8 @@ public class DataSourceSchemaIdentifierNameResolver { private String dataSourceName = DEFAULT_DATASOURCE_NAME; private String schemaName = DEFAULT_SCHEMA_NAME; - private String identifierName; - private DataSourceService dataSourceService; + private final String identifierName; + private final DataSourceService dataSourceService; private static final String DOT = "."; diff --git a/core/src/main/java/org/opensearch/sql/analysis/symbol/SymbolTable.java b/core/src/main/java/org/opensearch/sql/analysis/symbol/SymbolTable.java index 8bb6824a63..64a4fc4e09 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/symbol/SymbolTable.java +++ b/core/src/main/java/org/opensearch/sql/analysis/symbol/SymbolTable.java @@ -20,14 +20,14 @@ public class SymbolTable { /** Two-dimension hash table to manage symbols with type in different namespace. */ - private Map<Namespace, NavigableMap<String, ExprType>> tableByNamespace = + private final Map<Namespace, NavigableMap<String, ExprType>> tableByNamespace = new EnumMap<>(Namespace.class); /** * Two-dimension hash table to manage symbols with type in different namespace. Comparing with * tableByNamespace, orderedTable use the LinkedHashMap to keep the order of symbol. */ - private Map<Namespace, LinkedHashMap<String, ExprType>> orderedTable = + private final Map<Namespace, LinkedHashMap<String, ExprType>> orderedTable = new EnumMap<>(Namespace.class); /** diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Filter.java b/core/src/main/java/org/opensearch/sql/ast/tree/Filter.java index 6c57275db9..da98fef7be 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Filter.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Filter.java @@ -18,7 +18,7 @@ @EqualsAndHashCode(callSuper = false) @Getter public class Filter extends UnresolvedPlan { - private UnresolvedExpression condition; + private final UnresolvedExpression condition; private UnresolvedPlan child; public Filter(UnresolvedExpression condition) { diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Project.java b/core/src/main/java/org/opensearch/sql/ast/tree/Project.java index cffb4dfdce..bbe4d0e2ce 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Project.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Project.java @@ -22,7 +22,7 @@ @EqualsAndHashCode(callSuper = false) public class Project extends UnresolvedPlan { @Setter private List<UnresolvedExpression> projectList; - private List<Argument> argExprList; + private final List<Argument> argExprList; private UnresolvedPlan child; public Project(List<UnresolvedExpression> projectList) { diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java index 6efc7c935c..2282bc694a 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java @@ -30,36 +30,36 @@ public class DataSourceMetadata { public static final String DEFAULT_RESULT_INDEX = "query_execution_result"; public static final int MAX_RESULT_INDEX_NAME_SIZE = 255; - private static String DATASOURCE_NAME_REGEX = "[@*A-Za-z]+?[*a-zA-Z_\\-0-9]*"; + private static final String DATASOURCE_NAME_REGEX = "[@*A-Za-z]+?[*a-zA-Z_\\-0-9]*"; // OS doesn’t allow uppercase: https://tinyurl.com/yse2xdbx public static final String RESULT_INDEX_NAME_PATTERN = "[a-z0-9_-]+"; - public static String INVALID_RESULT_INDEX_NAME_SIZE = + public static final String INVALID_RESULT_INDEX_NAME_SIZE = "Result index name size must contains less than " + MAX_RESULT_INDEX_NAME_SIZE + " characters."; - public static String INVALID_CHAR_IN_RESULT_INDEX_NAME = + public static final String INVALID_CHAR_IN_RESULT_INDEX_NAME = "Result index name has invalid character. Valid characters are a-z, 0-9, -(hyphen) and" + " _(underscore)."; - public static String INVALID_RESULT_INDEX_PREFIX = + public static final String INVALID_RESULT_INDEX_PREFIX = "Result index must start with " + DEFAULT_RESULT_INDEX; - @JsonProperty private String name; + @JsonProperty private final String name; - @JsonProperty private String description; + @JsonProperty private final String description; @JsonProperty @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) - private DataSourceType connector; + private final DataSourceType connector; - @JsonProperty private List<String> allowedRoles; + @JsonProperty private final List<String> allowedRoles; - @JsonProperty private Map<String, String> properties; + @JsonProperty private final Map<String, String> properties; - @JsonProperty private String resultIndex; + @JsonProperty private final String resultIndex; - @JsonProperty private DataSourceStatus status; + @JsonProperty private final DataSourceStatus status; - public static Function<String, String> DATASOURCE_TO_RESULT_INDEX = + public static final Function<String, String> DATASOURCE_TO_RESULT_INDEX = datasourceName -> String.format("%s_%s", DEFAULT_RESULT_INDEX, datasourceName); private DataSourceMetadata(Builder builder) { diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceStatus.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceStatus.java index bca47217c1..a4282eb419 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceStatus.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceStatus.java @@ -10,7 +10,7 @@ public enum DataSourceStatus { ACTIVE("active"), DISABLED("disabled"); - private String text; + private final String text; DataSourceStatus(String text) { this.text = text; diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java index ac8ae1a5e1..442497094b 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java @@ -13,14 +13,14 @@ @RequiredArgsConstructor @EqualsAndHashCode public class DataSourceType { - public static DataSourceType PROMETHEUS = new DataSourceType("PROMETHEUS"); - public static DataSourceType OPENSEARCH = new DataSourceType("OPENSEARCH"); - public static DataSourceType SPARK = new DataSourceType("SPARK"); - public static DataSourceType S3GLUE = new DataSourceType("S3GLUE"); - public static DataSourceType SECURITY_LAKE = new DataSourceType("SECURITY_LAKE"); + public static final DataSourceType PROMETHEUS = new DataSourceType("PROMETHEUS"); + public static final DataSourceType OPENSEARCH = new DataSourceType("OPENSEARCH"); + public static final DataSourceType SPARK = new DataSourceType("SPARK"); + public static final DataSourceType S3GLUE = new DataSourceType("S3GLUE"); + public static final DataSourceType SECURITY_LAKE = new DataSourceType("SECURITY_LAKE"); // Map from uppercase DataSourceType name to DataSourceType object - private static Map<String, DataSourceType> knownValues = new HashMap<>(); + private static final Map<String, DataSourceType> knownValues = new HashMap<>(); static { register(PROMETHEUS, OPENSEARCH, SPARK, S3GLUE, SECURITY_LAKE); diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java index e439d93f6c..48975a5608 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java @@ -26,7 +26,7 @@ public class DefaultMetadataLog<T> implements MetadataLog<T> { private static final long MIN_ACCEPTABLE_ID = 0L; - private SortedMap<Long, T> metadataMap = new TreeMap<>(); + private final SortedMap<Long, T> metadataMap = new TreeMap<>(); @Override public boolean add(Long batchId, T metadata) { diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java index 96ff7785b7..6adc4fb2a3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java @@ -118,7 +118,7 @@ public static void register(BuiltinFunctionRepository repository) { * </tr> * </table> */ - private static Table<ExprValue, ExprValue, ExprValue> andTable = + private static final Table<ExprValue, ExprValue, ExprValue> andTable = new ImmutableTable.Builder<ExprValue, ExprValue, ExprValue>() .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_TRUE) .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_FALSE) @@ -193,7 +193,7 @@ public static void register(BuiltinFunctionRepository repository) { * </tr> * </table> */ - private static Table<ExprValue, ExprValue, ExprValue> orTable = + private static final Table<ExprValue, ExprValue, ExprValue> orTable = new ImmutableTable.Builder<ExprValue, ExprValue, ExprValue>() .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_TRUE) .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_TRUE) @@ -268,7 +268,7 @@ public static void register(BuiltinFunctionRepository repository) { * </tr> * </table> */ - private static Table<ExprValue, ExprValue, ExprValue> xorTable = + private static final Table<ExprValue, ExprValue, ExprValue> xorTable = new ImmutableTable.Builder<ExprValue, ExprValue, ExprValue>() .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_FALSE) .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_TRUE) diff --git a/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java b/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java index 8a5302070c..2c3bbf7efb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/text/TextFunctions.java @@ -39,7 +39,7 @@ */ @UtilityClass public class TextFunctions { - private static String EMPTY_STRING = ""; + private static final String EMPTY_STRING = ""; /** * Register String Functions. diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalNested.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalNested.java index e791a1fad1..089efe707e 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalNested.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalNested.java @@ -19,7 +19,7 @@ @Getter @ToString public class LogicalNested extends LogicalPlan { - private List<Map<String, ReferenceExpression>> fields; + private final List<Map<String, ReferenceExpression>> fields; private final List<NamedExpression> projectList; /** Constructor of LogicalNested. */ diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/NestedOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/NestedOperator.java index 8539df5463..fb5ec276ac 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/NestedOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/NestedOperator.java @@ -37,7 +37,7 @@ public class NestedOperator extends PhysicalPlan { @Getter private final Set<String> fields; // Needs to be a Set to match legacy implementation @Getter private final Map<String, List<String>> groupedPathsAndFields; @EqualsAndHashCode.Exclude private List<Map<String, ExprValue>> result = new ArrayList<>(); - @EqualsAndHashCode.Exclude private List<String> nonNestedFields = new ArrayList<>(); + @EqualsAndHashCode.Exclude private final List<String> nonNestedFields = new ArrayList<>(); @EqualsAndHashCode.Exclude private ListIterator<Map<String, ExprValue>> flattenedResult = result.listIterator(); diff --git a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java index 2487c651ad..c5c12584fd 100644 --- a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java +++ b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java @@ -17,7 +17,7 @@ * output column name is bindingName, the value is the ExprValue. */ public abstract class BindingTuple implements Environment<Expression, ExprValue> { - public static BindingTuple EMPTY = + public static final BindingTuple EMPTY = new BindingTuple() { @Override public ExprValue resolve(ReferenceExpression ref) { diff --git a/core/src/main/java/org/opensearch/sql/utils/ExpressionUtils.java b/core/src/main/java/org/opensearch/sql/utils/ExpressionUtils.java index f04bf3748f..8ae0c6ba88 100644 --- a/core/src/main/java/org/opensearch/sql/utils/ExpressionUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/ExpressionUtils.java @@ -14,7 +14,7 @@ @UtilityClass public class ExpressionUtils { - public static String PATH_SEP = "."; + public static final String PATH_SEP = "."; /** Format the list of {@link Expression}. */ public static String format(List<Expression> expressionList) { diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 0bf959a1b7..17f86cadba 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -148,15 +148,15 @@ protected Environment<Expression, ExprType> typeEnv() { }; } - protected AnalysisContext analysisContext = analysisContext(typeEnvironment(symbolTable())); + protected final AnalysisContext analysisContext = analysisContext(typeEnvironment(symbolTable())); - protected ExpressionAnalyzer expressionAnalyzer = expressionAnalyzer(); + protected final ExpressionAnalyzer expressionAnalyzer = expressionAnalyzer(); - protected Table table = table(); + protected final Table table = table(); - protected DataSourceService dataSourceService = dataSourceService(); + protected final DataSourceService dataSourceService = dataSourceService(); - protected Analyzer analyzer = analyzer(expressionAnalyzer(), dataSourceService); + protected final Analyzer analyzer = analyzer(expressionAnalyzer(), dataSourceService); protected Analyzer analyzer( ExpressionAnalyzer expressionAnalyzer, DataSourceService dataSourceService) { @@ -245,11 +245,11 @@ public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata( private class TestTableFunctionImplementation implements TableFunctionImplementation { - private FunctionName functionName; + private final FunctionName functionName; - private List<Expression> arguments; + private final List<Expression> arguments; - private Table table; + private final Table table; public TestTableFunctionImplementation( FunctionName functionName, List<Expression> arguments, Table table) { diff --git a/core/src/test/java/org/opensearch/sql/analysis/TypeEnvironmentTest.java b/core/src/test/java/org/opensearch/sql/analysis/TypeEnvironmentTest.java index 91677a901e..00e2f10469 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/TypeEnvironmentTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/TypeEnvironmentTest.java @@ -21,7 +21,7 @@ public class TypeEnvironmentTest { /** Use context class for push/pop. */ - private AnalysisContext context = new AnalysisContext(); + private final AnalysisContext context = new AnalysisContext(); @Test public void defineFieldSymbolInDifferentEnvironmentsShouldBeAbleToResolve() { diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index 92b6aac64f..6655640e28 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -33,7 +33,7 @@ public class TestConfig { public static final String STRING_TYPE_NULL_VALUE_FIELD = "string_null_value"; public static final String STRING_TYPE_MISSING_VALUE_FIELD = "string_missing_value"; - public static Map<String, ExprType> typeMapping = + public static final Map<String, ExprType> typeMapping = new ImmutableMap.Builder<String, ExprType>() .put("integer_value", ExprCoreType.INTEGER) .put(INT_TYPE_NULL_VALUE_FIELD, ExprCoreType.INTEGER) diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java index 0baf5052e4..9fe6347102 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java @@ -50,18 +50,18 @@ @DisplayName("Test Expression Value Utils") public class ExprValueUtilsTest { - private static LinkedHashMap<String, ExprValue> testTuple = new LinkedHashMap<>(); + private static final LinkedHashMap<String, ExprValue> testTuple = new LinkedHashMap<>(); static { testTuple.put("1", new ExprIntegerValue(1)); } - private static List<ExprValue> numberValues = + private static final List<ExprValue> numberValues = Stream.of((byte) 1, (short) 1, 1, 1L, 1f, 1D) .map(ExprValueUtils::fromObjectValue) .collect(Collectors.toList()); - private static List<ExprValue> nonNumberValues = + private static final List<ExprValue> nonNumberValues = Arrays.asList( new ExprStringValue("1"), ExprBooleanValue.of(true), @@ -72,10 +72,10 @@ public class ExprValueUtilsTest { new ExprTimestampValue("2012-08-07 18:00:00"), new ExprIntervalValue(Duration.ofSeconds(100))); - private static List<ExprValue> allValues = + private static final List<ExprValue> allValues = Lists.newArrayList(Iterables.concat(numberValues, nonNumberValues)); - private static List<Function<ExprValue, Object>> numberValueExtractor = + private static final List<Function<ExprValue, Object>> numberValueExtractor = Arrays.asList( ExprValueUtils::getByteValue, ExprValueUtils::getShortValue, @@ -83,24 +83,24 @@ public class ExprValueUtilsTest { ExprValueUtils::getLongValue, ExprValueUtils::getFloatValue, ExprValueUtils::getDoubleValue); - private static List<Function<ExprValue, Object>> nonNumberValueExtractor = + private static final List<Function<ExprValue, Object>> nonNumberValueExtractor = Arrays.asList( ExprValueUtils::getStringValue, ExprValueUtils::getBooleanValue, ExprValueUtils::getCollectionValue, ExprValueUtils::getTupleValue); - private static List<Function<ExprValue, Object>> dateAndTimeValueExtractor = + private static final List<Function<ExprValue, Object>> dateAndTimeValueExtractor = Arrays.asList( ExprValue::dateValue, ExprValue::timeValue, ExprValue::timestampValue, ExprValue::intervalValue); - private static List<Function<ExprValue, Object>> allValueExtractor = + private static final List<Function<ExprValue, Object>> allValueExtractor = Lists.newArrayList( Iterables.concat( numberValueExtractor, nonNumberValueExtractor, dateAndTimeValueExtractor)); - private static List<ExprCoreType> numberTypes = + private static final List<ExprCoreType> numberTypes = Arrays.asList( ExprCoreType.BYTE, ExprCoreType.SHORT, @@ -108,10 +108,11 @@ public class ExprValueUtilsTest { ExprCoreType.LONG, ExprCoreType.FLOAT, ExprCoreType.DOUBLE); - private static List<ExprCoreType> nonNumberTypes = Arrays.asList(STRING, BOOLEAN, ARRAY, STRUCT); - private static List<ExprCoreType> dateAndTimeTypes = + private static final List<ExprCoreType> nonNumberTypes = + Arrays.asList(STRING, BOOLEAN, ARRAY, STRUCT); + private static final List<ExprCoreType> dateAndTimeTypes = Arrays.asList(DATE, TIME, TIMESTAMP, INTERVAL); - private static List<ExprCoreType> allTypes = + private static final List<ExprCoreType> allTypes = Lists.newArrayList(Iterables.concat(numberTypes, nonNumberTypes, dateAndTimeTypes)); private static Stream<Arguments> getValueTestArgumentStream() { diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryManagerTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryManagerTest.java index 7f34d348bc..2d8d4d4ee2 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryManagerTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryManagerTest.java @@ -17,7 +17,7 @@ class QueryManagerTest { @Mock private QueryId queryId; - private QueryManager queryManager = + private final QueryManager queryManager = id -> { throw new UnsupportedOperationException(); }; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/IntervalTriggerExecutionTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/IntervalTriggerExecutionTest.java index 9eb99d37e3..030114749c 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/IntervalTriggerExecutionTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/IntervalTriggerExecutionTest.java @@ -32,7 +32,7 @@ Helper triggerTask(long interval) { class Helper implements Runnable { - private StreamingQueryPlan.IntervalTriggerExecution executionStrategy; + private final StreamingQueryPlan.IntervalTriggerExecution executionStrategy; private static final int START = 0; @@ -42,7 +42,7 @@ class Helper implements Runnable { private int state = START; - private long interval; + private final long interval; private long taskExecutionTime; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/StreamingQueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/StreamingQueryPlanTest.java index 2e8666aea4..c2cc606a88 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/StreamingQueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/StreamingQueryPlanTest.java @@ -77,7 +77,7 @@ Helper streamingQuery() { class Helper { - private StreamingQueryPlan queryPlan; + private final StreamingQueryPlan queryPlan; public Helper() { queryPlan = diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java index fd886cdda3..fc7e73d5b2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java @@ -36,7 +36,7 @@ public class ExpressionTestBase { - protected FunctionProperties functionProperties = new FunctionProperties(); + protected final FunctionProperties functionProperties = new FunctionProperties(); protected Environment<Expression, ExprType> typeEnv; diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java index f1a3a9d948..38aa263ddc 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java @@ -17,7 +17,7 @@ public class AggregationTest extends ExpressionTestBase { - protected static List<ExprValue> tuples = + protected static final List<ExprValue> tuples = Arrays.asList( ExprValueUtils.tupleValue( new ImmutableMap.Builder<String, Object>() @@ -95,7 +95,7 @@ public class AggregationTest extends ExpressionTestBase { "timestamp_value", "2040-01-01 07:00:00"))); - protected static List<ExprValue> tuples_with_duplicates = + protected static final List<ExprValue> tuples_with_duplicates = Arrays.asList( ExprValueUtils.tupleValue( ImmutableMap.of( @@ -138,7 +138,7 @@ public class AggregationTest extends ExpressionTestBase { "array_value", ImmutableList.of(1, 2)))); - protected static List<ExprValue> tuples_with_null_and_missing = + protected static final List<ExprValue> tuples_with_null_and_missing = Arrays.asList( ExprValueUtils.tupleValue( ImmutableMap.of("integer_value", 2, "string_value", "m", "double_value", 3d)), @@ -146,7 +146,7 @@ public class AggregationTest extends ExpressionTestBase { ImmutableMap.of("integer_value", 1, "string_value", "f", "double_value", 4d)), ExprValueUtils.tupleValue(Collections.singletonMap("double_value", null))); - protected static List<ExprValue> tuples_with_all_null_or_missing = + protected static final List<ExprValue> tuples_with_all_null_or_missing = Arrays.asList( ExprValueUtils.tupleValue(Collections.singletonMap("integer_value", null)), ExprValueUtils.tupleValue(Collections.singletonMap("double", null)), diff --git a/core/src/test/java/org/opensearch/sql/expression/function/FunctionSignatureTest.java b/core/src/test/java/org/opensearch/sql/expression/function/FunctionSignatureTest.java index 2fb5dc468e..b301cf7ba8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/FunctionSignatureTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/FunctionSignatureTest.java @@ -27,8 +27,8 @@ class FunctionSignatureTest { @Mock private FunctionSignature funcSignature; @Mock private List<ExprType> funcParamTypeList; - private FunctionName unresolvedFuncName = FunctionName.of("add"); - private List<ExprType> unresolvedParamTypeList = + private final FunctionName unresolvedFuncName = FunctionName.of("add"); + private final List<ExprType> unresolvedParamTypeList = Arrays.asList(ExprCoreType.INTEGER, ExprCoreType.FLOAT); @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java index 9de1e65108..3064ffcdee 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java @@ -36,7 +36,7 @@ import org.opensearch.sql.exception.ExpressionEvaluationException; class WideningTypeRuleTest { - private static Table<ExprCoreType, ExprCoreType, Integer> numberWidenRule = + private static final Table<ExprCoreType, ExprCoreType, Integer> numberWidenRule = new ImmutableTable.Builder<ExprCoreType, ExprCoreType, Integer>() .put(BYTE, SHORT, 1) .put(BYTE, INTEGER, 2) diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 19cbb4674e..3fc7f737f8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -57,7 +57,7 @@ class BinaryPredicateOperatorTest extends ExpressionTestBase { - private static List<StringPatternPair> STRING_PATTERN_PAIRS = + private static final List<StringPatternPair> STRING_PATTERN_PAIRS = ImmutableList.of( new StringPatternPair("Michael!", ".*"), new StringPatternPair("new*\\n*line", "new\\\\*.\\\\*line"), diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTest.java index d63ab35773..eb35c00d9e 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTest.java @@ -24,7 +24,7 @@ class PhysicalPlanTest { @Mock PhysicalPlan child; - private PhysicalPlan testPlan = + private final PhysicalPlan testPlan = new PhysicalPlan() { @Override public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java index 6399f945ed..397f241484 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java @@ -93,7 +93,7 @@ public class PhysicalPlanTestBase { ImmutableMap.of("ip", "74.125.19.106", "action", "POST", "response", 500))) .build(); - private static Map<String, ExprCoreType> typeMapping = + private static final Map<String, ExprCoreType> typeMapping = new ImmutableMap.Builder<String, ExprCoreType>() .put("ip", ExprCoreType.STRING) .put("action", ExprCoreType.STRING) diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java index a0c0f5e24d..d4c74c7b30 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java @@ -14,7 +14,7 @@ /** Error Message. */ public class ErrorMessage { - protected Throwable exception; + protected final Throwable exception; private final int status; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java index 95e6493e05..edf4d14e1e 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java @@ -30,8 +30,8 @@ public class TransportCreateDataSourceAction public static final ActionType<CreateDataSourceActionResponse> ACTION_TYPE = new ActionType<>(NAME, CreateDataSourceActionResponse::new); - private DataSourceService dataSourceService; - private org.opensearch.sql.opensearch.setting.OpenSearchSettings settings; + private final DataSourceService dataSourceService; + private final org.opensearch.sql.opensearch.setting.OpenSearchSettings settings; /** * TransportCreateDataSourceAction action for creating datasource. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java index 5578d40651..d17deb7df0 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java @@ -26,7 +26,7 @@ public class TransportDeleteDataSourceAction public static final ActionType<DeleteDataSourceActionResponse> ACTION_TYPE = new ActionType<>(NAME, DeleteDataSourceActionResponse::new); - private DataSourceService dataSourceService; + private final DataSourceService dataSourceService; /** * TransportDeleteDataSourceAction action for deleting datasource. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java index 34ad59c80f..e2da2a8e0c 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java @@ -29,7 +29,7 @@ public class TransportGetDataSourceAction public static final ActionType<GetDataSourceActionResponse> ACTION_TYPE = new ActionType<>(NAME, GetDataSourceActionResponse::new); - private DataSourceService dataSourceService; + private final DataSourceService dataSourceService; /** * TransportGetDataSourceAction action for getting datasource. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportPatchDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportPatchDataSourceAction.java index 303e905cec..dd55869df7 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportPatchDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportPatchDataSourceAction.java @@ -30,7 +30,7 @@ public class TransportPatchDataSourceAction public static final ActionType<PatchDataSourceActionResponse> ACTION_TYPE = new ActionType<>(NAME, PatchDataSourceActionResponse::new); - private DataSourceService dataSourceService; + private final DataSourceService dataSourceService; /** * TransportPatchDataSourceAction action for updating datasource. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java index fefd0f3a01..44e0625cf7 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java @@ -29,7 +29,7 @@ public class TransportUpdateDataSourceAction public static final ActionType<UpdateDataSourceActionResponse> ACTION_TYPE = new ActionType<>(NAME, UpdateDataSourceActionResponse::new); - private DataSourceService dataSourceService; + private final DataSourceService dataSourceService; /** * TransportUpdateDataSourceAction action for updating datasource. diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestReportTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestReportTest.java index 9ac5151b21..43f678f60e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestReportTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestReportTest.java @@ -22,7 +22,7 @@ /** Test for {@link TestReport} */ public class TestReportTest { - private TestReport report = new TestReport(); + private final TestReport report = new TestReport(); @Test public void testSuccessReport() { diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/testset/TestQuerySet.java b/integ-test/src/test/java/org/opensearch/sql/correctness/testset/TestQuerySet.java index 161d314c1d..b3dfbec259 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/testset/TestQuerySet.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/testset/TestQuerySet.java @@ -14,7 +14,7 @@ /** Test query set including SQL queries for comparison testing. */ public class TestQuerySet implements Iterable<String> { - private List<String> queries; + private final List<String> queries; /** * Construct by a test query file. diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java index 81edb54556..6cc4aba811 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/GetEndpointQueryIT.java @@ -16,7 +16,7 @@ /** Tests to cover requests with "?format=csv" parameter */ public class GetEndpointQueryIT extends SQLIntegTestCase { - @Rule public ExpectedException rule = ExpectedException.none(); + @Rule public final ExpectedException rule = ExpectedException.none(); @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java index 75b2b45df6..3933338f0a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java @@ -17,7 +17,7 @@ /** Test cases for writing missing join table aliases. */ public class JoinAliasWriterRuleIT extends SQLIntegTestCase { - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); protected void init() throws Exception { loadIndex(Index.ORDER); // opensearch-sql_test_index_order diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java index 84750f8a27..bee85ac314 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MultiQueryIT.java @@ -18,9 +18,9 @@ public class MultiQueryIT extends SQLIntegTestCase { - private static String MINUS_SCROLL_DEFAULT_HINT = + private static final String MINUS_SCROLL_DEFAULT_HINT = " /*! MINUS_SCROLL_FETCH_AND_RESULT_LIMITS(1000, 50, 100) */ "; - private static String MINUS_TERMS_OPTIMIZATION_HINT = + private static final String MINUS_TERMS_OPTIMIZATION_HINT = " /*! MINUS_USE_TERMS_OPTIMIZATION(true) */ "; @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java index c1d656628f..39abad92df 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java @@ -37,7 +37,7 @@ public class SubqueryIT extends SQLIntegTestCase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java index 7cc083cbb6..dca33b9f96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java @@ -43,9 +43,9 @@ public void resetTimeZone() { TimeZone.setDefault(testTz); } - private String functionCall; - private String name; - private Boolean expectedResult; + private final String functionCall; + private final String name; + private final Boolean expectedResult; public DateTimeComparisonIT( @Name("functionCall") String functionCall, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLPluginIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLPluginIT.java index 44f79a8944..96362a2fd2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLPluginIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLPluginIT.java @@ -27,7 +27,7 @@ import org.opensearch.sql.util.TestUtils; public class PPLPluginIT extends PPLIntegTestCase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); private static final String PERSISTENT = "persistent"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java index af3d81e374..d385b54dff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java @@ -50,9 +50,9 @@ public void resetTimeZone() { TimeZone.setDefault(testTz); } - private String functionCall; - private String name; - private Boolean expectedResult; + private final String functionCall; + private final String name; + private final Boolean expectedResult; public DateTimeComparisonIT( @Name("functionCall") String functionCall, diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ExpressionIT.java index be1471641e..45da456553 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ExpressionIT.java @@ -29,7 +29,7 @@ @Ignore public class ExpressionIT extends RestIntegTestCase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index e42b68631f..8e38af2a92 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -29,7 +29,7 @@ */ public class QueryValidationIT extends SQLIntegTestCase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Override protected void init() throws Exception { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/scope/SymbolTable.java b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/scope/SymbolTable.java index ee9f4545a6..0f65ee1b99 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/scope/SymbolTable.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/scope/SymbolTable.java @@ -20,7 +20,7 @@ public class SymbolTable { /** Two-dimension hash table to manage symbols with type in different namespace */ - private Map<Namespace, NavigableMap<String, TypeSupplier>> tableByNamespace = + private final Map<Namespace, NavigableMap<String, TypeSupplier>> tableByNamespace = new EnumMap<>(Namespace.class); /** diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java index 9cebf3dda6..5ede1aa1a5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java @@ -24,7 +24,7 @@ public enum AggregateFunction implements TypeExpression { AVG(func(T(NUMBER)).to(DOUBLE)), SUM(func(T(NUMBER)).to(T)); - private TypeExpressionSpec[] specifications; + private final TypeExpressionSpec[] specifications; AggregateFunction(TypeExpressionSpec... specifications) { this.specifications = specifications; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java index f86635910a..2c86ff00ad 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java @@ -52,15 +52,15 @@ public enum OPERATOR { NTERM, NREGEXP; - public static Map<String, OPERATOR> methodNameToOpear; + public static final Map<String, OPERATOR> methodNameToOpear; - public static Map<String, OPERATOR> operStringToOpear; + public static final Map<String, OPERATOR> operStringToOpear; - public static Map<String, OPERATOR> simpleOperStringToOpear; + public static final Map<String, OPERATOR> simpleOperStringToOpear; - private static BiMap<OPERATOR, OPERATOR> negatives; + private static final BiMap<OPERATOR, OPERATOR> negatives; - private static BiMap<OPERATOR, OPERATOR> simpleReverses; + private static final BiMap<OPERATOR, OPERATOR> simpleReverses; static { methodNameToOpear = new HashMap<>(); @@ -161,7 +161,7 @@ public Boolean isSimpleOperator() { private String name; - private SQLExpr nameExpr; + private final SQLExpr nameExpr; private Object value; @@ -173,7 +173,7 @@ public SQLExpr getValueExpr() { return valueExpr; } - private SQLExpr valueExpr; + private final SQLExpr valueExpr; private OPERATOR OPERATOR; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Having.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Having.java index a53fb0c275..b5a66d5eb5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Having.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Having.java @@ -44,7 +44,7 @@ public class Having { /** Conditions parsed out of HAVING clause */ private final List<Where> conditions; - private HavingParser havingParser; + private final HavingParser havingParser; public List<Field> getHavingFields() { return havingParser.getHavingFields(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/IndexStatement.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/IndexStatement.java index 2a5be5728c..b2efe99327 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/IndexStatement.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/IndexStatement.java @@ -8,8 +8,8 @@ /** Class used to differentiate SHOW and DESCRIBE statements */ public class IndexStatement implements QueryStatement { - private StatementType statementType; - private String query; + private final StatementType statementType; + private final String query; private String indexPattern; private String columnPattern; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/JoinSelect.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/JoinSelect.java index 211b33c68a..a7d18d7a41 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/JoinSelect.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/JoinSelect.java @@ -12,8 +12,8 @@ /** Created by Eliran on 20/8/2015. */ public class JoinSelect extends Query { - private TableOnJoinSelect firstTable; - private TableOnJoinSelect secondTable; + private final TableOnJoinSelect firstTable; + private final TableOnJoinSelect secondTable; private Where connectedWhere; private List<Hint> hints; private List<Condition> connectedConditions; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/KVValue.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/KVValue.java index d864cbac12..9488952c2b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/KVValue.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/KVValue.java @@ -7,7 +7,7 @@ public class KVValue implements Cloneable { public String key; - public Object value; + public final Object value; public KVValue(Object value) { this.value = value; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java index f593d6c428..57280b7df2 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java @@ -12,7 +12,7 @@ public class Order { private String nestedPath; private String name; private String type; - private Field sortField; + private final Field sortField; public boolean isScript() { return sortField != null && sortField.isScriptField(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Paramer.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Paramer.java index 38ca556199..e3b6f710b6 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Paramer.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Paramer.java @@ -30,7 +30,7 @@ public class Paramer { public String value; public Integer slop; - public Map<String, Float> fieldsBoosts = new HashMap<>(); + public final Map<String, Float> fieldsBoosts = new HashMap<>(); public String type; public Float tieBreaker; public Operator operator; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Query.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Query.java index 6f891e7fc5..0fe302cd22 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Query.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Query.java @@ -12,7 +12,7 @@ public abstract class Query implements QueryStatement { private Where where = null; - private List<From> from = new ArrayList<>(); + private final List<From> from = new ArrayList<>(); public Where getWhere() { return this.where; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java index 2faa8cc6e5..1de86f43dc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java @@ -36,11 +36,11 @@ public class Select extends Query { "PERCENTILES", "SCRIPTED_METRIC"); - private List<Hint> hints = new ArrayList<>(); - private List<Field> fields = new ArrayList<>(); - private List<List<Field>> groupBys = new ArrayList<>(); + private final List<Hint> hints = new ArrayList<>(); + private final List<Field> fields = new ArrayList<>(); + private final List<List<Field>> groupBys = new ArrayList<>(); private Having having; - private List<Order> orderBys = new ArrayList<>(); + private final List<Order> orderBys = new ArrayList<>(); private int offset; private Integer rowCount; private boolean containsSubQueries; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Where.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Where.java index d6f767203b..342d9e5521 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Where.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Where.java @@ -22,7 +22,7 @@ public static Where newInstance() { return new Where(CONN.AND); } - private LinkedList<Where> wheres = new LinkedList<>(); + private final LinkedList<Where> wheres = new LinkedList<>(); protected CONN conn; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/bucketpath/BucketPath.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/bucketpath/BucketPath.java index 635d0062a5..e2f6c6a7e0 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/bucketpath/BucketPath.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/bucketpath/BucketPath.java @@ -15,7 +15,7 @@ * <p>https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-pipeline.html#buckets-path-syntax */ public class BucketPath { - private Deque<Path> pathStack = new ArrayDeque<>(); + private final Deque<Path> pathStack = new ArrayDeque<>(); public BucketPath add(Path path) { if (pathStack.isEmpty()) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/hints/Hint.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/hints/Hint.java index b83c63aae1..7a6c445b83 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/hints/Hint.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/hints/Hint.java @@ -7,8 +7,8 @@ /** Created by Eliran on 5/9/2015. */ public class Hint { - private HintType type; - private Object[] params; + private final HintType type; + private final Object[] params; public Hint(HintType type, Object[] params) { this.type = type; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java index 58808ee8f3..fbe1cca6f3 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java @@ -24,7 +24,7 @@ /** Created by Eliran on 6/10/2015. */ public class GetIndexRequestRestListener extends RestBuilderListener<GetIndexResponse> { - private GetIndexRequest getIndexRequest; + private final GetIndexRequest getIndexRequest; public GetIndexRequestRestListener(RestChannel channel, GetIndexRequest getIndexRequest) { super(channel); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java index 222ca5d9fc..b94ef51b7d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java @@ -33,7 +33,7 @@ public class CursorCloseExecutor implements CursorRestExecutor { private static final String SUCCEEDED_TRUE = "{\"succeeded\":true}"; private static final String SUCCEEDED_FALSE = "{\"succeeded\":false}"; - private String cursorId; + private final String cursorId; public CursorCloseExecutor(String cursorId) { this.cursorId = cursorId; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index 4947d06b2f..14ca3aea62 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -41,8 +41,8 @@ public class CursorResultExecutor implements CursorRestExecutor { - private String cursorId; - private Format format; + private final String cursorId; + private final Format format; private static final Logger LOG = LogManager.getLogger(CursorResultExecutor.class); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DataRows.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DataRows.java index fc153afae8..514d016866 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DataRows.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DataRows.java @@ -11,9 +11,9 @@ public class DataRows implements Iterable<DataRows.Row> { - private long size; - private long totalHits; - private List<Row> rows; + private final long size; + private final long totalHits; + private final List<Row> rows; public DataRows(long size, long totalHits, List<Row> rows) { this.size = size; @@ -61,7 +61,7 @@ public void remove() { // Inner class for Row object public static class Row { - private Map<String, Object> data; + private final Map<String, Object> data; public Row(Map<String, Object> data) { this.data = data; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index feba13d139..f8dd261be4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -42,7 +42,7 @@ public class DateFieldFormatter { private final Map<String, List<String>> dateFieldFormatMap; private final Map<String, String> fieldAliasMap; - private Set<String> dateColumns; + private final Set<String> dateColumns; public DateFieldFormatter( String indexName, List<Schema.Column> columns, Map<String, String> fieldAliasMap) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFormat.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFormat.java index fc9237918c..abb04d7f81 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFormat.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFormat.java @@ -14,7 +14,7 @@ public class DateFormat { - private static Map<String, String> formatMap = new HashMap<>(); + private static final Map<String, String> formatMap = new HashMap<>(); static { // Special cases that are parsed separately @@ -104,31 +104,31 @@ public static String getFormattedDate(java.util.Date date, String dateFormat) { } private static class Date { - static String BASIC_DATE = "yyyyMMdd"; - static String BASIC_ORDINAL_DATE = "yyyyDDD"; - static String BASIC_WEEK_DATE = "YYYY'W'wwu"; + static final String BASIC_DATE = "yyyyMMdd"; + static final String BASIC_ORDINAL_DATE = "yyyyDDD"; + static final String BASIC_WEEK_DATE = "YYYY'W'wwu"; - static String DATE = "yyyy-MM-dd"; - static String ORDINAL_DATE = "yyyy-DDD"; + static final String DATE = "yyyy-MM-dd"; + static final String ORDINAL_DATE = "yyyy-DDD"; - static String YEAR = "yyyy"; - static String YEAR_MONTH = "yyyy-MM"; + static final String YEAR = "yyyy"; + static final String YEAR_MONTH = "yyyy-MM"; - static String WEEK_DATE = "YYYY-'W'ww-u"; - static String WEEKYEAR = "YYYY"; - static String WEEKYEAR_WEEK = "YYYY-'W'ww"; + static final String WEEK_DATE = "YYYY-'W'ww-u"; + static final String WEEKYEAR = "YYYY"; + static final String WEEKYEAR_WEEK = "YYYY-'W'ww"; } private static class Time { - static String T = "'T'"; - static String BASIC_TIME = "HHmmss"; - static String TIME = "HH:mm:ss"; + static final String T = "'T'"; + static final String BASIC_TIME = "HHmmss"; + static final String TIME = "HH:mm:ss"; - static String HOUR = "HH"; - static String HOUR_MINUTE = "HH:mm"; + static final String HOUR = "HH"; + static final String HOUR_MINUTE = "HH:mm"; - static String MILLIS = ".SSS"; - static String TZ = "Z"; - static String TZZ = "XX"; + static final String MILLIS = ".SSS"; + static final String TZ = "Z"; + static final String TZZ = "XX"; } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DeleteResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DeleteResultSet.java index 24afb0a7af..2e040d78fb 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DeleteResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DeleteResultSet.java @@ -13,8 +13,8 @@ import org.opensearch.sql.legacy.domain.Delete; public class DeleteResultSet extends ResultSet { - private Delete query; - private Object queryResult; + private final Delete query; + private final Object queryResult; public static final String DELETED = "deleted_rows"; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java index eba6db2453..74df2ddeeb 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java @@ -29,8 +29,8 @@ public class DescribeResultSet extends ResultSet { */ public static final String DEFAULT_OBJECT_DATATYPE = "object"; - private IndexStatement statement; - private Object queryResult; + private final IndexStatement statement; + private final Object queryResult; public DescribeResultSet(Client client, IndexStatement statement, Object queryResult) { this.client = client; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java index aa0d02bed8..b75689ba9c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java @@ -10,12 +10,12 @@ public class ErrorMessage<E extends Exception> { - protected E exception; + protected final E exception; - private int status; - private String type; - private String reason; - private String details; + private final int status; + private final String type; + private final String reason; + private final String details; public ErrorMessage(E exception, int status) { this.exception = exception; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java index e6ea767e17..95600d234a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java @@ -36,7 +36,7 @@ public class Protocol { static final int ERROR_STATUS = 500; private final String formatType; - private int status; + private final int status; private long size; private long total; private ResultSet resultSet; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Schema.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Schema.java index b29369f713..e2946e946b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Schema.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Schema.java @@ -19,7 +19,7 @@ public class Schema implements Iterable<Schema.Column> { private String indexName; private List<Column> columns; - private static Set<String> types; + private static final Set<String> types; static { types = getTypes(); @@ -121,10 +121,10 @@ public String nameLowerCase() { public static class Column { private final String name; - private String alias; + private final String alias; private final Type type; - private boolean identifiedByAlias; + private final boolean identifiedByAlias; public Column(String name, String alias, Type type, boolean identifiedByAlias) { this.name = name; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index bc5c1fb162..84b7c00857 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -70,7 +70,7 @@ public class SelectResultSet extends ResultSet { private final String formatType; private Query query; - private Object queryResult; + private final Object queryResult; private boolean selectAll; private String indexName; @@ -82,11 +82,11 @@ public class SelectResultSet extends ResultSet { private long totalHits; private long internalTotalHits; private List<DataRows.Row> rows; - private Cursor cursor; + private final Cursor cursor; private DateFieldFormatter dateFieldFormatter; // alias -> base field name - private Map<String, String> fieldAliasMap = new HashMap<>(); + private final Map<String, String> fieldAliasMap = new HashMap<>(); public SelectResultSet( Client client, diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ShowResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ShowResultSet.java index 263bf1e7db..9b7d15807b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ShowResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ShowResultSet.java @@ -22,8 +22,8 @@ public class ShowResultSet extends ResultSet { private static final String TABLE_TYPE = "BASE TABLE"; - private IndexStatement statement; - private Object queryResult; + private final IndexStatement statement; + private final Object queryResult; public ShowResultSet(Client client, IndexStatement statement, Object queryResult) { this.client = client; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index e5011d1af8..5ff52cf657 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -45,11 +45,11 @@ public abstract class ElasticJoinExecutor extends ElasticHitsExecutor { protected List<SearchHit> results; // Keep list to avoid copy to new array in SearchHits - protected MetaSearchResult metaResults; + protected final MetaSearchResult metaResults; protected final int MAX_RESULTS_ON_ONE_FETCH = 10000; - private Set<String> aliasesOnReturn; - private boolean allFieldsReturn; - protected String[] indices; + private final Set<String> aliasesOnReturn; + private final boolean allFieldsReturn; + protected final String[] indices; protected ElasticJoinExecutor(Client client, JoinRequestBuilder requestBuilder) { metaResults = new MetaSearchResult(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinComparisonStructure.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinComparisonStructure.java index 8216feac66..cf81a6fe49 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinComparisonStructure.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinComparisonStructure.java @@ -15,8 +15,8 @@ /** Created by Eliran on 2/11/2015. */ public class HashJoinComparisonStructure { - private HashMap<String, List<Map.Entry<Field, Field>>> comparisonIDtoComparisonFields; - private HashMap<String, HashMap<String, SearchHitsResult>> comparisonIDtoComparisonHash; + private final HashMap<String, List<Map.Entry<Field, Field>>> comparisonIDtoComparisonFields; + private final HashMap<String, HashMap<String, SearchHitsResult>> comparisonIDtoComparisonHash; public HashJoinComparisonStructure(List<List<Map.Entry<Field, Field>>> t1ToT2FieldsComparisons) { comparisonIDtoComparisonFields = new HashMap<>(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java index 0e33ab9eef..46b31b447d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java @@ -34,11 +34,11 @@ /** Created by Eliran on 22/8/2015. */ public class HashJoinElasticExecutor extends ElasticJoinExecutor { - private HashJoinElasticRequestBuilder requestBuilder; + private final HashJoinElasticRequestBuilder requestBuilder; private boolean useQueryTermsFilterOptimization = false; private final int MAX_RESULTS_FOR_FIRST_TABLE = 100000; - HashJoinComparisonStructure hashJoinComparisonStructure; - private Set<String> alreadyMatched; + final HashJoinComparisonStructure hashJoinComparisonStructure; + private final Set<String> alreadyMatched; public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requestBuilder) { super(client, requestBuilder); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index 9356a0058e..f4e4347e06 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -378,8 +378,8 @@ private void reverseOrderOfCondition(Condition cond, String t1Alias, String t2Al } private class FetchWithScrollResponse { - private SearchResponse response; - private boolean needScrollForFirstTable; + private final SearchResponse response; + private final boolean needScrollForFirstTable; private FetchWithScrollResponse(SearchResponse response, boolean needScrollForFirstTable) { this.response = response; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/ComperableHitResult.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/ComperableHitResult.java index fa3514600b..f799cd63d2 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/ComperableHitResult.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/ComperableHitResult.java @@ -15,10 +15,10 @@ /** Created by Eliran on 9/9/2016. */ public class ComperableHitResult { - private SearchHit hit; - private String comperator; + private final SearchHit hit; + private final String comperator; private boolean isAllNull; - private Map<String, Object> flattenMap; + private final Map<String, Object> flattenMap; public ComperableHitResult(SearchHit hit, String[] fieldsOrder, String seperator) { this.hit = hit; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index 06186d0695..a3f8596eb4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -42,9 +42,9 @@ /** Created by Eliran on 26/8/2016. */ public class MinusExecutor extends ElasticHitsExecutor { - private MultiQueryRequestBuilder builder; + private final MultiQueryRequestBuilder builder; private SearchHits minusHits; - private boolean useTermsOptimization; + private final boolean useTermsOptimization; private boolean termsOptimizationWithToLower; private boolean useScrolling; private int maxDocsToFetchOnFirstTable; @@ -52,7 +52,7 @@ public class MinusExecutor extends ElasticHitsExecutor { private int maxDocsToFetchOnEachScrollShard; private String[] fieldsOrderFirstTable; private String[] fieldsOrderSecondTable; - private String seperator; + private final String seperator; public MinusExecutor(Client client, MultiQueryRequestBuilder builder) { this.client = client; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java index 3d7206ab13..5843e0d931 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java @@ -10,8 +10,8 @@ /** Created by Eliran on 26/8/2016. */ class MinusOneFieldAndOptimizationResult { - private Set<Object> fieldValues; - private SearchHit someHit; + private final Set<Object> fieldValues; + private final SearchHit someHit; MinusOneFieldAndOptimizationResult(Set<Object> fieldValues, SearchHit someHit) { this.fieldValues = fieldValues; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java index 375c40a5c1..024d1bb4c3 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java @@ -25,7 +25,7 @@ /** Created by Eliran on 21/8/2016. */ public class UnionExecutor extends ElasticHitsExecutor { - private MultiQueryRequestBuilder multiQueryBuilder; + private final MultiQueryRequestBuilder multiQueryBuilder; private SearchHits results; private int currentId; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/BasicCounter.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/BasicCounter.java index 88d5f817e8..5c238521a0 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/BasicCounter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/BasicCounter.java @@ -9,7 +9,7 @@ public class BasicCounter implements Counter<Long> { - private LongAdder count = new LongAdder(); + private final LongAdder count = new LongAdder(); @Override public void increment() { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/GaugeMetric.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/GaugeMetric.java index 2f7c269351..150862e4d1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/GaugeMetric.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/GaugeMetric.java @@ -10,7 +10,7 @@ /** Gauge metric, an instant value like cpu usage, state and so on */ public class GaugeMetric<T> extends Metric<T> { - private Supplier<T> loadValue; + private final Supplier<T> loadValue; public GaugeMetric(String name, Supplier<T> supplier) { super(name); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metric.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metric.java index 956e0f558c..ea543cbd2d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metric.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metric.java @@ -9,7 +9,7 @@ public abstract class Metric<T> implements java.io.Serializable { private static final long serialVersionUID = 1L; - private String name; + private final String name; public Metric(String name) { this.name = name; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/MetricName.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/MetricName.java index 72960944b6..7c2a7cb824 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/MetricName.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/MetricName.java @@ -50,7 +50,7 @@ public enum MetricName { EMR_BATCH_QUERY_JOBS_CREATION_COUNT("emr_batch_jobs_creation_count"), STREAMING_JOB_HOUSEKEEPER_TASK_FAILURE_COUNT("streaming_job_housekeeper_task_failure_count"); - private String name; + private final String name; MetricName(String name) { this.name = name; @@ -64,7 +64,7 @@ public static List<String> getNames() { return Arrays.stream(MetricName.values()).map(v -> v.name).collect(Collectors.toList()); } - private static Set<MetricName> NUMERICAL_METRIC = + private static final Set<MetricName> NUMERICAL_METRIC = new ImmutableSet.Builder<MetricName>() .add(PPL_REQ_TOTAL) .add(PPL_REQ_COUNT_TOTAL) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metrics.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metrics.java index 858f9e5cef..a47f0e12bd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metrics.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/Metrics.java @@ -12,8 +12,9 @@ public class Metrics { - private static Metrics metrics = new Metrics(); - private ConcurrentHashMap<String, Metric> registeredMetricsByName = new ConcurrentHashMap<>(); + private static final Metrics metrics = new Metrics(); + private final ConcurrentHashMap<String, Metric> registeredMetricsByName = + new ConcurrentHashMap<>(); public static Metrics getInstance() { return metrics; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/NumericMetric.java b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/NumericMetric.java index ee6d373f8f..93bc9485bc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/metrics/NumericMetric.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/metrics/NumericMetric.java @@ -7,7 +7,7 @@ public class NumericMetric<T> extends Metric<T> { - private Counter<T> counter; + private final Counter<T> counter; public NumericMetric(String name, Counter counter) { super(name); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/CaseWhenParser.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/CaseWhenParser.java index d55ee64601..9b26bf4608 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/CaseWhenParser.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/CaseWhenParser.java @@ -21,9 +21,9 @@ /** Created by allwefantasy on 9/3/16. */ public class CaseWhenParser { - private SQLCaseExpr caseExpr; - private String alias; - private String tableAlias; + private final SQLCaseExpr caseExpr; + private final String alias; + private final String tableAlias; public CaseWhenParser(SQLCaseExpr caseExpr, String alias, String tableAlias) { this.alias = alias; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/ElasticSqlExprParser.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/ElasticSqlExprParser.java index be9c2f9652..ed9cc2cd3d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/ElasticSqlExprParser.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/ElasticSqlExprParser.java @@ -193,7 +193,7 @@ public SQLExpr primary() { return expr; } - public static String[] AGGREGATE_FUNCTIONS = { + public static final String[] AGGREGATE_FUNCTIONS = { "AVG", "COUNT", "GROUP_CONCAT", "MAX", "MIN", "STDDEV", "SUM" }; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java index da08f81453..ca0fbb8393 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java @@ -43,7 +43,7 @@ * @author ansj */ public class FieldMaker { - private SQLFunctions sqlFunctions = new SQLFunctions(); + private final SQLFunctions sqlFunctions = new SQLFunctions(); public Field makeField(SQLExpr expr, String alias, String tableAlias) throws SqlParseException { Field field = makeFieldImpl(expr, alias, tableAlias); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/SqlParser.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/SqlParser.java index 947533630b..c380ded176 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/SqlParser.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/SqlParser.java @@ -55,7 +55,7 @@ * @author ansj */ public class SqlParser { - private FieldMaker fieldMaker = new FieldMaker(); + private final FieldMaker fieldMaker = new FieldMaker(); public SqlParser() {} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/SubQueryExpression.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/SubQueryExpression.java index e9b0797d00..663de4f4b7 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/SubQueryExpression.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/SubQueryExpression.java @@ -11,7 +11,7 @@ public class SubQueryExpression { private Object[] values; private Select select; - private String returnField; + private final String returnField; public SubQueryExpression(Select innerSelect) { this.select = innerSelect; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/WhereParser.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/WhereParser.java index a329d1ed52..4ba25f9fad 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/WhereParser.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/WhereParser.java @@ -44,12 +44,12 @@ /** Created by allwefantasy on 9/2/16. */ public class WhereParser { - private FieldMaker fieldMaker; + private final FieldMaker fieldMaker; private MySqlSelectQueryBlock query; private SQLDeleteStatement delete; private SQLExpr where; - private SqlParser sqlParser; + private final SqlParser sqlParser; public WhereParser(SqlParser sqlParser, MySqlSelectQueryBlock query, FieldMaker fieldMaker) { this.sqlParser = sqlParser; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java index 8d61c03388..db3530e91e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImpl.java @@ -24,7 +24,7 @@ /** Handler for Point In Time */ public class PointInTimeHandlerImpl implements PointInTimeHandler { - private Client client; + private final Client client; private String[] indices; @Getter @Setter private String pitId; private static final Logger LOG = LogManager.getLogger(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java index 57af269001..c7a1f82df1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java @@ -41,7 +41,7 @@ public class AggregationQueryAction extends QueryAction { private final Select select; - private AggMaker aggMaker = new AggMaker(); + private final AggMaker aggMaker = new AggMaker(); private SearchRequestBuilder request; public AggregationQueryAction(Client client, Select select) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/QueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/QueryAction.java index c9b39d2f97..4d40701964 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/QueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/QueryAction.java @@ -36,8 +36,8 @@ */ public abstract class QueryAction { - protected Query query; - protected Client client; + protected final Query query; + protected final Client client; protected SqlRequest sqlRequest = SqlRequest.NULL; protected ColumnTypeProvider scriptColumnType; protected Format format; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java index 2203cbb39e..1ff35df531 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java @@ -13,7 +13,7 @@ /** Created by Eliran on 19/8/2015. */ public class SqlElasticDeleteByQueryRequestBuilder implements SqlElasticRequestBuilder { - DeleteByQueryRequestBuilder deleteByQueryRequestBuilder; + final DeleteByQueryRequestBuilder deleteByQueryRequestBuilder; public SqlElasticDeleteByQueryRequestBuilder( DeleteByQueryRequestBuilder deleteByQueryRequestBuilder) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlOpenSearchRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlOpenSearchRequestBuilder.java index 2beb16837b..d15debb0fd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlOpenSearchRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/SqlOpenSearchRequestBuilder.java @@ -11,7 +11,7 @@ /** Created by Eliran on 19/8/2015. */ public class SqlOpenSearchRequestBuilder implements SqlElasticRequestBuilder { - ActionRequestBuilder requestBuilder; + final ActionRequestBuilder requestBuilder; public SqlOpenSearchRequestBuilder(ActionRequestBuilder requestBuilder) { this.requestBuilder = requestBuilder; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/BackOffRetryStrategy.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/BackOffRetryStrategy.java index d767268cb1..3386298802 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/BackOffRetryStrategy.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/BackOffRetryStrategy.java @@ -31,9 +31,9 @@ public class BackOffRetryStrategy { private static final int threshold = 85; - private static IdentityHashMap<Object, Tuple<Long, Long>> memUse = new IdentityHashMap<>(); + private static final IdentityHashMap<Object, Tuple<Long, Long>> memUse = new IdentityHashMap<>(); - private static AtomicLong mem = new AtomicLong(0L); + private static final AtomicLong mem = new AtomicLong(0L); private static long lastTimeoutCleanTime = System.currentTimeMillis(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java index 82ebd1b225..0a9917a624 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java @@ -21,8 +21,8 @@ public class JoinRequestBuilder implements SqlElasticRequestBuilder { private MultiSearchRequest multi; - private TableInJoinRequestBuilder firstTable; - private TableInJoinRequestBuilder secondTable; + private final TableInJoinRequestBuilder firstTable; + private final TableInJoinRequestBuilder secondTable; private SQLJoinTableSource.JoinType joinType; private int totalLimit; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/OpenSearchJoinQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/OpenSearchJoinQueryAction.java index 7068ddf9a2..b317ef9740 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/OpenSearchJoinQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/OpenSearchJoinQueryAction.java @@ -22,7 +22,7 @@ /** Created by Eliran on 15/9/2015. */ public abstract class OpenSearchJoinQueryAction extends QueryAction { - protected JoinSelect joinSelect; + protected final JoinSelect joinSelect; public OpenSearchJoinQueryAction(Client client, JoinSelect joinSelect) { super(client, joinSelect); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java index dcb703cd33..75753ce24f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java @@ -65,7 +65,7 @@ public class AggMaker { /** The mapping bettwen group fieldName or Alias to the KVValue. */ - private Map<String, KVValue> groupMap = new HashMap<>(); + private final Map<String, KVValue> groupMap = new HashMap<>(); private Where where; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryAction.java index a9eb6113f7..549fdfc077 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryAction.java @@ -19,7 +19,7 @@ /** Created by Eliran on 19/8/2016. */ public class MultiQueryAction extends QueryAction { - private MultiQuerySelect multiQuerySelect; + private final MultiQuerySelect multiQuerySelect; public MultiQueryAction(Client client, MultiQuerySelect multiSelect) { super(client, null); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java index b4e92a8de6..a02f4037c7 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java @@ -27,10 +27,10 @@ public class MultiQueryRequestBuilder implements SqlElasticRequestBuilder { private SearchRequestBuilder firstSearchRequest; private SearchRequestBuilder secondSearchRequest; - private Map<String, String> firstTableFieldToAlias; - private Map<String, String> secondTableFieldToAlias; - private MultiQuerySelect multiQuerySelect; - private SQLUnionOperator relation; + private final Map<String, String> firstTableFieldToAlias; + private final Map<String, String> secondTableFieldToAlias; + private final MultiQuerySelect multiQuerySelect; + private final SQLUnionOperator relation; public MultiQueryRequestBuilder(MultiQuerySelect multiQuerySelect) { this.multiQuerySelect = multiQuerySelect; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQuerySelect.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQuerySelect.java index 72e7232a30..6a573c999f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQuerySelect.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQuerySelect.java @@ -10,9 +10,9 @@ /** Created by Eliran on 19/8/2016. */ public class MultiQuerySelect { - private SQLUnionOperator operation; - private Select firstSelect; - private Select secondSelect; + private final SQLUnionOperator operation; + private final Select firstSelect; + private final Select secondSelect; public MultiQuerySelect(SQLUnionOperator operation, Select firstSelect, Select secondSelect) { this.operation = operation; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/converter/SQLAggregationParser.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/converter/SQLAggregationParser.java index b54e260fd4..0944339939 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/converter/SQLAggregationParser.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/converter/SQLAggregationParser.java @@ -39,7 +39,7 @@ public class SQLAggregationParser { private final ColumnTypeProvider columnTypeProvider; private Context context; - @Getter private List<ColumnNode> columnNodes = new ArrayList<>(); + @Getter private final List<ColumnNode> columnNodes = new ArrayList<>(); public void parse(MySqlSelectQueryBlock queryBlock) { context = new Context(constructSQLExprAliasMapFromSelect(queryBlock)); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java index a8fb7cc53c..e4cb323e71 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java @@ -18,8 +18,8 @@ /** The definition of QueryPlanner which return the {@link BindingTuple} as result. */ public class BindingTupleQueryPlanner { - private PhysicalOperator<BindingTuple> physicalOperator; - @Getter private List<ColumnNode> columnNodes; + private final PhysicalOperator<BindingTuple> physicalOperator; + @Getter private final List<ColumnNode> columnNodes; public BindingTupleQueryPlanner( Client client, SQLQueryExpr sqlExpr, ColumnTypeProvider columnTypeProvider) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/ExecuteParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/ExecuteParams.java index c5ed48a514..86ab26876a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/ExecuteParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/core/ExecuteParams.java @@ -11,7 +11,7 @@ public class ExecuteParams { /** Mapping from type to parameters */ - private EnumMap<ExecuteParamType, Object> params = new EnumMap<>(ExecuteParamType.class); + private final EnumMap<ExecuteParamType, Object> params = new EnumMap<>(ExecuteParamType.class); public <T> void add(ExecuteParamType type, T param) { params.put(type, param); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/Group.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/Group.java index da94ae74da..79724db7d9 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/Group.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/logical/node/Group.java @@ -14,10 +14,10 @@ public class Group implements LogicalOperator { /** Optional pushed down projection */ - private Project<?> project; + private final Project<?> project; /** Optional pushed down filter (selection) */ - private Filter filter; + private final Filter filter; /** Required table scan operator */ private final TableScan tableScan; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/estimation/Estimation.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/estimation/Estimation.java index 72ffbd4652..982b400821 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/estimation/Estimation.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/estimation/Estimation.java @@ -23,7 +23,7 @@ public class Estimation<T> implements LogicalPlanVisitor { /** Optimal physical operator for logical operator based on completed estimation */ - private Map<LogicalOperator, PhysicalOperator<T>> optimalOps = new IdentityHashMap<>(); + private final Map<LogicalOperator, PhysicalOperator<T>> optimalOps = new IdentityHashMap<>(); /** Keep tracking of the operator that exit visit() */ private PhysicalOperator<T> root; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java index b1fb43441e..1682c14042 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java @@ -18,8 +18,8 @@ */ public class CombinedRow<T> { - private Row<T> rightRow; - private Collection<Row<T>> leftRows; + private final Row<T> rightRow; + private final Collection<Row<T>> leftRows; public CombinedRow(Row<T> rightRow, Collection<Row<T>> leftRows) { this.rightRow = rightRow; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java index 9f2c9e4174..0c0d50258d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java @@ -51,7 +51,7 @@ public abstract class JoinAlgorithm<T> extends BatchPhysicalOperator<T> { private final Set<Row<T>> leftMismatch; /** Hash table for right table probing */ - protected HashTable<T> hashTable; + protected final HashTable<T> hashTable; /** Execute params to reset right side for each left block */ protected ExecuteParams params; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java index baf0af8c86..37486045b7 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java @@ -15,7 +15,7 @@ /** List implementation to avoid normal hash table degrading into linked list. */ public class ListHashTable<T> implements HashTable<T> { - private List<Row<T>> rows = new ArrayList<>(); + private final List<Row<T>> rows = new ArrayList<>(); @Override public void add(Row<T> row) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/Stats.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/Stats.java index 3ff4662ce4..ac830d2595 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/Stats.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/Stats.java @@ -19,7 +19,7 @@ public class Stats { /** Client connection to OpenSearch cluster (unused now) */ - private Client client; + private final Client client; public Stats(Client client) { this.client = client; @@ -31,8 +31,8 @@ public MemStats collectMemStats() { /** Statistics data class for memory usage */ public static class MemStats { - private long free; - private long total; + private final long free; + private final long total; public MemStats(long free, long total) { this.free = free; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java index 339e326cc3..a5e7841d69 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java @@ -8,7 +8,7 @@ /** Adaptive block size calculator based on resource usage dynamically. */ public class AdaptiveBlockSize implements BlockSize { - private int upperLimit; + private final int upperLimit; public AdaptiveBlockSize(int upperLimit) { this.upperLimit = upperLimit; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java index 6e5a2703f4..154cc6122f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java @@ -18,7 +18,7 @@ public interface BlockSize { /** Default implementation with fixed block size */ class FixedBlockSize implements BlockSize { - private int blockSize; + private final int blockSize; public FixedBlockSize(int blockSize) { this.blockSize = blockSize; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/request/PreparedStatementRequest.java b/legacy/src/main/java/org/opensearch/sql/legacy/request/PreparedStatementRequest.java index c32e529157..c9ce288784 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/request/PreparedStatementRequest.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/request/PreparedStatementRequest.java @@ -10,8 +10,8 @@ public class PreparedStatementRequest extends SqlRequest { - private List<PreparedStatementParameter> parameters; - private String sqlTemplate; + private final List<PreparedStatementParameter> parameters; + private final String sqlTemplate; public PreparedStatementRequest( String sql, JSONObject payloadJson, List<PreparedStatementParameter> parameters) { @@ -117,7 +117,7 @@ public enum ParameterType { } public static class PreparedStatementParameter<T> { - protected T value; + protected final T value; public PreparedStatementParameter(T value) { this.value = value; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java index 884784ed42..0ce61966dc 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java @@ -205,10 +205,10 @@ public String getAlias() { } /** Table Name. */ - private String name; + private final String name; /** Table Alias. */ - private String alias; + private final String alias; Table(String name, String alias) { this.name = name; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java index 922e3b930e..d7a1000179 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java @@ -40,8 +40,8 @@ */ public class TermFieldRewriter extends MySqlASTVisitorAdapter { - private Deque<TermFieldScope> environment = new ArrayDeque<>(); - private TermRewriterFilter filterType; + private final Deque<TermFieldScope> environment = new ArrayDeque<>(); + private final TermRewriterFilter filterType; public TermFieldRewriter() { this.filterType = TermRewriterFilter.COMMA; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java index 46afbb8ca1..59f415f306 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java @@ -57,7 +57,7 @@ public class NestedFieldRewriter extends MySqlASTVisitorAdapter { * Scope stack to record the state (nested field names etc) for current query. In the case of * subquery, the active scope of current query is the top element of the stack. */ - private Deque<Scope> environment = new ArrayDeque<>(); + private final Deque<Scope> environment = new ArrayDeque<>(); /** * Rewrite FROM here to make sure FROM statement always be visited before other statement in diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/Scope.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/Scope.java index f65d7f166b..e9c2729716 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/Scope.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/nestedfield/Scope.java @@ -26,13 +26,13 @@ class Scope { * Mapping from nested field path alias to path full name in FROM. eg. e in {e => employees} in * "FROM t.employees e" */ - private Map<String, String> aliasFullPaths = new HashMap<>(); + private final Map<String, String> aliasFullPaths = new HashMap<>(); /** * Mapping from binary operation condition (in WHERE) to nested field tag (full path for nested, * EMPTY for non-nested field) */ - private Map<SQLBinaryOpExpr, String> conditionTags = new IdentityHashMap<>(); + private final Map<SQLBinaryOpExpr, String> conditionTags = new IdentityHashMap<>(); String getParentAlias() { return parentAlias; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java index ed853823ce..7a1ac49b0e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java @@ -68,13 +68,13 @@ private void changeOrdinalAliasInGroupAndOrderBy( root.accept( new MySqlASTVisitorAdapter() { - private String groupException = "Invalid ordinal [%s] specified in [GROUP BY %s]"; - private String orderException = "Invalid ordinal [%s] specified in [ORDER BY %s]"; + private final String groupException = "Invalid ordinal [%s] specified in [GROUP BY %s]"; + private final String orderException = "Invalid ordinal [%s] specified in [ORDER BY %s]"; - private List<SQLSelectItem> groupSelectList = + private final List<SQLSelectItem> groupSelectList = ((MySqlSelectQueryBlock) exprGroup.getSubQuery().getQuery()).getSelectList(); - private List<SQLSelectItem> orderSelectList = + private final List<SQLSelectItem> orderSelectList = ((MySqlSelectQueryBlock) exprOrder.getSubQuery().getQuery()).getSelectList(); @Override diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java index 5177b2d6d3..0fa55fe59d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java @@ -13,7 +13,7 @@ /** Subquery Rewriter Rule. */ public class SubQueryRewriteRule implements RewriteRule<SQLQueryExpr> { - private FindSubQuery findAllSubQuery = new FindSubQuery(); + private final FindSubQuery findAllSubQuery = new FindSubQuery(); @Override public boolean match(SQLQueryExpr expr) throws SQLFeatureNotSupportedException { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java index 7176bd030c..955cf258b1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java @@ -152,10 +152,10 @@ public String getAlias() { } /** Table Name. */ - private String name; + private final String name; /** Table Alias. */ - private String alias; + private final String alias; Table(String name, String alias) { this.name = name; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/BoundingBoxFilterParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/BoundingBoxFilterParams.java index fb62f60ae7..3fe292e94d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/BoundingBoxFilterParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/BoundingBoxFilterParams.java @@ -7,8 +7,8 @@ /** Created by Eliran on 1/8/2015. */ public class BoundingBoxFilterParams { - private Point topLeft; - private Point bottomRight; + private final Point topLeft; + private final Point bottomRight; public BoundingBoxFilterParams(Point topLeft, Point bottomRight) { this.topLeft = topLeft; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/CellFilterParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/CellFilterParams.java index 6c50c17467..8c169814cd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/CellFilterParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/CellFilterParams.java @@ -7,9 +7,9 @@ /** Created by Eliran on 15/8/2015. */ public class CellFilterParams { - private Point geohashPoint; - private int precision; - private boolean neighbors; + private final Point geohashPoint; + private final int precision; + private final boolean neighbors; public CellFilterParams(Point geohashPoint, int precision, boolean neighbors) { this.geohashPoint = geohashPoint; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/DistanceFilterParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/DistanceFilterParams.java index 8c419de58d..32027dec42 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/DistanceFilterParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/DistanceFilterParams.java @@ -7,8 +7,8 @@ /** Created by Eliran on 1/8/2015. */ public class DistanceFilterParams { - private String distance; - private Point from; + private final String distance; + private final Point from; public DistanceFilterParams(String distance, Point from) { this.distance = distance; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/Point.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/Point.java index f3f8639a1c..a5ed003c3d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/Point.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/Point.java @@ -7,8 +7,8 @@ /** Created by Eliran on 1/8/2015. */ public class Point { - private double lon; - private double lat; + private final double lon; + private final double lat; public Point(double lon, double lat) { this.lon = lon; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/PolygonFilterParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/PolygonFilterParams.java index 1aeddb24a4..edf2deced4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/PolygonFilterParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/PolygonFilterParams.java @@ -9,7 +9,7 @@ /** Created by Eliran on 15/8/2015. */ public class PolygonFilterParams { - private List<Point> polygon; + private final List<Point> polygon; public PolygonFilterParams(List<Point> polygon) { this.polygon = polygon; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/RangeDistanceFilterParams.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/RangeDistanceFilterParams.java index 0bdb01c3ce..6dd25ad2cf 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/RangeDistanceFilterParams.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/RangeDistanceFilterParams.java @@ -7,7 +7,7 @@ /** Created by Eliran on 15/8/2015. */ public class RangeDistanceFilterParams extends DistanceFilterParams { - private String distanceTo; + private final String distanceTo; public RangeDistanceFilterParams(String distanceFrom, String distanceTo, Point from) { super(distanceFrom, from); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/SpatialParamsFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/SpatialParamsFactory.java index 5e1102994e..d9de1e1a00 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/spatial/SpatialParamsFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/spatial/SpatialParamsFactory.java @@ -13,7 +13,7 @@ /** Created by Eliran on 1/8/2015. */ public class SpatialParamsFactory { - public static Set<String> allowedMethods; + public static final Set<String> allowedMethods; static { allowedMethods = new HashSet<>(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java index d46a80f6d3..a6a9199533 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java @@ -112,7 +112,7 @@ public class SQLFunctions { .flatMap(Set::stream) .collect(Collectors.toSet()); - private Map<String, Integer> generatedIds = new HashMap<>(); + private final Map<String, Integer> generatedIds = new HashMap<>(); /** * Generates next id for given method name. The id's are increasing for each method name, so diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/SyntaxAnalysisTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/SyntaxAnalysisTest.java index 765bb0616e..4275b6152e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/SyntaxAnalysisTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/SyntaxAnalysisTest.java @@ -22,9 +22,9 @@ public class SyntaxAnalysisTest { /** public accessor is required by @Rule annotation */ - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); - private OpenSearchLegacySqlAnalyzer analyzer = + private final OpenSearchLegacySqlAnalyzer analyzer = new OpenSearchLegacySqlAnalyzer(new SqlAnalysisConfig(true, true, 1000)); /** In reality exception occurs before reaching new parser for now */ diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java index 2b9a5e418c..79d42a05b7 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java @@ -19,7 +19,7 @@ /** Test cases for semantic analysis configuration */ public class SemanticAnalyzerConfigTest extends SemanticAnalyzerTestBase { - @Rule public ExpectedException exceptionWithoutSuggestion = ExpectedException.none(); + @Rule public final ExpectedException exceptionWithoutSuggestion = ExpectedException.none(); @Test public void noAnalysisShouldPerformForNonSelectStatement() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java index 403c2f49b7..390760af32 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java @@ -33,9 +33,9 @@ public abstract class SemanticAnalyzerTestBase { private static final String TEST_MAPPING_FILE = "mappings/semantics.json"; /** public accessor is required by @Rule annotation */ - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); - private OpenSearchLegacySqlAnalyzer analyzer = + private final OpenSearchLegacySqlAnalyzer analyzer = new OpenSearchLegacySqlAnalyzer(new SqlAnalysisConfig(true, true, 1000)); @SuppressWarnings("UnstableApiUsage") diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java index e6090117c1..57d8b3c7bb 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java @@ -14,7 +14,7 @@ import org.opensearch.sql.legacy.antlr.semantic.types.base.OpenSearchDataType; public class TypeSupplierTest { - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); @Test public void haveOneTypeShouldPass() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java index be4b5a5197..2baaa91980 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java @@ -27,7 +27,7 @@ /** Test cases for AntlrSqlParseTreeVisitor */ public class AntlrSqlParseTreeVisitorTest { - private TypeChecker analyzer = + private final TypeChecker analyzer = new TypeChecker(new SemanticContext()) { @Override public Type visitIndexName(String indexName) { @@ -47,7 +47,7 @@ public Type visitFieldName(String fieldName) { } }; - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void selectNumberShouldReturnNumberAsQueryVisitingResult() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java index 9be2517c4a..eea1c9a87a 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java @@ -43,7 +43,7 @@ public class AsyncRestExecutorTest { @Mock private Client client; - private Map<String, String> params = emptyMap(); + private final Map<String, String> params = emptyMap(); @Mock private QueryAction action; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java index eba4ae0346..d9fcf4e87e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -38,7 +38,7 @@ public class PointInTimeHandlerImplTest { @Mock private Client mockClient; - private String[] indices = {"index1", "index2"}; + private final String[] indices = {"index1", "index2"}; private PointInTimeHandlerImpl pointInTimeHandlerImpl; private final String PIT_ID = "testId"; private CreatePitResponse mockCreatePitResponse; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java index 09cd9e9efc..31baaced85 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java @@ -15,8 +15,8 @@ public class ErrorMessageFactoryTest { - private Throwable nonOpenSearchThrowable = new Throwable(); - private Throwable openSearchThrowable = new OpenSearchException(nonOpenSearchThrowable); + private final Throwable nonOpenSearchThrowable = new Throwable(); + private final Throwable openSearchThrowable = new OpenSearchException(nonOpenSearchThrowable); @Test public void openSearchExceptionShouldCreateEsErrorMessage() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java index 3c47832761..0d29b55106 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestParamTest.java @@ -19,7 +19,7 @@ import org.opensearch.sql.legacy.request.SqlRequestParam; public class SqlRequestParamTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void shouldReturnTrueIfPrettyParamsIsTrue() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java index acc0e9c60e..610930e0e7 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java @@ -20,7 +20,7 @@ @RunWith(MockitoJUnitRunner.class) public class BinaryExpressionTest extends ExpressionTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void addIntegerValueShouldPass() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java index e030e1c6cf..f049cd843a 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java @@ -20,7 +20,7 @@ @RunWith(MockitoJUnitRunner.class) public class UnaryExpressionTest extends ExpressionTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void absShouldPass() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java index 15fd72a522..846406879d 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java @@ -18,7 +18,7 @@ @RunWith(MockitoJUnitRunner.class) public class ExprValueUtilsTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void getIntegerValueWithIntegerExprValueShouldPass() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/BucketPathTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/BucketPathTest.java index c26740a04c..c88874c3f1 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/BucketPathTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/BucketPathTest.java @@ -15,7 +15,7 @@ public class BucketPathTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); private final Path agg1 = Path.getAggPath("projects@NESTED"); private final Path agg2 = Path.getAggPath("projects@FILTERED"); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SqlParserTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SqlParserTest.java index 38eefaaec1..a81e4f4dfd 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SqlParserTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SqlParserTest.java @@ -62,7 +62,7 @@ public void init() { parser = new SqlParser(); } - @Rule public ExpectedException thrown = ExpectedException.none(); + @Rule public final ExpectedException thrown = ExpectedException.none(); @Test public void whereConditionLeftFunctionRightPropertyGreatTest() throws Exception { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SubQueryParserTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SubQueryParserTest.java index 5713179b46..81e084e152 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SubQueryParserTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/parser/SubQueryParserTest.java @@ -19,7 +19,7 @@ public class SubQueryParserTest { - private static SqlParser parser = new SqlParser(); + private static final SqlParser parser = new SqlParser(); @Test public void selectFromSubqueryShouldPass() throws SqlParseException { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java index 0c77550a2f..bb3a960270 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java @@ -43,7 +43,7 @@ public class QueryPlannerBatchTest extends QueryPlannerTest { private static final String TEST_SQL3 = "ON d.id = e.departmentId " + " WHERE e.age <= 50"; - private SearchHit[] employees = { + private final SearchHit[] employees = { employee(1, "People 1", "A"), employee(2, "People 2", "A"), employee(3, "People 3", "A"), @@ -56,7 +56,7 @@ public class QueryPlannerBatchTest extends QueryPlannerTest { employee(10, "People 10", "F") }; - private SearchHit[] departments = { + private final SearchHit[] departments = { department(1, "A", "AWS"), department(2, "C", "Capital One"), department(3, "D", "Dell"), @@ -66,7 +66,7 @@ public class QueryPlannerBatchTest extends QueryPlannerTest { department(7, "U", "Uber"), }; - private Matcher[] matched = { + private final Matcher[] matched = { hit(kv("name", "People 1"), kv("id", "A"), kv("dep", "AWS")), hit(kv("name", "People 2"), kv("id", "A"), kv("dep", "AWS")), hit(kv("name", "People 3"), kv("id", "A"), kv("dep", "AWS")), @@ -76,25 +76,27 @@ public class QueryPlannerBatchTest extends QueryPlannerTest { hit(kv("name", "People 10"), kv("id", "F"), kv("dep", "Facebook")) }; - private Matcher[] mismatched1 = { + private final Matcher[] mismatched1 = { hit(kv("name", null), kv("id", "G"), kv("dep", "Google")), hit(kv("name", null), kv("id", "M"), kv("dep", "Microsoft")), hit(kv("name", null), kv("id", "U"), kv("dep", "Uber")) }; - private Matcher[] mismatched2 = { + private final Matcher[] mismatched2 = { hit(kv("name", "People 4"), kv("id", null), kv("dep", null)), hit(kv("name", "People 5"), kv("id", null), kv("dep", null)), hit(kv("name", "People 9"), kv("id", null), kv("dep", null)) }; - private Matcher<SearchHits> expectedInnerJoinResult = hits(matched); + private final Matcher<SearchHits> expectedInnerJoinResult = hits(matched); /** Department left join Employee */ - private Matcher<SearchHits> expectedLeftOuterJoinResult1 = hits(concat(matched, mismatched1)); + private final Matcher<SearchHits> expectedLeftOuterJoinResult1 = + hits(concat(matched, mismatched1)); /** Employee left join Department */ - private Matcher<SearchHits> expectedLeftOuterJoinResult2 = hits(concat(matched, mismatched2)); + private final Matcher<SearchHits> expectedLeftOuterJoinResult2 = + hits(concat(matched, mismatched2)); /** Parameterized test cases */ private final int blockSize; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java index 9b1d307ebc..6995743d06 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java @@ -34,7 +34,7 @@ public class QueryPlannerMonitorTest extends QueryPlannerTest { private static final long[] PERCENT_USAGE_24 = freeAndTotalMem(76, 100); private static final long[] PERCENT_USAGE_50 = freeAndTotalMem(50, 100); - @Spy private Stats stats = new Stats(client); + @Spy private final Stats stats = new Stats(client); @Test public void reachedLimitAndRecoverAt1stAttempt() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java index d6911ac2fc..6b93dbcaf4 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java @@ -35,7 +35,7 @@ @RunWith(MockitoJUnitRunner.class) public class SQLAggregationParserTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void parseAggWithoutExpressionShouldPass() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java index ac949eb0d7..100b5ebe47 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java @@ -33,7 +33,7 @@ @RunWith(MockitoJUnitRunner.class) public class SQLExprToExpressionConverterTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); private SQLExprToExpressionConverter converter; private SQLAggregationParser.Context context; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java index 30bbac861a..5a27420b12 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java @@ -65,8 +65,8 @@ private QueryAssertion query(String sql) { private static class QueryAssertion { - private UnquoteIdentifierRule rule = new UnquoteIdentifierRule(); - private SQLQueryExpr expr; + private final UnquoteIdentifierRule rule = new UnquoteIdentifierRule(); + private final SQLQueryExpr expr; QueryAssertion(String sql) { this.expr = SqlParserUtils.parse(sql); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java index d27967e361..d4bbc59eb5 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java @@ -18,7 +18,7 @@ /** Test cases for ordinal aliases in GROUP BY and ORDER BY */ public class OrdinalRewriterRuleTest { - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); @Test public void ordinalInGroupByShouldMatch() { @@ -118,8 +118,8 @@ private QueryAssertion query(String sql) { private static class QueryAssertion { - private OrdinalRewriterRule rule; - private SQLQueryExpr expr; + private final OrdinalRewriterRule rule; + private final SQLQueryExpr expr; QueryAssertion(String sql) { this.expr = SqlParserUtils.parse(sql); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java index 460b045ca0..3940ba1366 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java @@ -19,7 +19,7 @@ public class SQLExprParentSetterRuleTest { @Mock private SQLQueryExpr queryExpr; - private SQLExprParentSetterRule rule = new SQLExprParentSetterRule(); + private final SQLExprParentSetterRule rule = new SQLExprParentSetterRule(); @Test public void match() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java index dd15fd6683..e60eead33b 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java @@ -13,7 +13,7 @@ public class ExistsSubQueryRewriterTest extends SubQueryRewriterTestBase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void nonCorrelatedExists() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java index bb33baae7d..95d236460e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java @@ -13,7 +13,7 @@ public class InSubqueryRewriterTest extends SubQueryRewriterTestBase { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void nonCorrleatedIn() throws Exception { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java index 7bd3dd847e..0e0a30ef3a 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java @@ -19,7 +19,7 @@ public class SubQueryRewriteRuleTest { final SubQueryRewriteRule rewriteRule = new SubQueryRewriteRule(); - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void isInMatch() throws SQLFeatureNotSupportedException { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java index 7922d60647..561cafab81 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java @@ -21,7 +21,7 @@ import org.opensearch.sql.legacy.util.SqlParserUtils; public class TermFieldRewriterTest { - @Rule public ExpectedException exception = ExpectedException.none(); + @Rule public final ExpectedException exception = ExpectedException.none(); @Before public void setup() { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/SQLFunctionsTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/SQLFunctionsTest.java index 9fc2b6012d..983f10023e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/SQLFunctionsTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/SQLFunctionsTest.java @@ -34,7 +34,7 @@ public class SQLFunctionsTest { - private SQLFunctions sqlFunctions = new SQLFunctions(); + private final SQLFunctions sqlFunctions = new SQLFunctions(); @Rule public ExpectedException exceptionRule = ExpectedException.none(); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java index 8890a6853f..85b8cffb59 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java @@ -33,7 +33,7 @@ public class MultipleIndexClusterUtils { public static final String INDEX_ACCOUNT_2 = "account2"; public static final String INDEX_ACCOUNT_ALL = "account*"; - public static String INDEX_ACCOUNT_1_MAPPING = + public static final String INDEX_ACCOUNT_1_MAPPING = "{\n" + " \"field_mappings\": {\n" + " \"mappings\": {\n" @@ -93,7 +93,7 @@ public class MultipleIndexClusterUtils { * The difference with account1. 1. missing address. 2. age has different type. 3. * projects.started_year has different type. */ - public static String INDEX_ACCOUNT_2_MAPPING = + public static final String INDEX_ACCOUNT_2_MAPPING = "{\n" + " \"field_mappings\": {\n" + " \"mappings\": {\n" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java index bbcacc1d2c..fbe6d3cd72 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java @@ -12,7 +12,7 @@ /** Error Message. */ public class ErrorMessage { - protected Throwable exception; + protected final Throwable exception; private final int status; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java index f4b0b05256..08567b78c1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java @@ -42,7 +42,7 @@ @EqualsAndHashCode class OpenSearchIndexScanQueryBuilder implements PushDownQueryBuilder { - OpenSearchRequestBuilder requestBuilder; + final OpenSearchRequestBuilder requestBuilder; public OpenSearchIndexScanQueryBuilder(OpenSearchRequestBuilder requestBuilder) { this.requestBuilder = requestBuilder; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java index 7669b569d4..54e3477410 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java @@ -25,14 +25,14 @@ public class SortQueryBuilder { /** The mapping between Core Engine sort order and OpenSearch sort order. */ - private Map<Sort.SortOrder, SortOrder> sortOrderMap = + private final Map<Sort.SortOrder, SortOrder> sortOrderMap = new ImmutableMap.Builder<Sort.SortOrder, SortOrder>() .put(Sort.SortOrder.ASC, SortOrder.ASC) .put(Sort.SortOrder.DESC, SortOrder.DESC) .build(); /** The mapping between Core Engine null order and OpenSearch null order. */ - private Map<Sort.NullOrder, String> missingMap = + private final Map<Sort.NullOrder, String> missingMap = new ImmutableMap.Builder<Sort.NullOrder, String>() .put(Sort.NullOrder.NULL_FIRST, "_first") .put(Sort.NullOrder.NULL_LAST, "_last") diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java index defa97d8c8..1ff3321237 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java @@ -14,7 +14,8 @@ class OpenSearchExprGeoPointValueTest { - private OpenSearchExprGeoPointValue geoPointValue = new OpenSearchExprGeoPointValue(1.0, 1.0); + private final OpenSearchExprGeoPointValue geoPointValue = + new OpenSearchExprGeoPointValue(1.0, 1.0); @Test void value() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java index 0a3f56285f..fa328fd26c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java @@ -65,7 +65,7 @@ public class MLOperatorTest { private NodeClient nodeClient; private MLOperator mlOperator; - Map<String, Literal> arguments = new HashMap<>(); + final Map<String, Literal> arguments = new HashMap<>(); @Mock(answer = Answers.RETURNS_DEEP_STUBS) private MachineLearningNodeClient machineLearningNodeClient; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 217145a052..984c98f803 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -60,14 +60,14 @@ class OpenSearchResponseTest { @Mock private Aggregations aggregations; - private List<String> includes = List.of(); + private final List<String> includes = List.of(); @Mock private OpenSearchAggregationResponseParser parser; - private ExprTupleValue exprTupleValue1 = + private final ExprTupleValue exprTupleValue1 = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id1", new ExprIntegerValue(1))); - private ExprTupleValue exprTupleValue2 = + private final ExprTupleValue exprTupleValue2 = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id2", new ExprIntegerValue(2))); @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java index eb759233a8..6ffe6b275c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java @@ -13,8 +13,8 @@ import org.opensearch.core.rest.RestStatus; public class ErrorMessageFactoryTest { - private Throwable nonOpenSearchThrowable = new Throwable(); - private Throwable openSearchThrowable = new OpenSearchException(nonOpenSearchThrowable); + private final Throwable nonOpenSearchThrowable = new Throwable(); + private final Throwable openSearchThrowable = new OpenSearchException(nonOpenSearchThrowable); @Test public void openSearchExceptionShouldCreateEsErrorMessage() { diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java index 286ac20fed..1cce1ccc18 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java @@ -20,7 +20,7 @@ public class TransportPPLQueryRequestTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void testValidate() { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index aec22ac231..98c41027ff 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -63,7 +63,7 @@ public class AstExpressionBuilder extends OpenSearchPPLParserBaseVisitor<Unresol private static final int DEFAULT_TAKE_FUNCTION_SIZE_VALUE = 10; /** The function name mapping between fronted and core engine. */ - private static Map<String, String> FUNCTION_NAME_MAPPING = + private static final Map<String, String> FUNCTION_NAME_MAPPING = new ImmutableMap.Builder<String, String>() .put("isnull", IS_NULL.getName().getFunctionName()) .put("isnotnull", IS_NOT_NULL.getName().getFunctionName()) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 598f6691cb..34553823c4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -32,9 +32,9 @@ @RunWith(MockitoJUnitRunner.class) public class PPLServiceTest { - private static String QUERY = "/_plugins/_ppl"; + private static final String QUERY = "/_plugins/_ppl"; - private static String EXPLAIN = "/_plugins/_ppl/_explain"; + private static final String EXPLAIN = "/_plugins/_ppl/_explain"; private PPLService pplService; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 5601bda485..2645be3aca 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -18,7 +18,7 @@ public class PPLSyntaxParserTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void testSearchCommandShouldPass() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/domain/PPLQueryRequestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/domain/PPLQueryRequestTest.java index 29e6ff3298..f4e90395cb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/domain/PPLQueryRequestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/domain/PPLQueryRequestTest.java @@ -15,7 +15,7 @@ public class PPLQueryRequestTest { - @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @Test public void getRequestShouldPass() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 488cb7dc14..ac2bce9dbc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -66,7 +66,7 @@ public class AstBuilderTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); - private PPLSyntaxParser parser = new PPLSyntaxParser(); + private final PPLSyntaxParser parser = new PPLSyntaxParser(); @Test public void testSearchCommand() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java index 7d7b31e822..0b98ee6179 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java @@ -30,7 +30,7 @@ public class AstStatementBuilderTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); - private PPLSyntaxParser parser = new PPLSyntaxParser(); + private final PPLSyntaxParser parser = new PPLSyntaxParser(); @Test public void buildQueryStatement() { diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/QueryResultTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/QueryResultTest.java index e03169e9f8..fc3402e20a 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/QueryResultTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/QueryResultTest.java @@ -22,7 +22,7 @@ class QueryResultTest { - private ExecutionEngine.Schema schema = + private final ExecutionEngine.Schema schema = new ExecutionEngine.Schema( ImmutableList.of( new ExecutionEngine.Schema.Column("name", null, STRING), diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java b/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java index 3ef911c8d8..7b7fa1eadf 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java +++ b/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java @@ -34,7 +34,7 @@ public class EmrClientImpl implements SparkClient { private final FlintHelper flint; private final String sparkApplicationJar; private static final Logger logger = LogManager.getLogger(EmrClientImpl.class); - private SparkResponse sparkResponse; + private final SparkResponse sparkResponse; /** * Constructor for EMR Client Implementation. diff --git a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java index b124757069..dc920b248f 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java @@ -36,9 +36,9 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class SQLServiceTest { - private static String QUERY = "/_plugins/_sql"; + private static final String QUERY = "/_plugins/_sql"; - private static String EXPLAIN = "/_plugins/_sql/_explain"; + private static final String EXPLAIN = "/_plugins/_sql/_explain"; private SQLService sqlService; From 3e2cb1dd907d30ac7dc3e1058078adf1295bb6fe Mon Sep 17 00:00:00 2001 From: Peng Huo <penghuo@gmail.com> Date: Mon, 9 Dec 2024 14:37:01 -0800 Subject: [PATCH 89/96] Fix FilterOperator to cache next element and avoid repeated consumption on hasNext() calls (#3123) Signed-off-by: Peng Huo <penghuo@gmail.com> --- .../sql/planner/physical/FilterOperator.java | 32 +++++-- .../planner/physical/FilterOperatorTest.java | 84 +++++++++++++++++++ .../resources/correctness/bugfixes/3121.txt | 1 + 3 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 integ-test/src/test/resources/correctness/bugfixes/3121.txt diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java index 192ea5cb4f..088dd07f8d 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/FilterOperator.java @@ -28,6 +28,7 @@ public class FilterOperator extends PhysicalPlan { @Getter private final PhysicalPlan input; @Getter private final Expression conditions; @ToString.Exclude private ExprValue next = null; + @ToString.Exclude private boolean nextPrepared = false; @Override public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { @@ -41,19 +42,34 @@ public List<PhysicalPlan> getChild() { @Override public boolean hasNext() { + if (!nextPrepared) { + prepareNext(); + } + return next != null; + } + + @Override + public ExprValue next() { + if (!nextPrepared) { + prepareNext(); + } + ExprValue result = next; + next = null; + nextPrepared = false; + return result; + } + + private void prepareNext() { while (input.hasNext()) { ExprValue inputValue = input.next(); ExprValue exprValue = conditions.valueOf(inputValue.bindingTuples()); - if (!(exprValue.isNull() || exprValue.isMissing()) && (exprValue.booleanValue())) { + if (!(exprValue.isNull() || exprValue.isMissing()) && exprValue.booleanValue()) { next = inputValue; - return true; + nextPrepared = true; + return; } } - return false; - } - - @Override - public ExprValue next() { - return next; + next = null; + nextPrepared = true; } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/FilterOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/FilterOperatorTest.java index bfe3b323c4..ba2354b168 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/FilterOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/FilterOperatorTest.java @@ -8,14 +8,24 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.filter; import com.google.common.collect.ImmutableMap; import java.util.LinkedHashMap; import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; @@ -26,12 +36,22 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; @ExtendWith(MockitoExtension.class) @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class FilterOperatorTest extends PhysicalPlanTestBase { @Mock private PhysicalPlan inputPlan; + @Mock private Expression condition; + + private FilterOperator filterOperator; + + @BeforeEach + public void setup() { + filterOperator = filter(inputPlan, condition); + } + @Test public void filter_test() { FilterOperator plan = @@ -82,4 +102,68 @@ public void missing_value_should_been_ignored() { List<ExprValue> result = execute(plan); assertEquals(0, result.size()); } + + @Test + public void testHasNextWhenInputHasNoElements() { + when(inputPlan.hasNext()).thenReturn(false); + + assertFalse( + filterOperator.hasNext(), "hasNext() should return false when input has no elements"); + } + + @Test + public void testHasNextWithMatchingCondition() { + ExprValue inputValue = mock(ExprValue.class); + when(inputPlan.hasNext()).thenReturn(true).thenReturn(false); + when(inputPlan.next()).thenReturn(inputValue); + when(condition.valueOf(any())).thenReturn(LITERAL_TRUE); + + assertTrue(filterOperator.hasNext(), "hasNext() should return true when condition matches"); + assertEquals( + inputValue, filterOperator.next(), "next() should return the matching input value"); + } + + @Test + public void testHasNextWithNonMatchingCondition() { + ExprValue inputValue = mock(ExprValue.class); + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(inputValue); + when(condition.valueOf(any())).thenReturn(LITERAL_FALSE); + + assertFalse( + filterOperator.hasNext(), "hasNext() should return false if no values match the condition"); + } + + @Test + public void testMultipleCallsToHasNextDoNotConsumeInput() { + ExprValue inputValue = mock(ExprValue.class); + when(inputPlan.hasNext()).thenReturn(true); + when(inputPlan.next()).thenReturn(inputValue); + when(condition.valueOf(any())).thenReturn(LITERAL_TRUE); + + assertTrue( + filterOperator.hasNext(), + "First hasNext() call should return true if there is a matching value"); + verify(inputPlan, times(1)).next(); + assertTrue( + filterOperator.hasNext(), + "Subsequent hasNext() calls should still return true without advancing the input"); + verify(inputPlan, times(1)).next(); + assertEquals( + inputValue, filterOperator.next(), "next() should return the matching input value"); + verify(inputPlan, times(1)).next(); + } + + @Test + public void testNextWithoutCallingHasNext() { + ExprValue inputValue = mock(ExprValue.class); + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(inputValue); + when(condition.valueOf(any())).thenReturn(LITERAL_TRUE); + + assertEquals( + inputValue, + filterOperator.next(), + "next() should return the matching input value even if hasNext() was not called"); + } } diff --git a/integ-test/src/test/resources/correctness/bugfixes/3121.txt b/integ-test/src/test/resources/correctness/bugfixes/3121.txt new file mode 100644 index 0000000000..f60f724897 --- /dev/null +++ b/integ-test/src/test/resources/correctness/bugfixes/3121.txt @@ -0,0 +1 @@ +SELECT Origin, Dest FROM (SELECT * FROM opensearch_dashboards_sample_data_flights WHERE AvgTicketPrice > 100 GROUP BY Origin, Dest, AvgTicketPrice) AS flights WHERE AvgTicketPrice < 1000 ORDER BY AvgTicketPrice LIMIT 30 From ed0ca8ddf6a217ebafd24dbb3263db798bf15e27 Mon Sep 17 00:00:00 2001 From: James Duong <duong.james@gmail.com> Date: Thu, 12 Dec 2024 11:01:17 -0800 Subject: [PATCH 90/96] Add trendline PPL command (#3071) * Add trendline (With SWA) PPL command --------- Signed-off-by: James Duong <james.duong@improving.com> Signed-off-by: Andrew Carbonetto <andrew.carbonetto@improving.com> Co-authored-by: Andrew Carbonetto <andrew.carbonetto@improving.com> --- .../org/opensearch/sql/analysis/Analyzer.java | 94 ++++- .../sql/ast/AbstractNodeVisitor.java | 9 + .../org/opensearch/sql/ast/dsl/AstDSL.java | 14 + .../opensearch/sql/ast/tree/Trendline.java | 71 ++++ .../org/opensearch/sql/executor/Explain.java | 32 ++ .../sql/planner/DefaultImplementor.java | 7 + .../sql/planner/logical/LogicalPlanDSL.java | 7 + .../logical/LogicalPlanNodeVisitor.java | 4 + .../sql/planner/logical/LogicalTrendline.java | 42 ++ .../physical/PhysicalPlanNodeVisitor.java | 4 + .../planner/physical/TrendlineOperator.java | 317 ++++++++++++++ .../opensearch/sql/analysis/AnalyzerTest.java | 60 +++ .../opensearch/sql/executor/ExplainTest.java | 43 ++ .../sql/planner/DefaultImplementorTest.java | 21 + .../logical/LogicalPlanNodeVisitorTest.java | 14 +- .../physical/PhysicalPlanNodeVisitorTest.java | 31 +- .../physical/TrendlineOperatorTest.java | 398 ++++++++++++++++++ docs/category.json | 1 + docs/user/ppl/cmd/trendline.rst | 90 ++++ docs/user/ppl/index.rst | 2 + .../org/opensearch/sql/ppl/ExplainIT.java | 26 ++ .../sql/ppl/TrendlineCommandIT.java | 78 ++++ .../ppl/explain_trendline_push.json | 32 ++ .../ppl/explain_trendline_sort_push.json | 32 ++ .../OpenSearchExecutionProtector.java | 7 + .../OpenSearchExecutionProtectorTest.java | 21 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 4 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 14 + .../opensearch/sql/ppl/parser/AstBuilder.java | 16 + .../sql/ppl/parser/AstExpressionBuilder.java | 25 ++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 27 +- .../sql/ppl/parser/AstBuilderTest.java | 74 ++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 7 + 33 files changed, 1601 insertions(+), 23 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Trendline.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/logical/LogicalTrendline.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/TrendlineOperator.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/TrendlineOperatorTest.java create mode 100644 docs/user/ppl/cmd/trendline.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/TrendlineCommandIT.java create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 71db736f78..d0051568c4 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -10,7 +10,10 @@ import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST; import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; @@ -22,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -62,6 +66,7 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -100,6 +105,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.planner.logical.LogicalTrendline; import org.opensearch.sql.planner.logical.LogicalValues; import org.opensearch.sql.planner.physical.datasource.DataSourceTable; import org.opensearch.sql.storage.Table; @@ -469,23 +475,7 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) { @Override public LogicalPlan visitSort(Sort node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); - ExpressionReferenceOptimizer optimizer = - new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child); - - List<Pair<SortOption, Expression>> sortList = - node.getSortList().stream() - .map( - sortField -> { - var analyzed = expressionAnalyzer.analyze(sortField.getField(), context); - if (analyzed == null) { - throw new UnsupportedOperationException( - String.format("Invalid use of expression %s", sortField.getField())); - } - Expression expression = optimizer.optimize(analyzed, context); - return ImmutablePair.of(analyzeSortOption(sortField.getFieldArgs()), expression); - }) - .collect(Collectors.toList()); - return new LogicalSort(child, sortList); + return buildSort(child, context, node.getSortList()); } /** Build {@link LogicalDedupe}. */ @@ -594,6 +584,55 @@ public LogicalPlan visitML(ML node, AnalysisContext context) { return new LogicalML(child, node.getArguments()); } + /** Build {@link LogicalTrendline} for Trendline command. */ + @Override + public LogicalPlan visitTrendline(Trendline node, AnalysisContext context) { + final LogicalPlan child = node.getChild().get(0).accept(this, context); + + final TypeEnvironment currEnv = context.peek(); + final List<Trendline.TrendlineComputation> computations = node.getComputations(); + final ImmutableList.Builder<Pair<Trendline.TrendlineComputation, ExprCoreType>> + computationsAndTypes = ImmutableList.builder(); + computations.forEach( + computation -> { + final Expression resolvedField = + expressionAnalyzer.analyze(computation.getDataField(), context); + final ExprCoreType averageType; + // Duplicate the semantics of AvgAggregator#create(): + // - All numerical types have the DOUBLE type for the moving average. + // - All datetime types have the same datetime type for the moving average. + if (ExprCoreType.numberTypes().contains(resolvedField.type())) { + averageType = ExprCoreType.DOUBLE; + } else { + switch (resolvedField.type()) { + case DATE: + case TIME: + case TIMESTAMP: + averageType = (ExprCoreType) resolvedField.type(); + break; + default: + throw new SemanticCheckException( + String.format( + "Invalid field used for trendline computation %s. Source field %s had type" + + " %s but must be a numerical or datetime field.", + computation.getAlias(), + computation.getDataField().getChild().get(0), + resolvedField.type().typeName())); + } + } + currEnv.define(new Symbol(Namespace.FIELD_NAME, computation.getAlias()), averageType); + computationsAndTypes.add(Pair.of(computation, averageType)); + }); + + if (node.getSortByField().isEmpty()) { + return new LogicalTrendline(child, computationsAndTypes.build()); + } + + return new LogicalTrendline( + buildSort(child, context, Collections.singletonList(node.getSortByField().get())), + computationsAndTypes.build()); + } + @Override public LogicalPlan visitPaginate(Paginate paginate, AnalysisContext context) { LogicalPlan child = paginate.getChild().get(0).accept(this, context); @@ -612,6 +651,27 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context)); } + private LogicalSort buildSort( + LogicalPlan child, AnalysisContext context, List<Field> sortFields) { + ExpressionReferenceOptimizer optimizer = + new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child); + + List<Pair<SortOption, Expression>> sortList = + sortFields.stream() + .map( + sortField -> { + var analyzed = expressionAnalyzer.analyze(sortField.getField(), context); + if (analyzed == null) { + throw new UnsupportedOperationException( + String.format("Invalid use of expression %s", sortField.getField())); + } + Expression expression = optimizer.optimize(analyzed, context); + return ImmutablePair.of(analyzeSortOption(sortField.getFieldArgs()), expression); + }) + .collect(Collectors.toList()); + return new LogicalSort(child, sortList); + } + /** * The first argument is always "asc", others are optional. Given nullFirst argument, use its * value. Otherwise just use DEFAULT_ASC/DESC. diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index a0520dc70e..f27260dd5f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -60,6 +60,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Values; /** AST nodes visitor Defines the traverse path. */ @@ -110,6 +111,14 @@ public T visitFilter(Filter node, C context) { return visitChildren(node, context); } + public T visitTrendline(Trendline node, C context) { + return visitChildren(node, context); + } + + public T visitTrendlineComputation(Trendline.TrendlineComputation node, C context) { + return visitChildren(node, context); + } + public T visitProject(Project node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 8135731ff6..d9956609ec 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -62,6 +63,7 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; @@ -466,6 +468,18 @@ public static Limit limit(UnresolvedPlan input, Integer limit, Integer offset) { return new Limit(limit, offset).attach(input); } + public static Trendline trendline( + UnresolvedPlan input, + Optional<Field> sortField, + Trendline.TrendlineComputation... computations) { + return new Trendline(sortField, Arrays.asList(computations)).attach(input); + } + + public static Trendline.TrendlineComputation computation( + Integer numDataPoints, Field dataField, String alias, Trendline.TrendlineType type) { + return new Trendline.TrendlineComputation(numDataPoints, dataField, alias, type); + } + public static Parse parse( UnresolvedPlan input, ParseMethod parseMethod, diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Trendline.java b/core/src/main/java/org/opensearch/sql/ast/tree/Trendline.java new file mode 100644 index 0000000000..aa4fcc200d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Trendline.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Optional; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +@ToString +@Getter +@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class Trendline extends UnresolvedPlan { + + private UnresolvedPlan child; + private final Optional<Field> sortByField; + private final List<TrendlineComputation> computations; + + @Override + public Trendline attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List<? extends Node> getChild() { + return ImmutableList.of(child); + } + + @Override + public <T, C> T accept(AbstractNodeVisitor<T, C> visitor, C context) { + return visitor.visitTrendline(this, context); + } + + @Getter + public static class TrendlineComputation extends UnresolvedExpression { + + private final Integer numberOfDataPoints; + private final Field dataField; + private final String alias; + private final TrendlineType computationType; + + public TrendlineComputation( + Integer numberOfDataPoints, Field dataField, String alias, TrendlineType computationType) { + this.numberOfDataPoints = numberOfDataPoints; + this.dataField = dataField; + this.alias = alias; + this.computationType = computationType; + } + + @Override + public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) { + return nodeVisitor.visitTrendlineComputation(this, context); + } + } + + public enum TrendlineType { + SMA + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/Explain.java b/core/src/main/java/org/opensearch/sql/executor/Explain.java index fffbe6f693..31890a8090 100644 --- a/core/src/main/java/org/opensearch/sql/executor/Explain.java +++ b/core/src/main/java/org/opensearch/sql/executor/Explain.java @@ -8,12 +8,14 @@ import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.expression.Expression; @@ -31,6 +33,7 @@ import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; import org.opensearch.sql.planner.physical.TakeOrderedOperator; +import org.opensearch.sql.planner.physical.TrendlineOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.TableScanOperator; @@ -211,6 +214,21 @@ public ExplainResponseNode visitNested(NestedOperator node, Object context) { explanNode -> explanNode.setDescription(ImmutableMap.of("nested", node.getFields()))); } + @Override + public ExplainResponseNode visitTrendline(TrendlineOperator node, Object context) { + return explain( + node, + context, + explainNode -> + explainNode.setDescription( + ImmutableMap.of( + "computations", + describeTrendlineComputations( + node.getComputations().stream() + .map(Pair::getKey) + .collect(Collectors.toList()))))); + } + protected ExplainResponseNode explain( PhysicalPlan node, Object context, Consumer<ExplainResponseNode> doExplain) { ExplainResponseNode explainNode = new ExplainResponseNode(getOperatorName(node)); @@ -245,4 +263,18 @@ private Map<String, Map<String, String>> describeSortList( "sortOrder", p.getLeft().getSortOrder().toString(), "nullOrder", p.getLeft().getNullOrder().toString()))); } + + private List<Map<String, String>> describeTrendlineComputations( + List<Trendline.TrendlineComputation> computations) { + return computations.stream() + .map( + computation -> + ImmutableMap.of( + "computationType", + computation.getComputationType().name().toLowerCase(Locale.ROOT), + "numberOfDataPoints", computation.getNumberOfDataPoints().toString(), + "dataField", computation.getDataField().getChild().get(0).toString(), + "alias", computation.getAlias())) + .collect(Collectors.toList()); + } } diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index f962c3e4bf..c988084d1b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -23,6 +23,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.planner.logical.LogicalTrendline; import org.opensearch.sql.planner.logical.LogicalValues; import org.opensearch.sql.planner.logical.LogicalWindow; import org.opensearch.sql.planner.physical.AggregationOperator; @@ -39,6 +40,7 @@ import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; import org.opensearch.sql.planner.physical.TakeOrderedOperator; +import org.opensearch.sql.planner.physical.TrendlineOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.read.TableScanBuilder; @@ -166,6 +168,11 @@ public PhysicalPlan visitCloseCursor(LogicalCloseCursor node, C context) { return new CursorCloseOperator(visitChild(node, context)); } + @Override + public PhysicalPlan visitTrendline(LogicalTrendline plan, C context) { + return new TrendlineOperator(visitChild(plan, context), plan.getComputations()); + } + // Called when paging query requested without `FROM` clause only @Override public PhysicalPlan visitPaginate(LogicalPaginate plan, C context) { diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 2a886ba0ca..13c6d7a979 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -15,6 +15,8 @@ import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; @@ -130,6 +132,11 @@ public static LogicalPlan rareTopN( return new LogicalRareTopN(input, commandType, noOfResults, Arrays.asList(fields), groupByList); } + public static LogicalTrendline trendline( + LogicalPlan input, Pair<Trendline.TrendlineComputation, ExprCoreType>... computations) { + return new LogicalTrendline(input, Arrays.asList(computations)); + } + @SafeVarargs public LogicalPlan values(List<LiteralExpression>... values) { return new LogicalValues(Arrays.asList(values)); diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 156db35306..c9eedd8efc 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -104,6 +104,10 @@ public R visitAD(LogicalAD plan, C context) { return visitNode(plan, context); } + public R visitTrendline(LogicalTrendline plan, C context) { + return visitNode(plan, context); + } + public R visitPaginate(LogicalPaginate plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalTrendline.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalTrendline.java new file mode 100644 index 0000000000..3e992035e2 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalTrendline.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.data.type.ExprCoreType; + +/* + * Trendline logical plan. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class LogicalTrendline extends LogicalPlan { + private final List<Pair<Trendline.TrendlineComputation, ExprCoreType>> computations; + + /** + * Constructor of LogicalTrendline. + * + * @param child child logical plan + * @param computations the computations for this trendline call. + */ + public LogicalTrendline( + LogicalPlan child, List<Pair<Trendline.TrendlineComputation, ExprCoreType>> computations) { + super(Collections.singletonList(child)); + this.computations = computations; + } + + @Override + public <R, C> R accept(LogicalPlanNodeVisitor<R, C> visitor, C context) { + return visitor.visitTrendline(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 67d7a05135..66c7219e39 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -96,6 +96,10 @@ public R visitML(PhysicalPlan node, C context) { return visitNode(node, context); } + public R visitTrendline(TrendlineOperator node, C context) { + return visitNode(node, context); + } + public R visitCursorClose(CursorCloseOperator node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/TrendlineOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/TrendlineOperator.java new file mode 100644 index 0000000000..7bf10964cf --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/TrendlineOperator.java @@ -0,0 +1,317 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static java.time.temporal.ChronoUnit.MILLIS; + +import com.google.common.collect.EvictingQueue; +import com.google.common.collect.ImmutableMap.Builder; +import java.time.Instant; +import java.time.LocalTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.LiteralExpression; + +/** Trendline command implementation */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class TrendlineOperator extends PhysicalPlan { + @Getter private final PhysicalPlan input; + @Getter private final List<Pair<Trendline.TrendlineComputation, ExprCoreType>> computations; + @EqualsAndHashCode.Exclude private final List<TrendlineAccumulator> accumulators; + @EqualsAndHashCode.Exclude private final Map<String, Integer> fieldToIndexMap; + @EqualsAndHashCode.Exclude private final HashSet<String> aliases; + + public TrendlineOperator( + PhysicalPlan input, List<Pair<Trendline.TrendlineComputation, ExprCoreType>> computations) { + this.input = input; + this.computations = computations; + this.accumulators = computations.stream().map(TrendlineOperator::createAccumulator).toList(); + fieldToIndexMap = new HashMap<>(computations.size()); + aliases = new HashSet<>(computations.size()); + for (int i = 0; i < computations.size(); ++i) { + final Trendline.TrendlineComputation computation = computations.get(i).getKey(); + fieldToIndexMap.put(computation.getDataField().getChild().get(0).toString(), i); + aliases.add(computation.getAlias()); + } + } + + @Override + public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { + return visitor.visitTrendline(this, context); + } + + @Override + public List<PhysicalPlan> getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return getChild().getFirst().hasNext(); + } + + @Override + public ExprValue next() { + final ExprValue result; + final ExprValue next = input.next(); + final Map<String, ExprValue> inputStruct = consumeInputTuple(next); + final Builder<String, ExprValue> mapBuilder = new Builder<>(); + mapBuilder.putAll(inputStruct); + + // Add calculated trendline values, which might overwrite existing fields from the input. + for (int i = 0; i < accumulators.size(); ++i) { + final ExprValue calculateResult = accumulators.get(i).calculate(); + final String field = computations.get(i).getKey().getAlias(); + if (calculateResult != null) { + mapBuilder.put(field, calculateResult); + } + } + + result = ExprTupleValue.fromExprValueMap(mapBuilder.buildKeepingLast()); + return result; + } + + private Map<String, ExprValue> consumeInputTuple(ExprValue inputValue) { + final Map<String, ExprValue> tupleValue = ExprValueUtils.getTupleValue(inputValue); + for (String bindName : tupleValue.keySet()) { + final Integer index = fieldToIndexMap.get(bindName); + if (index != null) { + final ExprValue fieldValue = tupleValue.get(bindName); + if (!fieldValue.isNull()) { + accumulators.get(index).accumulate(fieldValue); + } + } + } + tupleValue.keySet().removeAll(aliases); + return tupleValue; + } + + private static TrendlineAccumulator createAccumulator( + Pair<Trendline.TrendlineComputation, ExprCoreType> computation) { + // Add a switch statement based on computation type to choose the accumulator when more + // types of computations are supported. + return new SimpleMovingAverageAccumulator(computation.getKey(), computation.getValue()); + } + + /** Maintains stateful information for calculating the trendline. */ + private interface TrendlineAccumulator { + void accumulate(ExprValue value); + + ExprValue calculate(); + + static ArithmeticEvaluator getEvaluator(ExprCoreType type) { + switch (type) { + case DOUBLE: + return NumericArithmeticEvaluator.INSTANCE; + case DATE: + return DateArithmeticEvaluator.INSTANCE; + case TIME: + return TimeArithmeticEvaluator.INSTANCE; + case TIMESTAMP: + return TimestampArithmeticEvaluator.INSTANCE; + } + throw new IllegalArgumentException( + String.format("Invalid type %s used for moving average.", type.typeName())); + } + } + + private static class SimpleMovingAverageAccumulator implements TrendlineAccumulator { + private final LiteralExpression dataPointsNeeded; + private final EvictingQueue<ExprValue> receivedValues; + private final ArithmeticEvaluator evaluator; + private Expression runningTotal = null; + + public SimpleMovingAverageAccumulator( + Trendline.TrendlineComputation computation, ExprCoreType type) { + dataPointsNeeded = DSL.literal(computation.getNumberOfDataPoints().doubleValue()); + receivedValues = EvictingQueue.create(computation.getNumberOfDataPoints()); + evaluator = TrendlineAccumulator.getEvaluator(type); + } + + @Override + public void accumulate(ExprValue value) { + if (dataPointsNeeded.valueOf().integerValue() == 1) { + runningTotal = evaluator.calculateFirstTotal(Collections.singletonList(value)); + receivedValues.add(value); + return; + } + + final ExprValue valueToRemove; + if (receivedValues.size() == dataPointsNeeded.valueOf().integerValue()) { + valueToRemove = receivedValues.remove(); + } else { + valueToRemove = null; + } + receivedValues.add(value); + + if (receivedValues.size() == dataPointsNeeded.valueOf().integerValue()) { + if (runningTotal != null) { + // We can use the previous calculation. + // Subtract the evicted value and add the new value. + // Refactored, that would be previous + (newValue - oldValue). + runningTotal = evaluator.add(runningTotal, value, valueToRemove); + } else { + // This is the first average calculation so sum the entire receivedValues dataset. + final List<ExprValue> data = receivedValues.stream().toList(); + runningTotal = evaluator.calculateFirstTotal(data); + } + } + } + + @Override + public ExprValue calculate() { + if (receivedValues.size() < dataPointsNeeded.valueOf().integerValue()) { + return null; + } else if (dataPointsNeeded.valueOf().integerValue() == 1) { + return receivedValues.peek(); + } + return evaluator.evaluate(runningTotal, dataPointsNeeded); + } + } + + private interface ArithmeticEvaluator { + Expression calculateFirstTotal(List<ExprValue> dataPoints); + + Expression add(Expression runningTotal, ExprValue incomingValue, ExprValue evictedValue); + + ExprValue evaluate(Expression runningTotal, LiteralExpression numberOfDataPoints); + } + + private static class NumericArithmeticEvaluator implements ArithmeticEvaluator { + private static final NumericArithmeticEvaluator INSTANCE = new NumericArithmeticEvaluator(); + + private NumericArithmeticEvaluator() {} + + @Override + public Expression calculateFirstTotal(List<ExprValue> dataPoints) { + Expression total = DSL.literal(0.0D); + for (ExprValue dataPoint : dataPoints) { + total = DSL.add(total, DSL.literal(dataPoint.doubleValue())); + } + return DSL.literal(total.valueOf().doubleValue()); + } + + @Override + public Expression add( + Expression runningTotal, ExprValue incomingValue, ExprValue evictedValue) { + return DSL.literal( + DSL.add(runningTotal, DSL.subtract(DSL.literal(incomingValue), DSL.literal(evictedValue))) + .valueOf() + .doubleValue()); + } + + @Override + public ExprValue evaluate(Expression runningTotal, LiteralExpression numberOfDataPoints) { + return DSL.divide(runningTotal, numberOfDataPoints).valueOf(); + } + } + + private static class DateArithmeticEvaluator implements ArithmeticEvaluator { + private static final DateArithmeticEvaluator INSTANCE = new DateArithmeticEvaluator(); + + private DateArithmeticEvaluator() {} + + @Override + public Expression calculateFirstTotal(List<ExprValue> dataPoints) { + return TimestampArithmeticEvaluator.INSTANCE.calculateFirstTotal(dataPoints); + } + + @Override + public Expression add( + Expression runningTotal, ExprValue incomingValue, ExprValue evictedValue) { + return TimestampArithmeticEvaluator.INSTANCE.add(runningTotal, incomingValue, evictedValue); + } + + @Override + public ExprValue evaluate(Expression runningTotal, LiteralExpression numberOfDataPoints) { + final ExprValue timestampResult = + TimestampArithmeticEvaluator.INSTANCE.evaluate(runningTotal, numberOfDataPoints); + return ExprValueUtils.dateValue(timestampResult.dateValue()); + } + } + + private static class TimeArithmeticEvaluator implements ArithmeticEvaluator { + private static final TimeArithmeticEvaluator INSTANCE = new TimeArithmeticEvaluator(); + + private TimeArithmeticEvaluator() {} + + @Override + public Expression calculateFirstTotal(List<ExprValue> dataPoints) { + Expression total = DSL.literal(0); + for (ExprValue dataPoint : dataPoints) { + total = DSL.add(total, DSL.literal(MILLIS.between(LocalTime.MIN, dataPoint.timeValue()))); + } + return DSL.literal(total.valueOf().longValue()); + } + + @Override + public Expression add( + Expression runningTotal, ExprValue incomingValue, ExprValue evictedValue) { + return DSL.literal( + DSL.add( + runningTotal, + DSL.subtract( + DSL.literal(MILLIS.between(LocalTime.MIN, incomingValue.timeValue())), + DSL.literal(MILLIS.between(LocalTime.MIN, evictedValue.timeValue())))) + .valueOf()); + } + + @Override + public ExprValue evaluate(Expression runningTotal, LiteralExpression numberOfDataPoints) { + return ExprValueUtils.timeValue( + LocalTime.MIN.plus( + DSL.divide(runningTotal, numberOfDataPoints).valueOf().longValue(), MILLIS)); + } + } + + private static class TimestampArithmeticEvaluator implements ArithmeticEvaluator { + private static final TimestampArithmeticEvaluator INSTANCE = new TimestampArithmeticEvaluator(); + + private TimestampArithmeticEvaluator() {} + + @Override + public Expression calculateFirstTotal(List<ExprValue> dataPoints) { + Expression total = DSL.literal(0); + for (ExprValue dataPoint : dataPoints) { + total = DSL.add(total, DSL.literal(dataPoint.timestampValue().toEpochMilli())); + } + return DSL.literal(total.valueOf().longValue()); + } + + @Override + public Expression add( + Expression runningTotal, ExprValue incomingValue, ExprValue evictedValue) { + return DSL.literal( + DSL.add( + runningTotal, + DSL.subtract( + DSL.literal(incomingValue.timestampValue().toEpochMilli()), + DSL.literal(evictedValue.timestampValue().toEpochMilli()))) + .valueOf()); + } + + @Override + public ExprValue evaluate(Expression runningTotal, LiteralExpression numberOfDataPoints) { + return ExprValueUtils.timestampValue( + Instant.ofEpochMilli(DSL.divide(runningTotal, numberOfDataPoints).valueOf().longValue())); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 4f06ce9d23..d6cb0544d8 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -18,6 +18,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.argument; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.computation; import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.filter; import static org.opensearch.sql.ast.dsl.AstDSL.filteredAggregate; @@ -33,6 +34,7 @@ import static org.opensearch.sql.ast.tree.Sort.SortOption; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; @@ -66,6 +68,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Disabled; @@ -89,6 +92,7 @@ import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; @@ -1481,6 +1485,62 @@ public void fillnull_various_values() { AstDSL.field("int_null_value"), AstDSL.intLiteral(1)))))); } + @Test + public void trendline() { + assertAnalyzeEqual( + LogicalPlanDSL.trendline( + LogicalPlanDSL.relation("schema", table), + Pair.of(computation(5, field("float_value"), "test_field_alias", SMA), DOUBLE), + Pair.of(computation(1, field("double_value"), "test_field_alias_2", SMA), DOUBLE)), + AstDSL.trendline( + AstDSL.relation("schema"), + Optional.empty(), + computation(5, field("float_value"), "test_field_alias", SMA), + computation(1, field("double_value"), "test_field_alias_2", SMA))); + } + + @Test + public void trendline_datetime_types() { + assertAnalyzeEqual( + LogicalPlanDSL.trendline( + LogicalPlanDSL.relation("schema", table), + Pair.of(computation(5, field("timestamp_value"), "test_field_alias", SMA), TIMESTAMP)), + AstDSL.trendline( + AstDSL.relation("schema"), + Optional.empty(), + computation(5, field("timestamp_value"), "test_field_alias", SMA))); + } + + @Test + public void trendline_illegal_type() { + assertThrows( + SemanticCheckException.class, + () -> + analyze( + AstDSL.trendline( + AstDSL.relation("schema"), + Optional.empty(), + computation(5, field("array_value"), "test_field_alias", SMA)))); + } + + @Test + public void trendline_with_sort() { + assertAnalyzeEqual( + LogicalPlanDSL.trendline( + LogicalPlanDSL.sort( + LogicalPlanDSL.relation("schema", table), + Pair.of( + new SortOption(SortOrder.ASC, NullOrder.NULL_FIRST), + DSL.ref("float_value", ExprCoreType.FLOAT))), + Pair.of(computation(5, field("float_value"), "test_field_alias", SMA), DOUBLE), + Pair.of(computation(1, field("double_value"), "test_field_alias_2", SMA), DOUBLE)), + AstDSL.trendline( + AstDSL.relation("schema"), + Optional.of(field("float_value", argument("asc", booleanLiteral(true)))), + computation(5, field("float_value"), "test_field_alias", SMA), + computation(1, field("double_value"), "test_field_alias_2", SMA))); + } + @Test public void ad_batchRCF_relation() { Map<String, Literal> argumentMap = diff --git a/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java b/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java index eaeae07242..febf662843 100644 --- a/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/ExplainTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.ast.tree.RareTopN.CommandType.TOP; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -31,6 +32,8 @@ import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.values; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.window; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -39,6 +42,7 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; @@ -52,6 +56,7 @@ import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.TrendlineOperator; import org.opensearch.sql.storage.TableScanOperator; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -256,6 +261,44 @@ void can_explain_nested() { explain.apply(plan)); } + @Test + void can_explain_trendline() { + PhysicalPlan plan = + new TrendlineOperator( + tableScan, + Arrays.asList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), DOUBLE), + Pair.of(AstDSL.computation(3, AstDSL.field("time"), "time_alias", SMA), DOUBLE))); + assertEquals( + new ExplainResponse( + new ExplainResponseNode( + "TrendlineOperator", + ImmutableMap.of( + "computations", + List.of( + ImmutableMap.of( + "computationType", + "sma", + "numberOfDataPoints", + "2", + "dataField", + "distance", + "alias", + "distance_alias"), + ImmutableMap.of( + "computationType", + "sma", + "numberOfDataPoints", + "3", + "dataField", + "time", + "alias", + "time_alias"))), + singletonList(tableScan.explainNode()))), + explain.apply(plan)); + } + private static class FakeTableScan extends TableScanOperator { @Override public boolean hasNext() { diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java index 8e71fc2bec..8ee0dd7e70 100644 --- a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java @@ -7,11 +7,13 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; @@ -44,8 +46,10 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.pagination.PlanSerializer; @@ -63,11 +67,13 @@ import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.logical.LogicalTrendline; import org.opensearch.sql.planner.logical.LogicalValues; import org.opensearch.sql.planner.physical.CursorCloseOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanDSL; import org.opensearch.sql.planner.physical.ProjectOperator; +import org.opensearch.sql.planner.physical.TrendlineOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; @@ -304,4 +310,19 @@ public void visitLimit_support_return_takeOrdered() { 5); assertEquals(physicalPlanTree, logicalLimit.accept(implementor, null)); } + + @Test + public void visitTrendline_should_build_TrendlineOperator() { + var logicalChild = mock(LogicalPlan.class); + var physicalChild = mock(PhysicalPlan.class); + when(logicalChild.accept(implementor, null)).thenReturn(physicalChild); + final Trendline.TrendlineComputation computation = + AstDSL.computation(1, AstDSL.field("field"), "alias", SMA); + var logicalPlan = + new LogicalTrendline( + logicalChild, Collections.singletonList(Pair.of(computation, ExprCoreType.DOUBLE))); + var implemented = logicalPlan.accept(implementor, null); + assertInstanceOf(TrendlineOperator.class, implemented); + assertSame(physicalChild, implemented.getChild().get(0)); + } } diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index f212749f48..43ce23ed56 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.named; @@ -25,9 +26,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; @@ -141,6 +144,14 @@ public TableWriteOperator build(PhysicalPlan child) { LogicalCloseCursor closeCursor = new LogicalCloseCursor(cursor); + LogicalTrendline trendline = + new LogicalTrendline( + relation, + Collections.singletonList( + Pair.of( + AstDSL.computation(1, AstDSL.field("testField"), "dummy", SMA), + ExprCoreType.DOUBLE))); + return Stream.of( relation, tableScanBuilder, @@ -163,7 +174,8 @@ public TableWriteOperator build(PhysicalPlan child) { paginate, nested, cursor, - closeCursor) + closeCursor, + trendline) .map(Arguments::of); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index 17fb128ace..26f288e6b6 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.expression.DSL.named; @@ -29,6 +30,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,6 +45,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.expression.DSL; @@ -65,7 +68,16 @@ public void print_physical_plan() { agg( rareTopN( filter( - limit(new TestScan(), 1, 1), + limit( + new TrendlineOperator( + new TestScan(), + Collections.singletonList( + Pair.of( + AstDSL.computation( + 1, AstDSL.field("field"), "alias", SMA), + DOUBLE))), + 1, + 1), DSL.equal(DSL.ref("response", INTEGER), DSL.literal(10))), CommandType.TOP, ImmutableList.of(), @@ -85,7 +97,8 @@ public void print_physical_plan() { + "\t\t\tAggregation->\n" + "\t\t\t\tRareTopN->\n" + "\t\t\t\t\tFilter->\n" - + "\t\t\t\t\t\tLimit->", + + "\t\t\t\t\t\tLimit->\n" + + "\t\t\t\t\t\t\tTrendline->", printer.print(plan)); } @@ -134,6 +147,12 @@ public static Stream<Arguments> getPhysicalPlanForTest() { PhysicalPlan cursorClose = new CursorCloseOperator(plan); + PhysicalPlan trendline = + new TrendlineOperator( + plan, + Collections.singletonList( + Pair.of(AstDSL.computation(1, AstDSL.field("field"), "alias", SMA), DOUBLE))); + return Stream.of( Arguments.of(filter, "filter"), Arguments.of(aggregation, "aggregation"), @@ -149,7 +168,8 @@ public static Stream<Arguments> getPhysicalPlanForTest() { Arguments.of(rareTopN, "rareTopN"), Arguments.of(limit, "limit"), Arguments.of(nested, "nested"), - Arguments.of(cursorClose, "cursorClose")); + Arguments.of(cursorClose, "cursorClose"), + Arguments.of(trendline, "trendline")); } @ParameterizedTest(name = "{1}") @@ -223,6 +243,11 @@ public String visitLimit(LimitOperator node, Integer tabs) { return name(node, "Limit->", tabs); } + @Override + public String visitTrendline(TrendlineOperator node, Integer tabs) { + return name(node, "Trendline->", tabs); + } + private String name(PhysicalPlan node, String current, int tabs) { String child = node.getChild().get(0).accept(this, tabs + 1); StringBuilder sb = new StringBuilder(); diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/TrendlineOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/TrendlineOperatorTest.java new file mode 100644 index 0000000000..ef2c2907ce --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/TrendlineOperatorTest.java @@ -0,0 +1,398 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; + +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.Arrays; +import java.util.Collections; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.dsl.AstDSL; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@ExtendWith(MockitoExtension.class) +public class TrendlineOperatorTest { + @Mock private PhysicalPlan inputPlan; + + @Test + public void calculates_simple_moving_average_one_field_one_sample() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(1, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 100, "time", 10, "distance_alias", 100)), + plan.next()); + } + + @Test + public void calculates_simple_moving_average_one_field_two_samples() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 200, "time", 10, "distance_alias", 150.0)), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void calculates_simple_moving_average_one_field_two_samples_three_rows() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 200, "time", 10, "distance_alias", 150.0)), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 200, "time", 10, "distance_alias", 200.0)), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void calculates_simple_moving_average_multiple_computations() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 20)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 20))); + + var plan = + new TrendlineOperator( + inputPlan, + Arrays.asList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE), + Pair.of( + AstDSL.computation(2, AstDSL.field("time"), "time_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "distance", 200, "time", 20, "distance_alias", 150.0, "time_alias", 15.0)), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "distance", 200, "time", 20, "distance_alias", 200.0, "time_alias", 20.0)), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void alias_overwrites_input_field() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "time", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 150.0)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 200.0)), plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void calculates_simple_moving_average_one_field_two_samples_three_rows_null_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 300, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals(ExprValueUtils.tupleValue(ImmutableMap.of("time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 300, "time", 10, "distance_alias", 250.0)), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void use_null_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", ExprNullValue.of(), "time", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(1, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.DOUBLE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals(ExprValueUtils.tupleValue(ImmutableMap.of("time", 10)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("distance", ExprNullValue.of(), "time", 10)), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 100, "time", 10, "distance_alias", 100)), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void use_illegal_core_type() { + assertThrows( + IllegalArgumentException.class, + () -> { + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("distance"), "distance_alias", SMA), + ExprCoreType.ARRAY))); + }); + } + + @Test + public void calculates_simple_moving_average_date() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("date", ExprValueUtils.dateValue(LocalDate.EPOCH))), + ExprValueUtils.tupleValue( + ImmutableMap.of("date", ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(6)))), + ExprValueUtils.tupleValue( + ImmutableMap.of("date", ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(12))))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("date"), "date_alias", SMA), + ExprCoreType.DATE))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of("date", ExprValueUtils.dateValue(LocalDate.EPOCH))), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "date", + ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(6)), + "date_alias", + ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(3)))), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "date", + ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(12)), + "date_alias", + ExprValueUtils.dateValue(LocalDate.EPOCH.plusDays(9)))), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void calculates_simple_moving_average_time() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("time", ExprValueUtils.timeValue(LocalTime.MIN))), + ExprValueUtils.tupleValue( + ImmutableMap.of("time", ExprValueUtils.timeValue(LocalTime.MIN.plusHours(6)))), + ExprValueUtils.tupleValue( + ImmutableMap.of("time", ExprValueUtils.timeValue(LocalTime.MIN.plusHours(12))))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("time"), "time_alias", SMA), + ExprCoreType.TIME))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals(ExprValueUtils.tupleValue(ImmutableMap.of("time", LocalTime.MIN)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "time", LocalTime.MIN.plusHours(6), "time_alias", LocalTime.MIN.plusHours(3))), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "time", LocalTime.MIN.plusHours(12), "time_alias", LocalTime.MIN.plusHours(9))), + plan.next()); + assertFalse(plan.hasNext()); + } + + @Test + public void calculates_simple_moving_average_timestamp() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("timestamp", ExprValueUtils.timestampValue(Instant.EPOCH))), + ExprValueUtils.tupleValue( + ImmutableMap.of( + "timestamp", ExprValueUtils.timestampValue(Instant.EPOCH.plusMillis(1000)))), + ExprValueUtils.tupleValue( + ImmutableMap.of( + "timestamp", ExprValueUtils.timestampValue(Instant.EPOCH.plusMillis(1500))))); + + var plan = + new TrendlineOperator( + inputPlan, + Collections.singletonList( + Pair.of( + AstDSL.computation(2, AstDSL.field("timestamp"), "timestamp_alias", SMA), + ExprCoreType.TIMESTAMP))); + + plan.open(); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue(ImmutableMap.of("timestamp", Instant.EPOCH)), plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "timestamp", + Instant.EPOCH.plusMillis(1000), + "timestamp_alias", + Instant.EPOCH.plusMillis(500))), + plan.next()); + assertTrue(plan.hasNext()); + assertEquals( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "timestamp", + Instant.EPOCH.plusMillis(1500), + "timestamp_alias", + Instant.EPOCH.plusMillis(1250))), + plan.next()); + assertFalse(plan.hasNext()); + } +} diff --git a/docs/category.json b/docs/category.json index aacfc43478..32f56cfb46 100644 --- a/docs/category.json +++ b/docs/category.json @@ -25,6 +25,7 @@ "user/ppl/cmd/sort.rst", "user/ppl/cmd/stats.rst", "user/ppl/cmd/syntax.rst", + "user/ppl/cmd/trendline.rst", "user/ppl/cmd/top.rst", "user/ppl/cmd/where.rst", "user/ppl/general/identifiers.rst", diff --git a/docs/user/ppl/cmd/trendline.rst b/docs/user/ppl/cmd/trendline.rst new file mode 100644 index 0000000000..166a3c056f --- /dev/null +++ b/docs/user/ppl/cmd/trendline.rst @@ -0,0 +1,90 @@ +============= +trendline +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``trendline`` command to calculate moving averages of fields. + +Syntax +============ +`TRENDLINE [sort <[+|-] sort-field>] SMA(number-of-datapoints, field) [AS alias] [SMA(number-of-datapoints, field) [AS alias]]...` + +* [+|-]: optional. The plus [+] stands for ascending order and NULL/MISSING first and a minus [-] stands for descending order and NULL/MISSING last. **Default:** ascending order and NULL/MISSING first. +* sort-field: mandatory when sorting is used. The field used to sort. +* number-of-datapoints: mandatory. The number of datapoints to calculate the moving average (must be greater than zero). +* field: mandatory. The name of the field the moving average should be calculated for. +* alias: optional. The name of the resulting column containing the moving average (defaults to the field name with "_trendline"). + +And the moment only the Simple Moving Average (SMA) type is supported. + +It is calculated like + + f[i]: The value of field 'f' in the i-th data-point + n: The number of data-points in the moving window (period) + t: The current time index + + SMA(t) = (1/n) * Σ(f[i]), where i = t-n+1 to t + +Example 1: Calculate the moving average on one field. +===================================================== + +The example shows how to calculate the moving average on one field. + +PPL query:: + + os> source=accounts | trendline sma(2, account_number) as an | fields an; + fetched rows / total rows = 4/4 + +------+ + | an | + |------| + | null | + | 3.5 | + | 9.5 | + | 15.5 | + +------+ + + +Example 2: Calculate the moving average on multiple fields. +=========================================================== + +The example shows how to calculate the moving average on multiple fields. + +PPL query:: + + os> source=accounts | trendline sma(2, account_number) as an sma(2, age) as age_trend | fields an, age_trend ; + fetched rows / total rows = 4/4 + +------+-----------+ + | an | age_trend | + |------+-----------| + | null | null | + | 3.5 | 34.0 | + | 9.5 | 32.0 | + | 15.5 | 30.5 | + +------+-----------+ + +Example 4: Calculate the moving average on one field without specifying an alias. +================================================================================= + +The example shows how to calculate the moving average on one field. + +PPL query:: + + os> source=accounts | trendline sma(2, account_number) | fields account_number_trendline; + fetched rows / total rows = 4/4 + +--------------------------+ + | account_number_trendline | + |--------------------------| + | null | + | 3.5 | + | 9.5 | + | 15.5 | + +--------------------------+ + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 9525874c59..ef8cff334e 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -74,6 +74,8 @@ The query start with search command and then flowing a set of command delimited - `stats command <cmd/stats.rst>`_ + - `trendline command <cmd/trendline.rst>`_ + - `where command <cmd/where.rst>`_ - `head command <cmd/head.rst>`_ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index b9c7f89ba0..531a24bad6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -100,6 +100,32 @@ public void testFillNullPushDownExplain() throws Exception { + " | fillnull with -1 in age,balance | fields age, balance")); } + @Test + public void testTrendlinePushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_trendline_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + "| head 5 " + + "| trendline sma(2, age) as ageTrend " + + "| fields ageTrend")); + } + + @Test + public void testTrendlineWithSortPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_trendline_sort_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + "| head 5 " + + "| trendline sort age sma(2, age) as ageTrend " + + "| fields ageTrend")); + } + String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TrendlineCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TrendlineCommandIT.java new file mode 100644 index 0000000000..38baa0f01f --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TrendlineCommandIT.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class TrendlineCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + } + + @Test + public void testTrendline() throws IOException { + final JSONObject result = + executeQuery( + String.format( + "source=%s | where balance > 39000 | sort balance | trendline sma(2, balance) as" + + " balance_trend | fields balance_trend", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(new Object[] {null}), rows(44313.0), rows(39882.5)); + } + + @Test + public void testTrendlineMultipleFields() throws IOException { + final JSONObject result = + executeQuery( + String.format( + "source=%s | where balance > 39000 | sort balance | trendline sma(2, balance) as" + + " balance_trend sma(2, account_number) as account_number_trend | fields" + + " balance_trend, account_number_trend", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(null, null), rows(44313.0, 28.5), rows(39882.5, 13.0)); + } + + @Test + public void testTrendlineOverwritesExistingField() throws IOException { + final JSONObject result = + executeQuery( + String.format( + "source=%s | where balance > 39000 | sort balance | trendline sma(2, balance) as" + + " age | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(new Object[] {null}), rows(44313.0), rows(39882.5)); + } + + @Test + public void testTrendlineNoAlias() throws IOException { + final JSONObject result = + executeQuery( + String.format( + "source=%s | where balance > 39000 | sort balance | trendline sma(2, balance) |" + + " fields balance_trendline", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(new Object[] {null}), rows(44313.0), rows(39882.5)); + } + + @Test + public void testTrendlineWithSort() throws IOException { + final JSONObject result = + executeQuery( + String.format( + "source=%s | where balance > 39000 | trendline sort balance sma(2, balance) |" + + " fields balance_trendline", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(new Object[] {null}), rows(44313.0), rows(39882.5)); + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json new file mode 100644 index 0000000000..754535dc32 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json @@ -0,0 +1,32 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[ageTrend]" + }, + "children": [ + { + "name": "TrendlineOperator", + "description": { + "computations": [ + { + "computationType" : "sma", + "numberOfDataPoints" : "2", + "dataField" : "age", + "alias" : "ageTrend" + } + ] + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" + }, + "children": [] + } + ] + } + ] + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json new file mode 100644 index 0000000000..6629434108 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json @@ -0,0 +1,32 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[ageTrend]" + }, + "children": [ + { + "name": "TrendlineOperator", + "description": { + "computations": [ + { + "computationType" : "sma", + "numberOfDataPoints" : "2", + "dataField" : "age", + "alias" : "ageTrend" + } + ] + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName\u003dopensearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":5,\"timeout\":\"1m\",\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean\u003dtrue, searchDone\u003dfalse, pitId\u003dnull, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" + }, + "children": [] + } + ] + } + ] + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index 28827b0a54..358bc10ab4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -24,6 +24,7 @@ import org.opensearch.sql.planner.physical.RenameOperator; import org.opensearch.sql.planner.physical.SortOperator; import org.opensearch.sql.planner.physical.TakeOrderedOperator; +import org.opensearch.sql.planner.physical.TrendlineOperator; import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.TableScanOperator; @@ -187,6 +188,12 @@ public PhysicalPlan visitML(PhysicalPlan node, Object context) { mlOperator.getNodeClient())); } + @Override + public PhysicalPlan visitTrendline(TrendlineOperator node, Object context) { + return doProtect( + new TrendlineOperator(visitInput(node.getInput(), context), node.getComputations())); + } + PhysicalPlan visitInput(PhysicalPlan node, Object context) { if (null == node) { return node; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index da06c1eb66..724178bd34 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.*; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -23,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,10 +39,12 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.node.NodeClient; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.expression.DSL; @@ -67,6 +71,7 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanDSL; import org.opensearch.sql.planner.physical.TakeOrderedOperator; +import org.opensearch.sql.planner.physical.TrendlineOperator; @ExtendWith(MockitoExtension.class) @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -318,6 +323,22 @@ public void test_visitTakeOrdered() { resourceMonitor(takeOrdered), executionProtector.visitTakeOrdered(takeOrdered, null)); } + @Test + public void test_visitTrendline() { + final TrendlineOperator trendlineOperator = + new TrendlineOperator( + PhysicalPlanDSL.values(emptyList()), + Collections.singletonList( + Pair.of( + new Trendline.TrendlineComputation( + 1, AstDSL.field("dummy"), "dummy_alias", SMA), + DOUBLE))); + + assertEquals( + resourceMonitor(trendlineOperator), + executionProtector.visitTrendline(trendlineOperator, null)); + } + PhysicalPlan resourceMonitor(PhysicalPlan input) { return new ResourceMonitorPlan(input, resourceMonitor); } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 3ba8da74f4..4a883fa656 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -36,6 +36,7 @@ KMEANS: 'KMEANS'; AD: 'AD'; ML: 'ML'; FILLNULL: 'FILLNULL'; +TRENDLINE: 'TRENDLINE'; // COMMAND ASSIST KEYWORDS AS: 'AS'; @@ -57,6 +58,9 @@ STR: 'STR'; IP: 'IP'; NUM: 'NUM'; +// TRENDLINE KEYWORDS +SMA: 'SMA'; + // ARGUMENT KEYWORDS KEEPEMPTY: 'KEEPEMPTY'; CONSECUTIVE: 'CONSECUTIVE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 89a32abe23..c9d0f2e110 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -50,6 +50,7 @@ commands | adCommand | mlCommand | fillnullCommand + | trendlineCommand ; searchCommand @@ -145,6 +146,18 @@ nullReplacementExpression : nullableField = fieldExpression EQUAL nullReplacement = valueExpression ; +trendlineCommand + : TRENDLINE (SORT sortField)? trendlineClause (trendlineClause)* + ; + +trendlineClause + : trendlineType LT_PRTHS numberOfDataPoints = integerLiteral COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? + ; + +trendlineType + : SMA + ; + kmeansCommand : KMEANS (kmeansParameter)* ; @@ -876,6 +889,7 @@ keywordsCanBeId | KMEANS | AD | ML + | TRENDLINE // commands assist keywords | SOURCE | INDEX diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 2fccb8e635..c3c31ee2e1 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -64,6 +64,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; @@ -421,6 +422,21 @@ public UnresolvedPlan visitFillNullWithFieldVariousValues( FillNull.ContainNullableFieldFill.ofVariousValue(replacementsBuilder.build())); } + /** trendline command. */ + @Override + public UnresolvedPlan visitTrendlineCommand(OpenSearchPPLParser.TrendlineCommandContext ctx) { + List<Trendline.TrendlineComputation> trendlineComputations = + ctx.trendlineClause().stream() + .map(expressionBuilder::visit) + .map(Trendline.TrendlineComputation.class::cast) + .collect(Collectors.toList()); + return Optional.ofNullable(ctx.sortField()) + .map(this::internalVisitExpression) + .map(Field.class::cast) + .map(sort -> new Trendline(Optional.of(sort), trendlineComputations)) + .orElse(new Trendline(Optional.empty(), trendlineComputations)); + } + /** Get original text in query. */ private String getTextInQuery(ParserRuleContext ctx) { Token start = ctx.getStart(); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 98c41027ff..8bc98c8eee 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -45,6 +45,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -52,6 +53,8 @@ import org.antlr.v4.runtime.RuleContext; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.*; +import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; @@ -75,6 +78,28 @@ public UnresolvedExpression visitEvalClause(EvalClauseContext ctx) { return new Let((Field) visit(ctx.fieldExpression()), visit(ctx.expression())); } + /** Trendline clause. */ + @Override + public Trendline.TrendlineComputation visitTrendlineClause( + OpenSearchPPLParser.TrendlineClauseContext ctx) { + final int numberOfDataPoints = Integer.parseInt(ctx.numberOfDataPoints.getText()); + if (numberOfDataPoints < 1) { + throw new SyntaxCheckException( + "Number of trendline data-points must be greater than or equal to 1"); + } + + final Field dataField = (Field) this.visitFieldExpression(ctx.field); + final String alias = + ctx.alias != null + ? ctx.alias.getText() + : dataField.getChild().get(0).toString() + "_trendline"; + + final Trendline.TrendlineType computationType = + Trendline.TrendlineType.valueOf(ctx.trendlineType().getText().toUpperCase(Locale.ROOT)); + return new Trendline.TrendlineComputation( + numberOfDataPoints, dataField, alias, computationType); + } + /** Logical expression excluding boolean, comparison. */ @Override public UnresolvedExpression visitLogicalNot(LogicalNotContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index a1ca0fd69a..96e21eafcd 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -43,6 +44,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.planner.logical.LogicalAggregation; @@ -221,14 +223,26 @@ public String visitHead(Head node, String context) { return StringUtils.format("%s | head %d", child, size); } + @Override + public String visitTrendline(Trendline node, String context) { + String child = node.getChild().get(0).accept(this, context); + String computations = visitExpressionList(node.getComputations(), " "); + return StringUtils.format("%s | trendline %s", child, computations); + } + private String visitFieldList(List<Field> fieldList) { return fieldList.stream().map(this::visitExpression).collect(Collectors.joining(",")); } - private String visitExpressionList(List<UnresolvedExpression> expressionList) { + private String visitExpressionList(List<? extends UnresolvedExpression> expressionList) { + return visitExpressionList(expressionList, ","); + } + + private String visitExpressionList( + List<? extends UnresolvedExpression> expressionList, String delimiter) { return expressionList.isEmpty() ? "" - : expressionList.stream().map(this::visitExpression).collect(Collectors.joining(",")); + : expressionList.stream().map(this::visitExpression).collect(Collectors.joining(delimiter)); } private String visitExpression(UnresolvedExpression expression) { @@ -344,5 +358,14 @@ public String visitAlias(Alias node, String context) { String expr = node.getDelegated().accept(this, context); return StringUtils.format("%s", expr); } + + @Override + public String visitTrendlineComputation(Trendline.TrendlineComputation node, String context) { + final String dataField = node.getDataField().accept(this, context); + final String aliasClause = " as " + node.getAlias(); + final String computationType = node.getComputationType().name().toLowerCase(Locale.ROOT); + return StringUtils.format( + "%s(%d, %s)%s", computationType, node.getNumberOfDataPoints(), dataField, aliasClause); + } } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index ac2bce9dbc..c6f4ed2044 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -7,12 +7,14 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.computation; import static org.opensearch.sql.ast.dsl.AstDSL.dedupe; import static org.opensearch.sql.ast.dsl.AstDSL.defaultDedupArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; @@ -38,13 +40,16 @@ import static org.opensearch.sql.ast.dsl.AstDSL.span; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.tableFunction; +import static org.opensearch.sql.ast.dsl.AstDSL.trendline; import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; +import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.Optional; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -60,6 +65,7 @@ import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; public class AstBuilderTest { @@ -692,6 +698,74 @@ public void testFillNullCommandVariousValues() { .build()))); } + public void testTrendline() { + assertEqual( + "source=t | trendline sma(5, test_field) as test_field_alias sma(1, test_field_2) as" + + " test_field_alias_2", + trendline( + relation("t"), + Optional.empty(), + computation(5, field("test_field"), "test_field_alias", SMA), + computation(1, field("test_field_2"), "test_field_alias_2", SMA))); + } + + @Test + public void testTrendlineSort() { + assertEqual( + "source=t | trendline sort test_field sma(5, test_field)", + trendline( + relation("t"), + Optional.of( + field( + "test_field", + argument("asc", booleanLiteral(true)), + argument("type", nullLiteral()))), + computation(5, field("test_field"), "test_field_trendline", SMA))); + } + + @Test + public void testTrendlineSortDesc() { + assertEqual( + "source=t | trendline sort - test_field sma(5, test_field)", + trendline( + relation("t"), + Optional.of( + field( + "test_field", + argument("asc", booleanLiteral(false)), + argument("type", nullLiteral()))), + computation(5, field("test_field"), "test_field_trendline", SMA))); + } + + @Test + public void testTrendlineSortAsc() { + assertEqual( + "source=t | trendline sort + test_field sma(5, test_field)", + trendline( + relation("t"), + Optional.of( + field( + "test_field", + argument("asc", booleanLiteral(true)), + argument("type", nullLiteral()))), + computation(5, field("test_field"), "test_field_trendline", SMA))); + } + + @Test + public void testTrendlineNoAlias() { + assertEqual( + "source=t | trendline sma(5, test_field)", + trendline( + relation("t"), + Optional.empty(), + computation(5, field("test_field"), "test_field_trendline", SMA))); + } + + @Test + public void testTrendlineTooFewSamples() { + assertThrows(SyntaxCheckException.class, () -> plan("source=t | trendline sma(0, test_field)")); + } + @Test public void testDescribeCommand() { assertEqual("describe t", relation(mappingTable("t"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index b5b4c97f13..06f8fbb061 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -89,6 +89,13 @@ public void testDedupCommand() { anonymize("source=t | dedup f1, f2")); } + @Test + public void testTrendlineCommand() { + assertEquals( + "source=t | trendline sma(2, date) as date_alias sma(3, time) as time_alias", + anonymize("source=t | trendline sma(2, date) as date_alias sma(3, time) as time_alias")); + } + @Test public void testHeadCommandWithNumber() { assertEquals("source=t | head 3", anonymize("source=t | head 3")); From 6911fafc0b7de807c8185dd5b36e9bdea4b602b7 Mon Sep 17 00:00:00 2001 From: Taylor Curran <taylor.curran@improving.com> Date: Fri, 13 Dec 2024 11:04:23 -0800 Subject: [PATCH 91/96] Fix broken linkchecker pipeline job (#3202) * Fix broken link Signed-off-by: currantw <taylor.curran@improving.com> * Cleanup up to reduce changes Signed-off-by: currantw <taylor.curran@improving.com> --------- Signed-off-by: currantw <taylor.curran@improving.com> --- MAINTAINERS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 1bc25b6b03..8012755450 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,7 +5,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje ## Current Maintainers | Maintainer | GitHub ID | Affiliation | -| ----------------- | ------------------------------------------------- | ----------- | +| ----------------- |-----------------------------------------------------| ----------- | | Eric Wei | [mengweieric](https://github.com/mengweieric) | Amazon | | Joshua Li | [joshuali925](https://github.com/joshuali925) | Amazon | | Shenoy Pratik | [ps48](https://github.com/ps48) | Amazon | @@ -16,7 +16,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Peter Fitzgibbons | [pjfitzgibbons](https://github.com/pjfitzgibbons) | Amazon | | Simeon Widdis | [swiddis](https://github.com/swiddis) | Amazon | | Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | -| Vamsi Manohar | [vamsi-amazon](https://github.com/vamsi-amazon) | Amazon | +| Vamsi Manohar | [vmmusings](https://github.com/vmmusings) | Amazon | | Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | | Sean Kao | [seankao-az](https://github.com/seankao-az) | Amazon | | Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | From 8bfa2e90d1f0c6da9cf2896dc9b0c2e1aa861e01 Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA <moritato@amazon.com> Date: Mon, 16 Dec 2024 10:13:26 -0800 Subject: [PATCH 92/96] Add validation for unsupported type/identifier/commands (#3195) Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --- .../validator/SQLQueryValidationVisitor.java | 35 +++++++++++ .../validator/SQLQueryValidatorTest.java | 60 ++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index f54ac49b4e..10fc48727a 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -30,6 +30,8 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropTablePartitionsContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.DropViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ErrorCapturingIdentifierContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ErrorCapturingIdentifierExtraContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ExplainContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.FunctionNameContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.HintContext; @@ -43,6 +45,7 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinRelationContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.JoinTypeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LateralViewContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LiteralTypeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.LoadDataContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ManageResourceContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.QueryOrganizationContext; @@ -77,7 +80,9 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TableValuedFunctionContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TypeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; +import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsupportedHiveNativeCommandsContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; /** This visitor validate grammar using GrammarElementValidator */ @@ -584,4 +589,34 @@ private void validateAllowed(SQLGrammarElement element) { throw new IllegalArgumentException(element + " is not allowed."); } } + + @Override + public Void visitErrorCapturingIdentifier(ErrorCapturingIdentifierContext ctx) { + ErrorCapturingIdentifierExtraContext extra = ctx.errorCapturingIdentifierExtra(); + if (extra.children != null) { + throw new IllegalArgumentException("Invalid identifier: " + ctx.getText()); + } + return super.visitErrorCapturingIdentifier(ctx); + } + + @Override + public Void visitLiteralType(LiteralTypeContext ctx) { + if (ctx.unsupportedType != null) { + throw new IllegalArgumentException("Unsupported typed literal: " + ctx.getText()); + } + return super.visitLiteralType(ctx); + } + + @Override + public Void visitType(TypeContext ctx) { + if (ctx.unsupportedType != null) { + throw new IllegalArgumentException("Unsupported data type: " + ctx.getText()); + } + return super.visitType(ctx); + } + + @Override + public Void visitUnsupportedHiveNativeCommands(UnsupportedHiveNativeCommandsContext ctx) { + throw new IllegalArgumentException("Unsupported command."); + } } diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 3e4eef52fd..ad73daa37f 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -571,6 +571,56 @@ void testValidateFlintExtensionQuery() { UUID.randomUUID().toString(), DataSourceType.SECURITY_LAKE)); } + @Test + void testInvalidIdentifier() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> true); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + v.ng("SELECT a.b.c as a-b-c FROM abc"); + v.ok("SELECT a.b.c as `a-b-c` FROM abc"); + v.ok("SELECT a.b.c as a_b_c FROM abc"); + + v.ng("SELECT a.b.c FROM a-b-c"); + v.ng("SELECT a.b.c FROM a.b-c"); + v.ok("SELECT a.b.c FROM b.c.`a-b-c`"); + v.ok("SELECT a.b.c FROM `a-b-c`"); + } + + @Test + void testUnsupportedType() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> true); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + + v.ng("SELECT cast ( a as DateTime ) FROM tbl"); + v.ok("SELECT cast ( a as DATE ) FROM tbl"); + v.ok("SELECT cast ( a as Date ) FROM tbl"); + v.ok("SELECT cast ( a as Timestamp ) FROM tbl"); + } + + @Test + void testUnsupportedTypedLiteral() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> true); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + + v.ng("SELECT DATETIME '2024-10-11'"); + v.ok("SELECT DATE '2024-10-11'"); + v.ok("SELECT TIMESTAMP '2024-10-11'"); + } + + @Test + void testUnsupportedHiveNativeCommand() { + when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> true); + VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.SPARK); + + v.ng("CREATE ROLE aaa"); + v.ng("SHOW GRANT"); + v.ng("EXPORT TABLE"); + v.ng("ALTER TABLE aaa NOT CLUSTERED"); + v.ng("START TRANSACTION"); + v.ng("COMMIT"); + v.ng("ROLLBACK"); + v.ng("DFS"); + } + @AllArgsConstructor private static class VerifyValidator { private final SQLQueryValidator validator; @@ -580,10 +630,18 @@ public void ok(TestElement query) { runValidate(query.getQueries()); } + public void ok(String query) { + runValidate(query); + } + public void ng(TestElement query) { + Arrays.stream(query.getQueries()).forEach(this::ng); + } + + public void ng(String query) { assertThrows( IllegalArgumentException.class, - () -> runValidate(query.getQueries()), + () -> runValidate(query), "The query should throw: query=`" + query.toString() + "`"); } From 3430deb57514a4c593813e8735d62484756c840d Mon Sep 17 00:00:00 2001 From: Taylor Curran <taylor.curran@improving.com> Date: Thu, 19 Dec 2024 09:19:20 -0800 Subject: [PATCH 93/96] #3145 Add IP Address Data Type (#3175) * Add `ExprIpValue` and `IP` data type Signed-off-by: currantw <taylor.curran@improving.com> * Add support for casting (`cast(field_name to ip)`) and remove existing unused sorting syntax. Signed-off-by: currantw <taylor.curran@improving.com> * Update comparison logic to compare in IPv6 Signed-off-by: currantw <taylor.curran@improving.com> * Fix bug casting to IP Signed-off-by: currantw <taylor.curran@improving.com> * Fix failing tests Signed-off-by: currantw <taylor.curran@improving.com> * Assert that comparison only valid if same type, update tests accordingly Signed-off-by: currantw <taylor.curran@improving.com> * Add additional tests to increase code coverage Signed-off-by: currantw <taylor.curran@improving.com> * Integrate `cidrmatch` changes Signed-off-by: currantw <taylor.curran@improving.com> * Remove `OpenSearchIPType` data type Signed-off-by: currantw <taylor.curran@improving.com> * Fix more failing tests Signed-off-by: currantw <taylor.curran@improving.com> * Minor cleanup Signed-off-by: currantw <taylor.curran@improving.com> * Add new tests for IP data type to `SortCommandIT`, and update `weblogs` test data. Signed-off-by: currantw <taylor.curran@improving.com> * Fixing IT test failure. Signed-off-by: currantw <taylor.curran@improving.com> * Spotless and update test to sort in SQL Signed-off-by: currantw <taylor.curran@improving.com> * Fix broken link Signed-off-by: currantw <taylor.curran@improving.com> * Fix failing code coverage Signed-off-by: currantw <taylor.curran@improving.com> * Fix failing doctest Signed-off-by: currantw <taylor.curran@improving.com> * Fix failing `ip.rst` doctest Signed-off-by: currantw <taylor.curran@improving.com> * Fix test failure due to merge. Signed-off-by: currantw <taylor.curran@improving.com> * Fix spotless Signed-off-by: currantw <taylor.curran@improving.com> * Add missing `url` field Signed-off-by: currantw <taylor.curran@improving.com> * Address minor review comments. Signed-off-by: currantw <taylor.curran@improving.com> * Revert sort syntax changes Signed-off-by: currantw <taylor.curran@improving.com> * Minor doc update Signed-off-by: currantw <taylor.curran@improving.com> * FIx failing `ip.rst` doctest Signed-off-by: currantw <taylor.curran@improving.com> * Add `IPComparisonIT` tests for comparison operators, rename modules and weblogs test index to make plural for consistency. Signed-off-by: currantw <taylor.curran@improving.com> --------- Signed-off-by: currantw <taylor.curran@improving.com> --- DEVELOPER_GUIDE.rst | 2 +- .../opensearch/sql/ast/expression/Cast.java | 2 + .../sql/data/model/ExprIpValue.java | 50 ++++++ .../opensearch/sql/data/model/ExprValue.java | 6 + .../sql/data/model/ExprValueUtils.java | 9 ++ .../sql/data/type/ExprCoreType.java | 3 + .../org/opensearch/sql/expression/DSL.java | 4 + .../function/BuiltinFunctionName.java | 1 + .../sql/expression/ip/IPFunctions.java | 69 ++------- .../operator/convert/TypeCastOperators.java | 10 ++ .../org/opensearch/sql/utils/IPUtils.java | 97 ++++++++++++ .../opensearch/sql/analysis/AnalyzerTest.java | 2 +- .../sql/data/model/ExprIpValueTest.java | 138 +++++++++++++++++ .../sql/data/model/ExprValueUtilsTest.java | 7 +- .../function/WideningTypeRuleTest.java | 2 + .../sql/expression/ip/IPFunctionTest.java | 120 --------------- .../sql/expression/ip/IPFunctionsTest.java | 93 +++++++++++ .../convert/TypeCastOperatorTest.java | 64 +++++++- docs/user/ppl/cmd/trendline.rst | 2 +- docs/user/ppl/functions/ip.rst | 24 +-- doctest/test_data/weblogs.json | 12 +- doctest/test_mapping/weblogs.json | 21 +++ .../org/opensearch/sql/legacy/JdbcTestIT.java | 5 +- .../sql/legacy/RestIntegTestCase.java | 4 +- .../sql/legacy/SQLIntegTestCase.java | 4 +- .../opensearch/sql/legacy/TestsConstants.java | 2 +- .../opensearch/sql/ppl/IPComparisonIT.java | 145 ++++++++++++++++++ .../{IPFunctionIT.java => IPFunctionsIT.java} | 27 ++-- .../org/opensearch/sql/ppl/SortCommandIT.java | 16 ++ .../weblogs_index_mapping.json | 5 +- integ-test/src/test/resources/weblogs.json | 12 +- .../data/type/OpenSearchDataType.java | 4 +- .../data/type/OpenSearchIpType.java | 34 ---- .../data/value/OpenSearchExprIpValue.java | 48 ------ .../value/OpenSearchExprValueFactory.java | 25 ++- .../script/filter/lucene/LuceneQuery.java | 6 + .../OpenSearchDataTypeRecognitionTest.java | 2 - .../data/type/OpenSearchDataTypeTest.java | 7 +- .../data/value/OpenSearchExprIpValueTest.java | 44 ------ .../value/OpenSearchExprValueFactoryTest.java | 40 +++-- .../script/filter/FilterQueryBuilderTest.java | 24 +++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 5 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 +- .../sql/ppl/parser/AstExpressionBuilder.java | 2 + 44 files changed, 803 insertions(+), 400 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java create mode 100644 core/src/main/java/org/opensearch/sql/utils/IPUtils.java create mode 100644 core/src/test/java/org/opensearch/sql/data/model/ExprIpValueTest.java delete mode 100644 core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionsTest.java create mode 100644 doctest/test_mapping/weblogs.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/IPComparisonIT.java rename integ-test/src/test/java/org/opensearch/sql/ppl/{IPFunctionIT.java => IPFunctionsIT.java} (53%) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index c0d2f85668..ec00c587a6 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -405,7 +405,7 @@ Sample test class: Doctest >>>>>>> -Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_. +Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_. Backports diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 2019346fb5..541dbedead 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -54,6 +55,7 @@ public class Cast extends UnresolvedExpression { .put("time", CAST_TO_TIME.getName()) .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) + .put("ip", CAST_TO_IP.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java new file mode 100644 index 0000000000..8bdbec4bb5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.data.model; + +import inet.ipaddr.IPAddress; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.utils.IPUtils; + +/** Expression IP Address Value. */ +public class ExprIpValue extends AbstractExprValue { + private final IPAddress value; + + public ExprIpValue(String addressString) { + value = IPUtils.toAddress(addressString); + } + + @Override + public String value() { + return value.toCanonicalString(); + } + + @Override + public ExprType type() { + return ExprCoreType.IP; + } + + @Override + public int compare(ExprValue other) { + return IPUtils.compare(value, ((ExprIpValue) other).value); + } + + @Override + public boolean equal(ExprValue other) { + return compare(other) == 0; + } + + @Override + public String toString() { + return String.format("IP %s", value()); + } + + @Override + public IPAddress ipValue() { + return value; + } +} diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java index 034ed22a75..da9c329f93 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java @@ -5,6 +5,7 @@ package org.opensearch.sql.data.model; +import inet.ipaddr.IPAddress; import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; @@ -102,6 +103,11 @@ default Double doubleValue() { "invalid to get doubleValue from value of type " + type()); } + /** Get IP address value. */ + default IPAddress ipValue() { + throw new ExpressionEvaluationException("invalid to get ipValue from value of type " + type()); + } + /** Get string value. */ default String stringValue() { throw new ExpressionEvaluationException( diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java index 20813045f2..890e0ef8d5 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java @@ -5,6 +5,7 @@ package org.opensearch.sql.data.model; +import inet.ipaddr.IPAddress; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -75,6 +76,10 @@ public static ExprValue timestampValue(Instant value) { return new ExprTimestampValue(value); } + public static ExprValue ipValue(String value) { + return new ExprIpValue(value); + } + /** {@link ExprTupleValue} constructor. */ public static ExprValue tupleValue(Map<String, Object> map) { LinkedHashMap<String, ExprValue> valueMap = new LinkedHashMap<>(); @@ -188,6 +193,10 @@ public static Map<String, ExprValue> getTupleValue(ExprValue exprValue) { return exprValue.tupleValue(); } + public static IPAddress getIpValue(ExprValue exprValue) { + return exprValue.ipValue(); + } + public static Boolean getBooleanValue(ExprValue exprValue) { return exprValue.booleanValue(); } diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java index cbc0c98255..6df2ba6390 100644 --- a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java @@ -45,6 +45,9 @@ public enum ExprCoreType implements ExprType { TIMESTAMP(STRING, DATE, TIME), INTERVAL(UNDEFINED), + /** IP Address. */ + IP(STRING), + /** Struct. */ STRUCT(UNDEFINED), diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 54bd35e70f..44ecc2bc86 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -835,6 +835,10 @@ public static FunctionExpression castTimestamp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_TIMESTAMP, value); } + public static FunctionExpression castIp(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index a67308c96a..f8e9cf7c5f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -231,6 +231,7 @@ public enum BuiltinFunctionName { CAST_TO_TIME(FunctionName.of("cast_to_time")), CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), + CAST_TO_IP(FunctionName.of("cast_to_ip")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java index b3e7fad211..8b3ee23014 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java @@ -6,14 +6,13 @@ package org.opensearch.sql.expression.ip; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; -import inet.ipaddr.AddressStringException; -import inet.ipaddr.IPAddressString; -import inet.ipaddr.IPAddressStringParameters; +import inet.ipaddr.IPAddress; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; @@ -21,6 +20,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.utils.IPUtils; /** Utility class that defines and registers IP functions. */ @UtilityClass @@ -31,20 +31,17 @@ public void register(BuiltinFunctionRepository repository) { } private DefaultFunctionResolver cidrmatch() { - - // TODO #3145: Add support for IP address data type. return define( BuiltinFunctionName.CIDRMATCH.getName(), - impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, STRING, STRING)); + impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, IP, STRING)); } /** * Returns whether the given IP address is within the specified inclusive CIDR IP address range. * Supports both IPv4 and IPv6 addresses. * - * @param addressExprValue IP address as a string (e.g. "198.51.100.14" or - * "2001:0db8::ff00:42:8329"). - * @param rangeExprValue IP address range in CIDR notation as a string (e.g. "198.51.100.0/24" or + * @param addressExprValue IP address (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @param rangeExprValue IP address range string in CIDR notation (e.g. "198.51.100.0/24" or * "2001:0db8::/32") * @return true if the address is in the range; otherwise false. * @throws SemanticCheckException if the address or range is not valid, or if they do not use the @@ -52,54 +49,12 @@ private DefaultFunctionResolver cidrmatch() { */ private ExprValue exprCidrMatch(ExprValue addressExprValue, ExprValue rangeExprValue) { - // TODO #3145: Update to support IP address data type. - String addressString = addressExprValue.stringValue(); - String rangeString = rangeExprValue.stringValue(); - - final IPAddressStringParameters validationOptions = - new IPAddressStringParameters.Builder() - .allowEmpty(false) - .setEmptyAsLoopback(false) - .allow_inet_aton(false) - .allowSingleSegment(false) - .toParams(); - - // Get and validate IP address. - IPAddressString address = - new IPAddressString(addressExprValue.stringValue(), validationOptions); - - try { - address.validate(); - } catch (AddressStringException e) { - String msg = - String.format( - "IP address '%s' is not valid. Error details: %s", addressString, e.getMessage()); - throw new SemanticCheckException(msg, e); - } - - // Get and validate CIDR IP address range. - IPAddressString range = new IPAddressString(rangeExprValue.stringValue(), validationOptions); - - try { - range.validate(); - } catch (AddressStringException e) { - String msg = - String.format( - "CIDR IP address range '%s' is not valid. Error details: %s", - rangeString, e.getMessage()); - throw new SemanticCheckException(msg, e); - } - - // Address and range must use the same IP version (IPv4 or IPv6). - if (address.isIPv4() ^ range.isIPv4()) { - String msg = - String.format( - "IP address '%s' and CIDR IP address range '%s' are not compatible. Both must be" - + " either IPv4 or IPv6.", - addressString, rangeString); - throw new SemanticCheckException(msg); - } + IPAddress address = addressExprValue.ipValue(); + IPAddress range = IPUtils.toRange(rangeExprValue.stringValue()); - return ExprValueUtils.booleanValue(range.contains(address)); + return (IPUtils.compare(address, range.getLower()) < 0) + || (IPUtils.compare(address, range.getUpper()) > 0) + ? ExprValueUtils.LITERAL_FALSE + : ExprValueUtils.LITERAL_TRUE; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 55e223d94c..b388f7d89a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -31,6 +32,7 @@ import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprShortValue; import org.opensearch.sql.data.model.ExprStringValue; @@ -54,6 +56,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToFloat()); repository.register(castToDouble()); repository.register(castToBoolean()); + repository.register(castToIp()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -173,6 +176,13 @@ private static DefaultFunctionResolver castToBoolean() { impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN)); } + private static DefaultFunctionResolver castToIp() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_IP.getName(), + impl(nullMissingHandling((v) -> new ExprIpValue(v.stringValue())), IP, STRING), + impl(nullMissingHandling((v) -> v), IP, IP)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/IPUtils.java b/core/src/main/java/org/opensearch/sql/utils/IPUtils.java new file mode 100644 index 0000000000..8874823a03 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/IPUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import inet.ipaddr.AddressStringException; +import inet.ipaddr.IPAddress; +import inet.ipaddr.IPAddressString; +import inet.ipaddr.IPAddressStringParameters; +import inet.ipaddr.ipv4.IPv4Address; +import inet.ipaddr.ipv6.IPv6Address; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.exception.SemanticCheckException; + +@UtilityClass +public class IPUtils { + + // Parameters for IP address strings. + private static final IPAddressStringParameters.Builder commonValidationOptions = + new IPAddressStringParameters.Builder() + .allowEmpty(false) + .allowMask(false) + .setEmptyAsLoopback(false) + .allowPrefixOnly(false) + .allow_inet_aton(false) + .allowSingleSegment(false); + + private static final IPAddressStringParameters ipAddressStringParameters = + commonValidationOptions.allowPrefix(false).toParams(); + private static final IPAddressStringParameters ipAddressRangeStringParameters = + commonValidationOptions.allowPrefix(true).toParams(); + + /** + * Builds and returns the {@link IPAddress} represented by the given IP address range string in + * CIDR (classless inter-domain routing) notation. Throws {@link SemanticCheckException} if it + * does not represent a valid IP address range. Supports both IPv4 and IPv6 address ranges. + */ + public static IPAddress toRange(String s) throws SemanticCheckException { + try { + IPAddress range = new IPAddressString(s, ipAddressRangeStringParameters).toAddress(); + + // Convert IPv6 mapped address range to IPv4. + if (range.isIPv4Convertible()) { + final int prefixLength = range.getPrefixLength(); + range = range.toIPv4().setPrefixLength(prefixLength, false); + } + + return range; + + } catch (AddressStringException e) { + final String errorFormat = "IP address range string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, s, e.getMessage()), e); + } + } + + /** + * Builds and returns the {@link IPAddress} represented to the given IP address string. Throws + * {@link SemanticCheckException} if it does not represent a valid IP address. Supports both IPv4 + * and IPv6 addresses. + */ + public static IPAddress toAddress(String s) throws SemanticCheckException { + try { + IPAddress address = new IPAddressString(s, ipAddressStringParameters).toAddress(); + + // Convert IPv6 mapped address to IPv4. + if (address.isIPv4Convertible()) { + address = address.toIPv4(); + } + + return address; + } catch (AddressStringException e) { + final String errorFormat = "IP address string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, s, e.getMessage()), e); + } + } + + /** + * Compares the given {@link IPAddress} objects for order. Returns a negative integer, zero, or a + * positive integer if the first {@link IPAddress} object is less than, equal to, or greater than + * the second one. IPv4 addresses are mapped to IPv6 for comparison. + */ + public static int compare(IPAddress a, IPAddress b) { + final IPv6Address ipv6A = toIPv6Address(a); + final IPv6Address ipv6B = toIPv6Address(b); + + return ipv6A.compareTo(ipv6B); + } + + /** Returns the {@link IPv6Address} corresponding to the given {@link IPAddress}. */ + private static IPv6Address toIPv6Address(IPAddress ipAddress) { + return ipAddress instanceof IPv4Address iPv4Address + ? iPv4Address.toIPv6() + : (IPv6Address) ipAddress; + } +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index d6cb0544d8..3f4752aa2e 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -163,7 +163,7 @@ public void filter_relation_with_invalid_qualifiedName_ExpressionEvaluationExcep assertEquals( "= function expected {[BYTE,BYTE],[SHORT,SHORT],[INTEGER,INTEGER],[LONG,LONG]," + "[FLOAT,FLOAT],[DOUBLE,DOUBLE],[STRING,STRING],[BOOLEAN,BOOLEAN],[DATE,DATE]," - + "[TIME,TIME],[TIMESTAMP,TIMESTAMP],[INTERVAL,INTERVAL]," + + "[TIME,TIME],[TIMESTAMP,TIMESTAMP],[INTERVAL,INTERVAL],[IP,IP]," + "[STRUCT,STRUCT],[ARRAY,ARRAY]}, but got [STRING,INTEGER]", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprIpValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprIpValueTest.java new file mode 100644 index 0000000000..b0ef598a5a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprIpValueTest.java @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.data.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.utils.IPUtils; + +public class ExprIpValueTest { + + private static final String ipv4String = "1.2.3.4"; + private static final String ipv6String = "2001:db7::ff00:42:8329"; + private static final String ipInvalidString = "INVALID"; + + private static final ExprValue exprIpv4Value = ExprValueUtils.ipValue(ipv4String); + private static final ExprValue exprIpv6Value = ExprValueUtils.ipValue(ipv6String); + + private static final List<String> ipv4LesserStrings = + List.of("1.2.3.3", "01.2.3.3", "::ffff:1.2.3.3", "::ffff:102:303"); + private static final List<String> ipv4EqualStrings = + List.of("1.2.3.4", "01.2.3.4", "::ffff:1.2.3.4", "::ffff:102:304"); + private static final List<String> ipv4GreaterStrings = + List.of("1.2.3.5", "01.2.3.5", "::ffff:1.2.3.5", "::ffff:102:305"); + + private static final List<String> ipv6LesserStrings = + List.of( + "2001:db7::ff00:42:8328", + "2001:0db7::ff00:0042:8328", + "2001:DB7::FF00:42:8328", + "2001:0db7:0000:0000:0000:ff00:0042:8328"); + private static final List<String> ipv6EqualStrings = + List.of( + "2001:db7::ff00:42:8329", + "2001:0db7::ff00:0042:8329", + "2001:DB7::FF00:42:8329", + "2001:0db7:0000:0000:0000:ff00:0042:8329"); + private static final List<String> ipv6GreaterStrings = + List.of( + "2001:db7::ff00:42:8330", + "2001:0db7::ff00:0042:8330", + "2001:DB7::FF00:42:8330", + "2001:0db7:0000:0000:0000:ff00:0042:8330"); + + @Test + public void testInvalid() { + assertThrows( + SemanticCheckException.class, + () -> ExprValueUtils.ipValue(ipInvalidString), + String.format("IP address string '%s' is not valid. Error details: .*", ipInvalidString)); + } + + @Test + public void testValue() { + ipv4EqualStrings.forEach((s) -> assertEquals(ipv4String, ExprValueUtils.ipValue(s).value())); + ipv6EqualStrings.forEach((s) -> assertEquals(ipv6String, ExprValueUtils.ipValue(s).value())); + } + + @Test + public void testType() { + assertEquals(ExprCoreType.IP, exprIpv4Value.type()); + assertEquals(ExprCoreType.IP, exprIpv6Value.type()); + } + + @Test + public void testCompare() { + + // Compare to IP address. + ipv4LesserStrings.forEach( + (s) -> assertTrue(exprIpv4Value.compareTo(ExprValueUtils.ipValue(s)) > 0)); + ipv4EqualStrings.forEach( + (s) -> assertEquals(0, exprIpv4Value.compareTo(ExprValueUtils.ipValue(s)))); + ipv4GreaterStrings.forEach( + (s) -> assertTrue(exprIpv4Value.compareTo(ExprValueUtils.ipValue(s)) < 0)); + ipv6LesserStrings.forEach( + (s) -> assertTrue(exprIpv6Value.compareTo(ExprValueUtils.ipValue(s)) > 0)); + ipv6EqualStrings.forEach( + (s) -> assertEquals(0, exprIpv6Value.compareTo(ExprValueUtils.ipValue(s)))); + ipv6GreaterStrings.forEach( + (s) -> assertTrue(exprIpv6Value.compareTo(ExprValueUtils.ipValue(s)) < 0)); + + // Compare to null/missing value. + assertThrows( + IllegalStateException.class, + () -> exprIpv4Value.compareTo(ExprValueUtils.LITERAL_NULL), + "[BUG] Unreachable, Comparing with NULL or MISSING is undefined"); + assertThrows( + IllegalStateException.class, + () -> exprIpv4Value.compareTo(ExprValueUtils.LITERAL_MISSING), + "[BUG] Unreachable, Comparing with NULL or MISSING is undefined"); + + // Compare to other data type. + assertThrows( + ExpressionEvaluationException.class, + () -> exprIpv4Value.compareTo(ExprValueUtils.LITERAL_TRUE), + "compare expected value have same type, but with [IP, BOOLEAN]"); + } + + @Test + public void testEquals() { + assertEquals(exprIpv4Value, exprIpv4Value); + assertNotEquals(exprIpv4Value, new Object()); + assertNotEquals(exprIpv4Value, ExprValueUtils.LITERAL_NULL); + assertNotEquals(exprIpv4Value, ExprValueUtils.LITERAL_MISSING); + + ipv4EqualStrings.forEach((s) -> assertEquals(exprIpv4Value, ExprValueUtils.ipValue(s))); + ipv6EqualStrings.forEach((s) -> assertEquals(exprIpv6Value, ExprValueUtils.ipValue(s))); + + ipv4LesserStrings.forEach((s) -> assertNotEquals(exprIpv4Value, ExprValueUtils.ipValue(s))); + ipv6GreaterStrings.forEach((s) -> assertNotEquals(exprIpv6Value, ExprValueUtils.ipValue(s))); + } + + @Test + public void testToString() { + ipv4EqualStrings.forEach( + (s) -> + assertEquals(String.format("IP %s", ipv4String), ExprValueUtils.ipValue(s).toString())); + ipv6EqualStrings.forEach( + (s) -> + assertEquals(String.format("IP %s", ipv6String), ExprValueUtils.ipValue(s).toString())); + } + + @Test + public void testIpValue() { + ipv4EqualStrings.forEach((s) -> assertEquals(IPUtils.toAddress(s), exprIpv4Value.ipValue())); + ipv6EqualStrings.forEach((s) -> assertEquals(IPUtils.toAddress(s), exprIpv6Value.ipValue())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java index 9fe6347102..48db530a94 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; @@ -47,6 +48,7 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.storage.bindingtuple.BindingTuple; +import org.opensearch.sql.utils.IPUtils; @DisplayName("Test Expression Value Utils") public class ExprValueUtilsTest { @@ -63,6 +65,7 @@ public class ExprValueUtilsTest { private static final List<ExprValue> nonNumberValues = Arrays.asList( + new ExprIpValue("1.2.3.4"), new ExprStringValue("1"), ExprBooleanValue.of(true), new ExprCollectionValue(ImmutableList.of(new ExprIntegerValue(1))), @@ -85,6 +88,7 @@ public class ExprValueUtilsTest { ExprValueUtils::getDoubleValue); private static final List<Function<ExprValue, Object>> nonNumberValueExtractor = Arrays.asList( + ExprValueUtils::getIpValue, ExprValueUtils::getStringValue, ExprValueUtils::getBooleanValue, ExprValueUtils::getCollectionValue, @@ -109,7 +113,7 @@ public class ExprValueUtilsTest { ExprCoreType.FLOAT, ExprCoreType.DOUBLE); private static final List<ExprCoreType> nonNumberTypes = - Arrays.asList(STRING, BOOLEAN, ARRAY, STRUCT); + Arrays.asList(IP, STRING, BOOLEAN, ARRAY, STRUCT); private static final List<ExprCoreType> dateAndTimeTypes = Arrays.asList(DATE, TIME, TIMESTAMP, INTERVAL); private static final List<ExprCoreType> allTypes = @@ -124,6 +128,7 @@ private static Stream<Arguments> getValueTestArgumentStream() { 1L, 1f, 1D, + IPUtils.toAddress("1.2.3.4"), "1", true, Arrays.asList(integerValue(1)), diff --git a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java index 3064ffcdee..d38be4c958 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -57,6 +58,7 @@ class WideningTypeRuleTest { .put(STRING, TIMESTAMP, 1) .put(STRING, DATE, 1) .put(STRING, TIME, 1) + .put(STRING, IP, 1) .put(DATE, TIMESTAMP, 1) .put(TIME, TIMESTAMP, 1) .put(UNDEFINED, BYTE, 1) diff --git a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java deleted file mode 100644 index b50bf9fd1f..0000000000 --- a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.ip; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.env.Environment; - -@ExtendWith(MockitoExtension.class) -public class IPFunctionTest { - - // IP range and address constants for testing. - private static final ExprValue IPv4Range = ExprValueUtils.stringValue("198.51.100.0/24"); - private static final ExprValue IPv6Range = ExprValueUtils.stringValue("2001:0db8::/32"); - - // TODO #3145: Add tests for IP address data type. - private static final ExprValue IPv4AddressBelow = ExprValueUtils.stringValue("198.51.99.1"); - private static final ExprValue IPv4AddressWithin = ExprValueUtils.stringValue("198.51.100.1"); - private static final ExprValue IPv4AddressAbove = ExprValueUtils.stringValue("198.51.101.2"); - - private static final ExprValue IPv6AddressBelow = - ExprValueUtils.stringValue("2001:0db7::ff00:42:8329"); - private static final ExprValue IPv6AddressWithin = - ExprValueUtils.stringValue("2001:0db8::ff00:42:8329"); - private static final ExprValue IPv6AddressAbove = - ExprValueUtils.stringValue("2001:0db9::ff00:42:8329"); - - // Mock value environment for testing. - @Mock private Environment<Expression, ExprValue> env; - - @Test - public void cidrmatch_invalid_address() { - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, - () -> execute(ExprValueUtils.stringValue("INVALID"), IPv4Range)); - assertTrue( - exception.getMessage().matches("IP address 'INVALID' is not valid. Error details: .*")); - } - - @Test - public void cidrmatch_invalid_range() { - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, - () -> execute(IPv4AddressWithin, ExprValueUtils.stringValue("INVALID"))); - assertTrue( - exception - .getMessage() - .matches("CIDR IP address range 'INVALID' is not valid. Error details: .*")); - } - - @Test - public void cidrmatch_different_versions() { - SemanticCheckException exception; - - exception = - assertThrows(SemanticCheckException.class, () -> execute(IPv4AddressWithin, IPv6Range)); - assertEquals( - "IP address '198.51.100.1' and CIDR IP address range '2001:0db8::/32' are not compatible." - + " Both must be either IPv4 or IPv6.", - exception.getMessage()); - - exception = - assertThrows(SemanticCheckException.class, () -> execute(IPv6AddressWithin, IPv4Range)); - assertEquals( - "IP address '2001:0db8::ff00:42:8329' and CIDR IP address range '198.51.100.0/24' are not" - + " compatible. Both must be either IPv4 or IPv6.", - exception.getMessage()); - } - - @Test - public void cidrmatch_valid_ipv4() { - assertEquals(LITERAL_FALSE, execute(IPv4AddressBelow, IPv4Range)); - assertEquals(LITERAL_TRUE, execute(IPv4AddressWithin, IPv4Range)); - assertEquals(LITERAL_FALSE, execute(IPv4AddressAbove, IPv4Range)); - } - - @Test - public void cidrmatch_valid_ipv6() { - assertEquals(LITERAL_FALSE, execute(IPv6AddressBelow, IPv6Range)); - assertEquals(LITERAL_TRUE, execute(IPv6AddressWithin, IPv6Range)); - assertEquals(LITERAL_FALSE, execute(IPv6AddressAbove, IPv6Range)); - } - - /** - * Builds and evaluates a CIDR function expression with the given field and range expression - * values, and returns the resulting value. - */ - private ExprValue execute(ExprValue field, ExprValue range) { - - final String fieldName = "ip_address"; - FunctionExpression exp = DSL.cidrmatch(DSL.ref(fieldName, STRING), DSL.literal(range)); - - // Mock the value environment to return the specified field - // expression as the value for the "ip_address" field. - when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(field); - - return exp.valueOf(env); - } -} diff --git a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionsTest.java new file mode 100644 index 0000000000..a74bbda3a1 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionsTest.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.ip; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.data.type.ExprCoreType.IP; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; + +@ExtendWith(MockitoExtension.class) +public class IPFunctionsTest { + + // IP range and address constants for testing. + private static final ExprValue IPv4Range = ExprValueUtils.stringValue("198.51.100.0/24"); + private static final ExprValue IPv4RangeMapped = + ExprValueUtils.stringValue("::ffff:198.51.100.0/24"); + private static final ExprValue IPv6Range = ExprValueUtils.stringValue("2001:0db8::/32"); + + private static final ExprValue IPv4AddressBelow = ExprValueUtils.ipValue("198.51.99.1"); + private static final ExprValue IPv4AddressWithin = ExprValueUtils.ipValue("198.51.100.1"); + private static final ExprValue IPv4AddressAbove = ExprValueUtils.ipValue("198.51.101.2"); + + private static final ExprValue IPv6AddressBelow = + ExprValueUtils.ipValue("2001:0db7::ff00:42:8329"); + private static final ExprValue IPv6AddressWithin = + ExprValueUtils.ipValue("2001:0db8::ff00:42:8329"); + private static final ExprValue IPv6AddressAbove = + ExprValueUtils.ipValue("2001:0db9::ff00:42:8329"); + + // Mock value environment for testing. + @Mock private Environment<Expression, ExprValue> env; + + @Test + public void cidrmatch_invalid_arguments() { + assertThrows( + SemanticCheckException.class, + () -> execute(ExprValueUtils.ipValue("INVALID"), IPv4Range), + "IP address string 'INVALID' is not valid. Error details: .*"); + assertThrows( + SemanticCheckException.class, + () -> execute(IPv4AddressWithin, ExprValueUtils.stringValue("INVALID")), + "IP address range string 'INVALID' is not valid. Error details: .*"); + } + + @Test + public void cidrmatch_valid_arguments() { + + assertEquals(LITERAL_FALSE, execute(IPv4AddressBelow, IPv4Range)); + assertEquals(LITERAL_TRUE, execute(IPv4AddressWithin, IPv4Range)); + assertEquals(LITERAL_FALSE, execute(IPv4AddressAbove, IPv4Range)); + + assertEquals(LITERAL_FALSE, execute(IPv4AddressBelow, IPv4RangeMapped)); + assertEquals(LITERAL_TRUE, execute(IPv4AddressWithin, IPv4RangeMapped)); + assertEquals(LITERAL_FALSE, execute(IPv4AddressAbove, IPv4RangeMapped)); + + assertEquals(LITERAL_FALSE, execute(IPv6AddressBelow, IPv6Range)); + assertEquals(LITERAL_TRUE, execute(IPv6AddressWithin, IPv6Range)); + assertEquals(LITERAL_FALSE, execute(IPv6AddressAbove, IPv6Range)); + } + + /** + * Builds and evaluates a {@code cidrmatch} function expression with the given address and range + * expression values, and returns the resulting value. + */ + private ExprValue execute(ExprValue address, ExprValue range) { + + final String fieldName = "ip_address"; + FunctionExpression exp = DSL.cidrmatch(DSL.ref(fieldName, IP), DSL.literal(range)); + + // Mock the value environment to return the specified field + // expression as the value for the "ip_address" field. + when(DSL.ref(fieldName, IP).valueOf(env)).thenReturn(address); + + return exp.valueOf(env); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 44a3ccabbd..fd579dfb47 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -7,12 +7,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -29,12 +31,17 @@ import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprMissingValue; +import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprShortValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -316,10 +323,6 @@ void castToTime() { assertEquals(TIME, expression.type()); assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); - expression = DSL.castTime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); - assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); - expression = DSL.castTime(DSL.literal(new ExprTimeValue("01:01:01"))); assertEquals(TIME, expression.type()); assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); @@ -334,9 +337,56 @@ void castToTimestamp() { expression = DSL.castTimestamp(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(TIMESTAMP, expression.type()); assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); + } - expression = DSL.castTimestamp(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); - assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); + @Test + void castToIp() { + FunctionExpression exp; + + final String ipv4String = "1.2.3.4"; + final String ipv6String = "2001:db7::ff00:42:8329"; + final String ipInvalidString = "INVALID"; + + final ExprValue exprIpv4Value = new ExprIpValue(ipv4String); + final ExprValue exprIpv6Value = new ExprIpValue(ipv6String); + + // From string + exp = DSL.castIp(DSL.literal(ipv4String)); + assertEquals(IP, exp.type()); + assertEquals(exprIpv4Value, exp.valueOf()); + + exp = DSL.castIp(DSL.literal(ipv6String)); + assertEquals(IP, exp.type()); + assertEquals(exprIpv6Value, exp.valueOf()); + + exp = DSL.castIp(DSL.literal(ipInvalidString)); + assertThrows( + SemanticCheckException.class, + exp::valueOf, + String.format("IP address string '%s' is not valid. Error details: .*", ipInvalidString)); + + // From IP address + exp = DSL.castIp(DSL.literal(exprIpv4Value)); + assertEquals(IP, exp.type()); + assertEquals(exprIpv4Value, exp.valueOf()); + + exp = DSL.castIp(DSL.literal(exprIpv6Value)); + assertEquals(IP, exp.type()); + assertEquals(exprIpv6Value, exp.valueOf()); + + // From invalid type + assertThrows( + ExpressionEvaluationException.class, + () -> DSL.castIp(DSL.literal(0)), + "cast_to_ip function expected {[IP],[STRING]}, but got [INTEGER]"); + + // From null or missing value + exp = DSL.castIp(DSL.literal(ExprNullValue.of())); + assertEquals(IP, exp.type()); + assertTrue(exp.valueOf().isNull()); + + exp = DSL.castIp(DSL.literal(ExprMissingValue.of())); + assertEquals(IP, exp.type()); + assertTrue(exp.valueOf().isMissing()); } } diff --git a/docs/user/ppl/cmd/trendline.rst b/docs/user/ppl/cmd/trendline.rst index 166a3c056f..e6df0d7a2c 100644 --- a/docs/user/ppl/cmd/trendline.rst +++ b/docs/user/ppl/cmd/trendline.rst @@ -23,7 +23,7 @@ Syntax * field: mandatory. The name of the field the moving average should be calculated for. * alias: optional. The name of the resulting column containing the moving average (defaults to the field name with "_trendline"). -And the moment only the Simple Moving Average (SMA) type is supported. +At the moment only the Simple Moving Average (SMA) type is supported. It is calculated like diff --git a/docs/user/ppl/functions/ip.rst b/docs/user/ppl/functions/ip.rst index 3387974af5..30cb9020b0 100644 --- a/docs/user/ppl/functions/ip.rst +++ b/docs/user/ppl/functions/ip.rst @@ -20,19 +20,19 @@ Argument type: STRING, STRING Return type: BOOLEAN -Example: +Example:: - os> source=weblogs | where cidrmatch(host, '199.120.110.0/24') | fields host - fetched rows / total rows = 1/1 - +----------------+ - | host | - |----------------| - | 199.120.110.21 | - +----------------+ + > source=weblogs | where cidrmatch(host, '1.2.3.0/24') | fields host, url + fetched rows / total rows = 2/2 + +---------+--------------------+ + | host | url | + |---------|--------------------| + | 1.2.3.4 | /history/voyager1/ | + | 1.2.3.5 | /history/voyager2/ | + +---------+--------------------+ Note: - - `ip` can be an IPv4 or an IPv6 address - - `cidr` can be an IPv4 or an IPv6 block - - `ip` and `cidr` must be either both IPv4 or both IPv6 - - `ip` and `cidr` must both be valid and non-empty/non-null + - `ip` can be an IPv4 or IPv6 address + - `cidr` can be an IPv4 or IPv6 block + - `ip` and `cidr` must both be valid and non-missing/non-null diff --git a/doctest/test_data/weblogs.json b/doctest/test_data/weblogs.json index 4228e9c4d2..afb1679e22 100644 --- a/doctest/test_data/weblogs.json +++ b/doctest/test_data/weblogs.json @@ -1,6 +1,6 @@ -{"index":{}} -{"host": "199.72.81.55", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} -{"index":{}} -{"host": "199.120.110.21", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} -{"index":{}} -{"host": "205.212.115.106", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} +{"host":"::1","method":"GET","url":"/history/apollo/","response":"200","bytes":"6245"} +{"host":"0.0.0.2","method":"GET","url":"/shuttle/missions/sts-73/mission-sts-73.html","response":"200","bytes":"4085"} +{"host":"::3","method":"GET","url":"/shuttle/countdown/countdown.html","response":"200","bytes":"3985"} +{"host":"::FFFF:1.2.3.4","method":"GET","url":"/history/voyager1/","response":"200","bytes":"1234"} +{"host":"1.2.3.5","method":"GET","url":"/history/voyager2/","response": "200","bytes":"4321"} +{"host":"::FFFF:1234","method":"GET","url":"/history/artemis/","response":"200","bytes": "9876"} diff --git a/doctest/test_mapping/weblogs.json b/doctest/test_mapping/weblogs.json new file mode 100644 index 0000000000..05b9784313 --- /dev/null +++ b/doctest/test_mapping/weblogs.json @@ -0,0 +1,21 @@ +{ + "mappings": { + "properties": { + "host": { + "type": "ip" + }, + "method": { + "type": "text" + }, + "url": { + "type": "text" + }, + "response": { + "type": "text" + }, + "bytes": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 005119a9bc..4ad88c632b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -155,10 +155,7 @@ public void dateFunctionNameCaseInsensitiveTest() { public void ipTypeShouldPassJdbcFormatter() { assertThat( executeQuery( - "SELECT host_ip AS hostIP FROM " - + TestsConstants.TEST_INDEX_WEBLOG - + " ORDER BY hostIP", - "jdbc"), + "SELECT host FROM " + TestsConstants.TEST_INDEX_WEBLOGS + " ORDER BY host", "jdbc"), containsString("\"type\": \"ip\"")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java index a94047c1e4..3d53b96668 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java @@ -273,8 +273,8 @@ public enum Index { getOrderIndexMapping(), "src/test/resources/order.json"), WEBLOG( - TestsConstants.TEST_INDEX_WEBLOG, - "weblog", + TestsConstants.TEST_INDEX_WEBLOGS, + "weblogs", getWeblogsIndexMapping(), "src/test/resources/weblogs.json"), DATE( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 5b956fb5d3..1728be74e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -661,8 +661,8 @@ public enum Index { getOrderIndexMapping(), "src/test/resources/order.json"), WEBLOG( - TestsConstants.TEST_INDEX_WEBLOG, - "weblog", + TestsConstants.TEST_INDEX_WEBLOGS, + "weblogs", getWeblogsIndexMapping(), "src/test/resources/weblogs.json"), DATE( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 73838feb4f..1e336f544e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -43,7 +43,7 @@ public class TestsConstants { public static final String TEST_INDEX_BANK_CSV_SANITIZE = TEST_INDEX_BANK + "_csv_sanitize"; public static final String TEST_INDEX_BANK_RAW_SANITIZE = TEST_INDEX_BANK + "_raw_sanitize"; public static final String TEST_INDEX_ORDER = TEST_INDEX + "_order"; - public static final String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; + public static final String TEST_INDEX_WEBLOGS = TEST_INDEX + "_weblogs"; public static final String TEST_INDEX_DATE = TEST_INDEX + "_date"; public static final String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; public static final String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/IPComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/IPComparisonIT.java new file mode 100644 index 0000000000..a19ea32a68 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/IPComparisonIT.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class IPComparisonIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(SQLIntegTestCase.Index.WEBLOG); + } + + @Test + public void test_equal() throws IOException { + JSONObject result; + final String operator = "="; + + result = executeComparisonQuery(operator, "1.2.3.4"); + verifyDataRows(result, rows("1.2.3.4")); + + result = executeComparisonQuery(operator, "::ffff:1.2.3.4"); + verifyDataRows(result, rows("1.2.3.4")); + + result = executeComparisonQuery(operator, "::1"); + verifyDataRows(result, rows("::1")); + + result = executeComparisonQuery(operator, "0000:0000:0000:0000:0000:0000:0000:0001"); + verifyDataRows(result, rows("::1")); + } + + @Test + public void test_not_equal() throws IOException { + JSONObject result; + final String operator = "!="; + + result = executeComparisonQuery(operator, "1.2.3.4"); + verifyDataRows( + result, rows("::1"), rows("0.0.0.2"), rows("::3"), rows("1.2.3.5"), rows("::ffff:1234")); + + result = executeComparisonQuery(operator, "::ffff:1.2.3.4"); + verifyDataRows( + result, rows("::1"), rows("0.0.0.2"), rows("::3"), rows("1.2.3.5"), rows("::ffff:1234")); + + result = executeComparisonQuery(operator, "::1"); + verifyDataRows( + result, + rows("0.0.0.2"), + rows("::3"), + rows("1.2.3.4"), + rows("1.2.3.5"), + rows("::ffff:1234")); + + result = executeComparisonQuery(operator, "0000:0000:0000:0000:0000:0000:0000:0001"); + verifyDataRows( + result, + rows("0.0.0.2"), + rows("::3"), + rows("1.2.3.4"), + rows("1.2.3.5"), + rows("::ffff:1234")); + } + + @Test + public void test_greater_than() throws IOException { + JSONObject result; + final String operator = ">"; + + result = executeComparisonQuery(operator, "1.2.3.3"); + verifyDataRows(result, rows("1.2.3.4"), rows("1.2.3.5")); + + result = executeComparisonQuery(operator, "1.2.3.4"); + verifyDataRows(result, rows("1.2.3.5")); + + result = executeComparisonQuery(operator, "1.2.3.5"); + verifyDataRows(result); + } + + @Test + public void test_greater_than_or_equal_to() throws IOException { + JSONObject result; + final String operator = ">="; + + result = executeComparisonQuery(operator, "1.2.3.4"); + verifyDataRows(result, rows("1.2.3.4"), rows("1.2.3.5")); + + result = executeComparisonQuery(operator, "1.2.3.5"); + verifyDataRows(result, rows("1.2.3.5")); + + result = executeComparisonQuery(operator, "1.2.3.6"); + verifyDataRows(result); + } + + @Test + public void test_less_than() throws IOException { + JSONObject result; + final String operator = "<"; + + result = executeComparisonQuery(operator, "::4"); + verifyDataRows(result, rows("::1"), rows("::3")); + + result = executeComparisonQuery(operator, "::3"); + verifyDataRows(result, rows("::1")); + + result = executeComparisonQuery(operator, "::1"); + verifyDataRows(result); + } + + @Test + public void test_less_than_or_equal_to() throws IOException { + JSONObject result; + final String operator = "<="; + + result = executeComparisonQuery(operator, "::3"); + verifyDataRows(result, rows("::1"), rows("::3")); + + result = executeComparisonQuery(operator, "::1"); + verifyDataRows(result, rows("::1")); + + result = executeComparisonQuery(operator, "::0"); + verifyDataRows(result); + } + + /** + * Executes a query comparison on the weblogs test index with the given comparison operator and IP + * address string, and returns the resulting {@link JSONObject}; + */ + private JSONObject executeComparisonQuery(String comparisonOperator, String addressString) + throws IOException { + String formatString = "source=%s | where host %s '%s' | fields host"; + String query = + String.format(formatString, TEST_INDEX_WEBLOGS, comparisonOperator, addressString); + return executeQuery(query); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionsIT.java similarity index 53% rename from integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java rename to integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionsIT.java index adb044d0d2..1b0dbf711c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionsIT.java @@ -5,7 +5,7 @@ package org.opensearch.sql.ppl; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOG; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class IPFunctionIT extends PPLIntegTestCase { +public class IPFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { @@ -25,34 +25,33 @@ public void init() throws IOException { @Test public void test_cidrmatch() throws IOException { - // TODO #3145: Add tests for IP address data type. JSONObject result; // No matches result = executeQuery( String.format( - "source=%s | where cidrmatch(host_string, '199.120.111.0/24') | fields host_string", - TEST_INDEX_WEBLOG)); - verifySchema(result, schema("host_string", null, "string")); + "source=%s | where cidrmatch(host, '250.0.0.0/24') | fields host", + TEST_INDEX_WEBLOGS)); + verifySchema(result, schema("host", null, "ip")); verifyDataRows(result); // One match result = executeQuery( String.format( - "source=%s | where cidrmatch(host_string, '199.120.110.0/24') | fields host_string", - TEST_INDEX_WEBLOG)); - verifySchema(result, schema("host_string", null, "string")); - verifyDataRows(result, rows("199.120.110.21")); + "source=%s | where cidrmatch(host, '0.0.0.0/24') | fields host", + TEST_INDEX_WEBLOGS)); + verifySchema(result, schema("host", null, "ip")); + verifyDataRows(result, rows("0.0.0.2")); // Multiple matches result = executeQuery( String.format( - "source=%s | where cidrmatch(host_string, '199.0.0.0/8') | fields host_string", - TEST_INDEX_WEBLOG)); - verifySchema(result, schema("host_string", null, "string")); - verifyDataRows(result, rows("199.72.81.55"), rows("199.120.110.21")); + "source=%s | where cidrmatch(host, '1.2.3.0/24') | fields host", + TEST_INDEX_WEBLOGS)); + verifySchema(result, schema("host", null, "ip")); + verifyDataRows(result, rows("1.2.3.4"), rows("1.2.3.5")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index 1061f0bd9d..b234dd032d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyOrder; @@ -28,6 +29,7 @@ public void init() throws IOException { loadIndex(Index.BANK); loadIndex(Index.BANK_WITH_NULL_VALUES); loadIndex(Index.DOG); + loadIndex(Index.WEBLOG); } @Test @@ -130,6 +132,20 @@ public void testSortStringField() throws IOException { rows("Ratliff")); } + @Test + public void testSortIpField() throws IOException { + final JSONObject result = + executeQuery(String.format("source=%s | fields host | sort host", TEST_INDEX_WEBLOGS)); + verifyOrder( + result, + rows("::1"), + rows("::3"), + rows("::ffff:1234"), + rows("0.0.0.2"), + rows("1.2.3.4"), + rows("1.2.3.5")); + } + @Test public void testSortMultipleFields() throws IOException { JSONObject result = diff --git a/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json index bff3e20bb9..05b9784313 100644 --- a/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json @@ -1,12 +1,9 @@ { "mappings": { "properties": { - "host_ip": { + "host": { "type": "ip" }, - "host_string": { - "type": "keyword" - }, "method": { "type": "text" }, diff --git a/integ-test/src/test/resources/weblogs.json b/integ-test/src/test/resources/weblogs.json index d2e9a968f8..27d39b83be 100644 --- a/integ-test/src/test/resources/weblogs.json +++ b/integ-test/src/test/resources/weblogs.json @@ -1,6 +1,12 @@ {"index":{}} -{"host_ip": "199.72.81.55", "host_string": "199.72.81.55", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} +{"host": "::1", "method": "GET", "url": "/history/apollo/", "response": "200", "bytes": "6245"} {"index":{}} -{"host_ip": "199.120.110.21", "host_string": "199.120.110.21", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} +{"host": "0.0.0.2", "method": "GET", "url": "/shuttle/missions/sts-73/mission-sts-73.html", "response": "200", "bytes": "4085"} {"index":{}} -{"host_ip": "205.212.115.106", "host_string": "205.212.115.106", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} +{"host": "::3", "method": "GET", "url": "/shuttle/countdown/countdown.html", "response": "200", "bytes": "3985"} +{"index":{}} +{"host": "::FFFF:1.2.3.4", "method": "GET", "url": "/history/voyager1/", "response": "200", "bytes": "1234"} +{"index":{}} +{"host": "1.2.3.5", "method": "GET", "url": "/history/voyager2/", "response": "200", "bytes": "4321"} +{"index":{}} +{"host": "::FFFF:1234", "method": "GET", "url": "/history/artemis/", "response": "200", "bytes": "9876"} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index c35eacfc72..6c8912be86 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -26,7 +26,7 @@ public enum MappingType { Invalid(null, ExprCoreType.UNKNOWN), Text("text", ExprCoreType.UNKNOWN), Keyword("keyword", ExprCoreType.STRING), - Ip("ip", ExprCoreType.UNKNOWN), + Ip("ip", ExprCoreType.IP), GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), Date("date", ExprCoreType.TIMESTAMP), @@ -160,8 +160,6 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object> return OpenSearchGeoPointType.of(); case Binary: return OpenSearchBinaryType.of(); - case Ip: - return OpenSearchIpType.of(); case Date: case DateNanos: // Default date formatter is used when "" is passed as the second parameter diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java deleted file mode 100644 index 22581ec28c..0000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.data.type; - -import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; - -import lombok.EqualsAndHashCode; - -/** - * The type of an ip value. See <a - * href="https://opensearch.org/docs/latest/opensearch/supported-field-types/ip/">doc</a> - */ -@EqualsAndHashCode(callSuper = false) -public class OpenSearchIpType extends OpenSearchDataType { - - private static final OpenSearchIpType instance = new OpenSearchIpType(); - - private OpenSearchIpType() { - super(MappingType.Ip); - exprCoreType = UNKNOWN; - } - - public static OpenSearchIpType of() { - return OpenSearchIpType.instance; - } - - @Override - protected OpenSearchDataType cloneEmpty() { - return instance; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java deleted file mode 100644 index 30b3784bfc..0000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.data.value; - -import java.util.Objects; -import lombok.RequiredArgsConstructor; -import org.opensearch.sql.data.model.AbstractExprValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; - -/** - * OpenSearch IP ExprValue<br> - * Todo, add this to avoid the unknown value type exception, the implementation will be changed. - */ -@RequiredArgsConstructor -public class OpenSearchExprIpValue extends AbstractExprValue { - - private final String ip; - - @Override - public Object value() { - return ip; - } - - @Override - public ExprType type() { - return OpenSearchIpType.of(); - } - - @Override - public int compare(ExprValue other) { - return ip.compareTo(((OpenSearchExprIpValue) other).ip); - } - - @Override - public boolean equal(ExprValue other) { - return ip.equals(((OpenSearchExprIpValue) other).ip); - } - - @Override - public int hashCode() { - return Objects.hashCode(ip); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 41d6667ded..68c6fda617 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; @@ -50,6 +51,7 @@ import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprShortValue; @@ -63,7 +65,6 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; -import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -133,8 +134,8 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) { OpenSearchDateType.of(TIMESTAMP), OpenSearchExprValueFactory::createOpenSearchDateType) .put( - OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), - (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) + OpenSearchDateType.of(OpenSearchDataType.MappingType.Ip), + (c, dt) -> new ExprIpValue(c.stringValue())) .put( OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary), (c, dt) -> new OpenSearchExprBinaryValue(c.stringValue())) @@ -202,14 +203,12 @@ private ExprValue parse( } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) || type == STRUCT) { return parseStruct(content, field, supportArrays); + } else if (typeActionMap.containsKey(type)) { + return typeActionMap.get(type).apply(content, type); } else { - if (typeActionMap.containsKey(type)) { - return typeActionMap.get(type).apply(content, type); - } else { - throw new IllegalStateException( - String.format( - "Unsupported type: %s for value: %s.", type.typeName(), content.objectValue())); - } + throw new IllegalStateException( + String.format( + "Unsupported type: %s for value: %s.", type.typeName(), content.objectValue())); } } @@ -418,10 +417,10 @@ private ExprValue parseGeoPoint(Content content, boolean supportArrays) { */ private ExprValue parseInnerArrayValue( Content content, String prefix, ExprType type, boolean supportArrays) { - if (type instanceof OpenSearchIpType - || type instanceof OpenSearchBinaryType - || type instanceof OpenSearchDateType) { + if (type instanceof OpenSearchBinaryType || type instanceof OpenSearchDateType) { return parse(content, prefix, Optional.of(type), supportArrays); + } else if (content.isString() && type.equals(OpenSearchDataType.of(IP))) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(IP)), supportArrays); } else if (content.isString()) { return parse(content, prefix, Optional.of(OpenSearchDataType.of(STRING)), supportArrays); } else if (content.isLong()) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index c9ef5bcca5..26ef56e576 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprShortValue; import org.opensearch.sql.data.model.ExprStringValue; @@ -211,6 +212,11 @@ private ExprValue cast(FunctionExpression castFunction, ReferenceExpression ref) return expr.valueOf(); } }) + .put( + BuiltinFunctionName.CAST_TO_IP.getName(), + (expr, ref) -> { + return new ExprIpValue(expr.valueOf().stringValue()); + }) .put( BuiltinFunctionName.CAST_TO_DATE.getName(), (expr, ref) -> { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java index 35ad6b7ea6..2e90004571 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java @@ -17,7 +17,6 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.opensearch.data.value.OpenSearchExprBinaryValue; import org.opensearch.sql.opensearch.data.value.OpenSearchExprGeoPointValue; -import org.opensearch.sql.opensearch.data.value.OpenSearchExprIpValue; import org.opensearch.sql.opensearch.data.value.OpenSearchExprTextValue; public class OpenSearchDataTypeRecognitionTest { @@ -33,7 +32,6 @@ private static Stream<Arguments> types() { return Stream.of( Arguments.of("TEXT", new OpenSearchExprTextValue("A"), "text without fields"), Arguments.of("BINARY", new OpenSearchExprBinaryValue("A"), "binary"), - Arguments.of("IP", new OpenSearchExprIpValue("A"), "ip"), Arguments.of("TEXT", new TestTextWithFieldValue("Hello World"), "text with fields"), Arguments.of("GEO_POINT", new OpenSearchExprGeoPointValue(0d, 0d), "geo point")); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 76fbbd6e65..77b905e228 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -108,9 +109,9 @@ private static Stream<Arguments> getTestDataWithType() { Arguments.of(MappingType.DateNanos, "timestamp", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), + Arguments.of(MappingType.Ip, "ip", IP), Arguments.of(MappingType.GeoPoint, "geo_point", OpenSearchGeoPointType.of()), - Arguments.of(MappingType.Binary, "binary", OpenSearchBinaryType.of()), - Arguments.of(MappingType.Ip, "ip", OpenSearchIpType.of())); + Arguments.of(MappingType.Binary, "binary", OpenSearchBinaryType.of())); } @ParameterizedTest(name = "{1}") @@ -188,13 +189,13 @@ public void types_but_clones_are_singletons_and_cached() { () -> assertSame(OpenSearchDataType.of(MappingType.Text), OpenSearchTextType.of()), () -> assertSame(OpenSearchDataType.of(MappingType.Binary), OpenSearchBinaryType.of()), () -> assertSame(OpenSearchDataType.of(MappingType.GeoPoint), OpenSearchGeoPointType.of()), - () -> assertSame(OpenSearchDataType.of(MappingType.Ip), OpenSearchIpType.of()), () -> assertNotSame( OpenSearchTextType.of(), OpenSearchTextType.of(Map.of("properties", OpenSearchDataType.of(INTEGER)))), () -> assertSame(OpenSearchDataType.of(INTEGER), OpenSearchDataType.of(INTEGER)), () -> assertSame(OpenSearchDataType.of(STRING), OpenSearchDataType.of(STRING)), + () -> assertSame(OpenSearchDataType.of(IP), OpenSearchDataType.of(IP)), () -> assertSame(OpenSearchDataType.of(STRUCT), OpenSearchDataType.of(STRUCT)), () -> assertNotSame( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java index 5ee175f304..8b13789179 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java @@ -1,45 +1 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.sql.opensearch.data.value; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; -import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; - -public class OpenSearchExprIpValueTest { - - private final String ipString = "192.168.0.1"; - private final OpenSearchExprIpValue ipValue = new OpenSearchExprIpValue(ipString); - - @Test - void testValue() { - assertEquals(ipString, ipValue.value()); - } - - @Test - void testType() { - assertEquals(OpenSearchIpType.of(), ipValue.type()); - } - - @Test - void testCompare() { - assertEquals(0, ipValue.compareTo(new OpenSearchExprIpValue(ipString))); - assertEquals(ipValue, new OpenSearchExprIpValue(ipString)); - } - - @Test - void testEqual() { - assertTrue(ipValue.equal(new OpenSearchExprIpValue(ipString))); - } - - @Test - void testHashCode() { - assertNotNull(ipValue.hashCode()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index d82926077e..89dfd4dbdb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; import static org.opensearch.sql.data.model.ExprValueUtils.floatValue; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.ipValue; import static org.opensearch.sql.data.model.ExprValueUtils.longValue; import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; import static org.opensearch.sql.data.model.ExprValueUtils.shortValue; @@ -51,6 +52,7 @@ import org.opensearch.geometry.utils.Geohash; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; @@ -215,6 +217,16 @@ public void constructString() { () -> assertEquals(stringValue("text"), constructFromObject("stringV", "text"))); } + @Test + public void constructIp() { + assertAll( + () -> assertEquals(ipValue("1.2.3.4"), tupleValue("{\"ipV\":\"1.2.3.4\"}").get("ipV")), + () -> + assertEquals( + ipValue("2001:db7::ff00:42:8329"), + constructFromObject("ipV", "2001:db7::ff00:42:8329"))); + } + @Test public void constructBoolean() { assertAll( @@ -659,17 +671,6 @@ public void constructArrayOfGeoPointsReturnsAll() { .get("geoV")); } - @Test - public void constructArrayOfIPsReturnsAll() { - final String ip1 = "192.168.0.1"; - final String ip2 = "192.168.0.2"; - - assertEquals( - new ExprCollectionValue( - List.of(new OpenSearchExprIpValue(ip1), new OpenSearchExprIpValue(ip2))), - tupleValue(String.format("{\"%s\":[\"%s\",\"%s\"]}", fieldIp, ip1, ip2)).get(fieldIp)); - } - @Test public void constructBinaryArrayReturnsAll() { assertEquals( @@ -681,6 +682,17 @@ public void constructBinaryArrayReturnsAll() { .get("binaryV")); } + @Test + public void constructArrayOfIPsReturnsAll() { + final String ipv4String = "1.2.3.4"; + final String ipv6String = "2001:db7::ff00:42:8329"; + + assertEquals( + new ExprCollectionValue(List.of(ipValue(ipv4String), ipValue(ipv6String))), + tupleValue(String.format("{\"%s\":[\"%s\",\"%s\"]}", fieldIp, ipv4String, ipv6String)) + .get(fieldIp)); + } + @Test public void constructArrayOfCustomEpochMillisReturnsAll() { assertEquals( @@ -743,10 +755,10 @@ public void constructStruct() { @Test public void constructIP() { - final String valueIp = "192.168.0.1"; + final String ipString = "192.168.0.1"; assertEquals( - new OpenSearchExprIpValue(valueIp), - tupleValue(String.format("{\"%s\":\"%s\"}", fieldIp, valueIp)).get(fieldIp)); + new ExprIpValue(ipString), + tupleValue(String.format("{\"%s\":\"%s\"}", fieldIp, ipString)).get(fieldIp)); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index bd2a9901ed..f8c43743ab 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -59,6 +60,10 @@ @ExtendWith(MockitoExtension.class) class FilterQueryBuilderTest { + private static Stream<LiteralExpression> ipCastSource() { + return Stream.of(literal("1.2.3.4"), literal("2001:db7::ff00:42:8329")); + } + private static Stream<LiteralExpression> numericCastSource() { return Stream.of( literal((byte) 1), @@ -1715,6 +1720,25 @@ void cast_to_boolean_false_in_filter(LiteralExpression expr) { json, buildQuery(DSL.equal(ref("boolean_value", BOOLEAN), DSL.castBoolean(expr)))); } + @ParameterizedTest(name = "castIp({0})") + @MethodSource({"ipCastSource"}) + void cast_to_ip_in_filter(LiteralExpression expr) { + String json = + String.format( + """ + { + "term" : { + "ip_value" : { + "value" : "%s", + "boost" : 1.0 + } + } + }""", + expr.valueOf().stringValue()); + + assertJsonEquals(json, buildQuery(DSL.equal(ref("ip_value", IP), DSL.castIp(expr)))); + } + @Test void cast_from_boolean() { Expression booleanExpr = literal(false); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4a883fa656..053ec530db 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -52,10 +52,10 @@ WITH: 'WITH'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; -// FIELD KEYWORDS +// SORT FIELD KEYWORDS +// TODO #3180: Fix broken sort functionality AUTO: 'AUTO'; STR: 'STR'; -IP: 'IP'; NUM: 'NUM'; // TRENDLINE KEYWORDS @@ -142,6 +142,7 @@ LONG: 'LONG'; FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; +IP: 'IP'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index c9d0f2e110..27f7e4014b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -408,6 +408,7 @@ convertedDataType | typeName = FLOAT | typeName = STRING | typeName = BOOLEAN + | typeName = IP ; evalFunctionName @@ -897,7 +898,8 @@ keywordsCanBeId | DATASOURCES // CLAUSEKEYWORDS | SORTBY - // FIELDKEYWORDSAUTO + // SORT FIELD KEYWORDS + | AUTO | STR | IP | NUM diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 8bc98c8eee..5a7522683a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -161,6 +161,8 @@ public UnresolvedExpression visitWcFieldExpression(WcFieldExpressionContext ctx) @Override public UnresolvedExpression visitSortField(SortFieldContext ctx) { + + // TODO #3180: Fix broken sort functionality return new Field( visit(ctx.sortFieldExpression().fieldExpression().qualifiedName()), ArgumentFactory.getArgumentList(ctx)); From 44250d06b31373d4ae5b6e91b76f0a7ea3796e4f Mon Sep 17 00:00:00 2001 From: Simeon Widdis <sawiddis@gmail.com> Date: Fri, 27 Dec 2024 09:46:32 -0800 Subject: [PATCH 94/96] Add stalled PRs workflow (#3221) * Add stalled PRs workflow Signed-off-by: Simeon Widdis <sawiddis@amazon.com> * Update repo target Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --------- Signed-off-by: Simeon Widdis <sawiddis@amazon.com> --- .github/workflows/stalled.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/stalled.yml diff --git a/.github/workflows/stalled.yml b/.github/workflows/stalled.yml new file mode 100644 index 0000000000..2b10140bbd --- /dev/null +++ b/.github/workflows/stalled.yml @@ -0,0 +1,28 @@ +name: Label Stalled PRs +on: + schedule: + - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT +permissions: + pull-requests: write +jobs: + stale: + if: github.repository == 'opensearch-project/sql' + runs-on: ubuntu-latest + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v2.1.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Stale PRs + uses: actions/stale@v9 + with: + repo-token: ${{ steps.github_app_token.outputs.token }} + stale-pr-label: 'stalled' + stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.' + days-before-pr-stale: 30 + days-before-issue-stale: -1 + days-before-pr-close: -1 + days-before-issue-close: -1 From 696d4217edb0eeb2139a249257294cb69ba4148e Mon Sep 17 00:00:00 2001 From: Rupal Mahajan <maharup@amazon.com> Date: Fri, 27 Dec 2024 10:41:46 -0800 Subject: [PATCH 95/96] Remove spark-sql-application (#1916) Signed-off-by: Rupal Mahajan <maharup@amazon.com> --- spark-sql-application/.gitignore | 14 --- spark-sql-application/README.md | 109 ----------------- spark-sql-application/build.sbt | 28 ----- .../project/build.properties | 1 - spark-sql-application/project/plugins.sbt | 6 - spark-sql-application/scalastyle-config.xml | 106 ----------------- .../scala/org/opensearch/sql/SQLJob.scala | 110 ------------------ .../scala/org/opensearch/sql/SQLJobTest.scala | 56 --------- 8 files changed, 430 deletions(-) delete mode 100644 spark-sql-application/.gitignore delete mode 100644 spark-sql-application/README.md delete mode 100644 spark-sql-application/build.sbt delete mode 100644 spark-sql-application/project/build.properties delete mode 100644 spark-sql-application/project/plugins.sbt delete mode 100644 spark-sql-application/scalastyle-config.xml delete mode 100644 spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala delete mode 100644 spark-sql-application/src/test/scala/org/opensearch/sql/SQLJobTest.scala diff --git a/spark-sql-application/.gitignore b/spark-sql-application/.gitignore deleted file mode 100644 index ec13a702be..0000000000 --- a/spark-sql-application/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -# Compiled output -target/ -project/target/ - -# sbt-specific files -.sbtserver -.sbt/ -.bsp/ - -# Miscellaneous -.DS_Store -*.class -*.log -*.zip \ No newline at end of file diff --git a/spark-sql-application/README.md b/spark-sql-application/README.md deleted file mode 100644 index 6422f294cd..0000000000 --- a/spark-sql-application/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Spark SQL Application - -This application execute sql query and store the result in OpenSearch index in following format -``` -"stepId":"<emr-step-id>", -"applicationId":"<spark-application-id>" -"schema": "json blob", -"result": "json blob" -``` - -## Prerequisites - -+ Spark 3.3.1 -+ Scala 2.12.15 -+ flint-spark-integration - -## Usage - -To use this application, you can run Spark with Flint extension: - -``` -./bin/spark-submit \ - --class org.opensearch.sql.SQLJob \ - --jars <flint-spark-integration-jar> \ - sql-job.jar \ - <spark-sql-query> \ - <opensearch-index> \ - <opensearch-host> \ - <opensearch-port> \ - <opensearch-scheme> \ - <opensearch-auth> \ - <opensearch-region> \ -``` - -## Result Specifications - -Following example shows how the result is written to OpenSearch index after query execution. - -Let's assume sql query result is -``` -+------+------+ -|Letter|Number| -+------+------+ -|A |1 | -|B |2 | -|C |3 | -+------+------+ -``` -OpenSearch index document will look like -```json -{ - "_index" : ".query_execution_result", - "_id" : "A2WOsYgBMUoqCqlDJHrn", - "_score" : 1.0, - "_source" : { - "result" : [ - "{'Letter':'A','Number':1}", - "{'Letter':'B','Number':2}", - "{'Letter':'C','Number':3}" - ], - "schema" : [ - "{'column_name':'Letter','data_type':'string'}", - "{'column_name':'Number','data_type':'integer'}" - ], - "stepId" : "s-JZSB1139WIVU", - "applicationId" : "application_1687726870985_0003" - } -} -``` - -## Build - -To build and run this application with Spark, you can run: - -``` -sbt clean publishLocal -``` - -## Test - -To run tests, you can use: - -``` -sbt test -``` - -## Scalastyle - -To check code with scalastyle, you can run: - -``` -sbt scalastyle -``` - -## Code of Conduct - -This project has adopted an [Open Source Code of Conduct](../CODE_OF_CONDUCT.md). - -## Security - -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. - -## License - -See the [LICENSE](../LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. - -## Copyright - -Copyright OpenSearch Contributors. See [NOTICE](../NOTICE) for details. \ No newline at end of file diff --git a/spark-sql-application/build.sbt b/spark-sql-application/build.sbt deleted file mode 100644 index 79d69a30d1..0000000000 --- a/spark-sql-application/build.sbt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -name := "sql-job" - -version := "1.0" - -scalaVersion := "2.12.15" - -val sparkVersion = "3.3.2" - -mainClass := Some("org.opensearch.sql.SQLJob") - -artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) => - "sql-job.jar" -} - -resolvers ++= Seq( - ("apache-snapshots" at "http://repository.apache.org/snapshots/").withAllowInsecureProtocol(true) -) - -libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-core" % sparkVersion % "provided", - "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - "org.scalatest" %% "scalatest" % "3.2.15" % Test -) diff --git a/spark-sql-application/project/build.properties b/spark-sql-application/project/build.properties deleted file mode 100644 index 46e43a97ed..0000000000 --- a/spark-sql-application/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.8.2 diff --git a/spark-sql-application/project/plugins.sbt b/spark-sql-application/project/plugins.sbt deleted file mode 100644 index 4d14ba6c10..0000000000 --- a/spark-sql-application/project/plugins.sbt +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") \ No newline at end of file diff --git a/spark-sql-application/scalastyle-config.xml b/spark-sql-application/scalastyle-config.xml deleted file mode 100644 index 37b1978cd7..0000000000 --- a/spark-sql-application/scalastyle-config.xml +++ /dev/null @@ -1,106 +0,0 @@ -<scalastyle> - <name>Scalastyle standard configuration</name> - <check level="warning" class="org.scalastyle.file.FileTabChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.FileLengthChecker" enabled="true"> - <parameters> - <parameter name="maxFileLength"><![CDATA[800]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true"> - <parameters> - <parameter name="header"><![CDATA[/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.FileLineLengthChecker" enabled="true"> - <parameters> - <parameter name="maxLineLength"><![CDATA[160]]></parameter> - <parameter name="tabSize"><![CDATA[4]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true"> - <parameters> - <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true"> - <parameters> - <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true"> - <parameters> - <parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true"> - <parameters> - <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true"> - <parameters> - <parameter name="maxParameters"><![CDATA[8]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true"> - <parameters> - <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.RegexChecker" enabled="true"> - <parameters> - <parameter name="regex"><![CDATA[println]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true"> - <parameters> - <parameter name="maxTypes"><![CDATA[30]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true"> - <parameters> - <parameter name="maximum"><![CDATA[10]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true"> - <parameters> - <parameter name="singleLineAllowed"><![CDATA[true]]></parameter> - <parameter name="doubleLineAllowed"><![CDATA[false]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true"> - <parameters> - <parameter name="maxLength"><![CDATA[50]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true"> - <parameters> - <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true"> - <parameters> - <parameter name="maxMethods"><![CDATA[30]]></parameter> - </parameters> - </check> - <check level="warning" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check> - <check level="warning" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check> -</scalastyle> \ No newline at end of file diff --git a/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala b/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala deleted file mode 100644 index 98a3a08134..0000000000 --- a/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql - -import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, SparkSession, Row} -import org.apache.spark.sql.types._ - -/** - * Spark SQL Application entrypoint - * - * @param args(0) - * sql query - * @param args(1) - * opensearch index name - * @param args(2-6) - * opensearch connection values required for flint-integration jar. host, port, scheme, auth, region respectively. - * @return - * write sql query result to given opensearch index - */ -object SQLJob { - def main(args: Array[String]) { - // Get the SQL query and Opensearch Config from the command line arguments - val query = args(0) - val index = args(1) - val host = args(2) - val port = args(3) - val scheme = args(4) - val auth = args(5) - val region = args(6) - - val conf: SparkConf = new SparkConf() - .setAppName("SQLJob") - .set("spark.sql.extensions", "org.opensearch.flint.spark.FlintSparkExtensions") - .set("spark.datasource.flint.host", host) - .set("spark.datasource.flint.port", port) - .set("spark.datasource.flint.scheme", scheme) - .set("spark.datasource.flint.auth", auth) - .set("spark.datasource.flint.region", region) - - // Create a SparkSession - val spark = SparkSession.builder().config(conf).enableHiveSupport().getOrCreate() - - try { - // Execute SQL query - val result: DataFrame = spark.sql(query) - - // Get Data - val data = getFormattedData(result, spark) - - // Write data to OpenSearch index - val aos = Map( - "host" -> host, - "port" -> port, - "scheme" -> scheme, - "auth" -> auth, - "region" -> region) - - data.write - .format("flint") - .options(aos) - .mode("append") - .save(index) - - } finally { - // Stop SparkSession - spark.stop() - } - } - - /** - * Create a new formatted dataframe with json result, json schema and EMR_STEP_ID. - * - * @param result - * sql query result dataframe - * @param spark - * spark session - * @return - * dataframe with result, schema and emr step id - */ - def getFormattedData(result: DataFrame, spark: SparkSession): DataFrame = { - // Create the schema dataframe - val schemaRows = result.schema.fields.map { field => - Row(field.name, field.dataType.typeName) - } - val resultSchema = spark.createDataFrame(spark.sparkContext.parallelize(schemaRows), StructType(Seq( - StructField("column_name", StringType, nullable = false), - StructField("data_type", StringType, nullable = false)))) - - // Define the data schema - val schema = StructType(Seq( - StructField("result", ArrayType(StringType, containsNull = true), nullable = true), - StructField("schema", ArrayType(StringType, containsNull = true), nullable = true), - StructField("stepId", StringType, nullable = true), - StructField("applicationId", StringType, nullable = true))) - - // Create the data rows - val rows = Seq(( - result.toJSON.collect.toList.map(_.replaceAll("'", "\\\\'").replaceAll("\"", "'")), - resultSchema.toJSON.collect.toList.map(_.replaceAll("\"", "'")), - sys.env.getOrElse("EMR_STEP_ID", "unknown"), - spark.sparkContext.applicationId)) - - // Create the DataFrame for data - spark.createDataFrame(rows).toDF(schema.fields.map(_.name): _*) - } -} diff --git a/spark-sql-application/src/test/scala/org/opensearch/sql/SQLJobTest.scala b/spark-sql-application/src/test/scala/org/opensearch/sql/SQLJobTest.scala deleted file mode 100644 index 7ec4e45450..0000000000 --- a/spark-sql-application/src/test/scala/org/opensearch/sql/SQLJobTest.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql - -import org.scalatest.funsuite.AnyFunSuite -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.apache.spark.sql.types.{ArrayType, IntegerType, StringType, StructField, StructType} - - -class SQLJobTest extends AnyFunSuite{ - - val spark = SparkSession.builder().appName("Test").master("local").getOrCreate() - - // Define input dataframe - val inputSchema = StructType(Seq( - StructField("Letter", StringType, nullable = false), - StructField("Number", IntegerType, nullable = false) - )) - val inputRows = Seq( - Row("A", 1), - Row("B", 2), - Row("C", 3) - ) - val input: DataFrame = spark.createDataFrame(spark.sparkContext.parallelize(inputRows), inputSchema) - - test("Test getFormattedData method") { - // Define expected dataframe - val expectedSchema = StructType(Seq( - StructField("result", ArrayType(StringType, containsNull = true), nullable = true), - StructField("schema", ArrayType(StringType, containsNull = true), nullable = true), - StructField("stepId", StringType, nullable = true), - StructField("applicationId", StringType, nullable = true) - )) - val expectedRows = Seq( - Row( - Array("{'Letter':'A','Number':1}","{'Letter':'B','Number':2}", "{'Letter':'C','Number':3}"), - Array("{'column_name':'Letter','data_type':'string'}", "{'column_name':'Number','data_type':'integer'}"), - "unknown", - spark.sparkContext.applicationId - ) - ) - val expected: DataFrame = spark.createDataFrame(spark.sparkContext.parallelize(expectedRows), expectedSchema) - - // Compare the result - val result = SQLJob.getFormattedData(input, spark) - assertEqualDataframe(expected, result) - } - - def assertEqualDataframe(expected: DataFrame, result: DataFrame): Unit ={ - assert(expected.schema === result.schema) - assert(expected.collect() === result.collect()) - } -} From 4224f318871d4bf8cb35764fb7f589530449a0de Mon Sep 17 00:00:00 2001 From: Lantao Jin <ltjin@amazon.com> Date: Mon, 30 Dec 2024 12:50:01 +0800 Subject: [PATCH 96/96] Function str_to_date should work with two-digits year (#2841) Signed-off-by: Lantao Jin <ltjin@amazon.com> --- .../sql/expression/datetime/DateTimeFormatterUtil.java | 2 +- .../opensearch/sql/expression/datetime/StrToDateTest.java | 7 +++++++ .../java/org/opensearch/sql/ppl/DateTimeFunctionIT.java | 8 ++++++++ .../java/org/opensearch/sql/sql/DateTimeFunctionIT.java | 4 ++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java index d23cbc2df3..10507533bd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java @@ -173,7 +173,7 @@ interface DateTimeFormatHandler { .put("%T", "HH:mm:ss") // %T => HH:mm:ss .put("%W", "EEEE") // %W => EEEE - Weekday name (Sunday..Saturday) .put("%Y", "u") // %Y => yyyy - Year, numeric, 4 digits - .put("%y", "u") // %y => yy - Year, numeric, 2 digits + .put("%y", "uu") // %y => yy - Year, numeric, 2 digits .put("%f", "n") // %f => n - Nanoseconds // The following have been implemented but cannot be aligned with // MySQL due to the limitations of the DatetimeFormatter diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/StrToDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/StrToDateTest.java index 7f0861d9c3..d0943d54b9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/StrToDateTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/StrToDateTest.java @@ -53,6 +53,13 @@ private static Stream<Arguments> getTestDataForStrToDate() { "%Y,%m,%d,%h,%i", new ExprTimestampValue("2000-01-01 10:11:00"), TIMESTAMP), + Arguments.of( + "01-May-13", "%d-%b-%y", new ExprTimestampValue("2013-05-01 00:00:00"), TIMESTAMP), + Arguments.of( + "01-Jan-00", "%d-%b-%y", new ExprTimestampValue("2000-01-01 00:00:00"), TIMESTAMP), + // Behavior consistent with OpenSearch Core + Arguments.of( + "31-Jul-99", "%d-%b-%y", new ExprTimestampValue("2099-07-31 00:00:00"), TIMESTAMP), // Invalid Arguments (should return null) Arguments.of("a09:30:17", "a%h:%i:%s", ExprNullValue.of(), UNDEFINED), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 3ea6897087..f688e5a42c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -1486,6 +1486,14 @@ public void testStrToDate() throws IOException { TEST_INDEX_DATE, "%d,%m,%Y")); verifySchema(result, schema("f", null, "timestamp")); verifySome(result.getJSONArray("datarows"), rows("2013-05-01 00:00:00")); + // two digits year case + result = + executeQuery( + String.format( + "source=%s | eval f = str_to_date('1-May-13', '%s') | fields f", + TEST_INDEX_DATE, "%d-%b-%y")); + verifySchema(result, schema("f", null, "timestamp")); + verifySome(result.getJSONArray("datarows"), rows("2013-05-01 00:00:00")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 0ec77f9f31..56ff2fafc1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -920,6 +920,10 @@ public void testStrToDate() throws IOException { "SELECT str_to_date(firstname," + " '%%Y-%%m-%%d %%h:%%i:%%s') FROM %s LIMIT 2", TEST_INDEX_BANK)); verifyDataRows(result, rows((Object) null), rows((Object) null)); + + // two digits year case + result = executeQuery("SELECT str_to_date('23-Oct-17 00:00:00', '%d-%b-%y %h:%i:%s')"); + verifyDataRows(result, rows("2017-10-23 00:00:00")); } @Test