Skip to content

Commit

Permalink
test in DeltaTableReadsSuite.scala added
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilicmarkodb
ilicmarkodb committed Sep 24, 2024
1 parent d9e1396 commit b30ce18
Showing 1 changed file with 293 additions and 1 deletion.
294 changes: 293 additions & 1 deletion kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableReadsSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ import io.delta.kernel.defaults.utils.{TestRow, TestUtils}
import io.delta.kernel.internal.fs.Path
import io.delta.kernel.internal.util.InternalUtils.daysSinceEpoch
import io.delta.kernel.internal.util.{DateTimeConstants, FileNames}
import io.delta.kernel.types.{LongType, StructType}
import io.delta.kernel.types.{CollationIdentifier, LongType, StructType}
import io.delta.kernel.Table
import io.delta.kernel.expressions.{And, CollatedPredicate, Column, Literal, Or, Predicate}
import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils
import org.apache.spark.sql.functions.col
import org.scalatest.funsuite.AnyFunSuite
Expand Down Expand Up @@ -586,6 +587,297 @@ class DeltaTableReadsSuite extends AnyFunSuite with TestUtils {
)
}

test("check collated predicate") {
val defaultCollationIdentifier = CollationIdentifier.fromString("SPARK.UTF8_BINARY")
val firstFileC1Data = Seq(
TestRow("A"),
TestRow("B"),
TestRow("C")
)
val secondFileC1Data = Seq(
TestRow("a"),
TestRow("b"),
TestRow("c")
)
val thirdFileC1Data = Seq(
TestRow(Seq(null): _*),
TestRow(Seq(null): _*),
TestRow("C")
)
val allFilesC1Data = firstFileC1Data ++ secondFileC1Data ++ thirdFileC1Data


Seq(
// (predicate, expected results)
(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("b"),
defaultCollationIdentifier
),
allFilesC1Data
),
(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
firstFileC1Data
),
(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
),
allFilesC1Data
),
(
new And(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
firstFileC1Data
),
(
new And(
new Predicate(
"<",
new Column("c1"),
new Column("c2")
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new And(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new And(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
new Predicate(
"<",
new Column("c2"),
Literal.ofString("1")
)
),
Seq()
),
(
new Or(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
new CollatedPredicate(
"<",
new Column("c2"),
Literal.ofString("1"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new Predicate(
"NOT",
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new Predicate("NOT",
new Or(
new Predicate(
">",
new Column("c1"),
Literal.ofString("B")
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
),
firstFileC1Data
),
(
new Predicate("NOT",
new And(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
),
allFilesC1Data
),
(
new Predicate(
"NOT",
new Predicate("IS_NOT_NULL",
new And(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
)
),
allFilesC1Data
),
(
new Predicate(
"NOT",
new Predicate("IS_NULL",
new And(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
)
),
allFilesC1Data
),
(
new Predicate("IS_NULL",
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new Predicate("IS_NULL",
new And(
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
),
allFilesC1Data
),
(
new Predicate("IS_NOT_NULL",
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
),
allFilesC1Data
),
(
new Predicate("IS_NOT_NULL",
new And(
new Predicate(
"<",
new Column("c1"),
Literal.ofString("B")
),
new CollatedPredicate(
"<",
new Column("c1"),
Literal.ofString("B"),
defaultCollationIdentifier
)
)
),
allFilesC1Data
)
).foreach {
case(predicate, expectedAnswer) =>
checkTable(
path = goldenTablePath("data-skipping-basic-stats-collated-data"),
expectedAnswer = expectedAnswer,
readCols = Seq("c1"),
filter = predicate,
expectedRemainingFilter = predicate
)
}
}

test("error - version not contiguous") {
val e = intercept[InvalidTableException] {
latestSnapshot(goldenTablePath("versions-not-contiguous"))
Expand Down

0 comments on commit b30ce18

Please sign in to comment.