Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix error message when no columns found in schema #72

Merged
merged 1 commit into from
Mar 17, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,9 @@ case class ParquetMetastoreSupport() extends MetastoreSupport with Logging {
// fetch specified columns, inferred fields should be the same as requested columns
val inferredSchema = StructType(fileStruct.filter { field => columns.contains(field.name) })
columns.foreach { name =>
val containsField = inferredSchema.exists { _.name == name }
if (!containsField) {
throw new IllegalArgumentException("Failed to select indexed columns. " +
s"Column $name does not exist in inferred schema ${inferredSchema.simpleString}")
if (!inferredSchema.exists { _.name == name }) {
throw new IllegalArgumentException(s"Failed to select indexed column '$name' in " +
s"inferred/original schema ${fileStruct.simpleString}")
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ object ParquetSchemaUtils {
schema.getFields.asScala.map { field =>
if (uniqueColumns.contains(field.getName)) {
throw new IllegalArgumentException(s"""
| Found field [$field] with duplicate column name ${field.getName}.
| Found field [$field] with duplicate column name '${field.getName}'.
| Schema $schema
| This situation is currently not supported, ensure that names of all top level columns
| in schema are unique""".stripMargin)
Expand Down
4 changes: 2 additions & 2 deletions src/test/scala/org/apache/spark/sql/IndexSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ class IndexSuite extends UnitTestSuite with SparkLocal {
val err = intercept[IllegalArgumentException] {
spark.index.create.indexBy("str", "id").parquet(dir.toString / "test")
}
assert(err.getMessage.contains("Failed to select indexed columns. Column str does not " +
"exist in inferred schema struct<id:bigint>"))
assert(err.getMessage.contains("Failed to select indexed column 'str' in " +
"inferred/original schema struct<id:bigint>"))
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ class ParquetSchemaUtilsSuite extends UnitTestSuite {
val err = intercept[IllegalArgumentException] {
ParquetSchemaUtils.topLevelUniqueColumns(schema)
}
assert(err.getMessage.contains("[required int32 id] with duplicate column name id"))
assert(err.getMessage.contains("[required int32 id] with duplicate column name 'id'"))
}

test("merge - two identical schemas") {
Expand Down