Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for virtual columns (#42)
* Add support for virtual columns * Improve docs slightly
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| /// Expression virtual columns use Druid's native expression system to allow defining query time transforms of inputs from one or more columns. | ||
| /// | ||
| /// https://druid.apache.org/docs/latest/querying/math-expr | ||
| public struct ExpressionVirtualColumn: Codable, Hashable, Equatable { | ||
| public init(name: String, expression: String, outputType: String? = nil) { | ||
| self.name = name | ||
| self.expression = expression | ||
| self.outputType = outputType | ||
| } | ||
|
|
||
| public let name: String | ||
| public let expression: String | ||
| public let outputType: String? | ||
| } | ||
|
|
||
|
|
||
|
Check warning on line 16 in Sources/DataTransferObjects/Query/Virtual Column/ExpressionVirtualColumn.swift
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| /// This virtual column provides an alternative way to use 'list filtered' dimension spec as a virtual column. It has optimized access to the underlying column value indexes that can provide a small performance improvement in some cases. | ||
| public struct ListFilteredVirtualColumn: Codable, Hashable, Equatable { | ||
| public init(name: String, delegate: String, values: [String], isAllowList: Bool? = nil) { | ||
| self.name = name | ||
| self.delegate = delegate | ||
| self.values = values | ||
| self.isAllowList = isAllowList | ||
| } | ||
|
|
||
| /// The output name of the virtual column | ||
| public let name: String | ||
|
|
||
| /// The name of the multi-value STRING input column to filter | ||
| public let delegate: String | ||
|
|
||
| /// Set of STRING values to allow or deny | ||
| public let values: [String] | ||
|
|
||
| /// If true, the output of the virtual column will be limited to the set specified by values, | ||
| /// else it will provide all values except those specified. | ||
| public let isAllowList: Bool? | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| public indirect enum VirtualColumn: Codable, Hashable, Equatable { | ||
| case expression(ExpressionVirtualColumn) | ||
| case listFiltered(ListFilteredVirtualColumn) | ||
|
|
||
| enum CodingKeys: String, CodingKey { | ||
| case type | ||
| } | ||
|
|
||
| public init(from decoder: Decoder) throws { | ||
| let values = try decoder.container(keyedBy: CodingKeys.self) | ||
| let type = try values.decode(String.self, forKey: .type) | ||
|
|
||
| switch type { | ||
| case "expression": | ||
| self = try .expression(ExpressionVirtualColumn(from: decoder)) | ||
| case "mv-filtered": | ||
| self = try .listFiltered(ListFilteredVirtualColumn(from: decoder)) | ||
| default: | ||
| throw EncodingError.invalidValue("Invalid type", .init(codingPath: [CodingKeys.type], debugDescription: "Invalid Type: \(type)", underlyingError: nil)) | ||
| } | ||
| } | ||
|
|
||
| public func encode(to encoder: Encoder) throws { | ||
| var container = encoder.container(keyedBy: CodingKeys.self) | ||
|
|
||
| switch self { | ||
| case let .expression(virtualColumn): | ||
| try container.encode("expression", forKey: .type) | ||
| try virtualColumn.encode(to: encoder) | ||
| case let .listFiltered(virtualColumn): | ||
| try container.encode("mv-filtered", forKey: .type) | ||
| try virtualColumn.encode(to: encoder) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| @testable import DataTransferObjects | ||
| import XCTest | ||
|
|
||
| final class VirtualColumnTests: XCTestCase { | ||
| let tdValueString = """ | ||
| { | ||
| "aggregations": [ | ||
| { "fieldName": "clientUser", "name": "count", "type": "thetaSketch" } | ||
| ], | ||
| "dimension": { | ||
| "dimension": "calculatedSystemVersion", | ||
| "outputName": "fooPage", | ||
| "type": "default" | ||
| }, | ||
| "granularity": "all", | ||
| "metric": { "metric": "count", "type": "numeric" }, | ||
| "queryType": "topN", | ||
| "threshold": 10, | ||
| "virtualColumns": [ | ||
| { | ||
| "expression": "nvl(majorMinorSystemVersion,concat('TelemetryDeck.Device.operatingSystem+'-'+nvl(OSVersion,'unknown')))", | ||
| "name": "calculatedSystemVersion", | ||
| "outputType": "STRING", | ||
| "type": "expression" | ||
| } | ||
| ] | ||
| } | ||
| """ | ||
| .filter { !$0.isWhitespace } | ||
|
|
||
| let tdValue = CustomQuery( | ||
| queryType: .topN, | ||
| virtualColumns: [ | ||
| .expression( | ||
| .init( | ||
| name: "calculatedSystemVersion", | ||
| expression: "nvl(majorMinorSystemVersion,concat('TelemetryDeck.Device.operatingSystem+'-'+nvl(OSVersion,'unknown')))", | ||
| outputType: "STRING" | ||
| ) | ||
| ) | ||
| ], | ||
| granularity: .all, | ||
| aggregations: [ | ||
| .thetaSketch(.init(type: .thetaSketch, name: "count", fieldName: "clientUser")) | ||
| ], | ||
| threshold: 10, | ||
| metric: .numeric(.init(metric: "count")), | ||
| dimension: .default(.init(dimension: "calculatedSystemVersion", outputName: "fooPage")) | ||
| ) | ||
|
|
||
| let testedType = CustomQuery.self | ||
|
|
||
| func testDecodingDocsExample() throws { | ||
| let decodedValue = try JSONDecoder.telemetryDecoder.decode(testedType, from: tdValueString.data(using: .utf8)!) | ||
| XCTAssertEqual(tdValue, decodedValue) | ||
| } | ||
|
|
||
| func testEncodingDocsExample() throws { | ||
| let encodedValue = try JSONEncoder.telemetryEncoder.encode(tdValue) | ||
| XCTAssertEqual(tdValueString, String(data: encodedValue, encoding: .utf8)!) | ||
| } | ||
| } | ||
|
|
||
| final class ExpressionVirtualColumnTests: XCTestCase { | ||
| let docsValueString = """ | ||
| { | ||
| "expression": "<rowexpression>", | ||
| "name": "<nameofthevirtualcolumn>", | ||
| "outputType": "FLOAT", | ||
| "type": "expression" | ||
| } | ||
| """ | ||
| .filter { !$0.isWhitespace } | ||
|
|
||
| let docsValue = VirtualColumn.expression( | ||
| ExpressionVirtualColumn(name: "<nameofthevirtualcolumn>", expression: "<rowexpression>", outputType: "FLOAT") | ||
| ) | ||
|
|
||
| let testedType = VirtualColumn.self | ||
|
|
||
| func testDecodingDocsExample() throws { | ||
| let decodedValue = try JSONDecoder.telemetryDecoder.decode(testedType, from: docsValueString.data(using: .utf8)!) | ||
| XCTAssertEqual(docsValue, decodedValue) | ||
| } | ||
|
|
||
| func testEncodingDocsExample() throws { | ||
| let encodedValue = try JSONEncoder.telemetryEncoder.encode(docsValue) | ||
| XCTAssertEqual(docsValueString, String(data: encodedValue, encoding: .utf8)!) | ||
| } | ||
| } | ||
|
|
||
| final class ListFilteredVirtualColumnTests: XCTestCase { | ||
| let docsValueString = """ | ||
| { | ||
| "delegate": "dim3", | ||
| "isAllowList": true, | ||
| "name": "filteredDim3", | ||
| "type": "mv-filtered", | ||
| "values": ["hello", "world"] | ||
| } | ||
| """ | ||
| .filter { !$0.isWhitespace } | ||
|
|
||
| let docsValue = VirtualColumn.listFiltered( | ||
| .init( | ||
| name: "filteredDim3", | ||
| delegate: "dim3", | ||
| values: ["hello", "world"], | ||
| isAllowList: true | ||
| ) | ||
| ) | ||
|
|
||
| let testedType = VirtualColumn.self | ||
|
|
||
| func testDecodingDocsExample() throws { | ||
| let decodedValue = try JSONDecoder.telemetryDecoder.decode(testedType, from: docsValueString.data(using: .utf8)!) | ||
| XCTAssertEqual(docsValue, decodedValue) | ||
| } | ||
|
|
||
| func testEncodingDocsExample() throws { | ||
| let encodedValue = try JSONEncoder.telemetryEncoder.encode(docsValue) | ||
| XCTAssertEqual(docsValueString, String(data: encodedValue, encoding: .utf8)!) | ||
| } | ||
| } |