diff --git a/Package.resolved b/Package.resolved index 53e65e7..02f0d8f 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "e36efc60ff6513f9fd73eec6246018b16a077ae5221e7e54d785cf0a8c172a1a", + "originHash" : "cf3317819e9dd07905aa1240016b12528f485e5811ed0edacee39014b35850b9", "pins" : [ { "identity" : "gis-tools", "kind" : "remoteSourceControl", "location" : "https://github.com/Outdooractive/gis-tools", "state" : { - "revision" : "a8118bcd5e715a69f640476446fbf058fe39973f", - "version" : "1.8.3" + "revision" : "96aeecd98b0b2871e5f654d8888c7ea60fea8575", + "version" : "1.8.4" } }, { diff --git a/Package.swift b/Package.swift index 899d04d..b3f7fa1 100644 --- a/Package.swift +++ b/Package.swift @@ -24,7 +24,7 @@ let package = Package( targets: ["MVTTools"]), ], dependencies: [ - .package(url: "https://github.com/Outdooractive/gis-tools", from: "1.8.3"), + .package(url: "https://github.com/Outdooractive/gis-tools", from: "1.8.4"), .package(url: "https://github.com/1024jp/GzipSwift.git", from: "5.2.0"), .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.6.1"), diff --git a/README.md b/README.md index 8ced33f..e2380d2 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ mvt query Tests/MVTToolsTests/TestData/14_8716_8015.geojson "3.87324,11.53731,10 } ``` --- -**Example 3**: Query a tile for properties. +**Example 3**: Query Feature properties in a tile. ```bash mvt query -p Tests/MVTToolsTests/TestData/14_8716_8015.vector.mvt ".area > 40000 and .class == 'hospital'" @@ -346,9 +346,10 @@ mvt query -p Tests/MVTToolsTests/TestData/14_8716_8015.vector.mvt ".area > 40000 } ``` -The query language is loosely modeled after the jq query language. Here is an overview. +The query language is very loosely modeled after the jq query language. +The output will contain all features where the query returns `true`. -Example: +Here is an overview. Example: ``` "properties": { "foo": {"bar": 1}, @@ -359,13 +360,14 @@ Example: ``` Values are retrieved by putting a `.` in front of the property name. The property name must be quoted -if it is a number or contains any non-alphabetic characters. Elements in arrays can be -accesses either by simply using the array index after the dot, or by wrapping it in brackets. +if it is a number or contains non-alphabetic characters. Elements in arrays can be +accessed either by simply using the array index after the dot, or by wrapping it in brackets. ``` .foo // true, property "foo" exists .foo.bar // true, property "foo" is a dictionary containing "bar" ."foo"."bar" // true, same as above but quoted +.'foo'.'bar' // true, same as above but quoted .foo.x // false, "foo" doesn't contain "x" ."foo.bar" // false, property "foo.bar" doesn't exist .foo.[0] // false, "foo" is not an array @@ -375,7 +377,6 @@ accesses either by simply using the array index after the dot, or by wrapping it ``` Comparisons can be expressed like this: - ``` .value == "bar" // false .value == 1 // true @@ -386,20 +387,52 @@ Comparisons can be expressed like this: .value <= 1 // true .string =~ /[Ss]ome/ // true .string =~ /some/ // false -.string =~ /some/i // true, case insensitive -.string =~ "^Some" // true +.string =~ /some/i // true, case insensitive regexp +.string =~ "^Some" // true, can also use quotes ``` Conditions (evaluated left to right): - ``` .foo.bar == 1 and .value == 1 // true .foo == 1 or .bar == 2 // false .foo == 1 or .value == 1 // true -.foo not // true if foo does not exist -.foo and .bar not // true if foo and bar don't exist together -.foo or .bar not // true if neither foo nor bar exist -.foo.bar not // true if "bar" in dictionary "foo" doesn't exist +.foo not // false, true if foo does not exist +.foo and .bar not // true, foo and bar don't exist together +.foo or .bar not // false, true if neither foo nor bar exist +.foo.bar not // false, true if "bar" in dictionary "foo" doesn't exist +``` + +Other: +``` +near(latitude,longitude,tolerance) // true if the feature is within "tolerance" around the coordinate +``` + +Some complete examples: +``` +// Can use single quotes for strings +mvt query -p 14_8716_8015.vector.mvt ".area > 20000 and .class == 'hospital'" + +// ... or double quotes, but they must be escaped +mvt query -p 14_8716_8015.vector.mvt ".area > 20000 and .class == \"hospital\"" + +// No need to quote the query if it doesn't conflict with your shell +// Print all features that have an "area" property +mvt query -p 14_8716_8015.vector.mvt .area +// Features which don't have "area" and "name" properties +mvt query -p 14_8716_8015.vector.mvt .area and .name not + +// Case insensitive regular expression +vt query -p 14_8716_8015.vector.mvt ".name =~ /hopital/i" + +// Case sensitive regular expression +mvt query -p 14_8716_8015.vector.mvt ".name =~ /Recherches?/" +// Can also use quotes instead of slashes +mvt query -p 14_8716_8015.vector.mvt ".name =~ 'Recherches?'" + +// Features around a coordinate +mvt query -p 14_8716_8015.vector.mvt "near(3.87324,11.53731,1000)" +// With other conditions +mvt query -p 14_8716_8015.vector.mvt ".name =~ /^lac/i and near(3.87324,11.53731,10000)" ``` --- diff --git a/Sources/MVTTools/Extensions/StringExtensions.swift b/Sources/MVTTools/Extensions/StringExtensions.swift index e3e208a..c335fca 100644 --- a/Sources/MVTTools/Extensions/StringExtensions.swift +++ b/Sources/MVTTools/Extensions/StringExtensions.swift @@ -4,6 +4,16 @@ extension String { var isNotEmpty: Bool { !isEmpty } + /// Trims white space and new line characters + public mutating func trim() { + self = self.trimmed() + } + + /// Trims white space and new line characters, returns a new string + public func trimmed() -> String { + self.trimmingCharacters(in: .whitespacesAndNewlines) + } + func matches(_ regex: String) -> Bool { var options: String.CompareOptions = .regularExpression diff --git a/Sources/MVTTools/Query.swift b/Sources/MVTTools/Query.swift index 7508931..8774926 100644 --- a/Sources/MVTTools/Query.swift +++ b/Sources/MVTTools/Query.swift @@ -62,11 +62,11 @@ extension VectorTile { if let queryParser, let properties = feature.properties as? [String: AnyHashable] { - return queryParser.evaluate(on: properties) + return queryParser.evaluate(on: properties, coordinate: feature.geometry.centroid?.coordinate) } else { for value in feature.properties.values.compactMap({ $0 as? String }) { - if value.contains(term) { + if value.localizedCaseInsensitiveContains(term) { return true } } diff --git a/Sources/MVTTools/QueryParser.swift b/Sources/MVTTools/QueryParser.swift index 2b9e008..dec1569 100644 --- a/Sources/MVTTools/QueryParser.swift +++ b/Sources/MVTTools/QueryParser.swift @@ -1,4 +1,5 @@ import Foundation +import GISTools public struct QueryParser { @@ -30,6 +31,8 @@ public struct QueryParser { case comparison(Comparison) case condition(Condition) case literal(AnyHashable) + case near(Coordinate3D, Double) + case searchValues(String) case value([KeyOrIndex]) } @@ -37,10 +40,11 @@ public struct QueryParser { private(set) var pipeline: [Expression]? public init?(string: String) { - guard string.hasPrefix(".") else { return nil } - self.reader = Reader(characters: Array(string.utf8)) - self.parseQuery() + + if !self.parseQuery() { + return nil + } } public init(pipeline: [Expression]) { @@ -49,43 +53,17 @@ public struct QueryParser { } // Works in a reverse polish notation - public func evaluate(on properties: [String: AnyHashable]) -> Bool { + public func evaluate( + on properties: [String: AnyHashable], + coordinate featureCoordinate: Coordinate3D?) + -> Bool + { guard let pipeline else { return false } var stack: [AnyHashable?] = [] for expression in pipeline { switch expression { - case let .literal(value): - stack.insert(value, at: 0) - - case let .value(keys): - var current: AnyHashable? = properties - - for keyOrIndex in keys { - switch keyOrIndex { - case let .key(key): - if let object = current as? [String: AnyHashable] { - current = object[key] - } - else { - current = nil - break - } - - case let .index(index): - if let array = current as? [AnyHashable] { - current = array.get(at: index) - } - else { - current = nil - break - } - } - } - - stack.insert(current, at: 0) - case let .comparison(comparison): switch comparison { case .equals, .notEquals: @@ -143,6 +121,53 @@ public struct QueryParser { stack.insert(!valueIsTrue, at: 0) } + + case let .literal(value): + stack.insert(value, at: 0) + + case let .near(coordinate, tolerance): + var result = false + if let featureCoordinate { + result = coordinate.distance(from: featureCoordinate) <= tolerance + } + stack.insert(result, at: 0) + + case let .searchValues(searchString): + var result = false + for value in properties.values.compactMap({ $0 as? String }) { + if value.localizedCaseInsensitiveContains(searchString) { + result = true + break + } + } + stack.insert(result, at: 0) + + case let .value(keys): + var current: AnyHashable? = properties + + for keyOrIndex in keys { + switch keyOrIndex { + case let .key(key): + if let object = current as? [String: AnyHashable] { + current = object[key] + } + else { + current = nil + break + } + + case let .index(index): + if let array = current as? [AnyHashable] { + current = array.get(at: index) + } + else { + current = nil + break + } + } + } + + stack.insert(current, at: 0) } } @@ -231,29 +256,27 @@ public struct QueryParser { } } - private mutating func parseQuery() { - // skipWhitespace returns the first non-whitespace character, - // which must be a '.' - guard var reader, - let firstCharacter = reader.skipWhitespace(), - firstCharacter == UInt8(ascii: ".") - else { return } + private mutating func parseQuery() -> Bool { + guard var reader else { return false } + + reader.skipWhitespace() pipeline = [] var terms: [Expression] = [] var comparison: Expression? var condition: Expression? - var isBeginningOfTerm = false + var isBeginningOfTerm = true outer: while let char = reader.peek() { // Check for: // - and, or, not // - ==, !=, >, >=, <, <=, =~ if isBeginningOfTerm { - let hasAnd = reader.peekString("and", caseInsensitive: true) - let hasOr = reader.peekString("or", caseInsensitive: true) - let hasNot = reader.peekString("not", caseInsensitive: true) + let hasAnd = reader.peekWord("and") + let hasOr = reader.peekWord("or") + let hasNot = reader.peekWord("not") + let hasNear = reader.peekString("near(", caseInsensitive: true) if hasAnd || hasOr || hasNot { pipeline?.append(contentsOf: terms) @@ -276,7 +299,7 @@ public struct QueryParser { condition = .condition(.or) reader.moveIndex(by: 2) } - else { + else if hasNot { pipeline?.append(.condition(.not)) reader.moveIndex(by: 3) } @@ -284,6 +307,15 @@ public struct QueryParser { continue } + if hasNear { + guard let term = reader.readNear() else { return false } + + isBeginningOfTerm = false + terms.append(term) + + continue + } + // Must be in the middle, otherwise it's just some literal value if terms.count == 1, let term = reader.readComparisonExpression() @@ -301,12 +333,12 @@ public struct QueryParser { continue case UInt8(ascii: "."): - guard let term = reader.readValueExpression() else { return } + guard let term = reader.readValueExpression() else { return false } isBeginningOfTerm = false terms.append(term) default: - guard let term = reader.readLiteralExpression() else { return } + guard let term = reader.readLiteralExpression() else { return false } isBeginningOfTerm = false terms.append(term) } @@ -319,6 +351,22 @@ public struct QueryParser { if let condition { pipeline?.append(condition) } + + // Only literal values -> global search + if pipeline?.allSatisfy({ if case .literal = $0 { true } else { false } }) ?? false, + let searchString = pipeline? + .compactMap({ expression in + if case let .literal(value) = expression { + return value as? String + } + return nil + }) + .joined(separator: " ") + { + pipeline = [.searchValues(searchString)] + } + + return true } // MARK: - Reader @@ -354,13 +402,23 @@ public struct QueryParser { return characters[index + offset] } - func peekString(_ string: String, caseInsensitive: Bool) -> Bool { + func peekWord(_ string: String) -> Bool { + peekString(string, caseInsensitive: true, checkWordBoundary: true) + } + + func peekString( + _ string: String, + caseInsensitive: Bool = true, + checkWordBoundary: Bool = false) + -> Bool + { guard index + string.count <= characters.endIndex else { return false } let peekString = caseInsensitive ? string.lowercased() : string for (offset, char) in peekString.utf8.enumerated() { var c = characters[index + offset] + // only ASCII A-Z if caseInsensitive, c >= 65, c <= 90 { c += 32 } @@ -368,6 +426,12 @@ public struct QueryParser { if c != char { return false } } + if checkWordBoundary { + guard index + string.count == characters.endIndex + || characters[index + string.count] == UInt8(ascii: " ") + else { return false } + } + return true } @@ -621,6 +685,40 @@ public struct QueryParser { return nil } + mutating func readNear() -> Expression? { + guard peekString("near(", caseInsensitive: true) else { return nil } + + moveIndex(by: 5) + + let startIndex = index + var offset = 0 + + while let char = peek(withOffset: offset) { + switch char { + case UInt8(ascii: ")"): + moveIndex(by: offset + 1) + + guard let current = String(bytes: characters[startIndex ..< startIndex + offset], encoding: .utf8) else { + return nil + } + + let components = current.components(separatedBy: ",").compactMap({ $0.trimmed() }) + guard components.count == 3, + let latitude = Double(components[0]), + let longitude = Double(components[1]), + let tolerance = Double(components[2]) + else { return nil } + + return .near(Coordinate3D(latitude: latitude, longitude: longitude), tolerance) + + default: + offset += 1 + } + } + + return nil + } + } } diff --git a/Tests/MVTToolsTests/QueryParserTests.swift b/Tests/MVTToolsTests/QueryParserTests.swift index 84b5909..49eafff 100644 --- a/Tests/MVTToolsTests/QueryParserTests.swift +++ b/Tests/MVTToolsTests/QueryParserTests.swift @@ -1,3 +1,4 @@ +import GISTools import XCTest @testable import MVTTools @@ -18,7 +19,9 @@ final class QueryParserTests: XCTestCase { ] private func result(for pipeline: [QueryParser.Expression]) -> Bool { - QueryParser(pipeline: pipeline).evaluate(on: QueryParserTests.properties as! [String: AnyHashable]) + QueryParser(pipeline: pipeline).evaluate( + on: QueryParserTests.properties as! [String: AnyHashable], + coordinate: nil) } private func pipeline(for query: String) -> [QueryParser.Expression] { @@ -34,6 +37,12 @@ final class QueryParserTests: XCTestCase { XCTAssertTrue(result(for: [.value([.key("some"), .index(0)])])) } + func testNear() throws { + XCTAssertEqual(pipeline(for: "near(10.0, 20.0, 1000)"), [ + .near(Coordinate3D(latitude: 10.0, longitude: 20.0), 1000.0), + ]) + } + func testComparisons() throws { XCTAssertFalse(result(for: [.value([.key("value")]), .literal("bar"), .comparison(.equals)])) XCTAssertTrue(result(for: [.value([.key("value")]), .literal(1), .comparison(.equals)])) @@ -195,6 +204,11 @@ final class QueryParserTests: XCTestCase { .value([.key("foo"), .key("bar")]), .condition(.not), ]) + XCTAssertEqual(pipeline(for: ".foo == 'not'"), [ + .value([.key("foo")]), + .literal("not"), + .comparison(.equals), + ]) } }