From 6fa081b488da1c1a21aaa90eeb03ec3126570a5c Mon Sep 17 00:00:00 2001 From: Jerry Marino Date: Fri, 6 Dec 2019 14:24:23 -0800 Subject: [PATCH] Simplify streaming BEP binary protobuf for Xcode progress bar (#17) * Simplify streaming BEP binary protobuf for Xcode progress bar Move BEP event reading to allocating an InputStream per read, via NSFileHandle.readabilityHandler. Bazel works by appending content to a file, specifically, Java'sBufferedOutputStream. Naievely using an input stream for the path and waiting for available data will simply does not work with whatever BufferedOutputStream.flush() is doing internally. Reference: https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java Perhaps, SwiftProtobuf can come up with a better solution to read from files or upstream similar code https://github.com/apple/swift-protobuf/issues/130 Logic: - If there's already a file at the path remove it - Create a few file - When the build starts, Bazel will attempt to reuse the inode, and stream to it. Then, - Via NSFileHandle, wait for data to be available and read all the bytes --- Examples/BazelBuildService/BEPStream.swift | 85 +++++++++++----------- Examples/BazelBuildService/main.swift | 1 - 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/Examples/BazelBuildService/BEPStream.swift b/Examples/BazelBuildService/BEPStream.swift index 267edfa..709f0c5 100644 --- a/Examples/BazelBuildService/BEPStream.swift +++ b/Examples/BazelBuildService/BEPStream.swift @@ -6,11 +6,8 @@ import XCBProtocol public typealias BEPReadHandler = (BuildEventStream_BuildEvent) -> Void public class BEPStream { - private let readQueue = DispatchQueue(label: "com.bkbuildservice.bepstream") private let path: String - private var input: InputStream! - private var lastMTime: TimeInterval? - private var hitLastMessage: Bool = false + private var fileHandle: FileHandle? /// @param path - Binary BEP file /// this is passed to Bazel via --build_event_binary_file @@ -22,52 +19,58 @@ public class BEPStream { /// @param eventAvailableHandler - this is called with _every_ BEP event /// available public func read(eventAvailableHandler handler: @escaping BEPReadHandler) throws { - input = InputStream(fileAtPath: path)! - readQueue.async { - self.input.open() - self.readLoop(eventAvailableHandler: handler) + let fm = FileManager.default + // Bazel works by appending content to a file, specifically, + // Java'sBufferedOutputStream. + // Naievely using an input stream for the path and waiting for available + // data will simply does not work with whatever + // BufferedOutputStream.flush() is doing internally. + // + // Reference: + // https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java + // Perhaps, SwiftProtobuf can come up with a better solution to read + // from files or upstream similar code + // https://github.com/apple/swift-protobuf/issues/130 + // + // Logic: + // - If there's already a file at the path remove it + // - Create a few file + // - When the build starts, Bazel will attempt to reuse the inode, and + // stream to it. + // + // Then, + // - Via NSFileHandle, wait for data to be available and read all the + // bytes + try? fm.removeItem(atPath: path) + try fm.createFile(atPath: path, contents: Data()) + + guard let fileHandle = FileHandle(forReadingAtPath: path) else { + log("BEPStream: failed to allocate \(path)") + return } - } + self.fileHandle = fileHandle + fileHandle.readabilityHandler = { + handle in + let data = fileHandle.availableData + guard data.count > 0 else { + return + } - private func readLoop(eventAvailableHandler handler: @escaping BEPReadHandler) { - while !hitLastMessage { - if input.hasBytesAvailable { + // Wrap the file handle in an InputStream for SwiftProtobuf to read + // we read the stream until the end of the file + let input = InputStream(data: data) + input.open() + while input.hasBytesAvailable { do { let info = try BinaryDelimited.parse(messageType: BuildEventStream_BuildEvent.self, from: input) handler(info) - - // When we hit the last message close the stream and end - if info.lastMessage { - hitLastMessage = true - input.close() - break - } + log("BEPStream read event \(fileHandle.offsetInFile)") } catch { - log("BEPReadError" + error.localizedDescription) - input.close() + log("BEPStream read error: " + error.localizedDescription) + break } - } else { - // Wait until the BEP file is available - // FIXME: replace polling with kqueue or better - if hasChanged() { - try! read(eventAvailableHandler: handler) - return - } - sleep(1) } } } - - private func hasChanged() -> Bool { - let url = URL(fileURLWithPath: path) - let resourceValues = try? url.resourceValues(forKeys: - Set([.contentModificationDateKey])) - let mTime = resourceValues?.contentModificationDate?.timeIntervalSince1970 ?? 0 - if mTime != lastMTime { - lastMTime = mTime - return true - } - return false - } } diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index 79f95a5..d4faba1 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -17,7 +17,6 @@ var gStream: BEPStream? enum BasicMessageHandler { static func startStream(bepPath: String, startBuildInput: XCBInputStream, bkservice: BKBuildService) throws { log("startStream " + String(describing: startBuildInput)) - try? FileManager.default.removeItem(atPath: bepPath) let stream = try BEPStream(path: bepPath) var progressView: ProgressView? try stream.read {