Skip to content

Commit

Permalink
Simplify streaming BEP binary protobuf for Xcode progress bar (#17)
Browse files Browse the repository at this point in the history
* Simplify streaming BEP binary protobuf for Xcode progress bar

Move BEP event reading to allocating an InputStream per read, via
NSFileHandle.readabilityHandler.

Bazel works by appending content to a file, specifically,
Java'sBufferedOutputStream.  Naievely using an input stream for the path
and waiting for available data will simply does not work with whatever
BufferedOutputStream.flush() is doing internally.

Reference:
https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java

Perhaps, SwiftProtobuf can come up with a better solution to read
from files or upstream similar code
apple/swift-protobuf#130

Logic:
- If there's already a file at the path remove it
- Create a few file
- When the build starts, Bazel will attempt to reuse the inode, and
  stream to it.
  Then,
- Via NSFileHandle, wait for data to be available and read all the
  bytes
  • Loading branch information
jerrymarino authored Dec 6, 2019
1 parent a97f644 commit 6fa081b
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 42 deletions.
85 changes: 44 additions & 41 deletions Examples/BazelBuildService/BEPStream.swift
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,8 @@ import XCBProtocol
public typealias BEPReadHandler = (BuildEventStream_BuildEvent) -> Void

public class BEPStream {
private let readQueue = DispatchQueue(label: "com.bkbuildservice.bepstream")
private let path: String
private var input: InputStream!
private var lastMTime: TimeInterval?
private var hitLastMessage: Bool = false
private var fileHandle: FileHandle?

/// @param path - Binary BEP file
/// this is passed to Bazel via --build_event_binary_file
Expand All @@ -22,52 +19,58 @@ public class BEPStream {
/// @param eventAvailableHandler - this is called with _every_ BEP event
/// available
public func read(eventAvailableHandler handler: @escaping BEPReadHandler) throws {
input = InputStream(fileAtPath: path)!
readQueue.async {
self.input.open()
self.readLoop(eventAvailableHandler: handler)
let fm = FileManager.default
// Bazel works by appending content to a file, specifically,
// Java'sBufferedOutputStream.
// Naievely using an input stream for the path and waiting for available
// data will simply does not work with whatever
// BufferedOutputStream.flush() is doing internally.
//
// Reference:
// https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java
// Perhaps, SwiftProtobuf can come up with a better solution to read
// from files or upstream similar code
// https://github.com/apple/swift-protobuf/issues/130
//
// Logic:
// - If there's already a file at the path remove it
// - Create a few file
// - When the build starts, Bazel will attempt to reuse the inode, and
// stream to it.
//
// Then,
// - Via NSFileHandle, wait for data to be available and read all the
// bytes
try? fm.removeItem(atPath: path)
try fm.createFile(atPath: path, contents: Data())

guard let fileHandle = FileHandle(forReadingAtPath: path) else {
log("BEPStream: failed to allocate \(path)")
return
}
}
self.fileHandle = fileHandle
fileHandle.readabilityHandler = {
handle in
let data = fileHandle.availableData
guard data.count > 0 else {
return
}

private func readLoop(eventAvailableHandler handler: @escaping BEPReadHandler) {
while !hitLastMessage {
if input.hasBytesAvailable {
// Wrap the file handle in an InputStream for SwiftProtobuf to read
// we read the stream until the end of the file
let input = InputStream(data: data)
input.open()
while input.hasBytesAvailable {
do {
let info = try BinaryDelimited.parse(messageType:
BuildEventStream_BuildEvent.self, from: input)
handler(info)

// When we hit the last message close the stream and end
if info.lastMessage {
hitLastMessage = true
input.close()
break
}
log("BEPStream read event \(fileHandle.offsetInFile)")
} catch {
log("BEPReadError" + error.localizedDescription)
input.close()
log("BEPStream read error: " + error.localizedDescription)
break
}
} else {
// Wait until the BEP file is available
// FIXME: replace polling with kqueue or better
if hasChanged() {
try! read(eventAvailableHandler: handler)
return
}
sleep(1)
}
}
}

private func hasChanged() -> Bool {
let url = URL(fileURLWithPath: path)
let resourceValues = try? url.resourceValues(forKeys:
Set([.contentModificationDateKey]))
let mTime = resourceValues?.contentModificationDate?.timeIntervalSince1970 ?? 0
if mTime != lastMTime {
lastMTime = mTime
return true
}
return false
}
}
1 change: 0 additions & 1 deletion Examples/BazelBuildService/main.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ var gStream: BEPStream?
enum BasicMessageHandler {
static func startStream(bepPath: String, startBuildInput: XCBInputStream, bkservice: BKBuildService) throws {
log("startStream " + String(describing: startBuildInput))
try? FileManager.default.removeItem(atPath: bepPath)
let stream = try BEPStream(path: bepPath)
var progressView: ProgressView?
try stream.read {
Expand Down

0 comments on commit 6fa081b

Please sign in to comment.