add AWSS3 as a package extension

Project.toml

@@ -1,8 +1,7 @@
 name = "Onda"
 uuid = "e853f5be-6863-11e9-128d-476edb89bfb5"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.15.2"
-
+version = "0.15.3"
 
 [deps]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
@@ -17,22 +16,32 @@ TimeSpans = "bb34ddd2-327f-4c4a-bfb0-c98fc494ece1"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
+[weakdeps]
+AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
+
+[extensions]
+OndaAWSS3Ext = "AWSS3"
+
 [compat]
 Arrow = "1.6.2, 2"
+AWSS3 = "0.9, 0.10, 0.11"
 CodecZstd = "0.6, 0.7"
 Compat = "3.32, 4"
 DataFrames = "1.2"
 FLAC_jll = "1.3.3"
 Legolas = "0.5"
+Minio = "0.2"
 Tables = "1.4"
 TimeSpans = "0.3.4"
 TranscodingStreams = "0.9"
 julia = "1.6"
 
 [extras]
+AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 FLAC_jll = "1d38b3a6-207b-531b-80e8-c83f48dafa73"
+Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["FLAC_jll", "DataFrames", "Test"]
+test = ["AWSS3", "FLAC_jll", "DataFrames", "Minio", "Test"]

ext/OndaAWSS3Ext.jl

@@ -0,0 +1,24 @@
+module OndaAWSS3Ext
+
+using AWSS3: S3Path
+using Onda: Onda
+
+"""
+    Onda.read_byte_range(path::S3Path, byte_offset, byte_count)
+
+Implement method needed for Onda to read a byte range from an S3 path.  Uses
+`AWSS3.s3_get` under the hood.
+
+"""
+function Onda.read_byte_range(path::S3Path, byte_offset, byte_count)
+    # s3_get byte_range is 1-indexed, so we need to add one
+    byte_range = range(byte_offset + 1; length=byte_count)
+    return read(path; byte_range)
+end
+
+# avoid method ambiguity
+function Onda.read_byte_range(path::S3Path, ::Missing, ::Missing)
+    return read(path)
+end
+
+end # module

test/awss3.jl

@@ -0,0 +1,63 @@
+function minio_server(body, dirs=[mktempdir()]; address="localhost:9005")
+    server = Minio.Server(dirs; address)
+
+    try
+        run(server; wait=false)
+        sleep(0.5)  # give the server just a bit of time, though it is amazingly fast to start
+
+        config = MinioConfig(
+            "http://$address"; username="minioadmin", password="minioadmin"
+        )
+        body(config)
+    finally
+        # Make sure we kill the server even if a test failed.
+        kill(server)
+    end
+end
+
+# Test we are loading the `OndaAWSS3Ext` extension in the tests here
+if VERSION >= v"1.9"
+    @test Base.get_extension(Onda, :OndaAWSS3Ext) isa Module
+end
+
+@testset "AWSS3 usage" begin
+    minio_server() do config
+        s3_create_bucket(config, "test-bucket")
+
+        for (file_format, exc) in (("lpcm", AWSException), ("lpcm.zst", InexactError))
+            file_path = S3Path("s3://test-bucket/prefix/samples.$(file_format)"; config)
+            recording_uuid = uuid4()
+            start = Second(0)
+
+            info = SamplesInfoV2(sensor_type="eeg",
+                                 channels=["a", "b"],
+                                 sample_unit="unit",
+                                 sample_resolution_in_unit=1.0,
+                                 sample_offset_in_unit=0.0,
+                                 sample_type=Int16,
+                                 sample_rate=100.0)
+            samples = Samples(rand(sample_type(info), 2, 300), info, true)
+
+            signal = Onda.store(file_path, file_format, samples, recording_uuid, start)
+            @test signal.file_path isa S3Path
+
+            loaded_samples = Onda.load(signal; encoded=true)
+            @test samples == loaded_samples
+
+            # Load subspan to exercise method
+            span = TimeSpan(0, Second(1))
+            loaded_span = Onda.load(signal, span; encoded=true)
+            @test loaded_samples[:, span] == loaded_span
+
+            if VERSION >= v"1.9" # This test requires the package extension to work correctly
+                bad_span = TimeSpan(stop(signal.span) + Nanosecond(Second(1)),
+                                    stop(signal.span) + Nanosecond(Second(2)))
+                # this throws a BoundsError without our extension (since Onda falls back to
+                # loading EVERYTHING and then indexing.  with our utils, it passes the
+                # byte range to AWS which says it's invalid.
+                # For compressed data, Onda does byte range requests.
+                @test_throws exc Onda.load(signal, bad_span)
+            end
+        end
+    end
+end

test/runtests.jl

@@ -1,6 +1,8 @@
 using Compat: @compat
 using Test, UUIDs, Dates, Onda, Legolas, Arrow, Tables, TimeSpans, DataFrames, Random
 using Tables: rowmerge
+using AWSS3, Minio # for testing AWSS3 package extension
+using AWSS3.AWS: AWSException
 
 function has_rows(a, b)
     for name in propertynames(b)
@@ -17,5 +19,6 @@ include("signals.jl")
 include("serialization.jl")
 include("samples.jl")
 include("deprecations.jl")
+include("awss3.jl")
 include(joinpath(dirname(@__DIR__), "examples", "flac.jl"))
 include(joinpath(dirname(@__DIR__), "examples", "tour.jl"))