diff --git a/Project.toml b/Project.toml index b836c490..db62ec83 100644 --- a/Project.toml +++ b/Project.toml @@ -1,8 +1,7 @@ name = "Onda" uuid = "e853f5be-6863-11e9-128d-476edb89bfb5" authors = ["Beacon Biosignals, Inc."] -version = "0.15.2" - +version = "0.15.3" [deps] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" @@ -17,22 +16,32 @@ TimeSpans = "bb34ddd2-327f-4c4a-bfb0-c98fc494ece1" TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +[weakdeps] +AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" + +[extensions] +OndaAWSS3Ext = "AWSS3" + [compat] Arrow = "1.6.2, 2" +AWSS3 = "0.9, 0.10, 0.11" CodecZstd = "0.6, 0.7" Compat = "3.32, 4" DataFrames = "1.2" FLAC_jll = "1.3.3" Legolas = "0.5" +Minio = "0.2" Tables = "1.4" TimeSpans = "0.3.4" TranscodingStreams = "0.9" julia = "1.6" [extras] +AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" FLAC_jll = "1d38b3a6-207b-531b-80e8-c83f48dafa73" +Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["FLAC_jll", "DataFrames", "Test"] +test = ["AWSS3", "FLAC_jll", "DataFrames", "Minio", "Test"] diff --git a/ext/OndaAWSS3Ext.jl b/ext/OndaAWSS3Ext.jl new file mode 100644 index 00000000..6c2c0bce --- /dev/null +++ b/ext/OndaAWSS3Ext.jl @@ -0,0 +1,24 @@ +module OndaAWSS3Ext + +using AWSS3: S3Path +using Onda: Onda + +""" + Onda.read_byte_range(path::S3Path, byte_offset, byte_count) + +Implement method needed for Onda to read a byte range from an S3 path. Uses +`AWSS3.s3_get` under the hood. + +""" +function Onda.read_byte_range(path::S3Path, byte_offset, byte_count) + # s3_get byte_range is 1-indexed, so we need to add one + byte_range = range(byte_offset + 1; length=byte_count) + return read(path; byte_range) +end + +# avoid method ambiguity +function Onda.read_byte_range(path::S3Path, ::Missing, ::Missing) + return read(path) +end + +end # module diff --git a/test/awss3.jl b/test/awss3.jl new file mode 100644 index 00000000..9de3ba87 --- /dev/null +++ b/test/awss3.jl @@ -0,0 +1,63 @@ +function minio_server(body, dirs=[mktempdir()]; address="localhost:9005") + server = Minio.Server(dirs; address) + + try + run(server; wait=false) + sleep(0.5) # give the server just a bit of time, though it is amazingly fast to start + + config = MinioConfig( + "http://$address"; username="minioadmin", password="minioadmin" + ) + body(config) + finally + # Make sure we kill the server even if a test failed. + kill(server) + end +end + +# Test we are loading the `OndaAWSS3Ext` extension in the tests here +if VERSION >= v"1.9" + @test Base.get_extension(Onda, :OndaAWSS3Ext) isa Module +end + +@testset "AWSS3 usage" begin + minio_server() do config + s3_create_bucket(config, "test-bucket") + + for (file_format, exc) in (("lpcm", AWSException), ("lpcm.zst", InexactError)) + file_path = S3Path("s3://test-bucket/prefix/samples.$(file_format)"; config) + recording_uuid = uuid4() + start = Second(0) + + info = SamplesInfoV2(sensor_type="eeg", + channels=["a", "b"], + sample_unit="unit", + sample_resolution_in_unit=1.0, + sample_offset_in_unit=0.0, + sample_type=Int16, + sample_rate=100.0) + samples = Samples(rand(sample_type(info), 2, 300), info, true) + + signal = Onda.store(file_path, file_format, samples, recording_uuid, start) + @test signal.file_path isa S3Path + + loaded_samples = Onda.load(signal; encoded=true) + @test samples == loaded_samples + + # Load subspan to exercise method + span = TimeSpan(0, Second(1)) + loaded_span = Onda.load(signal, span; encoded=true) + @test loaded_samples[:, span] == loaded_span + + if VERSION >= v"1.9" # This test requires the package extension to work correctly + bad_span = TimeSpan(stop(signal.span) + Nanosecond(Second(1)), + stop(signal.span) + Nanosecond(Second(2))) + # this throws a BoundsError without our extension (since Onda falls back to + # loading EVERYTHING and then indexing. with our utils, it passes the + # byte range to AWS which says it's invalid. + # For compressed data, Onda does byte range requests. + @test_throws exc Onda.load(signal, bad_span) + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 5490b070..1ca2034b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,8 @@ using Compat: @compat using Test, UUIDs, Dates, Onda, Legolas, Arrow, Tables, TimeSpans, DataFrames, Random using Tables: rowmerge +using AWSS3, Minio # for testing AWSS3 package extension +using AWSS3.AWS: AWSException function has_rows(a, b) for name in propertynames(b) @@ -17,5 +19,6 @@ include("signals.jl") include("serialization.jl") include("samples.jl") include("deprecations.jl") +include("awss3.jl") include(joinpath(dirname(@__DIR__), "examples", "flac.jl")) include(joinpath(dirname(@__DIR__), "examples", "tour.jl"))