Add support for InfrastructureSystems methods for time series and supplemental attributes transactions

docs/src/how_to/improve_ts_performance.md

@@ -83,26 +83,16 @@ add_time_series!(sys, generator, forecast_max_reactive_power)
 
 By default, the call to [`add_time_series!`](@ref) will open the HDF5 file, write the data to the file,
 and close the file. It will also add a row to an SQLite database. These operations have overhead.
-If you will add thousands of time series arrays, consider using [`bulk_add_time_series!`](@ref).
-All arrays will be written with one file handle. The
-bulk SQLite operations are much more efficient. As a fallback option, use
-[`open_time_series_store!`](@ref) if timeseries must be added one at a time.
+If you will add thousands of time series arrays, consider using [`begin_time_series_update`](@ref).
+All arrays will be written with one file handle. The bulk SQLite operations are much more
+efficient.
 
 ```julia
-# Assumes `read_time_series` will return data appropriate for Deterministic forecasts
-# based on the generator name and the filenames match the component and time series names.
-resolution = Dates.Hour(1)
-associations = (
-    IS.TimeSeriesAssociation(
-        gen,
-        Deterministic(;
-            data = read_time_series(get_name(gen) * ".csv"),
-            name = "get_max_active_power",
-            resolution = resolution),
-    )
-    for gen in get_components(ThermalStandard, sys)
-)
-bulk_add_time_series!(sys, associations)
+begin_time_series_update(sys) do
+    add_time_series!(sys, component1, time_series1)
+    add_time_series!(sys, component2, time_series2)
+    add_time_series!(sys, component3, time_series3)
+end
 ```
 
 ## Using Forecast Caches for Simulations

src/PowerSystems.jl

@@ -313,6 +313,7 @@ export parse_file
 export open_time_series_store!
 export add_time_series!
 export bulk_add_time_series!
+export begin_time_series_update
 export remove_time_series!
 export check_time_series_consistency
 export clear_time_series!
@@ -357,6 +358,7 @@ export get_supplemental_attribute
 export get_supplemental_attributes
 export has_supplemental_attributes
 export iterate_supplemental_attributes
+export begin_supplemental_attributes_update
 export get_time_series
 export get_time_series_type
 export get_time_series_array

src/base.jl

@@ -852,10 +852,10 @@ open_time_series_store!(sys, "r+") do
     end
 end
 ```
-You can also use this function to make reads faster. Change the mode from `"r+"` to `"r"` to open
-the file read-only.
+You can also use this function to make reads faster.
+Change the mode from `"r+"` to `"r"` to open the file read-only.
 
-See also: [`bulk_add_time_series!`](@ref)
+See also: [`begin_time_series_update!`](@ref)
 """
 function open_time_series_store!(
     func::Function,
@@ -867,6 +867,25 @@ function open_time_series_store!(
     IS.open_time_series_store!(func, sys.data, mode, args...; kwargs...)
 end
 
+"""
+Begin an update of time series. Use this function when adding many time series arrays
+in order to improve performance.
+
+If an error occurs during the update, changes will be reverted.
+
+Using this function to remove time series is currently not supported.
+
+# Examples
+```julia
+begin_time_series_update(sys) do
+    add_time_series!(sys, component1, time_series1)
+    add_time_series!(sys, component2, time_series2)
+end
+```
+"""
+begin_time_series_update(func::Function, sys::System) =
+    IS.begin_time_series_update(func, sys.data.time_series_manager)
+
 """
 Add time series data from a metadata file or metadata descriptors.
 
@@ -1383,17 +1402,9 @@ function add_time_series!(
 end
 
 """
-Add many time series in bulk
+Add time series in bulk.
 
-This method is advantageous when adding thousands of time
-series arrays because of the overhead in writing the time series to the underlying storage.
-
-# Arguments
-- `sys::System`: system
-- `associations`: Iterable of [`TimeSeriesAssociation`](@ref) instances. Using a Vector is not
-  recommended. Pass a Generator or Iterator to avoid loading all time series data into
-  system memory at once.
-- `batch_size::Int`: (Default = 100) Number of time series to add per batch.
+Prefer use of [`begin_time_series_update!`](@ref).
 
 # Examples
 ```julia
@@ -1412,9 +1423,6 @@ associations = (
 )
 bulk_add_time_series!(sys, associations)
 ```
-
-See also: [`open_time_series_store!`](@ref) to minimize HDF5 file handle overhead if you
-must add time series arrays one at a time
 """
 function bulk_add_time_series!(
     sys::System,
@@ -1630,6 +1638,23 @@ function add_supplemental_attribute!(
     return IS.add_supplemental_attribute!(sys.data, component, attribute)
 end
 
+"""
+Begin an update of supplemental attributes. Use this function when adding
+or removing many supplemental attributes in order to improve performance.
+
+If an error occurs during the update, changes will be reverted.
+
+# Examples
+```julia
+begin_supplemental_attributes_update(sys) do
+    add_supplemental_attribute!(sys, component1, attribute1)
+    add_supplemental_attribute!(sys, component2, attribute2)
+end
+```
+"""
+begin_supplemental_attributes_update(func::Function, sys::System) =
+    IS.begin_supplemental_attributes_update(func, sys.data.supplemental_attribute_manager)
+
 """
 Remove the supplemental attribute from the component. The attribute will be removed from the
 system if it is not attached to any other component.

test/common.jl

@@ -175,33 +175,37 @@ function create_system_with_outages()
     gen2 = gens[2]
     geo1 = GeographicInfo(; geo_json = Dict("x" => 1.0, "y" => 2.0))
     geo2 = GeographicInfo(; geo_json = Dict("x" => 3.0, "y" => 4.0))
-    add_supplemental_attribute!(sys, gen1, geo1)
-    add_supplemental_attribute!(sys, gen1.bus, geo1)
-    add_supplemental_attribute!(sys, gen2, geo2)
-    add_supplemental_attribute!(sys, gen2.bus, geo2)
-    initial_time = Dates.DateTime("2020-01-01T00:00:00")
-    end_time = Dates.DateTime("2020-01-01T23:00:00")
-    dates = collect(initial_time:Dates.Hour(1):end_time)
-    fo1 = GeometricDistributionForcedOutage(;
-        mean_time_to_recovery = 1.0,
-        outage_transition_probability = 0.5,
-    )
-    fo2 = GeometricDistributionForcedOutage(;
-        mean_time_to_recovery = 2.0,
-        outage_transition_probability = 0.5,
-    )
-    po1 = PlannedOutage(; outage_schedule = "1")
-    po2 = PlannedOutage(; outage_schedule = "2")
-    add_supplemental_attribute!(sys, gen1, fo1)
-    add_supplemental_attribute!(sys, gen1, po1)
-    add_supplemental_attribute!(sys, gen2, fo2)
-    add_supplemental_attribute!(sys, gen2, po2)
-    for (i, outage) in enumerate((fo1, fo2, po1, po2))
-        data = collect(i:(i + 23))
-        ta = TimeSeries.TimeArray(dates, data, ["1"])
-        name = "ts_$(i)"
-        ts = SingleTimeSeries(; name = name, data = ta)
-        add_time_series!(sys, outage, ts)
+    begin_time_series_update(sys) do
+        begin_supplemental_attributes_update(sys) do
+            add_supplemental_attribute!(sys, gen1, geo1)
+            add_supplemental_attribute!(sys, gen1.bus, geo1)
+            add_supplemental_attribute!(sys, gen2, geo2)
+            add_supplemental_attribute!(sys, gen2.bus, geo2)
+            initial_time = Dates.DateTime("2020-01-01T00:00:00")
+            end_time = Dates.DateTime("2020-01-01T23:00:00")
+            dates = collect(initial_time:Dates.Hour(1):end_time)
+            fo1 = GeometricDistributionForcedOutage(;
+                mean_time_to_recovery = 1.0,
+                outage_transition_probability = 0.5,
+            )
+            fo2 = GeometricDistributionForcedOutage(;
+                mean_time_to_recovery = 2.0,
+                outage_transition_probability = 0.5,
+            )
+            po1 = PlannedOutage(; outage_schedule = "1")
+            po2 = PlannedOutage(; outage_schedule = "2")
+            add_supplemental_attribute!(sys, gen1, fo1)
+            add_supplemental_attribute!(sys, gen1, po1)
+            add_supplemental_attribute!(sys, gen2, fo2)
+            add_supplemental_attribute!(sys, gen2, po2)
+            for (i, outage) in enumerate((fo1, fo2, po1, po2))
+                data = collect(i:(i + 23))
+                ta = TimeSeries.TimeArray(dates, data, ["1"])
+                name = "ts_$(i)"
+                ts = SingleTimeSeries(; name = name, data = ta)
+                add_time_series!(sys, outage, ts)
+            end
+        end
     end
 
     return sys

test/test_system.jl

@@ -289,6 +289,39 @@ end
     end
 end
 
+@testset "Test begin_time_series_update" begin
+    sys = System(100.0)
+    bus = ACBus(nothing)
+    bus.bustype = ACBusTypes.REF
+    add_component!(sys, bus)
+    components = []
+    len = 2
+    component = ThermalStandard(nothing)
+    component.name = "gen"
+    component.bus = bus
+    add_component!(sys, component)
+    initial_time = Dates.DateTime("2020-09-01")
+    resolution = Dates.Hour(1)
+    len = 24
+    timestamps = range(initial_time; length = len, step = resolution)
+    arrays = [TimeSeries.TimeArray(timestamps, rand(len)) for _ in 1:5]
+    ts_name = "test"
+
+    begin_time_series_update(sys) do
+        for (i, ta) in enumerate(arrays)
+            ts = SingleTimeSeries(; data = ta, name = "$(ts_name)_$(i)")
+            add_time_series!(sys, component, ts)
+        end
+    end
+
+    open_time_series_store!(sys, "r") do
+        for (i, expected_array) in enumerate(arrays)
+            ts = IS.get_time_series(IS.SingleTimeSeries, component, "$(ts_name)_$(i)")
+            @test ts.data == expected_array
+        end
+    end
+end
+
 @testset "Test set_name! of system component" begin
     sys = System(100.0)
     bus = ACBus(nothing)