WIP

cimsparql/data_models.py

@@ -22,7 +22,8 @@ class FullModelSchema(JsonSchemaOut):
 
     @pa.dataframe_check
     def unique_time(cls, df: DataFrame) -> bool:
-        return len(df["time"].unique()) == 1
+        n = len(df["time"].unique())
+        return n == 1 or n == 0
 
 
 FullModelDataFrame = DataFrame[FullModelSchema]
@@ -76,8 +77,6 @@ class BusDataSchema(JsonSchemaOut):
 
 class LoadsSchema(NamedResourceSchema):
     node: Series[str] = pa.Field()
-    substation_mrid: Series[str] = pa.Field()
-    bidzone: Series[str] = pa.Field(nullable=True)
     status: Series[bool] = pa.Field()
     p: Series[float] = pa.Field(nullable=True)
     q: Series[float] = pa.Field(nullable=True)
@@ -87,7 +86,7 @@ class LoadsSchema(NamedResourceSchema):
 LoadsDataFrame = DataFrame[LoadsSchema]
 
 
-class WindGeneratingUnitsSchema(NamedMarketResourceSchema):
+class WindGeneratingUnitsSchema(NamedResourceSchema):
     station_group: Series[str] = pa.Field(nullable=True)
     min_p: Series[float] = pa.Field()
     max_p: Series[float] = pa.Field()
@@ -97,7 +96,7 @@ class WindGeneratingUnitsSchema(NamedMarketResourceSchema):
 WindGeneratingUnitsDataFrame = DataFrame[WindGeneratingUnitsSchema]
 
 
-class SynchronousMachinesSchema(NamedMarketResourceSchema):
+class SynchronousMachinesSchema(NamedResourceSchema):
     node: Series[str] = pa.Field()
     status: Series[bool] = pa.Field()
     station_group: Series[str] = pa.Field(nullable=True)
@@ -106,7 +105,6 @@ class SynchronousMachinesSchema(NamedMarketResourceSchema):
     maxP: Series[float] = pa.Field(nullable=True)  # noqa N815
     minP: Series[float] = pa.Field(nullable=True)  # noqa N815
     MO: Series[float] = pa.Field(nullable=True)
-    bidzone: Series[str] = pa.Field(nullable=True)
     sn: Series[float] = pa.Field()
     p: Series[float] = pa.Field(nullable=True)
     q: Series[float] = pa.Field(nullable=True)
@@ -174,8 +172,6 @@ class CoordinatesSchema(JsonSchemaOut):
 
 
 class BranchComponentSchema(NamedResourceSchema):
-    bidzone_1: Series[str] = pa.Field(nullable=True)
-    bidzone_2: Series[str] = pa.Field(nullable=True)
     node_1: Series[str] = pa.Field()
     node_2: Series[str] = pa.Field()
     ploss_1: Series[float] = pa.Field(nullable=True)
@@ -290,9 +286,13 @@ class StationGroupCodeNameSchema(JsonSchemaOut):
 StationGroupCodeNameDataFrame = DataFrame[StationGroupCodeNameSchema]
 
 
-class HVDCBidzonesSchema(MridResourceSchema):
-    from_area: Series[str] = pa.Field()
-    to_area: Series[str] = pa.Field()
+class HVDCBidzonesSchema(JsonSchemaOut):
+    bidzone_1: Series[str] = pa.Field()
+    bidzone_2: Series[str] = pa.Field()
+    converter_1: Series[str] = pa.Field()
+    converter_2: Series[str] = pa.Field()
+    name_1: Series[str] = pa.Field()
+    name_2: Series[str] = pa.Field()
 
 
 HVDCBidzonesDataFrame = DataFrame[HVDCBidzonesSchema]

cimsparql/model.py

@@ -526,9 +526,8 @@ async def dc_active_flow(self, region: str | None = None) -> DcActiveFlowDataFra
         query = self.dc_active_flow_query(region)
         df = await self.get_table_and_convert(query)
         # Unable to group on max within the sparql query so we do it here.
-        df = df.iloc[df.groupby("mrid")["p"].idxmax()].set_index("mrid")
-        df["p"] *= df["direction"]
-        return DcActiveFlowDataFrame(df.drop(columns="direction"))
+        df = df.iloc[df.eval("p_abs = abs(p)").groupby("mrid")["p_abs"].idxmax()].set_index("mrid")
+        return DcActiveFlowDataFrame(df)
 
     @property
     def sv_injection_query(self) -> str:
@@ -568,7 +567,7 @@ async def hvdc_converter_bidzones(self) -> HVDCBidzonesDataFrame:
         Fetching mrid of converters placed on HVDC exchange corridors together with
         to/from bidzone
         """
-        df = await self.get_table_and_convert(self.hvdc_converter_bidzone_query, index="mrid")
+        df = await self.get_table_and_convert(self.hvdc_converter_bidzone_query)
         return HVDCBidzonesDataFrame(df)
 
     @property

cimsparql/sparql/ac_lines.sparql

@@ -1,7 +1,7 @@
 # Name: AC Lines
 PREFIX cim: <${cim}>
-PREFIX SN: <${SN}>
 PREFIX xsd: <${xsd}>
+PREFIX pti:<${pti}>
 select
 # Below a lot of max aggregation is performed. For variables without any suffix (_1 og _2) all values within the
 # group are equal and we might as well just pick the maximum value. For variables with the suffix _1 og _2, the
@@ -11,8 +11,6 @@ select
 # thus max-aggregation will pick True if it exists in the group
 (max(?mrid) as ?mrid)
 (max(?name) as ?name)
-(max(?bidzone_1) as ?bidzone_1)
-(max(?bidzone_2) as ?bidzone_2)
 (max(?node_1) as ?node_1)
 (max(?node_2) as ?node_2)
 
@@ -43,19 +41,16 @@ where {
 
   ?terminal cim:Terminal.ConductingEquipment ?acline;
             cim:Terminal.ConnectivityNode ?con_node;
-            cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber ?nr .
+            cim:ACDCTerminal.sequenceNumber ?nr .
 
   # Find substation of each connectivity node of the terminals above
   ?con_node cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation ?substation .
 
   # Find area and optionally bidzone for each substation
   ?substation cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area .
-  optional {
-    ?substation SN:Substation.MarketDeliveryPoint/SN:MarketDeliveryPoint.BiddingArea/SN:BiddingArea.marketCode ?bidzone .
-  }
 
   optional {?acline cim:ACLineSegment.gch ?g .}
-  optional {?acline SN:Equipment.networkAnalysisEnable ?analysis_enabled .}
+  optional {?acline pti:Equipment.excludeFromCase ?exclude_from_case} .
 
   # If exists, extract active power limit for acline
   optional {
@@ -83,15 +78,12 @@ where {
   bind(if(?nr = 1, str(?p), '') as ?p_1)
   bind(if(?nr = 1, ?node, '') as ?node_1)
   bind(if(?nr = 1, ?connected, False) as ?connected_1)
-  bind(if(?nr = 1, ?bidzone, '') as ?bidzone_1)
   bind(if(?nr = 2, str(?p), '') as ?p_2)
   bind(if(?nr = 2, ?node, '') as ?node_2)
   bind(if(?nr = 2, ?connected, False) as ?connected_2)
-  bind(if(?nr = 2, ?bidzone, '') as  ?bidzone_2)
   filter(regex(?area, '${region}'))
 } group by ?acline
 # Filtration rules
 # 1) We don't need lines connecting nodes to themselves
 # 2) Only extract lines where at least two nodes exist
-# 3) Only extract lines where SN:Equipment.networkAnalysisEnable is True (if the field exists)
-having((max(?node_1) != max(?node_2)) && (max(?node_1) != "") && (max(?node_2) != "") && (count(*) > 1) && coalesce(max(?analysis_enabled), True))
+having((max(?node_1) != max(?node_2)) && (max(?node_1) != "") && (max(?node_2) != "") && (count(*) > 1) && coalesce(max(!?exclude_from_case), true))

cimsparql/sparql/branch_node_withdraw.sparql

@@ -1,6 +1,6 @@
 # Name: Branch node withdraw
 PREFIX cim:<${cim}>
-PREFIX SN:<${SN}>
+PREFIX pti:<${pti}>
 select ?mrid ?node ?p ?q
 where {
   # Extract properties of each terminal from the SV/TP/SSH profile.
@@ -18,7 +18,7 @@ where {
     # Specify components wer are interested in
     values ?rdf_type {cim:ACLineSegment cim:SeriesCompensator cim:PowerTransformer} .
     ?component a ?rdf_type .
-    optional {?component SN:Equipment.networkAnalysisEnable ?_analysis_enabled .}
+    optional {?component pti:Equipment.excludeFromCase ?_exclude_from_case} .
     ?terminal cim:Terminal.ConductingEquipment ?component;
             cim:IdentifiedObject.mRID ?mrid;
             cim:Terminal.ConnectivityNode ?con_node.
@@ -29,13 +29,13 @@ where {
   }
 
   # Assign analysis enabled (all True if not exist)
-  bind(coalesce(?_analysis_enabled, True) as ?analysis_enabled)
+  bind(coalesce(?_exclude_from_case, false) as ?exclude_from_case)
 
   # Assign an mRID to the ?node variable. The first that exists is used
   # 1) mRID of the topological node associated with the connectivity node the terminal
   # 2) mrID of the topoligical node associated with the terminel
   # 3) mRID of the terminal
   bind(coalesce(?con_top_node_mrid, ?term_top_node_mrid, ?mrid) as ?node)
-  filter(?analysis_enabled)
+  filter(!?exclude_from_case)
   filter(?connected)
 }

cimsparql/sparql/bus.sparql

@@ -1,6 +1,8 @@
 # Name: Bus
 PREFIX cim:<${cim}>
-PREFIX SN:<${SN}>
+PREFIX entsoe:<${entsoe}>
+PREFIX entsoe2:<${entsoe2}>
+PREFIX pti:<${pti}>
 select (?mrid as ?node) ?busname (?substation_name as ?substation) ?un ?substation_mrid ?bidzone ?sv_voltage ?island ?is_swing_bus
 where {
   # Extract propertoes from the TP/SV/SSH profile for each topological node
@@ -28,6 +30,6 @@ where {
 	       cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area .
 
   # Extract the bidzone of each substation if it exists
-  optional {?_substation SN:Substation.MarketDeliveryPoint/SN:MarketDeliveryPoint.BiddingArea/SN:BiddingArea.marketCode ?bidzone} .
+  optional {?_substation pti:Substation.EnergySchedulingArea/entsoe2:EnergySchedulingArea.EnergyCongestionZone/entsoe:IdentifiedObject.energyIdentCodeEic ?bidzone} .
   FILTER regex(?area, '${region}')
 }

cimsparql/sparql/converter_hvdc_bidzones.sparql

@@ -1,19 +1,26 @@
 # Name: Converter HVDC bidzones
 PREFIX cim:<${cim}>
-PREFIX SN:<${SN}>
-PREFIX ALG:<${ALG}>
+PREFIX pti:<${pti}>
+PREFIX entsoe2:<${entsoe2}>
+PREFIX entsoe:<${entsoe}>
+select distinct ?bidzone_1 ?bidzone_2 ?converter_1 ?converter_2 ?name_1 ?name_2
+where {
+  ?line a cim:DCLineSegment;
+        cim:IdentifiedObject.name ?name.
 
-select distinct ?mrid ?from_area ?to_area
-where
-{
-  ?converter ALG:DCConverter.DCPole | ALG:VoltageSourceConverter.DCPole ?pole ;
-      cim:IdentifiedObject.mRID ?mrid .
-  ?pole cim:Equipment.EquipmentContainer/SN:Substation.MarketDeliveryPoint ?delivery_point.
+  ?terminal_1 cim:DCTerminal.DCConductingEquipment ?line;
+              cim:Terminal.sequenceNumber 1;
+              cim:DCBaseTerminal.DCNode/^cim:DCBaseTerminal.DCNode/cim:ACDCConverterDCTerminal.DCConductingEquipment ?conducting_equipment_1.
+  ?conducting_equipment_1 ^cim:Terminal.ConductingEquipment/cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/pti:Substation.EnergySchedulingArea/entsoe2:EnergySchedulingArea.EnergyCongestionZone/entsoe:IdentifiedObject.energyIdentCodeEic ?bidzone_1;
+			  cim:IdentifiedObject.mRID ?converter_1;
+     cim:IdentifiedObject.name ?name_1.
 
-  ?delivery_point SN:MarketDeliveryPoint.BiddingArea/SN:BiddingArea.marketCode ?from_area .
+  ?terminal_2 cim:DCTerminal.DCConductingEquipment ?line;
+              cim:Terminal.sequenceNumber 2;
+              cim:DCBaseTerminal.DCNode/^cim:DCBaseTerminal.DCNode/cim:ACDCConverterDCTerminal.DCConductingEquipment ?conducting_equipment_2.
+  ?conducting_equipment_2 ^cim:Terminal.ConductingEquipment/cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/pti:Substation.EnergySchedulingArea/entsoe2:EnergySchedulingArea.EnergyCongestionZone/entsoe:IdentifiedObject.energyIdentCodeEic ?bidzone_2;
+			  cim:IdentifiedObject.mRID ?converter_2;
+     cim:IdentifiedObject.name ?name_2.
 
-  # Extract the destination point from the line that has one end matching from_area
-  ?line SN:Line.FromMarketDeliveryPoint ?delivery_point;
-        SN:Line.ToMarketDeliveryPoint/SN:MarketDeliveryPoint.BiddingArea/SN:BiddingArea.marketCode ?to_area
-  filter(?from_area != ?to_area)
+  filter (?bidzone_1 != ?bidzone_2)
 }

cimsparql/sparql/converters.sparql

@@ -1,8 +1,8 @@
 # Name: Converters
 PREFIX cim:<${cim}>
+PREFIX pti:<${pti}>
 PREFIX xsd:<${xsd}>
-PREFIX SN:<${SN}>
-select ?mrid ?name ?alias ?p ?q ?substation_mrid ?status ?node
+select ?mrid ?name ?alias ?p ?q ?substation_mrid ?status ?node ?area
 where {
   # Extract active and reactive power for the converter
   ?converter cim:ACDCConverter.p ?p;
@@ -18,25 +18,25 @@ where {
   optional {?con_node cim:ConnectivityNode.TopologicalNode/cim:IdentifiedObject.mRID ?con_top_node_mrid .}
 
   service <${eq_repo}> {
-  # Extract mrid, name, substation and optionally aliasName of the converter
-  ?converter cim:IdentifiedObject.mRID ?mrid;
-             cim:IdentifiedObject.name ?name;
-             cim:Equipment.EquipmentContainer/cim:VoltageLevel.Substation ?substation .
-  optional {?converter cim:IdentifiedObject.aliasName ?alias .}
-
-  # Find area and mRID for the substation
-  ?substation cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area;
-              cim:IdentifiedObject.mRID ?substation_mrid .
-
-  # Extract properties for the terminals for the converter
-  ?terminal cim:Terminal.ConductingEquipment ?converter;
-            cim:Terminal.ConnectivityNode ?con_node;
-            cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber 1 .
-  optional {?converter SN:Equipment.networkAnalysisEnable ?_analysis_enabled .}
-  filter regex(?area, '${region}')
-  bind(coalesce(?_analysis_enabled, True) as ?analysis_enabled)
-  filter(?analysis_enabled)
+    # Extract mrid, name, substation and optionally aliasName of the converter
+    ?converter cim:IdentifiedObject.mRID ?mrid;
+               cim:IdentifiedObject.name ?name;
+               cim:Equipment.EquipmentContainer/cim:DCConverterUnit.Substation ?substation .
+    optional {?converter cim:IdentifiedObject.aliasName ?alias}
+
+    # Find area and mRID for the substation
+    ?substation cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area;
+		cim:IdentifiedObject.mRID ?substation_mrid .
+
+    # Extract properties for the terminals for the converter
+    ?terminal cim:Terminal.ConductingEquipment ?converter;
+              cim:Terminal.ConnectivityNode ?con_node .
+    filter regex(?area, '${region}')
   }
+  optional {?converter pti:Equipment.excludeFromCase ?_exclude_from_case} .
+  bind(coalesce(?_exclude_from_case, false) as ?exclude_from_case)
+  filter (!?exclude_from_case)
+
 
   # Asiign an mrid to the ?node variable. The first that exists is choden
   # 1) mRID of the topological node associated with the connectivity node