Skip to content

Commit

Permalink
added moisture support to coerce_estimates
Browse files Browse the repository at this point in the history
  • Loading branch information
@elphick
elphick committed Oct 27, 2024
1 parent 37e6f5b commit 0738ec6
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 121 deletions.
28 changes: 27 additions & 1 deletion elphick/geomet/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ def data(self, value):
mass_wet=self.mass_wet_var, mass_dry=self.mass_dry_var,
moisture_column_name=self.moisture_column,
component_columns=composition.columns,
composition_units=self.composition_units)
composition_units=self.composition_units,
return_moisture=False)
self._logger.debug(f"Data has been set.")

else:
Expand All @@ -138,6 +139,22 @@ def mass_data(self, value):
# Recalculate the aggregate whenever the data changes
self.aggregate = self.weight_average()

def get_mass_data(self, include_moisture: bool = True) -> pd.DataFrame:
"""Get the mass data
Args:
include_moisture: If True (and moisture is in scope), include the moisture mass column
Returns:
"""
if include_moisture and self.moisture_in_scope:
moisture_mass = self._mass_data[self.mass_wet_var] - self._mass_data[self.mass_dry_var]
mass_data: pd.DataFrame = self._mass_data.copy()
mass_data.insert(loc=2, column=self.moisture_column, value=moisture_mass)
return mass_data
return self._mass_data

@property
def aggregate(self) -> pd.DataFrame:
if self._aggregate is None and self._mass_data is not None:
Expand Down Expand Up @@ -551,6 +568,10 @@ def add(self, other: MC, name: Optional[str] = None,

res: MC = self.create_congruent_object(name=name, include_mc_data=True,
include_supp_data=include_supplementary_data)

if set(self._mass_data.columns) != set(other._mass_data.columns):
raise ValueError(f"Mass data columns do not match: {set(self._mass_data.columns)} != "
f"{set(other._mass_data.columns)}")
res.update_mass_data(self._mass_data + other._mass_data)

# Ensure self and other are Stream objects
Expand Down Expand Up @@ -578,6 +599,11 @@ def sub(self, other: MC, name: Optional[str] = None,
"""
res = self.create_congruent_object(name=name, include_mc_data=True,
include_supp_data=include_supplementary_data)

if set(self._mass_data.columns) != set(other._mass_data.columns):
raise ValueError(f"Mass data columns do not match: {set(self._mass_data.columns)} != "
f"{set(other._mass_data.columns)}")

res.update_mass_data(self._mass_data - other._mass_data)

# Ensure self and other are Stream objects
Expand Down
135 changes: 32 additions & 103 deletions elphick/geomet/utils/estimates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
from elphick.geomet.base import MassComposition
from elphick.geomet.flowsheet import Flowsheet
from elphick.geomet.flowsheet.stream import Stream
from elphick.geomet.utils.moisture import solve_mass_moisture
from elphick.geomet.utils.pandas import composition_to_mass


def coerce_output_estimates(estimate_stream: Stream, input_stream: Stream,
recovery_bounds: tuple[float, float] = (0.01, 0.99),
complement_name: str = 'complement',
show_plot: bool = False) -> Stream:
def coerce_estimates(estimate_stream: Stream, input_stream: Stream,
recovery_bounds: tuple[float, float] = (0.01, 0.99),
complement_name: str = 'complement',
show_plot: bool = False) -> Stream:
"""Coerce output estimates within recovery and the component range.
Estimates contain error and at times can exceed the specified component range, or can consume more component
Expand Down Expand Up @@ -40,21 +41,38 @@ def coerce_output_estimates(estimate_stream: Stream, input_stream: Stream,
The coerced estimate stream
"""

if show_plot:
complement_stream: MassComposition = input_stream.sub(estimate_stream, name=complement_name)
fs: Flowsheet = Flowsheet.from_objects([input_stream, estimate_stream, complement_stream])
fig = fs.table_plot(plot_type='network')
fig.update_layout(title=f"{fs.name}: Balance prior to coercion").show()

if estimate_stream.status.ok is False:
logging.info(str(estimate_stream.status))

if input_stream.status.ok is False:
raise ValueError('Input stream is not OK')

# calculate the recovery
cols: list[str] = [estimate_stream.mass_dry_var] + estimate_stream.composition_columns
recovery: pd.DataFrame = estimate_stream.mass_data[cols] / input_stream.mass_data[cols]
if estimate_stream.moisture_in_scope:
recovery: pd.DataFrame = estimate_stream.get_mass_data().drop(
columns=estimate_stream.mass_wet_var) / input_stream.get_mass_data().drop(columns=input_stream.mass_wet_var)
else:
recovery: pd.DataFrame = estimate_stream._mass_data / input_stream._mass_data

# limit the recovery to the bounds
recovery = recovery.clip(lower=recovery_bounds[0], upper=recovery_bounds[1])

# recalculate the estimate from the bound recovery
new_mass: pd.DataFrame = recovery * input_stream.mass_data[cols]
if estimate_stream.moisture_in_scope:
new_mass: pd.DataFrame = recovery * input_stream.get_mass_data()[recovery.columns]
# calculate wmt and drop h2o, to match what is stored in _mass_data
new_mass.insert(loc=0, column=estimate_stream.mass_wet_var,
value=new_mass[estimate_stream.mass_dry_var] + new_mass[estimate_stream.moisture_column])
new_mass.drop(columns=estimate_stream.moisture_column, inplace=True)
else:
new_mass: pd.DataFrame = recovery * input_stream.mass_data

estimate_stream.update_mass_data(new_mass)

if estimate_stream.status.ok is False:
Expand All @@ -65,10 +83,12 @@ def coerce_output_estimates(estimate_stream: Stream, input_stream: Stream,
if complement_stream.status.ok is False:

# adjust the complement to be within the component
new_complement_composition = complement_stream.data[complement_stream.composition_columns]
cols = [col for col in complement_stream.data.columns if col not in [complement_stream.mass_wet_var]]
new_complement_composition = complement_stream.data[cols]
for comp, comp_range in complement_stream.status.ranges.items():
new_complement_composition[comp] = new_complement_composition[comp].clip(comp_range[0], comp_range[1])
new_component_mass: pd.DataFrame = composition_to_mass(new_complement_composition,
mass_wet=complement_stream.mass_wet_var,
mass_dry=complement_stream.mass_dry_var)
complement_stream.update_mass_data(new_component_mass)

Expand All @@ -79,104 +99,13 @@ def coerce_output_estimates(estimate_stream: Stream, input_stream: Stream,
if estimate_stream.status.ok is False:
raise ValueError('Estimate stream is not OK after adjustment')

fs: Flowsheet = Flowsheet.from_objects([input_stream, estimate_stream, complement_stream])

if show_plot:
fs.table_plot(plot_type='network').show()

if fs.balanced is False:
raise ValueError('Flowsheet is not balanced after adjustment')

return estimate_stream


def coerce_input_estimates(estimate_stream: Stream, input_stream: Stream,
recovery_bounds: tuple[float, float] = (0.01, 0.99),
output_name: str = 'output',
show_plot: bool = False) -> Stream:
"""Coerce input estimates within recovery and the component range.
Estimates contain error and at times can exceed the specified component range, or can consume more component
mass than is available in the feed. This function modifies (coerces) only the non-compliant estimate records
in order to balance the node and keep all components within range.
estimate_stream (supplied)
\
+ ──> output_stream
/
input_stream (supplied)
1. limits the estimate to within the recovery bounds,
2. ensures the estimate is within the component range,
3. solves the output, and ensures it is in range,
4. if the output is out of range, it is adjusted and the estimate adjusted to maintain the balance.
Args:
estimate_stream: The estimated object, which is a node output
input_stream: The input object, which is a node input
recovery_bounds: The bounds for the recovery, default is 0.01 to 0.99
output_name: The name of the output stream
show_plot: If True, show the network plot
Returns:
The coerced estimate stream
"""

if estimate_stream.status.ok is False:
logging.info(str(estimate_stream.status))

if input_stream.status.ok is False:
raise ValueError('Input stream is not OK')

output_stream: Stream = input_stream.add(estimate_stream, name=output_name)

fs: Flowsheet = Flowsheet.from_objects([input_stream, estimate_stream, output_stream])

if show_plot:
fig = fs.table_plot(plot_type='network')
fig.update_layout(title=f"{fs.name}: Balanced={fs.balanced}").show()

# calculate the recovery of the estimate relative to the output
cols: list[str] = [estimate_stream.mass_dry_var] + estimate_stream.composition_columns
recovery: pd.DataFrame = estimate_stream.mass_data[cols] / output_stream.mass_data[cols]

# limit the recovery to the bounds
recovery = recovery.clip(lower=recovery_bounds[0], upper=recovery_bounds[1])

# recalculate the estimate from the bound recovery
new_mass: pd.DataFrame = recovery * output_stream.mass_data[cols]
estimate_stream.update_mass_data(new_mass)

if estimate_stream.status.ok is False:
raise ValueError('Estimate stream is not OK - it should be after bounding recovery')

# re-calculate the output
output_stream: Stream = input_stream.add(estimate_stream, name=output_name)
if output_stream.status.ok is False:

# adjust the complement to be within the component without creating a new method
new_output_composition = output_stream.data[output_stream.composition_columns]
for comp, comp_range in output_stream.status.ranges.items():
new_output_composition[comp] = new_output_composition[comp].clip(comp_range[0], comp_range[1])
new_component_mass: pd.DataFrame = composition_to_mass(new_output_composition,
mass_dry=output_stream.mass_dry_var)
output_stream.update_mass_data(new_component_mass)

# adjust the estimate to maintain the balance
estimate_stream = output_stream.sub(input_stream, name=estimate_stream.name,
include_supplementary_data=True)

if estimate_stream.status.ok is False:
raise ValueError('Estimate stream is not OK after adjustment')

fs_out: Flowsheet = Flowsheet.from_objects([input_stream, estimate_stream, output_stream])
fs2: Flowsheet = Flowsheet.from_objects([input_stream, estimate_stream, complement_stream])

if show_plot:
fig = fs_out.table_plot(plot_type='network')
fig.update_layout(title=f"Coerced {fs_out.name}: Balanced={fs_out.balanced}").show()
fig = fs2.table_plot(plot_type='network')
fig.update_layout(title=f"{fs2.name}: Coerced Estimates").show()

if fs_out.balanced is False:
if fs2.balanced is False:
raise ValueError('Flowsheet is not balanced after adjustment')

return estimate_stream
39 changes: 22 additions & 17 deletions tests/test_014_coerce_estimates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,47 +2,52 @@

from elphick.geomet import Sample
from elphick.geomet.flowsheet.stream import Stream
from elphick.geomet.utils.estimates import coerce_input_estimates, coerce_output_estimates
from elphick.geomet.utils.estimates import coerce_estimates
from fixtures import sample_data as test_data


def test_coerce_output_estimate(test_data):
def test_coerce_output_estimate_with_moisture(test_data):
data: pd.DataFrame = test_data

obj_input: Stream = Stream(data=data, name='feed', moisture_in_scope=False, mass_dry_var='mass_dry')
obj_input: Stream = Stream(data=data, name='feed', moisture_in_scope=True, mass_dry_var='mass_dry')

df_est: pd.DataFrame = data.copy()
df_est['mass_dry'] = df_est['mass_dry'] * 0.95
df_est[['wet_mass', 'mass_dry']] = df_est[['wet_mass', 'mass_dry']] * 0.95
df_est['Fe'] = df_est['Fe'] * 1.3
df_est[['SiO2', 'Al2O3', 'LOI']] = df_est[['SiO2', 'Al2O3', 'LOI']] * 0.8
obj_est: Stream = Stream(data=df_est, name='estimate', moisture_in_scope=False, mass_dry_var='mass_dry')
obj_est: Stream = Stream(data=df_est, name='estimate', moisture_in_scope=True, mass_dry_var='mass_dry')

obj_coerced: Stream = coerce_output_estimates(estimate_stream=obj_est, input_stream=obj_input, show_plot=True)
obj_coerced: Stream = coerce_estimates(estimate_stream=obj_est, input_stream=obj_input, show_plot=True)

expected: pd.DataFrame = pd.DataFrame.from_dict(
{'mass_dry': {0: 85.5, 1: 76.0, 2: 85.5}, 'Fe': {0: 59.4, 1: 61.4842105263158, 2: 63.56842105263157},
'SiO2': {0: 4.16, 1: 2.4800000000000004, 2: 1.7599999999999998},
{'wet_mass': {0: 95.0, 1: 85.5, 2: 104.5}, 'mass_dry': {0: 85.5, 1: 76.0, 2: 85.5},
'H2O': {0: 10.0, 1: 11.11111111111111, 2: 18.181818181818183},
'Fe': {0: 59.4, 1: 61.4842105263158, 2: 63.56842105263157}, 'SiO2': {0: 4.16, 1: 2.4800000000000004, 2: 1.76},
'Al2O3': {0: 2.4, 1: 1.36, 2: 0.7200000000000002}, 'LOI': {0: 4.0, 1: 3.2000000000000006, 2: 2.4},
'group': {0: 'grp_1', 1: 'grp_1', 2: 'grp_2'}})
expected.index.name = 'index'

pd.testing.assert_frame_equal(obj_coerced.data, expected)


def test_coerce_input_estimate(test_data):
def test_coerce_output_estimate_no_moisture(test_data):
data: pd.DataFrame = test_data

obj_input: Stream = Stream(data=data, name='input', moisture_in_scope=False, mass_dry_var='mass_dry')
obj_input: Stream = Stream(data=data, name='feed', moisture_in_scope=False, mass_dry_var='mass_dry')

df_est: pd.DataFrame = data.copy().drop(columns='wet_mass')
df_est['mass_dry'] = df_est['mass_dry'] * 0.15
df_est['Fe'] = df_est['Fe'] * 0.8
df_est[['SiO2', 'Al2O3', 'LOI']] = df_est[['SiO2', 'Al2O3', 'LOI']] * 10
df_est: pd.DataFrame = data.copy()
df_est['mass_dry'] = df_est['mass_dry'] * 0.95
df_est['Fe'] = df_est['Fe'] * 1.3
df_est[['SiO2', 'Al2O3', 'LOI']] = df_est[['SiO2', 'Al2O3', 'LOI']] * 0.8
obj_est: Stream = Stream(data=df_est, name='estimate', moisture_in_scope=False, mass_dry_var='mass_dry')

obj_coerced: Stream = coerce_input_estimates(estimate_stream=obj_est, input_stream=obj_input, show_plot=True)
obj_coerced: Stream = coerce_estimates(estimate_stream=obj_est, input_stream=obj_input, show_plot=True)

# we know we did not have to change anything (in this particular test)
expected: pd.DataFrame = df_est.copy()
expected: pd.DataFrame = pd.DataFrame.from_dict(
{'mass_dry': {0: 85.5, 1: 76.0, 2: 85.5}, 'Fe': {0: 59.4, 1: 61.4842105263158, 2: 63.56842105263157},
'SiO2': {0: 4.16, 1: 2.4800000000000004, 2: 1.7599999999999998},
'Al2O3': {0: 2.4, 1: 1.36, 2: 0.7200000000000002}, 'LOI': {0: 4.0, 1: 3.2000000000000006, 2: 2.4},
'group': {0: 'grp_1', 1: 'grp_1', 2: 'grp_2'}})
expected.index.name = 'index'

pd.testing.assert_frame_equal(obj_coerced.data, expected)

0 comments on commit 0738ec6

Please sign in to comment.