Skip to content

Commit

Permalink
Give warning if runpath disk space is close to full on ert startup
Browse files Browse the repository at this point in the history
  • Loading branch information
larsevj committed Nov 12, 2024
1 parent b5d3671 commit 678f04d
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 2 deletions.
38 changes: 37 additions & 1 deletion src/ert/config/model_config.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,25 @@
from __future__ import annotations

import contextlib
import logging
import os.path
import shutil
from datetime import datetime
from pathlib import Path
from typing import List, Optional, no_type_check

from pydantic import field_validator
from pydantic.dataclasses import dataclass

from .parsing import ConfigDict, ConfigKeys, ConfigValidationError, HistorySource
from ert.shared.status.utils import byte_with_unit

from .parsing import (
ConfigDict,
ConfigKeys,
ConfigValidationError,
ConfigWarning,
HistorySource,
)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -37,6 +48,9 @@ def str_to_datetime(date_str: str) -> datetime:
DEFAULT_JOBNAME_FORMAT = "<CONFIG_FILE>-<IENS>"
DEFAULT_ECLBASE_FORMAT = "ECLBASE<IENS>"

FULL_DISK_PERCENTAGE_THRESHOLD = 0.97
MINIMUM_BYTES_LEFT_ON_DISK = 200 * 1024**3 # 200 GB required


@dataclass
class ModelConfig:
Expand Down Expand Up @@ -75,6 +89,19 @@ def validate_runpath(cls, runpath_format_string: str) -> str:
f"`{runpath_format_string}`. Valid example: "
f"`{DEFAULT_RUNPATH}` "
)
with contextlib.suppress(Exception):
mount_dir = _get_mount_directory(runpath_format_string)
total, used, free = shutil.disk_usage(mount_dir)
percentage_used = used / total
if (
percentage_used > FULL_DISK_PERCENTAGE_THRESHOLD
and free < MINIMUM_BYTES_LEFT_ON_DISK
):
msg = (
f"Little space left in runpath, only {byte_with_unit(free)} free on {mount_dir !s}."
" Consider cleaning up disk before running simulations."
)
ConfigWarning.warn(msg)
return result

@field_validator("jobname_format_string", mode="before")
Expand Down Expand Up @@ -134,3 +161,12 @@ def _replace_runpath_format(format_string: str) -> str:
format_string = format_string.replace("%d", "<IENS>", 1)
format_string = format_string.replace("%d", "<ITER>", 1)
return format_string


def _get_mount_directory(runpath: str) -> Path:
path = Path(runpath).absolute()

while not path.is_mount():
path = path.parent

return path
15 changes: 14 additions & 1 deletion tests/ert/unit_tests/config/test_model_config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
from pathlib import Path
from unittest.mock import patch

import pytest

from ert.config import ModelConfig
from ert.config.parsing import ConfigKeys, ConfigValidationError
from ert.config.parsing import ConfigKeys, ConfigValidationError, ConfigWarning


def test_default_model_config_run_path(tmpdir):
Expand Down Expand Up @@ -61,3 +64,13 @@ def test_that_invalid_time_map_file_raises_config_validation_error(tmpdir):

with pytest.raises(ConfigValidationError, match="Could not read timemap file"):
_ = ModelConfig.from_dict({ConfigKeys.TIME_MAP: "time_map.txt"})


def test_warning_when_full_disk(tmp_path):
Path(tmp_path / "simulations").mkdir()
runpath = "simulations/realization-%d/iter-%d"
msg = "Little space left in runpath, only 2.00 B free on"
with patch(
"ert.config.model_config.shutil.disk_usage", return_value=(100, 98, 2)
), pytest.warns(ConfigWarning, match=msg):
_ = ModelConfig(num_realizations=1, runpath_format_string=runpath)

0 comments on commit 678f04d

Please sign in to comment.