From 89ddc2be9d4a226ad697bc6bb68f42bfcc2d162b Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 18 Dec 2025 19:20:47 +0000 Subject: [PATCH 01/13] Add preliminary structure for compression jobs. --- .../tests/data/jobs/compression/__init__.py | 1 + .../data/jobs/compression/compression_jobs.py | 19 +++++++++++++++ .../tests/data/logs/json-multifile/README.md | 16 +++++++++++++ .../logs/sts-135-2011-07-08.jsonl | 8 +++++++ .../logs/sts-135-2011-07-09.jsonl | 8 +++++++ .../logs/sts-135-2011-07-11.jsonl | 8 +++++++ .../logs/sts-135-2011-07-19.jsonl | 8 +++++++ .../logs/sts-135-2011-07-21.jsonl | 8 +++++++ integration-tests/tests/utils/config.py | 24 +++++++++++++++++++ 9 files changed, 100 insertions(+) create mode 100644 integration-tests/tests/data/jobs/compression/__init__.py create mode 100644 integration-tests/tests/data/jobs/compression/compression_jobs.py create mode 100644 integration-tests/tests/data/logs/json-multifile/README.md create mode 100644 integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl create mode 100644 integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl create mode 100644 integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl create mode 100644 integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl create mode 100644 integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl diff --git a/integration-tests/tests/data/jobs/compression/__init__.py b/integration-tests/tests/data/jobs/compression/__init__.py new file mode 100644 index 0000000000..2f46b40375 --- /dev/null +++ b/integration-tests/tests/data/jobs/compression/__init__.py @@ -0,0 +1 @@ +"""Compression jobs to run on the CLP package.""" diff --git a/integration-tests/tests/data/jobs/compression/compression_jobs.py b/integration-tests/tests/data/jobs/compression/compression_jobs.py new file mode 100644 index 0000000000..88f6125fc0 --- /dev/null +++ b/integration-tests/tests/data/jobs/compression/compression_jobs.py @@ -0,0 +1,19 @@ +"""Compression job definitions for CLP package tests.""" + +from pathlib import Path + +from tests.utils.config import PackageCompressionJob + +PACKAGE_COMPRESSION_JOBS: dict[str, PackageCompressionJob] = { + "compress-json-multifile": PackageCompressionJob( + job_name="compress-json-multifile", + mode="clp-json", + script_path=Path("sbin") / "compress.sh", + log_path=Path("json-multifile") / "logs", + flags={ + "--timestamp-key": "timestamp", + "--dataset": "json_multifile", + }, + args=None, + ), +} diff --git a/integration-tests/tests/data/logs/json-multifile/README.md b/integration-tests/tests/data/logs/json-multifile/README.md new file mode 100644 index 0000000000..dd501c1ac4 --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/README.md @@ -0,0 +1,16 @@ +# Description of logs + +**Type** +JSON + +**Number of files** +5 + +**Number of events per file** +8 + +**Beginning timestamp (milliseconds)** +1310138944000 + +**End timestamp (milliseconds)** +1311208074120 diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl new file mode 100644 index 0000000000..1ba45d0ec1 --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310138944000,"mission":"STS-135","mission_day_index":0,"event":"SRB_IGNITION_CONFIRMED","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters report stable ignition and rising thrust","line_index":0} +{"timestamp":1310139148373,"mission":"STS-135","mission_day_index":0,"event":"CLEAR_OF_TOWER","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle cleared tower, roll program initiated, tracking nominal trajectory","line_index":1} +{"timestamp":1310139352746,"mission":"STS-135","mission_day_index":0,"event":"ROLL_PROGRAM_COMPLETE","subsystem":"GUIDANCE","level":"INFO","detail":"Roll program complete, heads down attitude achieved for ascent","line_index":2} +{"timestamp":1310139557119,"mission":"STS-135","mission_day_index":0,"event":"MAX_Q_THROTTLE","subsystem":"PROP","level":"INFO","detail":"Main engines throttled to manage maximum dynamic pressure on stack","line_index":3} +{"timestamp":1310139761493,"mission":"STS-135","mission_day_index":0,"event":"SRB_SEPARATION","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters separation confirmed, separation motors firing nominally","line_index":4} +{"timestamp":1310139965866,"mission":"STS-135","mission_day_index":0,"event":"PRESS_TO_MECO","subsystem":"GUIDANCE","level":"INFO","detail":"Performance marks indicate press to main engine cutoff achieved","line_index":5} +{"timestamp":1310140170239,"mission":"STS-135","mission_day_index":0,"event":"MAIN_ENGINE_CUTOFF","subsystem":"PROP","level":"INFO","detail":"All three SSME engines shutdown, engine parameters within expected bands","line_index":6} +{"timestamp":1310140374613,"mission":"STS-135","mission_day_index":0,"event":"EXTERNAL_TANK_SEPARATION","subsystem":"STRUCTURES","level":"INFO","detail":"External tank separation command executed, umbilical disconnect nominal","line_index":7} diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl new file mode 100644 index 0000000000..89650afd46 --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310169600000,"mission":"STS-135","mission_day_index":1,"event":"RNDZ_BURN","subsystem":"GUIDANCE","level":"INFO","detail":"Rendezvous correction burn complete, relative motion profile matches ISS plan","line_index":0} +{"timestamp":1310170175999,"mission":"STS-135","mission_day_index":1,"event":"RPM_MANEUVER","subsystem":"GUIDANCE","level":"INFO","detail":"R bar pitch maneuver complete for station imaging and TPS assessment","line_index":1} +{"timestamp":1310170751999,"mission":"STS-135","mission_day_index":1,"event":"FINAL_APPROACH","subsystem":"GUIDANCE","level":"INFO","detail":"Final approach corridor maintained, closing rate within docking constraints","line_index":2} +{"timestamp":1310171327999,"mission":"STS-135","mission_day_index":1,"event":"SOFT_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Soft capture confirmed at forward port, relative motion damped","line_index":3} +{"timestamp":1310171903999,"mission":"STS-135","mission_day_index":1,"event":"HARD_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Structural latches engaged, ISS and shuttle in hard dock configuration","line_index":4} +{"timestamp":1310172479999,"mission":"STS-135","mission_day_index":1,"event":"HATCH_OPEN","subsystem":"STRUCTURES","level":"INFO","detail":"Hatch open for ingress, pressure equalization verified within safe band","line_index":5} +{"timestamp":1310173055999,"mission":"STS-135","mission_day_index":1,"event":"MPLM_BERTHING","subsystem":"ROBOTICS","level":"INFO","detail":"Multipurpose logistics module berthed to Node, structural attach points verified","line_index":6} +{"timestamp":1310173631999,"mission":"STS-135","mission_day_index":1,"event":"CARGO_TRANSFER","subsystem":"PAYLOAD","level":"INFO","detail":"Cargo transfer underway according to integrated ISS and shuttle transfer list","line_index":7} diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl new file mode 100644 index 0000000000..cdbabf0945 --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310342400000,"mission":"STS-135","mission_day_index":3,"event":"JOINT_OPERATIONS_PLANNING","subsystem":"TIMELINE","level":"INFO","detail":"Daily planning conference complete, joint tasks synchronized","line_index":0} +{"timestamp":1310342975999,"mission":"STS-135","mission_day_index":3,"event":"EVA_PREP","subsystem":"EVA","level":"INFO","detail":"Extravehicular mobility units serviced, prebreathe protocols initiated","line_index":1} +{"timestamp":1310343551999,"mission":"STS-135","mission_day_index":3,"event":"EVA_TASK_STATUS","subsystem":"EVA","level":"INFO","detail":"Truss and payload bay tasks progressing, consumables within expected usage","line_index":2} +{"timestamp":1310344127999,"mission":"STS-135","mission_day_index":3,"event":"MPLM_UNBERTH","subsystem":"ROBOTICS","level":"INFO","detail":"Logistics module unberthed and reinstalled in payload bay","line_index":3} +{"timestamp":1310344703999,"mission":"STS-135","mission_day_index":3,"event":"ISS_SYSTEMS_HANDOFF","subsystem":"COMMS","level":"INFO","detail":"Command and telemetry handoff between shuttle and station completed","line_index":4} +{"timestamp":1310345279999,"mission":"STS-135","mission_day_index":3,"event":"MIDDECK_PAYLOAD_CHECK","subsystem":"PAYLOAD","level":"INFO","detail":"Middeck experiment racks powered and data recording verified","line_index":5} +{"timestamp":1310345855999,"mission":"STS-135","mission_day_index":3,"event":"ENVIRONMENTAL_MONITORING","subsystem":"ECLSS","level":"INFO","detail":"Atmospheric sampling shows stable CO2 and trace contaminant levels","line_index":6} +{"timestamp":1310346431999,"mission":"STS-135","mission_day_index":3,"event":"JOINT_CREW_MEETING","subsystem":"TIMELINE","level":"INFO","detail":"Joint crew status review completed, no blocking issues reported","line_index":7} diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl new file mode 100644 index 0000000000..c60ff4341a --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1311033600000,"mission":"STS-135","mission_day_index":11,"event":"UNDOCK_COMMAND","subsystem":"DOCKING","level":"INFO","detail":"Undock command executed, structural latches release verified","line_index":0} +{"timestamp":1311034175999,"mission":"STS-135","mission_day_index":11,"event":"SEPARATION_BURNS","subsystem":"GUIDANCE","level":"INFO","detail":"Separation maneuvers complete, increasing range rate from station","line_index":1} +{"timestamp":1311034751999,"mission":"STS-135","mission_day_index":11,"event":"TPS_INSPECTION","subsystem":"THERMAL","level":"INFO","detail":"Thermal protection system sensor data and imagery show no critical damage","line_index":2} +{"timestamp":1311035327999,"mission":"STS-135","mission_day_index":11,"event":"DEORBIT_BURN","subsystem":"PROP","level":"INFO","detail":"Deorbit burn complete, entry interface conditions within design envelope","line_index":3} +{"timestamp":1311035903999,"mission":"STS-135","mission_day_index":11,"event":"ENTRY_INTERFACE","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle at entry interface, guidance following nominal drag corridor","line_index":4} +{"timestamp":1311036479999,"mission":"STS-135","mission_day_index":11,"event":"COMM_BLACKOUT","subsystem":"COMMS","level":"INFO","detail":"Expected communications blackout region entered, tracking via ground radar only","line_index":5} +{"timestamp":1311037055999,"mission":"STS-135","mission_day_index":11,"event":"SUBSONIC_TRANSITION","subsystem":"GUIDANCE","level":"INFO","detail":"Shuttle transitions to subsonic flight, control surfaces active","line_index":6} +{"timestamp":1311037631999,"mission":"STS-135","mission_day_index":11,"event":"WHEELS_STOP","subsystem":"TIMELINE","level":"INFO","detail":"Vehicle stopped on runway, rollout distance within predicted range","line_index":7} diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl new file mode 100644 index 0000000000..f2b7aa9ec8 --- /dev/null +++ b/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1311206400000,"mission":"STS-135","mission_day_index":13,"event":"PAYLOAD_UNLOAD","subsystem":"PAYLOAD","level":"INFO","detail":"Post flight payload processing under way, canisters removed from bay","line_index":0} +{"timestamp":1311206639160,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_SAFE","subsystem":"GROUND","level":"INFO","detail":"Hazardous systems safed, access teams cleared for orbiter entry","line_index":1} +{"timestamp":1311206878320,"mission":"STS-135","mission_day_index":13,"event":"DATA_ARCHIVE","subsystem":"GROUND","level":"INFO","detail":"Flight data downlinked and archived for post mission analysis","line_index":2} +{"timestamp":1311207117480,"mission":"STS-135","mission_day_index":13,"event":"CREW_DEBRIEF","subsystem":"TIMELINE","level":"INFO","detail":"Crew debrief with mission control completed without anomalies","line_index":3} +{"timestamp":1311207356640,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_TOW","subsystem":"GROUND","level":"INFO","detail":"Atlantis towed from runway to Orbiter Processing Facility","line_index":4} +{"timestamp":1311207595800,"mission":"STS-135","mission_day_index":13,"event":"RETIREMENT_CONFIG","subsystem":"GROUND","level":"INFO","detail":"Systems configured for extended safe power down and display conversion","line_index":5} +{"timestamp":1311207834960,"mission":"STS-135","mission_day_index":13,"event":"PUBLIC_EVENT","subsystem":"OUTREACH","level":"INFO","detail":"Crew participates in public event describing program closeout activities","line_index":6} +{"timestamp":1311208074120,"mission":"STS-135","mission_day_index":13,"event":"HARDWARE_REUSE_ASSESSMENT","subsystem":"ENGINEERING","level":"INFO","detail":"Component assessments identify candidates for museum display","line_index":7} diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index ee9fc94a5a..6cef6c85b2 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -5,6 +5,7 @@ import re from dataclasses import dataclass, field, InitVar from pathlib import Path +from typing import Any import yaml from clp_py_utils.clp_config import ( @@ -120,6 +121,29 @@ def stop_script_path(self) -> Path: return self.clp_package_dir / "sbin" / "stop-clp.sh" +@dataclass(frozen=True) +class PackageCompressionJob: + """A compression job for a package test.""" + + # The name of the job. + job_name: str + + # The mode the job should be running in. + mode: str + + # Script path (relative to the CLP package). + script_path: Path + + # The path to the logs relative to tests/data/logs (either a file or directory). + log_path: Path + + # Flags to specify in the command. + flags: dict[str, Any] | None + + # Arguments to specify in the command. + args: list[str] | None + + @dataclass(frozen=True) class PackageConfig: """Metadata for a specific configuration of the CLP package.""" From 6c1facbafd0f295d49c310b6fd4038503313c9b9 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 18 Dec 2025 19:24:59 +0000 Subject: [PATCH 02/13] Add init. --- integration-tests/tests/data/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 integration-tests/tests/data/__init__.py diff --git a/integration-tests/tests/data/__init__.py b/integration-tests/tests/data/__init__.py new file mode 100644 index 0000000000..890528a804 --- /dev/null +++ b/integration-tests/tests/data/__init__.py @@ -0,0 +1 @@ +"""Static data storage relevant to CLP integration tests.""" From 35795366cb1675e902ad1e2359696df8268da6d8 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 18 Dec 2025 19:55:14 +0000 Subject: [PATCH 03/13] Assemble compression job list. --- integration-tests/tests/conftest.py | 11 +++++ .../tests/fixtures/package_config.py | 12 +++++ .../tests/fixtures/package_instance.py | 15 +++++- integration-tests/tests/test_package_start.py | 19 ++++++++ .../tests/utils/clp_job_utils.py | 48 +++++++++++++++++++ integration-tests/tests/utils/config.py | 10 ++++ 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 integration-tests/tests/utils/clp_job_utils.py diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index c1af599c98..62be1d2bf3 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -23,3 +23,14 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="55000", help="Base port for CLP package integration tests.", ) + parser.addoption( + "--job-name-contains", + dest="JOB_NAME_CONTAINS", + help="Filter CLP jobs by their name.", + ) + parser.addoption( + "--no-jobs", + action="store_true", + dest="NO_JOBS", + help="Only validate CLP package start and stop. Do not create or run any test jobs.", + ) diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index 8db533dcf2..9ad6b577f8 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -1,9 +1,13 @@ """Fixtures that create and remove temporary config files for CLP packages.""" +import logging from collections.abc import Iterator import pytest +from tests.utils.clp_job_utils import ( + build_package_job_list, +) from tests.utils.clp_mode_utils import ( get_clp_config_from_mode, get_required_component_list, @@ -11,6 +15,8 @@ from tests.utils.config import PackageConfig, PackagePathConfig from tests.utils.port_utils import assign_ports_from_base +logger = logging.getLogger(__name__) + @pytest.fixture def fixt_package_config( @@ -39,6 +45,11 @@ def fixt_package_config( required_components = get_required_component_list(clp_config_obj) + # Build the job list for this mode and the current job filter. + no_jobs: bool = bool(request.config.option.NO_JOBS) + job_filter: str = request.config.option.JOB_NAME_CONTAINS or "" + package_job_list = None if no_jobs else build_package_job_list(mode_name, job_filter) + # Construct PackageConfig. package_config = PackageConfig( path_config=fixt_package_path_config, @@ -46,6 +57,7 @@ def fixt_package_config( component_list=required_components, clp_config=clp_config_obj, base_port=base_port, + package_job_list=package_job_list, ) try: diff --git a/integration-tests/tests/fixtures/package_instance.py b/integration-tests/tests/fixtures/package_instance.py index b2ea7ef4b3..c776b3eeb3 100644 --- a/integration-tests/tests/fixtures/package_instance.py +++ b/integration-tests/tests/fixtures/package_instance.py @@ -1,5 +1,6 @@ """Fixtures that start and stop CLP package instances for integration tests.""" +import logging from collections.abc import Iterator import pytest @@ -13,16 +14,28 @@ stop_clp_package, ) +logger = logging.getLogger(__name__) + @pytest.fixture -def fixt_package_instance(fixt_package_config: PackageConfig) -> Iterator[PackageInstance]: +def fixt_package_instance( + fixt_package_config: PackageConfig, + request: pytest.FixtureRequest, +) -> Iterator[PackageInstance]: """ Starts a CLP package instance for the given configuration and stops it during teardown. :param fixt_package_config: + :param request: :return: Iterator that yields the running package instance. """ mode_name = fixt_package_config.mode_name + no_jobs: bool = bool(request.config.option.NO_JOBS) + package_job_list = fixt_package_config.package_job_list + + # Do not start this mode if there are no jobs and the '--no-jobs' flag wasn't specified by user. + if package_job_list is None and not no_jobs: + pytest.skip(f"No jobs to run for mode '{mode_name}' with current job filter.") try: start_clp_package(fixt_package_config) diff --git a/integration-tests/tests/test_package_start.py b/integration-tests/tests/test_package_start.py index 0603cf2cb0..8dc970c981 100644 --- a/integration-tests/tests/test_package_start.py +++ b/integration-tests/tests/test_package_start.py @@ -1,5 +1,7 @@ """Integration tests verifying that the CLP package can be started and stopped.""" +import logging + import pytest from tests.utils.asserting_utils import ( @@ -11,6 +13,8 @@ TEST_MODES = CLP_MODE_CONFIGS.keys() +logger = logging.getLogger(__name__) + @pytest.mark.package @pytest.mark.parametrize("fixt_package_config", TEST_MODES, indirect=True) @@ -25,3 +29,18 @@ def test_clp_package(fixt_package_instance: PackageInstance) -> None: # Ensure that the package is running in the correct mode. validate_running_mode_correct(fixt_package_instance) + + # TODO: write function `dispatch_test_jobs` and call it here; remove the logging statement. + package_job_list = fixt_package_instance.package_config.package_job_list + if package_job_list is not None: + mode_name = fixt_package_instance.package_config.mode_name + compression_job_list = package_job_list.package_compression_jobs + + compression_job_names = [job.job_name for job in compression_job_list] + compression_job_names_text = ", ".join(compression_job_names) + + log_msg = ( + f"Compression jobs listed to run for the '{mode_name}' package test:" + f" {compression_job_names_text}" + ) + logger.info(log_msg) diff --git a/integration-tests/tests/utils/clp_job_utils.py b/integration-tests/tests/utils/clp_job_utils.py new file mode 100644 index 0000000000..35ba131dcc --- /dev/null +++ b/integration-tests/tests/utils/clp_job_utils.py @@ -0,0 +1,48 @@ +"""Provides utilities related to the test jobs for the CLP package.""" + +import logging + +from tests.data.jobs.compression.compression_jobs import PACKAGE_COMPRESSION_JOBS +from tests.utils.config import ( + PackageCompressionJob, + PackageJobList, +) + +logger = logging.getLogger(__name__) + + +def _matches_keyword(job_name: str, keyword_filter: str) -> bool: + """Return True if this job should be included given the current -k filter.""" + if not keyword_filter: + return True + return keyword_filter.lower() in job_name.lower() + + +def _build_compression_jobs_list( + mode_name: str, + job_filter: str, +) -> list[PackageCompressionJob]: + package_compression_jobs: list[PackageCompressionJob] = [] + for job_name, package_compression_job in PACKAGE_COMPRESSION_JOBS.items(): + if package_compression_job.mode == mode_name and _matches_keyword(job_name, job_filter): + package_compression_jobs.append(package_compression_job) + return package_compression_jobs + + +def build_package_job_list(mode_name: str, job_filter: str) -> PackageJobList | None: + """ + Builds the list of package jobs for this test run. + + :param mode_name: + :param job_filter: + :return: PackageJobList if there are jobs for this mode, None if not. + """ + # Get compression jobs. + package_compression_jobs: list[PackageCompressionJob] = _build_compression_jobs_list( + mode_name, job_filter + ) + + if not package_compression_jobs: + return None + + return PackageJobList(package_compression_jobs=package_compression_jobs) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 6cef6c85b2..d1af2ecd8d 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -144,6 +144,13 @@ class PackageCompressionJob: args: list[str] | None +@dataclass(frozen=True) +class PackageJobList: + """List of jobs to run during a package test.""" + + package_compression_jobs: list[PackageCompressionJob] + + @dataclass(frozen=True) class PackageConfig: """Metadata for a specific configuration of the CLP package.""" @@ -163,6 +170,9 @@ class PackageConfig: #: The base port from which all ports for the components are derived. base_port: int + #: The list of jobs that this package will run during the test. + package_job_list: PackageJobList | None + def __post_init__(self) -> None: """Write the temporary config file for this package.""" self._write_temp_config_file() From 3aacf69f80eefb543aa6ef9162edd2910e4429cf Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 18 Dec 2025 20:10:21 +0000 Subject: [PATCH 04/13] Rename 'logs' to 'datasets'. --- .../tests/data/{logs => datasets}/json-multifile/README.md | 0 .../json-multifile/logs/sts-135-2011-07-08.jsonl | 0 .../json-multifile/logs/sts-135-2011-07-09.jsonl | 0 .../json-multifile/logs/sts-135-2011-07-11.jsonl | 0 .../json-multifile/logs/sts-135-2011-07-19.jsonl | 0 .../json-multifile/logs/sts-135-2011-07-21.jsonl | 0 .../tests/data/jobs/compression/compression_jobs.py | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename integration-tests/tests/data/{logs => datasets}/json-multifile/README.md (100%) rename integration-tests/tests/data/{logs => datasets}/json-multifile/logs/sts-135-2011-07-08.jsonl (100%) rename integration-tests/tests/data/{logs => datasets}/json-multifile/logs/sts-135-2011-07-09.jsonl (100%) rename integration-tests/tests/data/{logs => datasets}/json-multifile/logs/sts-135-2011-07-11.jsonl (100%) rename integration-tests/tests/data/{logs => datasets}/json-multifile/logs/sts-135-2011-07-19.jsonl (100%) rename integration-tests/tests/data/{logs => datasets}/json-multifile/logs/sts-135-2011-07-21.jsonl (100%) diff --git a/integration-tests/tests/data/logs/json-multifile/README.md b/integration-tests/tests/data/datasets/json-multifile/README.md similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/README.md rename to integration-tests/tests/data/datasets/json-multifile/README.md diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl b/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-08.jsonl similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-08.jsonl rename to integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-08.jsonl diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl b/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-09.jsonl similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-09.jsonl rename to integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-09.jsonl diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl b/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-11.jsonl similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-11.jsonl rename to integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-11.jsonl diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl b/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-19.jsonl similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-19.jsonl rename to integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-19.jsonl diff --git a/integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl b/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-21.jsonl similarity index 100% rename from integration-tests/tests/data/logs/json-multifile/logs/sts-135-2011-07-21.jsonl rename to integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-21.jsonl diff --git a/integration-tests/tests/data/jobs/compression/compression_jobs.py b/integration-tests/tests/data/jobs/compression/compression_jobs.py index 88f6125fc0..8574ec0b71 100644 --- a/integration-tests/tests/data/jobs/compression/compression_jobs.py +++ b/integration-tests/tests/data/jobs/compression/compression_jobs.py @@ -9,7 +9,7 @@ job_name="compress-json-multifile", mode="clp-json", script_path=Path("sbin") / "compress.sh", - log_path=Path("json-multifile") / "logs", + log_path=Path("json-multifile") / "datasets", flags={ "--timestamp-key": "timestamp", "--dataset": "json_multifile", From 134c9b9d896bff57f342bfba3f3e87cf13a42fda Mon Sep 17 00:00:00 2001 From: Quinn Date: Fri, 19 Dec 2025 16:43:21 +0000 Subject: [PATCH 05/13] Rabbit. --- integration-tests/tests/conftest.py | 2 +- .../tests/data/jobs/compression/compression_jobs.py | 2 +- integration-tests/tests/fixtures/package_config.py | 3 --- integration-tests/tests/fixtures/package_instance.py | 3 --- integration-tests/tests/utils/config.py | 2 +- 5 files changed, 3 insertions(+), 9 deletions(-) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 62be1d2bf3..1fbc01e3e6 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -26,7 +26,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--job-name-contains", dest="JOB_NAME_CONTAINS", - help="Filter CLP jobs by their name.", + help="Filter CLP jobs by their name (case-insensitive substring match).", ) parser.addoption( "--no-jobs", diff --git a/integration-tests/tests/data/jobs/compression/compression_jobs.py b/integration-tests/tests/data/jobs/compression/compression_jobs.py index 8574ec0b71..88f6125fc0 100644 --- a/integration-tests/tests/data/jobs/compression/compression_jobs.py +++ b/integration-tests/tests/data/jobs/compression/compression_jobs.py @@ -9,7 +9,7 @@ job_name="compress-json-multifile", mode="clp-json", script_path=Path("sbin") / "compress.sh", - log_path=Path("json-multifile") / "datasets", + log_path=Path("json-multifile") / "logs", flags={ "--timestamp-key": "timestamp", "--dataset": "json_multifile", diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index 9ad6b577f8..258fec1a3e 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -1,6 +1,5 @@ """Fixtures that create and remove temporary config files for CLP packages.""" -import logging from collections.abc import Iterator import pytest @@ -15,8 +14,6 @@ from tests.utils.config import PackageConfig, PackagePathConfig from tests.utils.port_utils import assign_ports_from_base -logger = logging.getLogger(__name__) - @pytest.fixture def fixt_package_config( diff --git a/integration-tests/tests/fixtures/package_instance.py b/integration-tests/tests/fixtures/package_instance.py index c776b3eeb3..919e4f660b 100644 --- a/integration-tests/tests/fixtures/package_instance.py +++ b/integration-tests/tests/fixtures/package_instance.py @@ -1,6 +1,5 @@ """Fixtures that start and stop CLP package instances for integration tests.""" -import logging from collections.abc import Iterator import pytest @@ -14,8 +13,6 @@ stop_clp_package, ) -logger = logging.getLogger(__name__) - @pytest.fixture def fixt_package_instance( diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index d1af2ecd8d..d2f35936fe 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -134,7 +134,7 @@ class PackageCompressionJob: # Script path (relative to the CLP package). script_path: Path - # The path to the logs relative to tests/data/logs (either a file or directory). + # The path to the logs relative to tests/data/datasets (either a file or directory). log_path: Path # Flags to specify in the command. From 9f9263346a4cdc4596d489ae0104f7943639d5c5 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 16:07:58 +0000 Subject: [PATCH 06/13] Remove fragments of old implementation. --- integration-tests/tests/conftest.py | 11 ----- integration-tests/tests/data/__init__.py | 1 - .../tests/data/jobs/compression/__init__.py | 1 - .../data/jobs/compression/compression_jobs.py | 19 -------- .../tests/fixtures/package_test_config.py | 1 - .../sample_datasets}/json-multifile/README.md | 0 .../logs/sts-135-2011-07-08.jsonl | 0 .../logs/sts-135-2011-07-09.jsonl | 0 .../logs/sts-135-2011-07-11.jsonl | 0 .../logs/sts-135-2011-07-19.jsonl | 0 .../logs/sts-135-2011-07-21.jsonl | 0 .../tests/utils/clp_job_utils.py | 48 ------------------- integration-tests/tests/utils/config.py | 4 -- 13 files changed, 85 deletions(-) delete mode 100644 integration-tests/tests/data/__init__.py delete mode 100644 integration-tests/tests/data/jobs/compression/__init__.py delete mode 100644 integration-tests/tests/data/jobs/compression/compression_jobs.py rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/README.md (100%) rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/logs/sts-135-2011-07-08.jsonl (100%) rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/logs/sts-135-2011-07-09.jsonl (100%) rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/logs/sts-135-2011-07-11.jsonl (100%) rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/logs/sts-135-2011-07-19.jsonl (100%) rename integration-tests/tests/{data/datasets => package_tests/clp_json/sample_datasets}/json-multifile/logs/sts-135-2011-07-21.jsonl (100%) delete mode 100644 integration-tests/tests/utils/clp_job_utils.py diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 30ec42a498..1741453b3a 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -23,14 +23,3 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="55000", help="Base port for CLP package integration tests.", ) - parser.addoption( - "--job-name-contains", - dest="JOB_NAME_CONTAINS", - help="Filter CLP jobs by their name (case-insensitive substring match).", - ) - parser.addoption( - "--no-jobs", - action="store_true", - dest="NO_JOBS", - help="Only validate CLP package start and stop. Do not create or run any test jobs.", - ) diff --git a/integration-tests/tests/data/__init__.py b/integration-tests/tests/data/__init__.py deleted file mode 100644 index 890528a804..0000000000 --- a/integration-tests/tests/data/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Static data storage relevant to CLP integration tests.""" diff --git a/integration-tests/tests/data/jobs/compression/__init__.py b/integration-tests/tests/data/jobs/compression/__init__.py deleted file mode 100644 index 2f46b40375..0000000000 --- a/integration-tests/tests/data/jobs/compression/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compression jobs to run on the CLP package.""" diff --git a/integration-tests/tests/data/jobs/compression/compression_jobs.py b/integration-tests/tests/data/jobs/compression/compression_jobs.py deleted file mode 100644 index 88f6125fc0..0000000000 --- a/integration-tests/tests/data/jobs/compression/compression_jobs.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Compression job definitions for CLP package tests.""" - -from pathlib import Path - -from tests.utils.config import PackageCompressionJob - -PACKAGE_COMPRESSION_JOBS: dict[str, PackageCompressionJob] = { - "compress-json-multifile": PackageCompressionJob( - job_name="compress-json-multifile", - mode="clp-json", - script_path=Path("sbin") / "compress.sh", - log_path=Path("json-multifile") / "logs", - flags={ - "--timestamp-key": "timestamp", - "--dataset": "json_multifile", - }, - args=None, - ), -} diff --git a/integration-tests/tests/fixtures/package_test_config.py b/integration-tests/tests/fixtures/package_test_config.py index c5d6690f95..e10ae7f3b9 100644 --- a/integration-tests/tests/fixtures/package_test_config.py +++ b/integration-tests/tests/fixtures/package_test_config.py @@ -38,7 +38,6 @@ def fixt_package_test_config( path_config=fixt_package_path_config, mode_config=mode_config, base_port=base_port, - package_job_list=package_job_list, ) try: diff --git a/integration-tests/tests/data/datasets/json-multifile/README.md b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/README.md similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/README.md rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/README.md diff --git a/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-08.jsonl b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-08.jsonl similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-08.jsonl rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-08.jsonl diff --git a/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-09.jsonl b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-09.jsonl similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-09.jsonl rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-09.jsonl diff --git a/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-11.jsonl b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-11.jsonl similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-11.jsonl rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-11.jsonl diff --git a/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-19.jsonl b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-19.jsonl similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-19.jsonl rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-19.jsonl diff --git a/integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-21.jsonl b/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-21.jsonl similarity index 100% rename from integration-tests/tests/data/datasets/json-multifile/logs/sts-135-2011-07-21.jsonl rename to integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-21.jsonl diff --git a/integration-tests/tests/utils/clp_job_utils.py b/integration-tests/tests/utils/clp_job_utils.py deleted file mode 100644 index 35ba131dcc..0000000000 --- a/integration-tests/tests/utils/clp_job_utils.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Provides utilities related to the test jobs for the CLP package.""" - -import logging - -from tests.data.jobs.compression.compression_jobs import PACKAGE_COMPRESSION_JOBS -from tests.utils.config import ( - PackageCompressionJob, - PackageJobList, -) - -logger = logging.getLogger(__name__) - - -def _matches_keyword(job_name: str, keyword_filter: str) -> bool: - """Return True if this job should be included given the current -k filter.""" - if not keyword_filter: - return True - return keyword_filter.lower() in job_name.lower() - - -def _build_compression_jobs_list( - mode_name: str, - job_filter: str, -) -> list[PackageCompressionJob]: - package_compression_jobs: list[PackageCompressionJob] = [] - for job_name, package_compression_job in PACKAGE_COMPRESSION_JOBS.items(): - if package_compression_job.mode == mode_name and _matches_keyword(job_name, job_filter): - package_compression_jobs.append(package_compression_job) - return package_compression_jobs - - -def build_package_job_list(mode_name: str, job_filter: str) -> PackageJobList | None: - """ - Builds the list of package jobs for this test run. - - :param mode_name: - :param job_filter: - :return: PackageJobList if there are jobs for this mode, None if not. - """ - # Get compression jobs. - package_compression_jobs: list[PackageCompressionJob] = _build_compression_jobs_list( - mode_name, job_filter - ) - - if not package_compression_jobs: - return None - - return PackageJobList(package_compression_jobs=package_compression_jobs) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 0cc32018eb..76a8e2d43a 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -5,7 +5,6 @@ import re from dataclasses import dataclass, field, InitVar from pathlib import Path -from typing import Any import yaml from clp_py_utils.clp_config import ( @@ -148,9 +147,6 @@ class PackageTestConfig: #: The base port from which all port assignments are derived. base_port: int - #: The list of jobs that this package will run during the test. - package_job_list: PackageJobList | None - def __post_init__(self) -> None: """Write the temporary config file for this package test.""" self._write_temp_config_file() From a8b0fbfaf9affb9b336f7a8d333f59316e799736 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 18:07:36 +0000 Subject: [PATCH 07/13] Re-implement compression with regards to the new modular approach. --- integration-tests/.pytest.ini | 1 + .../package_tests/clp_json/test_clp_json.py | 41 ++++++++++++--- .../tests/utils/asserting_utils.py | 51 ++++++++++++++++++- integration-tests/tests/utils/config.py | 31 +++++++++++ .../tests/utils/package_utils.py | 40 ++++++++++++++- 5 files changed, 155 insertions(+), 9 deletions(-) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 6124409099..9aa12e3868 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -11,6 +11,7 @@ env = D:CLP_BUILD_DIR=../build D:CLP_CORE_BINS_DIR=../build/core D:CLP_PACKAGE_DIR=../build/clp-package + D:INTEGRATION_TESTS_DIR=./ log_cli = True log_cli_date_format = %Y-%m-%d %H:%M:%S,%f log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index d3c48af72f..bea8de5269 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -12,9 +12,12 @@ from tests.utils.asserting_utils import ( validate_package_instance, + verify_compression, ) from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS -from tests.utils.config import PackageInstance, PackageModeConfig +from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig +from tests.utils.package_utils import run_package_compression_script +from tests.utils.utils import resolve_path_env_var logger = logging.getLogger(__name__) @@ -54,20 +57,42 @@ def test_clp_json_startup(fixt_package_instance: PackageInstance) -> None: @pytest.mark.compression -def test_clp_json_compression(fixt_package_instance: PackageInstance) -> None: +def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInstance) -> None: """ - Validate that the `clp-json` package successfully compresses some dataset. + Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. :param fixt_package_instance: """ validate_package_instance(fixt_package_instance) - # TODO: compress some dataset and check the correctness of compression. - assert True + # Compress a dataset. + compression_job = PackageCompressionJob( + path_to_dataset=( + resolve_path_env_var("INTEGRATION_TESTS_DIR") + / "tests" + / "package_tests" + / "clp_json" + / "sample_datasets" + / "json-multifile" + / "logs" + ), + options={ + "--timestamp-key": "timestamp", + "--dataset": "json_multifile", + }, + positional_args=None, + ) + package_test_config = fixt_package_instance.package_test_config + run_package_compression_script(compression_job, package_test_config) + + # Check the correctness of compression. + verify_compression(compression_job, package_test_config) log_msg = "test_clp_json_compression was successful." logger.info(log_msg) + # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp + @pytest.mark.search def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: @@ -78,7 +103,9 @@ def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: """ validate_package_instance(fixt_package_instance) - # TODO: compress some dataset and check the correctness of compression. + # TODO: compress a dataset + + # TODO: check the correctness of the compression # TODO: search through that dataset and check the correctness of the search results. @@ -86,3 +113,5 @@ def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: log_msg = "test_clp_json_search was successful." logger.info(log_msg) + + # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index 0a9b78e424..bbefef98d2 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -10,9 +10,9 @@ from pydantic import ValidationError from tests.utils.clp_mode_utils import compare_mode_signatures -from tests.utils.config import PackageInstance +from tests.utils.config import PackageCompressionJob, PackageInstance, PackageTestConfig from tests.utils.docker_utils import list_running_services_in_compose_project -from tests.utils.utils import load_yaml_to_dict +from tests.utils.utils import is_dir_tree_content_equal, load_yaml_to_dict, unlink logger = logging.getLogger(__name__) @@ -104,3 +104,50 @@ def _validate_running_mode_correct(package_instance: PackageInstance) -> None: if not compare_mode_signatures(intended_config, running_config): pytest.fail("Mode mismatch: running configuration does not match intended configuration.") + + +def verify_compression( + compression_job: PackageCompressionJob, + package_test_config: PackageTestConfig, +) -> None: + """ + Verify that the compression_job has been executed correctly by decompressing the contents of + `clp-package/var/data/archives` and comparing the decompressed logs to the originals. + + :param compression_job: + :param package_test_config: + """ + mode = package_test_config.mode_config.mode_name + + if mode in ("clp-json", "clp-presto"): + # TODO: Waiting for PR 1299 to be merged. + assert True + elif mode == "clp-text": + # Decompress the contents of `clp-package/var/data/archives`. + path_config = package_test_config.path_config + decompress_script_path = path_config.decompress_script_path + decompression_dir = path_config.package_decompression_dir + temp_config_file_path = package_test_config.temp_config_file_path + decompress_cmd = [ + str(decompress_script_path), + "--config", + str(temp_config_file_path), + "x", + "--extraction-dir", + str(decompression_dir), + ] + + # Run decompression command and assert that it succeeds. + run_and_assert(decompress_cmd) + + # Verify content equality. + decompression_dir = package_test_config.path_config.package_decompression_dir + path_to_dataset = compression_job.path_to_dataset + output_path = decompression_dir / path_to_dataset + + assert is_dir_tree_content_equal( + path_to_dataset, + output_path, + ), f"Mismatch between clp input {path_to_dataset} and output {output_path}." + + unlink(decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 76a8e2d43a..6e48973660 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -5,6 +5,7 @@ import re from dataclasses import dataclass, field, InitVar from pathlib import Path +from typing import Any import yaml from clp_py_utils.clp_config import ( @@ -76,6 +77,9 @@ class PackagePathConfig: #: Directory to store temporary package config files. temp_config_dir: Path = field(init=False, repr=True) + #: Directory where decompressed logs will be stored. + package_decompression_dir: Path = field(init=False, repr=True) + #: Directory where the CLP package writes logs. clp_log_dir: Path = field(init=False, repr=True) @@ -97,6 +101,9 @@ def __post_init__(self, test_root_dir: Path) -> None: # Initialize directory for package tests. validate_dir_exists(test_root_dir) object.__setattr__(self, "temp_config_dir", test_root_dir / "temp_config_files") + object.__setattr__( + self, "package_decompression_dir", test_root_dir / "package-decompressed-logs" + ) # Initialize log directory for the package. object.__setattr__( @@ -119,6 +126,30 @@ def stop_script_path(self) -> Path: """:return: The absolute path to the package stop script.""" return self.clp_package_dir / "sbin" / "stop-clp.sh" + @property + def compress_script_path(self) -> Path: + """:return: The absolute path to the package compress script.""" + return self.clp_package_dir / "sbin" / "compress.sh" + + @property + def decompress_script_path(self) -> Path: + """:return: The absolute path to the package decompress script.""" + return self.clp_package_dir / "sbin" / "decompress.sh" + + +@dataclass(frozen=True) +class PackageCompressionJob: + """A compression job for a package test.""" + + #: The absolute path to the dataset (either a file or directory). + path_to_dataset: Path + + #: Options to specify in the compression command, with their corresponding values. + options: dict[str, Any] | None + + #: Positional arguments to specify in the compression command (do not put paths to compress) + positional_args: list[str] | None + @dataclass(frozen=True) class PackageModeConfig: diff --git a/integration-tests/tests/utils/package_utils.py b/integration-tests/tests/utils/package_utils.py index 31254b8791..9c39e514cd 100644 --- a/integration-tests/tests/utils/package_utils.py +++ b/integration-tests/tests/utils/package_utils.py @@ -1,7 +1,10 @@ """Provides utility functions related to the CLP package used across `integration-tests`.""" from tests.utils.asserting_utils import run_and_assert -from tests.utils.config import PackageTestConfig +from tests.utils.config import ( + PackageCompressionJob, + PackageTestConfig, +) DEFAULT_CMD_TIMEOUT_SECONDS = 120.0 @@ -44,3 +47,38 @@ def stop_clp_package(package_test_config: PackageTestConfig) -> None: ] # fmt: on run_and_assert(stop_cmd, timeout=DEFAULT_CMD_TIMEOUT_SECONDS) + + +def run_package_compression_script( + compression_job: PackageCompressionJob, + package_test_config: PackageTestConfig, +) -> None: + """ + Constructs and runs a compression command on the CLP package. + + :param compression_job: + :param package_test_config: + """ + path_config = package_test_config.path_config + compress_script_path = path_config.compress_script_path + temp_config_file_path = package_test_config.temp_config_file_path + + compress_cmd = [ + str(compress_script_path), + "--config", + str(temp_config_file_path), + ] + + if compression_job.options is not None: + for option, option_value in compression_job.options.items(): + compress_cmd.append(option) + if option_value is not None: + compress_cmd.append(str(option_value)) + + if compression_job.positional_args is not None: + compress_cmd.extend(compression_job.positional_args) + + compress_cmd.append(str(compression_job.path_to_dataset)) + + # Run compression command for this job and assert that it succeeds. + run_and_assert(compress_cmd, timeout=DEFAULT_CMD_TIMEOUT_SECONDS) From 03f1f838bc5599abdb0ef1e2e3d0614609136868 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 19:07:33 +0000 Subject: [PATCH 08/13] Add clp-text compression test. --- .../package_tests/clp_json/test_clp_json.py | 6 ++- .../sample_datasets/text-multifile/README.md | 16 +++++++ .../text-multifile/logs/apollo-17_day01.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day02.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day03.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day04.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day05.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day06.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day07.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day08.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day09.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day10.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day11.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day12.txt | 15 +++++++ .../text-multifile/logs/apollo-17_day13.txt | 15 +++++++ .../package_tests/clp_text/test_clp_text.py | 42 ++++++++++++++++++- integration-tests/tests/utils/config.py | 6 +++ integration-tests/tests/utils/utils.py | 13 ++++++ 18 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt create mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index bea8de5269..2ecac629de 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -88,10 +88,12 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta # Check the correctness of compression. verify_compression(compression_job, package_test_config) - log_msg = "test_clp_json_compression was successful." + log_msg = "test_clp_json_compression_json_multifile was successful." logger.info(log_msg) - # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp + # Clean up. + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() @pytest.mark.search diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md new file mode 100644 index 0000000000..aae1f0c4b3 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md @@ -0,0 +1,16 @@ +# Description of logs + +**Type** +Unstructured text + +**Number of files** +13 + +**Number of events per file** +15 + +**Beginning timestamp (milliseconds)** +92572380000 + +**End timestamp (milliseconds)** +93614008000 diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt new file mode 100644 index 0000000000..ad77f1a53d --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt @@ -0,0 +1,15 @@ +1972-12-07T05:33:00.000 apollo-17 d01 e000 countdown sequencer resumes after hold and verifies launch commit criteria; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T05:42:35.999 apollo-17 d01 e001 terminal countdown proceeds at launch complex 39A with all stations reporting go; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T05:52:11.998 apollo-17 d01 e002 Saturn V first stage ignition and liftoff from Kennedy Space Center; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:01:47.997 apollo-17 d01 e003 tower cleared and roll program places vehicle on proper azimuth; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:11:23.996 apollo-17 d01 e004 pitch program guides ascent trajectory toward planned parking orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:20:59.995 apollo-17 d01 e005 first stage shutdown and staging event with clean separation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:30:35.994 apollo-17 d01 e006 second stage ignition with stable thrust and guidance lock; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:40:11.993 apollo-17 d01 e007 parking orbit achieved around Earth after third stage cutoff; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:49:47.992 apollo-17 d01 e008 systems checkout in Earth orbit confirms command and service module health; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:59:23.991 apollo-17 d01 e009 translunar injection burn ignition on third stage SIVB; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T07:08:59.990 apollo-17 d01 e010 translunar injection burn complete with nominal velocity for lunar transfer; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T07:18:35.989 apollo-17 d01 e011 command and service module separates and performs transposition and docking; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T07:28:11.988 apollo-17 d01 e012 lunar module Challenger extraction from SIVB and tunnel pressurization; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T07:37:47.987 apollo-17 d01 e013 spacecraft stack confirmed on translunar trajectory departing Earth orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T07:47:23.986 apollo-17 d01 e014 SIVB disposal maneuver targets heliocentric trajectory for spent stage; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt new file mode 100644 index 0000000000..b7ba290470 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt @@ -0,0 +1,15 @@ +1972-12-08T05:33:00.000 apollo-17 d02 e000 translunar coast period with spacecraft thermal control slow roll established; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T05:42:35.999 apollo-17 d02 e001 star sightings used for navigation alignment with onboard optics; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T05:52:11.998 apollo-17 d02 e002 midcourse correction burn refines trajectory toward lunar encounter; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:01:47.997 apollo-17 d02 e003 deep space network tracking confirms updated trajectory within limits; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:11:23.996 apollo-17 d02 e004 crew reports good rest periods and stable cabin environmental systems; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:20:59.995 apollo-17 d02 e005 fuel cell performance monitored with reactant usage in expected range; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:30:35.994 apollo-17 d02 e006 guidance platform alignment update using selected star pair; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:40:11.993 apollo-17 d02 e007 S-band communications checks with multiple ground stations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:49:47.992 apollo-17 d02 e008 medical telemetry review indicates crew in good physical condition; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T06:59:23.991 apollo-17 d02 e009 onboard housekeeping tasks completed and cabin configuration secured; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T07:08:59.990 apollo-17 d02 e010 navigation program update loaded and verified in guidance computer; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T07:18:35.989 apollo-17 d02 e011 oxygen and water consumption rates remain within mission planning margins; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T07:28:11.988 apollo-17 d02 e012 small attitude thruster pulses manage propellant settling and pointing; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T07:37:47.987 apollo-17 d02 e013 crew photography documents Earth receding in spacecraft windows; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-08T07:47:23.986 apollo-17 d02 e014 radiation dosimeter readings recorded for translunar environment; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt new file mode 100644 index 0000000000..7a9d38e879 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt @@ -0,0 +1,15 @@ +1972-12-09T05:33:00.000 apollo-17 d03 e000 spacecraft continues translunar coast approaching lunar sphere of influence; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T05:42:35.999 apollo-17 d03 e001 navigation solutions confirm free return option remains available if needed; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T05:52:11.998 apollo-17 d03 e002 crew conducts additional star sightings to refine state vector; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:01:47.997 apollo-17 d03 e003 communications delay slightly increases with growing range from Earth; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:11:23.996 apollo-17 d03 e004 midcourse correction option evaluated and determined not required; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:20:59.995 apollo-17 d03 e005 environmental control system maintains cabin pressure and temperature; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:30:35.994 apollo-17 d03 e006 crew conducts equipment inspections and stowage checks in cabin; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:40:11.993 apollo-17 d03 e007 guidance computer memory verified for upcoming lunar orbit programs; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:49:47.992 apollo-17 d03 e008 telemetry shows stable power margins on all fuel cells; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T06:59:23.991 apollo-17 d03 e009 attitude control jets fire in brief pulses to maintain thermal attitudes; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T07:08:59.990 apollo-17 d03 e010 flight controllers review contingency procedures for lunar orbit insertion; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T07:18:35.989 apollo-17 d03 e011 crew reads up on lunar geology targets in Taurus Littrow valley; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T07:28:11.988 apollo-17 d03 e012 range and velocity data show trajectory within narrow corridor; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T07:37:47.987 apollo-17 d03 e013 deep space network antennas perform handovers maintaining communication; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-09T07:47:23.986 apollo-17 d03 e014 crew sleep period begins with spacecraft in passive thermal control mode; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt new file mode 100644 index 0000000000..f94e4f9940 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt @@ -0,0 +1,15 @@ +1972-12-10T05:33:00.000 apollo-17 d04 e000 spacecraft enters lunar sphere of influence with gravity reference shifting; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T05:42:35.999 apollo-17 d04 e001 navigation update computes parameters for lunar orbit insertion burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T05:52:11.998 apollo-17 d04 e002 service propulsion system gimbals exercised in preparation for major burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:01:47.997 apollo-17 d04 e003 lunar orbit insertion burn ignition places spacecraft behind the Moon; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:11:23.996 apollo-17 d04 e004 loss of signal period begins as spacecraft passes behind lunar limb; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:20:59.995 apollo-17 d04 e005 lunar orbit insertion burn complete before acquisition of signal; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:30:35.994 apollo-17 d04 e006 acquisition of signal resumes with telemetry confirming stable lunar orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:40:11.993 apollo-17 d04 e007 initial lunar orbit tracking verifies altitude and period close to plan; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:49:47.992 apollo-17 d04 e008 crew performs visual inspection of lunar surface features from orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:59:23.991 apollo-17 d04 e009 lunar module powered up for initial systems checkout in lunar orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T07:08:59.990 apollo-17 d04 e010 radar and communications links between command and lunar modules verified; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T07:18:35.989 apollo-17 d04 e011 flight controllers review descent trajectory to Taurus Littrow landing site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T07:28:11.988 apollo-17 d04 e012 orbital trim maneuver planned to adjust pericynthion for descent path; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T07:37:47.987 apollo-17 d04 e013 crew rehearses undocking sequence and activation timeline for Challenger; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T07:47:23.986 apollo-17 d04 e014 lunar orbit mapping and photography program begins from command module; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt new file mode 100644 index 0000000000..a5577eca41 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt @@ -0,0 +1,15 @@ +1972-12-11T05:33:00.000 apollo-17 d05 e000 lunar module Challenger fully powered and configured for undocking; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T05:42:35.999 apollo-17 d05 e001 commander and lunar module pilot transfer from command module America; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T05:52:11.998 apollo-17 d05 e002 pressure checks on docking tunnel and hatches completed successfully; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:01:47.997 apollo-17 d05 e003 lunar module undocks cleanly and performs separation maneuver; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:11:23.996 apollo-17 d05 e004 descent orbit insertion burn shapes path toward Taurus Littrow site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:20:59.995 apollo-17 d05 e005 powered descent initiation with main engine throttle ramp sequence; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:30:35.994 apollo-17 d05 e006 landing radar acquisition confirms altitude and rate data availability; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:40:11.993 apollo-17 d05 e007 final approach over Taurus Littrow with manual targeting adjustments; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:49:47.992 apollo-17 d05 e008 contact light indicates lunar surface reached and engine shutdown commanded; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T06:59:23.991 apollo-17 d05 e009 touchdown on lunar surface confirmed with stable attitude and no tipping; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T07:08:59.990 apollo-17 d05 e010 post landing checklist executed with guidance system remaining powered; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T07:18:35.989 apollo-17 d05 e011 cabin depressurization planned for first extravehicular activity timeline; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T07:28:11.988 apollo-17 d05 e012 lunar module systems show stable temperatures in landed configuration; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T07:37:47.987 apollo-17 d05 e013 surface communications link established through relay in command module; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-11T07:47:23.986 apollo-17 d05 e014 crew reports initial impressions of local lunar terrain and lighting; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt new file mode 100644 index 0000000000..45d800d5fd --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt @@ -0,0 +1,15 @@ +1972-12-12T05:33:00.000 apollo-17 d06 e000 cabin depressurization begins in preparation for first surface excursion; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T05:42:35.999 apollo-17 d06 e001 commander egresses to lunar surface and descends the ladder; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T05:52:11.998 apollo-17 d06 e002 lunar module pilot joins commander on surface after hatch operations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:01:47.997 apollo-17 d06 e003 American flag deployment completed near lunar module with photography; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:11:23.996 apollo-17 d06 e004 lunar rover deployment and checkout concluded with mobility test drive; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:20:59.995 apollo-17 d06 e005 ALSEP experiment package deployed at planned distance from lunar module; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:30:35.994 apollo-17 d06 e006 gravimeter and seismic instruments positioned and powered on; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:40:11.993 apollo-17 d06 e007 crew collects documented geological samples near landing site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:49:47.992 apollo-17 d06 e008 panoramic photography of Taurus Littrow valley recorded for analysis; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T06:59:23.991 apollo-17 d06 e009 communications relay confirms voice and biomedical data throughout EVA; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T07:08:59.990 apollo-17 d06 e010 lunar dust accumulation noted on suits and hardware during traverses; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T07:18:35.989 apollo-17 d06 e011 crew returns to lunar module and performs post EVA debrief with control; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T07:28:11.988 apollo-17 d06 e012 airlock and cabin repressurization sequence restores shirt sleeve environment; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T07:37:47.987 apollo-17 d06 e013 consumables usage for first EVA compared against mission allocations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-12T07:47:23.986 apollo-17 d06 e014 sleep period scheduled following extensive surface activity and reporting; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt new file mode 100644 index 0000000000..93384cddd6 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt @@ -0,0 +1,15 @@ +1972-12-13T05:33:00.000 apollo-17 d07 e000 crew prepares for second surface excursion with suit and system checks; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T05:42:35.999 apollo-17 d07 e001 cabin depressurization and hatch opening procedures completed again; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T05:52:11.998 apollo-17 d07 e002 lunar rover driven toward designated geology stations away from landing site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:01:47.997 apollo-17 d07 e003 sampling at Station Two documents dark mantle deposits and boulders; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:11:23.996 apollo-17 d07 e004 crew deploys additional experiment packages and marker flags along traverse; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:20:59.995 apollo-17 d07 e005 detailed core samples extracted for stratigraphic study of regolith; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:30:35.994 apollo-17 d07 e006 camera panoramas taken at multiple stations for context documentation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:40:11.993 apollo-17 d07 e007 navigation data from rover used to refine local lunar topographic maps; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:49:47.992 apollo-17 d07 e008 communications remain stable during extended traverse distance from LM; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:59:23.991 apollo-17 d07 e009 crew notes color contrasts and layering in valley walls and boulders; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T07:08:59.990 apollo-17 d07 e010 time limits require planned truncation of some sampling stops; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T07:18:35.989 apollo-17 d07 e011 return drive to lunar module occurs with continual systems monitoring; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T07:28:11.988 apollo-17 d07 e012 post EVA cleanup efforts manage dust and stow tools back in cabin; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T07:37:47.987 apollo-17 d07 e013 experiment readouts show expected power levels and data acquisition; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T07:47:23.986 apollo-17 d07 e014 overnight monitoring of deployed instruments continues from Earth; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt new file mode 100644 index 0000000000..5300c781d5 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt @@ -0,0 +1,15 @@ +1972-12-14T05:33:00.000 apollo-17 d08 e000 third surface excursion begins with focus on remaining geology stations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T05:42:35.999 apollo-17 d08 e001 lunar rover driven to Station Six near base of massifs for sampling; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T05:52:11.998 apollo-17 d08 e002 crew collects orange soil and unusual samples for laboratory study; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:01:47.997 apollo-17 d08 e003 final traverses complete extended coverage of Taurus Littrow features; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:11:23.996 apollo-17 d08 e004 lunar rover parked at planned position for post mission photography; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:20:59.995 apollo-17 d08 e005 ALSEP instruments verified before crew departure from surface site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:30:35.994 apollo-17 d08 e006 flags and commemorative plaques documented in high resolution imagery; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:40:11.993 apollo-17 d08 e007 crew performs final walkaround inspection of lunar module exterior; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:49:47.992 apollo-17 d08 e008 ingress to lunar module followed by transfer of samples and film; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T06:59:23.991 apollo-17 d08 e009 cabin repressurization and suit doffing conclude surface operations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T07:08:59.990 apollo-17 d08 e010 stowage operations secure samples and equipment for ascent flight; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T07:18:35.989 apollo-17 d08 e011 systems configured for lunar module ascent and rendezvous sequence; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T07:28:11.988 apollo-17 d08 e012 final lunar surface status report transmitted to mission control; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T07:37:47.987 apollo-17 d08 e013 crew rests briefly prior to ascent timeline for launch from Moon; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-14T07:47:23.986 apollo-17 d08 e014 flight controllers review rendezvous profile with command module; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt new file mode 100644 index 0000000000..013db85a62 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt @@ -0,0 +1,15 @@ +1972-12-15T05:33:00.000 apollo-17 d09 e000 lunar module ascent engine ignition lifts Challenger from lunar surface; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T05:42:35.999 apollo-17 d09 e001 ascent trajectory tracks planned path toward command module America; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T05:52:11.998 apollo-17 d09 e002 staging separation of descent stage confirmed with clean telemetry; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:01:47.997 apollo-17 d09 e003 rendezvous radar locks onto command module for navigation updates; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:11:23.996 apollo-17 d09 e004 crew performs midcourse corrections during ascent orbit insertion; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:20:59.995 apollo-17 d09 e005 terminal phase initiation maneuver refines closing velocity and range; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:30:35.994 apollo-17 d09 e006 final approach brings two spacecraft into docking corridor alignment; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:40:11.993 apollo-17 d09 e007 soft dock capture achieved followed by hard dock and latching sequence; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:49:47.992 apollo-17 d09 e008 pressure equalization and hatch opening allow crew transfer to America; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T06:59:23.991 apollo-17 d09 e009 samples and experiment results moved from lunar module to command module; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T07:08:59.990 apollo-17 d09 e010 lunar module systems safed in preparation for eventual jettison; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T07:18:35.989 apollo-17 d09 e011 post rendezvous orbital parameters verified for trans Earth injection; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T07:28:11.988 apollo-17 d09 e012 crew reports satisfaction with completion of surface objectives; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T07:37:47.987 apollo-17 d09 e013 flight controllers perform detailed review of ascent phase performance; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-15T07:47:23.986 apollo-17 d09 e014 lunar module ascent stage targeted for controlled impact or disposal; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt new file mode 100644 index 0000000000..82a8ec499c --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt @@ -0,0 +1,15 @@ +1972-12-16T05:33:00.000 apollo-17 d10 e000 service propulsion system ignition performs trans Earth injection burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T05:42:35.999 apollo-17 d10 e001 trajectory after burn shows proper corridor for atmospheric reentry; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T05:52:11.998 apollo-17 d10 e002 spacecraft exits lunar orbit and begins long coast back to Earth; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:01:47.997 apollo-17 d10 e003 navigation sightings compare well with ground based tracking solutions; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:11:23.996 apollo-17 d10 e004 crew conducts deep space science experiments during return flight; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:20:59.995 apollo-17 d10 e005 thermal control roll re-established for uniform heating on spacecraft; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:30:35.994 apollo-17 d10 e006 communications schedule alternates between worldwide tracking stations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:40:11.993 apollo-17 d10 e007 crew performs housekeeping and stowage reconfiguration for entry phase; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:49:47.992 apollo-17 d10 e008 biomedical data indicates crew adapting well after lunar surface stay; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:59:23.991 apollo-17 d10 e009 fuel cell reactant status supports remaining mission with margin; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T07:08:59.990 apollo-17 d10 e010 navigation computer updated with latest state vector from ground; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T07:18:35.989 apollo-17 d10 e011 small correction burn considered and judged unnecessary at this time; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T07:28:11.988 apollo-17 d10 e012 camera imagery captures Moon receding into distance from command module; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T07:37:47.987 apollo-17 d10 e013 experiment results packaged and secured for Earth recovery and study; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T07:47:23.986 apollo-17 d10 e014 sleep periods adjusted to align with planned reentry ground support; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt new file mode 100644 index 0000000000..dc935ef308 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt @@ -0,0 +1,15 @@ +1972-12-17T05:33:00.000 apollo-17 d11 e000 midcourse correction opportunity evaluated using latest navigation data; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T05:42:35.999 apollo-17 d11 e001 small correction burn executed to tighten entry corridor parameters; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T05:52:11.998 apollo-17 d11 e002 flight dynamics officers confirm predicted splashdown point in Pacific; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:01:47.997 apollo-17 d11 e003 crew conducts equipment checks on reentry suits and harnesses; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:11:23.996 apollo-17 d11 e004 cabin atmosphere sampled and readings remain within nominal limits; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:20:59.995 apollo-17 d11 e005 communications checks performed with recovery forces coordination centers; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:30:35.994 apollo-17 d11 e006 command module interior rearranged to facilitate entry operations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:40:11.993 apollo-17 d11 e007 crew records audio commentary on mission highlights and observations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:49:47.992 apollo-17 d11 e008 instrument readouts for deep space experiments logged for scientists; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T06:59:23.991 apollo-17 d11 e009 guidance system alignments repeated to maintain precise attitude knowledge; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T07:08:59.990 apollo-17 d11 e010 thruster usage tracked to ensure adequate reserves for attitude control; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T07:18:35.989 apollo-17 d11 e011 radiation exposure totals calculated from dosimeter instrument data; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T07:28:11.988 apollo-17 d11 e012 galley and waste management systems checked before final flight phase; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T07:37:47.987 apollo-17 d11 e013 flight controllers review contingency entry options with crew procedures; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-17T07:47:23.986 apollo-17 d11 e014 Earth appears larger in spacecraft windows as range continues to close; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt new file mode 100644 index 0000000000..3e17323006 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt @@ -0,0 +1,15 @@ +1972-12-18T05:33:00.000 apollo-17 d12 e000 final midcourse update confirms entry interface conditions within limits; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T05:42:35.999 apollo-17 d12 e001 service module systems evaluated ahead of planned jettison sequence; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T05:52:11.998 apollo-17 d12 e002 crew configures command module for standalone entry operations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:01:47.997 apollo-17 d12 e003 communications tests conducted with recovery ships and aircraft; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:11:23.996 apollo-17 d12 e004 reentry checklist reviewed step by step with mission control; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:20:59.995 apollo-17 d12 e005 photography documents Earth weather systems along ground track; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:30:35.994 apollo-17 d12 e006 crew secures loose items in cabin to prevent free floating during G loads; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:40:11.993 apollo-17 d12 e007 instrument packages stowed in locations compatible with landing loads; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:49:47.992 apollo-17 d12 e008 guidance computer switched to appropriate reentry programs; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T06:59:23.991 apollo-17 d12 e009 service module jettison countdown initiated by crew on mission timers; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T07:08:59.990 apollo-17 d12 e010 physical conditioning exercises performed to prepare for gravity return; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T07:18:35.989 apollo-17 d12 e011 flight controllers monitor final trajectory refinements in real time; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T07:28:11.988 apollo-17 d12 e012 spacecraft attitude adjusted for correct heat shield orientation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T07:37:47.987 apollo-17 d12 e013 entry corridor displays show predicted deceleration profile for crew; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-18T07:47:23.986 apollo-17 d12 e014 crew reports readiness for high speed entry and ocean splashdown; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt new file mode 100644 index 0000000000..5362ca002c --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt @@ -0,0 +1,15 @@ +1972-12-19T05:33:00.000 apollo-17 d13 e000 service module jettison executed leaving command module alone for entry; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T05:38:44.786 apollo-17 d13 e001 final attitude adjustments bring heat shield into entry orientation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T05:44:29.572 apollo-17 d13 e002 radio blackout interval begins as plasma sheath forms around capsule; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T05:50:14.358 apollo-17 d13 e003 communications restored after blackout with voice contact to recovery forces; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T05:55:59.144 apollo-17 d13 e004 deceleration loads peak within planned values on crew couches; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:01:43.930 apollo-17 d13 e005 drogue parachute deployment sequence initiates to stabilize capsule; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:07:28.716 apollo-17 d13 e006 main parachutes deploy and show symmetrical inflation on visual checks; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:13:13.502 apollo-17 d13 e007 command module descends toward Pacific splashdown zone near recovery ship; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:18:58.288 apollo-17 d13 e008 impact with ocean surface recorded within predicted time and location; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:24:43.074 apollo-17 d13 e009 uprighting system inflates to place capsule in correct flotation attitude; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:30:27.860 apollo-17 d13 e010 recovery swimmers deploy from helicopters and secure lines to spacecraft; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:36:12.646 apollo-17 d13 e011 hatch opening occurs after cabin ventilation checks and safety verifications; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:41:57.432 apollo-17 d13 e012 crew exits command module and boards recovery helicopter for transport; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:47:42.218 apollo-17 d13 e013 command module hoisted onto recovery ship deck for post flight inspection; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:53:27.004 apollo-17 d13 e014 mission debriefing begins with engineering and science team representatives; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index dbbc862f4c..0fdd764f59 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -12,9 +12,12 @@ from tests.utils.asserting_utils import ( validate_package_instance, + verify_compression, ) from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS -from tests.utils.config import PackageInstance, PackageModeConfig +from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig +from tests.utils.package_utils import run_package_compression_script +from tests.utils.utils import resolve_path_env_var logger = logging.getLogger(__name__) @@ -49,3 +52,40 @@ def test_clp_text_startup(fixt_package_instance: PackageInstance) -> None: log_msg = "test_clp_text_startup was successful." logger.info(log_msg) + + +@pytest.mark.compression +def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInstance) -> None: + """ + Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. + + :param fixt_package_instance: + """ + validate_package_instance(fixt_package_instance) + + # Compress a dataset. + compression_job = PackageCompressionJob( + path_to_dataset=( + resolve_path_env_var("INTEGRATION_TESTS_DIR") + / "tests" + / "package_tests" + / "clp_text" + / "sample_datasets" + / "text-multifile" + / "logs" + ), + options=None, + positional_args=None, + ) + package_test_config = fixt_package_instance.package_test_config + run_package_compression_script(compression_job, package_test_config) + + # Check the correctness of compression. + verify_compression(compression_job, package_test_config) + + log_msg = "test_clp_text_compression_text_multifile was successful." + logger.info(log_msg) + + # Clean up. + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 6e48973660..c1dcaa1026 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -15,6 +15,7 @@ ) from tests.utils.utils import ( + clean_directory, unlink, validate_dir_exists, validate_file_exists, @@ -136,6 +137,11 @@ def decompress_script_path(self) -> Path: """:return: The absolute path to the package decompress script.""" return self.clp_package_dir / "sbin" / "decompress.sh" + def clear_package_archives(self) -> None: + """Removes the contents of `clp-package/var/data/archives`.""" + archives_dir = self.clp_package_dir / "var" / "data" / "archives" + clean_directory(archives_dir) + @dataclass(frozen=True) class PackageCompressionJob: diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index f58d3c70da..710bc3907a 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -10,6 +10,19 @@ import yaml +def clean_directory(directory: Path) -> None: + """ + Removes the contents of `directory` without removing `directory` itself. + + :param directory: + """ + for item in directory.iterdir(): + if item.is_file() or item.is_symlink(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + def get_binary_path(name: str) -> str: """ :param name: Name of the program binary to locate. From 1890061b3344331e07f2ea87090f0c6208a0de40 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 20:13:49 +0000 Subject: [PATCH 09/13] Reduce the size of text-multifile. --- .../sample_datasets/text-multifile/README.md | 4 ++-- .../text-multifile/logs/apollo-17_day01.txt | 5 ----- .../text-multifile/logs/apollo-17_day02.txt | 15 --------------- .../text-multifile/logs/apollo-17_day03.txt | 15 --------------- .../text-multifile/logs/apollo-17_day04.txt | 5 ----- .../text-multifile/logs/apollo-17_day05.txt | 15 --------------- .../text-multifile/logs/apollo-17_day06.txt | 15 --------------- .../text-multifile/logs/apollo-17_day07.txt | 5 ----- .../text-multifile/logs/apollo-17_day08.txt | 15 --------------- .../text-multifile/logs/apollo-17_day09.txt | 15 --------------- .../text-multifile/logs/apollo-17_day10.txt | 5 ----- .../text-multifile/logs/apollo-17_day11.txt | 15 --------------- .../text-multifile/logs/apollo-17_day12.txt | 15 --------------- .../text-multifile/logs/apollo-17_day13.txt | 5 ----- 14 files changed, 2 insertions(+), 147 deletions(-) delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt delete mode 100644 integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md index aae1f0c4b3..4ed344d5f1 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md @@ -4,10 +4,10 @@ Unstructured text **Number of files** -13 +5 **Number of events per file** -15 +10 **Beginning timestamp (milliseconds)** 92572380000 diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt index ad77f1a53d..6900c6d822 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt @@ -8,8 +8,3 @@ 1972-12-07T06:40:11.993 apollo-17 d01 e007 parking orbit achieved around Earth after third stage cutoff; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-07T06:49:47.992 apollo-17 d01 e008 systems checkout in Earth orbit confirms command and service module health; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-07T06:59:23.991 apollo-17 d01 e009 translunar injection burn ignition on third stage SIVB; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-07T07:08:59.990 apollo-17 d01 e010 translunar injection burn complete with nominal velocity for lunar transfer; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-07T07:18:35.989 apollo-17 d01 e011 command and service module separates and performs transposition and docking; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-07T07:28:11.988 apollo-17 d01 e012 lunar module Challenger extraction from SIVB and tunnel pressurization; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-07T07:37:47.987 apollo-17 d01 e013 spacecraft stack confirmed on translunar trajectory departing Earth orbit; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-07T07:47:23.986 apollo-17 d01 e014 SIVB disposal maneuver targets heliocentric trajectory for spent stage; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt deleted file mode 100644 index b7ba290470..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day02.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-08T05:33:00.000 apollo-17 d02 e000 translunar coast period with spacecraft thermal control slow roll established; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T05:42:35.999 apollo-17 d02 e001 star sightings used for navigation alignment with onboard optics; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T05:52:11.998 apollo-17 d02 e002 midcourse correction burn refines trajectory toward lunar encounter; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:01:47.997 apollo-17 d02 e003 deep space network tracking confirms updated trajectory within limits; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:11:23.996 apollo-17 d02 e004 crew reports good rest periods and stable cabin environmental systems; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:20:59.995 apollo-17 d02 e005 fuel cell performance monitored with reactant usage in expected range; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:30:35.994 apollo-17 d02 e006 guidance platform alignment update using selected star pair; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:40:11.993 apollo-17 d02 e007 S-band communications checks with multiple ground stations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:49:47.992 apollo-17 d02 e008 medical telemetry review indicates crew in good physical condition; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T06:59:23.991 apollo-17 d02 e009 onboard housekeeping tasks completed and cabin configuration secured; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T07:08:59.990 apollo-17 d02 e010 navigation program update loaded and verified in guidance computer; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T07:18:35.989 apollo-17 d02 e011 oxygen and water consumption rates remain within mission planning margins; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T07:28:11.988 apollo-17 d02 e012 small attitude thruster pulses manage propellant settling and pointing; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T07:37:47.987 apollo-17 d02 e013 crew photography documents Earth receding in spacecraft windows; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-08T07:47:23.986 apollo-17 d02 e014 radiation dosimeter readings recorded for translunar environment; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt deleted file mode 100644 index 7a9d38e879..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day03.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-09T05:33:00.000 apollo-17 d03 e000 spacecraft continues translunar coast approaching lunar sphere of influence; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T05:42:35.999 apollo-17 d03 e001 navigation solutions confirm free return option remains available if needed; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T05:52:11.998 apollo-17 d03 e002 crew conducts additional star sightings to refine state vector; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:01:47.997 apollo-17 d03 e003 communications delay slightly increases with growing range from Earth; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:11:23.996 apollo-17 d03 e004 midcourse correction option evaluated and determined not required; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:20:59.995 apollo-17 d03 e005 environmental control system maintains cabin pressure and temperature; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:30:35.994 apollo-17 d03 e006 crew conducts equipment inspections and stowage checks in cabin; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:40:11.993 apollo-17 d03 e007 guidance computer memory verified for upcoming lunar orbit programs; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:49:47.992 apollo-17 d03 e008 telemetry shows stable power margins on all fuel cells; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T06:59:23.991 apollo-17 d03 e009 attitude control jets fire in brief pulses to maintain thermal attitudes; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T07:08:59.990 apollo-17 d03 e010 flight controllers review contingency procedures for lunar orbit insertion; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T07:18:35.989 apollo-17 d03 e011 crew reads up on lunar geology targets in Taurus Littrow valley; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T07:28:11.988 apollo-17 d03 e012 range and velocity data show trajectory within narrow corridor; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T07:37:47.987 apollo-17 d03 e013 deep space network antennas perform handovers maintaining communication; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-09T07:47:23.986 apollo-17 d03 e014 crew sleep period begins with spacecraft in passive thermal control mode; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt index f94e4f9940..cfec6ff379 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt @@ -8,8 +8,3 @@ 1972-12-10T06:40:11.993 apollo-17 d04 e007 initial lunar orbit tracking verifies altitude and period close to plan; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-10T06:49:47.992 apollo-17 d04 e008 crew performs visual inspection of lunar surface features from orbit; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-10T06:59:23.991 apollo-17 d04 e009 lunar module powered up for initial systems checkout in lunar orbit; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-10T07:08:59.990 apollo-17 d04 e010 radar and communications links between command and lunar modules verified; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-10T07:18:35.989 apollo-17 d04 e011 flight controllers review descent trajectory to Taurus Littrow landing site; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-10T07:28:11.988 apollo-17 d04 e012 orbital trim maneuver planned to adjust pericynthion for descent path; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-10T07:37:47.987 apollo-17 d04 e013 crew rehearses undocking sequence and activation timeline for Challenger; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-10T07:47:23.986 apollo-17 d04 e014 lunar orbit mapping and photography program begins from command module; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt deleted file mode 100644 index a5577eca41..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day05.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-11T05:33:00.000 apollo-17 d05 e000 lunar module Challenger fully powered and configured for undocking; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T05:42:35.999 apollo-17 d05 e001 commander and lunar module pilot transfer from command module America; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T05:52:11.998 apollo-17 d05 e002 pressure checks on docking tunnel and hatches completed successfully; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:01:47.997 apollo-17 d05 e003 lunar module undocks cleanly and performs separation maneuver; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:11:23.996 apollo-17 d05 e004 descent orbit insertion burn shapes path toward Taurus Littrow site; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:20:59.995 apollo-17 d05 e005 powered descent initiation with main engine throttle ramp sequence; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:30:35.994 apollo-17 d05 e006 landing radar acquisition confirms altitude and rate data availability; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:40:11.993 apollo-17 d05 e007 final approach over Taurus Littrow with manual targeting adjustments; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:49:47.992 apollo-17 d05 e008 contact light indicates lunar surface reached and engine shutdown commanded; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T06:59:23.991 apollo-17 d05 e009 touchdown on lunar surface confirmed with stable attitude and no tipping; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T07:08:59.990 apollo-17 d05 e010 post landing checklist executed with guidance system remaining powered; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T07:18:35.989 apollo-17 d05 e011 cabin depressurization planned for first extravehicular activity timeline; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T07:28:11.988 apollo-17 d05 e012 lunar module systems show stable temperatures in landed configuration; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T07:37:47.987 apollo-17 d05 e013 surface communications link established through relay in command module; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-11T07:47:23.986 apollo-17 d05 e014 crew reports initial impressions of local lunar terrain and lighting; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt deleted file mode 100644 index 45d800d5fd..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day06.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-12T05:33:00.000 apollo-17 d06 e000 cabin depressurization begins in preparation for first surface excursion; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T05:42:35.999 apollo-17 d06 e001 commander egresses to lunar surface and descends the ladder; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T05:52:11.998 apollo-17 d06 e002 lunar module pilot joins commander on surface after hatch operations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:01:47.997 apollo-17 d06 e003 American flag deployment completed near lunar module with photography; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:11:23.996 apollo-17 d06 e004 lunar rover deployment and checkout concluded with mobility test drive; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:20:59.995 apollo-17 d06 e005 ALSEP experiment package deployed at planned distance from lunar module; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:30:35.994 apollo-17 d06 e006 gravimeter and seismic instruments positioned and powered on; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:40:11.993 apollo-17 d06 e007 crew collects documented geological samples near landing site; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:49:47.992 apollo-17 d06 e008 panoramic photography of Taurus Littrow valley recorded for analysis; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T06:59:23.991 apollo-17 d06 e009 communications relay confirms voice and biomedical data throughout EVA; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T07:08:59.990 apollo-17 d06 e010 lunar dust accumulation noted on suits and hardware during traverses; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T07:18:35.989 apollo-17 d06 e011 crew returns to lunar module and performs post EVA debrief with control; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T07:28:11.988 apollo-17 d06 e012 airlock and cabin repressurization sequence restores shirt sleeve environment; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T07:37:47.987 apollo-17 d06 e013 consumables usage for first EVA compared against mission allocations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-12T07:47:23.986 apollo-17 d06 e014 sleep period scheduled following extensive surface activity and reporting; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt index 93384cddd6..c30eb3129b 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt @@ -8,8 +8,3 @@ 1972-12-13T06:40:11.993 apollo-17 d07 e007 navigation data from rover used to refine local lunar topographic maps; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-13T06:49:47.992 apollo-17 d07 e008 communications remain stable during extended traverse distance from LM; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-13T06:59:23.991 apollo-17 d07 e009 crew notes color contrasts and layering in valley walls and boulders; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-13T07:08:59.990 apollo-17 d07 e010 time limits require planned truncation of some sampling stops; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-13T07:18:35.989 apollo-17 d07 e011 return drive to lunar module occurs with continual systems monitoring; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-13T07:28:11.988 apollo-17 d07 e012 post EVA cleanup efforts manage dust and stow tools back in cabin; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-13T07:37:47.987 apollo-17 d07 e013 experiment readouts show expected power levels and data acquisition; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-13T07:47:23.986 apollo-17 d07 e014 overnight monitoring of deployed instruments continues from Earth; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt deleted file mode 100644 index 5300c781d5..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day08.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-14T05:33:00.000 apollo-17 d08 e000 third surface excursion begins with focus on remaining geology stations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T05:42:35.999 apollo-17 d08 e001 lunar rover driven to Station Six near base of massifs for sampling; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T05:52:11.998 apollo-17 d08 e002 crew collects orange soil and unusual samples for laboratory study; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:01:47.997 apollo-17 d08 e003 final traverses complete extended coverage of Taurus Littrow features; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:11:23.996 apollo-17 d08 e004 lunar rover parked at planned position for post mission photography; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:20:59.995 apollo-17 d08 e005 ALSEP instruments verified before crew departure from surface site; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:30:35.994 apollo-17 d08 e006 flags and commemorative plaques documented in high resolution imagery; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:40:11.993 apollo-17 d08 e007 crew performs final walkaround inspection of lunar module exterior; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:49:47.992 apollo-17 d08 e008 ingress to lunar module followed by transfer of samples and film; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T06:59:23.991 apollo-17 d08 e009 cabin repressurization and suit doffing conclude surface operations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T07:08:59.990 apollo-17 d08 e010 stowage operations secure samples and equipment for ascent flight; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T07:18:35.989 apollo-17 d08 e011 systems configured for lunar module ascent and rendezvous sequence; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T07:28:11.988 apollo-17 d08 e012 final lunar surface status report transmitted to mission control; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T07:37:47.987 apollo-17 d08 e013 crew rests briefly prior to ascent timeline for launch from Moon; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-14T07:47:23.986 apollo-17 d08 e014 flight controllers review rendezvous profile with command module; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt deleted file mode 100644 index 013db85a62..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day09.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-15T05:33:00.000 apollo-17 d09 e000 lunar module ascent engine ignition lifts Challenger from lunar surface; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T05:42:35.999 apollo-17 d09 e001 ascent trajectory tracks planned path toward command module America; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T05:52:11.998 apollo-17 d09 e002 staging separation of descent stage confirmed with clean telemetry; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:01:47.997 apollo-17 d09 e003 rendezvous radar locks onto command module for navigation updates; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:11:23.996 apollo-17 d09 e004 crew performs midcourse corrections during ascent orbit insertion; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:20:59.995 apollo-17 d09 e005 terminal phase initiation maneuver refines closing velocity and range; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:30:35.994 apollo-17 d09 e006 final approach brings two spacecraft into docking corridor alignment; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:40:11.993 apollo-17 d09 e007 soft dock capture achieved followed by hard dock and latching sequence; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:49:47.992 apollo-17 d09 e008 pressure equalization and hatch opening allow crew transfer to America; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T06:59:23.991 apollo-17 d09 e009 samples and experiment results moved from lunar module to command module; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T07:08:59.990 apollo-17 d09 e010 lunar module systems safed in preparation for eventual jettison; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T07:18:35.989 apollo-17 d09 e011 post rendezvous orbital parameters verified for trans Earth injection; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T07:28:11.988 apollo-17 d09 e012 crew reports satisfaction with completion of surface objectives; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T07:37:47.987 apollo-17 d09 e013 flight controllers perform detailed review of ascent phase performance; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-15T07:47:23.986 apollo-17 d09 e014 lunar module ascent stage targeted for controlled impact or disposal; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt index 82a8ec499c..5255c9f241 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt @@ -8,8 +8,3 @@ 1972-12-16T06:40:11.993 apollo-17 d10 e007 crew performs housekeeping and stowage reconfiguration for entry phase; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-16T06:49:47.992 apollo-17 d10 e008 biomedical data indicates crew adapting well after lunar surface stay; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-16T06:59:23.991 apollo-17 d10 e009 fuel cell reactant status supports remaining mission with margin; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-16T07:08:59.990 apollo-17 d10 e010 navigation computer updated with latest state vector from ground; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-16T07:18:35.989 apollo-17 d10 e011 small correction burn considered and judged unnecessary at this time; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-16T07:28:11.988 apollo-17 d10 e012 camera imagery captures Moon receding into distance from command module; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-16T07:37:47.987 apollo-17 d10 e013 experiment results packaged and secured for Earth recovery and study; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-16T07:47:23.986 apollo-17 d10 e014 sleep periods adjusted to align with planned reentry ground support; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt deleted file mode 100644 index dc935ef308..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day11.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-17T05:33:00.000 apollo-17 d11 e000 midcourse correction opportunity evaluated using latest navigation data; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T05:42:35.999 apollo-17 d11 e001 small correction burn executed to tighten entry corridor parameters; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T05:52:11.998 apollo-17 d11 e002 flight dynamics officers confirm predicted splashdown point in Pacific; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:01:47.997 apollo-17 d11 e003 crew conducts equipment checks on reentry suits and harnesses; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:11:23.996 apollo-17 d11 e004 cabin atmosphere sampled and readings remain within nominal limits; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:20:59.995 apollo-17 d11 e005 communications checks performed with recovery forces coordination centers; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:30:35.994 apollo-17 d11 e006 command module interior rearranged to facilitate entry operations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:40:11.993 apollo-17 d11 e007 crew records audio commentary on mission highlights and observations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:49:47.992 apollo-17 d11 e008 instrument readouts for deep space experiments logged for scientists; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T06:59:23.991 apollo-17 d11 e009 guidance system alignments repeated to maintain precise attitude knowledge; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T07:08:59.990 apollo-17 d11 e010 thruster usage tracked to ensure adequate reserves for attitude control; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T07:18:35.989 apollo-17 d11 e011 radiation exposure totals calculated from dosimeter instrument data; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T07:28:11.988 apollo-17 d11 e012 galley and waste management systems checked before final flight phase; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T07:37:47.987 apollo-17 d11 e013 flight controllers review contingency entry options with crew procedures; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-17T07:47:23.986 apollo-17 d11 e014 Earth appears larger in spacecraft windows as range continues to close; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt deleted file mode 100644 index 3e17323006..0000000000 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day12.txt +++ /dev/null @@ -1,15 +0,0 @@ -1972-12-18T05:33:00.000 apollo-17 d12 e000 final midcourse update confirms entry interface conditions within limits; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T05:42:35.999 apollo-17 d12 e001 service module systems evaluated ahead of planned jettison sequence; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T05:52:11.998 apollo-17 d12 e002 crew configures command module for standalone entry operations; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:01:47.997 apollo-17 d12 e003 communications tests conducted with recovery ships and aircraft; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:11:23.996 apollo-17 d12 e004 reentry checklist reviewed step by step with mission control; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:20:59.995 apollo-17 d12 e005 photography documents Earth weather systems along ground track; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:30:35.994 apollo-17 d12 e006 crew secures loose items in cabin to prevent free floating during G loads; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:40:11.993 apollo-17 d12 e007 instrument packages stowed in locations compatible with landing loads; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:49:47.992 apollo-17 d12 e008 guidance computer switched to appropriate reentry programs; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T06:59:23.991 apollo-17 d12 e009 service module jettison countdown initiated by crew on mission timers; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T07:08:59.990 apollo-17 d12 e010 physical conditioning exercises performed to prepare for gravity return; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T07:18:35.989 apollo-17 d12 e011 flight controllers monitor final trajectory refinements in real time; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T07:28:11.988 apollo-17 d12 e012 spacecraft attitude adjusted for correct heat shield orientation; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T07:37:47.987 apollo-17 d12 e013 entry corridor displays show predicted deceleration profile for crew; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-18T07:47:23.986 apollo-17 d12 e014 crew reports readiness for high speed entry and ocean splashdown; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt index 5362ca002c..994b0febdb 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt @@ -1,8 +1,3 @@ -1972-12-19T05:33:00.000 apollo-17 d13 e000 service module jettison executed leaving command module alone for entry; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-19T05:38:44.786 apollo-17 d13 e001 final attitude adjustments bring heat shield into entry orientation; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-19T05:44:29.572 apollo-17 d13 e002 radio blackout interval begins as plasma sheath forms around capsule; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-19T05:50:14.358 apollo-17 d13 e003 communications restored after blackout with voice contact to recovery forces; all primary systems nominal; communications lock; navigation solution within expected error -1972-12-19T05:55:59.144 apollo-17 d13 e004 deceleration loads peak within planned values on crew couches; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-19T06:01:43.930 apollo-17 d13 e005 drogue parachute deployment sequence initiates to stabilize capsule; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-19T06:07:28.716 apollo-17 d13 e006 main parachutes deploy and show symmetrical inflation on visual checks; all primary systems nominal; communications lock; navigation solution within expected error 1972-12-19T06:13:13.502 apollo-17 d13 e007 command module descends toward Pacific splashdown zone near recovery ship; all primary systems nominal; communications lock; navigation solution within expected error From 95739fbb6000829c1595c3c44fbb6f62750f1962 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 20:40:42 +0000 Subject: [PATCH 10/13] Rabbit comments. --- .../tests/package_tests/clp_json/test_clp_json.py | 9 ++++++--- .../clp_text/sample_datasets/text-multifile/README.md | 2 +- .../tests/package_tests/clp_text/test_clp_text.py | 11 +++++++---- integration-tests/tests/utils/asserting_utils.py | 9 ++++----- integration-tests/tests/utils/utils.py | 3 +++ 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 2ecac629de..d9726966fc 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -65,6 +65,11 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta """ validate_package_instance(fixt_package_instance) + # Clear archives before compressing. + package_test_config = fixt_package_instance.package_test_config + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() + # Compress a dataset. compression_job = PackageCompressionJob( path_to_dataset=( @@ -82,7 +87,6 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta }, positional_args=None, ) - package_test_config = fixt_package_instance.package_test_config run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. @@ -91,8 +95,7 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta log_msg = "test_clp_json_compression_json_multifile was successful." logger.info(log_msg) - # Clean up. - package_path_config = package_test_config.path_config + # Clear archives. package_path_config.clear_package_archives() diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md index 4ed344d5f1..d3a910a18f 100644 --- a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md +++ b/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md @@ -1,7 +1,7 @@ # Description of logs **Type** -Unstructured text +Unstructured text **Number of files** 5 diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index 0fdd764f59..fa6fd28e36 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -57,12 +57,17 @@ def test_clp_text_startup(fixt_package_instance: PackageInstance) -> None: @pytest.mark.compression def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInstance) -> None: """ - Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. + Validate that the `clp-text` package successfully compresses the `text-multifile` dataset. :param fixt_package_instance: """ validate_package_instance(fixt_package_instance) + # Clear archives before compressing. + package_test_config = fixt_package_instance.package_test_config + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() + # Compress a dataset. compression_job = PackageCompressionJob( path_to_dataset=( @@ -77,7 +82,6 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta options=None, positional_args=None, ) - package_test_config = fixt_package_instance.package_test_config run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. @@ -86,6 +90,5 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta log_msg = "test_clp_text_compression_text_multifile was successful." logger.info(log_msg) - # Clean up. - package_path_config = package_test_config.path_config + # Clear archives. package_path_config.clear_package_archives() diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index bbefef98d2..dc8f49f016 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -143,11 +143,10 @@ def verify_compression( # Verify content equality. decompression_dir = package_test_config.path_config.package_decompression_dir path_to_dataset = compression_job.path_to_dataset - output_path = decompression_dir / path_to_dataset + output_path = decompression_dir / path_to_dataset.relative_to(path_to_dataset.anchor) - assert is_dir_tree_content_equal( - path_to_dataset, - output_path, - ), f"Mismatch between clp input {path_to_dataset} and output {output_path}." + if not is_dir_tree_content_equal(path_to_dataset, output_path): + err_msg = f"Mismatch between clp input {path_to_dataset} and output {output_path}." + pytest.fail(err_msg) unlink(decompression_dir) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 710bc3907a..ab447a945c 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -16,6 +16,9 @@ def clean_directory(directory: Path) -> None: :param directory: """ + if not directory.exists(): + return + for item in directory.iterdir(): if item.is_file() or item.is_symlink(): item.unlink() From a73d6e1eb931a819b618d9640382989c53c0b70e Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 29 Jan 2026 21:07:26 +0000 Subject: [PATCH 11/13] Rabbit. --- integration-tests/tests/utils/asserting_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index dc8f49f016..49d2bb6c62 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -141,7 +141,6 @@ def verify_compression( run_and_assert(decompress_cmd) # Verify content equality. - decompression_dir = package_test_config.path_config.package_decompression_dir path_to_dataset = compression_job.path_to_dataset output_path = decompression_dir / path_to_dataset.relative_to(path_to_dataset.anchor) From 7a2824dfd3de7b418d9f8cc5a7b86bc196ca0fc6 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 2 Feb 2026 19:57:40 +0000 Subject: [PATCH 12/13] Address Bill's comments. --- integration-tests/.pytest.ini | 2 +- .../json-multifile/README.md | 0 .../logs/sts-135-2011-07-08.jsonl | 0 .../logs/sts-135-2011-07-09.jsonl | 0 .../logs/sts-135-2011-07-11.jsonl | 0 .../logs/sts-135-2011-07-19.jsonl | 0 .../logs/sts-135-2011-07-21.jsonl | 0 .../package_tests/clp_json/test_clp_json.py | 8 +++---- .../text-multifile/README.md | 0 .../text-multifile/logs/apollo-17_day01.txt | 0 .../text-multifile/logs/apollo-17_day04.txt | 0 .../text-multifile/logs/apollo-17_day07.txt | 0 .../text-multifile/logs/apollo-17_day10.txt | 0 .../text-multifile/logs/apollo-17_day13.txt | 0 .../package_tests/clp_text/test_clp_text.py | 8 +++---- .../tests/utils/asserting_utils.py | 23 +++++++++++------- integration-tests/tests/utils/config.py | 4 ++-- integration-tests/tests/utils/utils.py | 24 +++++++++++++++---- 18 files changed, 44 insertions(+), 25 deletions(-) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/README.md (100%) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/logs/sts-135-2011-07-08.jsonl (100%) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/logs/sts-135-2011-07-09.jsonl (100%) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/logs/sts-135-2011-07-11.jsonl (100%) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/logs/sts-135-2011-07-19.jsonl (100%) rename integration-tests/tests/package_tests/clp_json/{sample_datasets => data}/json-multifile/logs/sts-135-2011-07-21.jsonl (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/README.md (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/logs/apollo-17_day01.txt (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/logs/apollo-17_day04.txt (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/logs/apollo-17_day07.txt (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/logs/apollo-17_day10.txt (100%) rename integration-tests/tests/package_tests/clp_text/{sample_datasets => data}/text-multifile/logs/apollo-17_day13.txt (100%) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 9aa12e3868..e1cf224243 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -11,7 +11,7 @@ env = D:CLP_BUILD_DIR=../build D:CLP_CORE_BINS_DIR=../build/core D:CLP_PACKAGE_DIR=../build/clp-package - D:INTEGRATION_TESTS_DIR=./ + D:INTEGRATION_TESTS_PROJECT_ROOT=./ log_cli = True log_cli_date_format = %Y-%m-%d %H:%M:%S,%f log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/README.md b/integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/README.md rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-08.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-08.jsonl rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-09.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-09.jsonl rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-11.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-11.jsonl rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-19.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-19.jsonl rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl diff --git a/integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-21.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl similarity index 100% rename from integration-tests/tests/package_tests/clp_json/sample_datasets/json-multifile/logs/sts-135-2011-07-21.jsonl rename to integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index d9726966fc..df9e324574 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -12,7 +12,7 @@ from tests.utils.asserting_utils import ( validate_package_instance, - verify_compression, + verify_package_compression, ) from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig @@ -73,11 +73,11 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta # Compress a dataset. compression_job = PackageCompressionJob( path_to_dataset=( - resolve_path_env_var("INTEGRATION_TESTS_DIR") + resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") / "tests" / "package_tests" / "clp_json" - / "sample_datasets" + / "data" / "json-multifile" / "logs" ), @@ -90,7 +90,7 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. - verify_compression(compression_job, package_test_config) + verify_package_compression(compression_job, package_test_config) log_msg = "test_clp_json_compression_json_multifile was successful." logger.info(log_msg) diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md b/integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/README.md rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day01.txt similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day01.txt rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day01.txt diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day04.txt similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day04.txt rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day04.txt diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day07.txt similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day07.txt rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day07.txt diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day10.txt similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day10.txt rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day10.txt diff --git a/integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day13.txt similarity index 100% rename from integration-tests/tests/package_tests/clp_text/sample_datasets/text-multifile/logs/apollo-17_day13.txt rename to integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day13.txt diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index fa6fd28e36..e6e9467e2d 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -12,7 +12,7 @@ from tests.utils.asserting_utils import ( validate_package_instance, - verify_compression, + verify_package_compression, ) from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig @@ -71,11 +71,11 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta # Compress a dataset. compression_job = PackageCompressionJob( path_to_dataset=( - resolve_path_env_var("INTEGRATION_TESTS_DIR") + resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") / "tests" / "package_tests" / "clp_text" - / "sample_datasets" + / "data" / "text-multifile" / "logs" ), @@ -85,7 +85,7 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. - verify_compression(compression_job, package_test_config) + verify_package_compression(compression_job, package_test_config) log_msg = "test_clp_text_compression_text_multifile was successful." logger.info(log_msg) diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index 49d2bb6c62..d85a4b2639 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -6,13 +6,14 @@ from typing import Any import pytest +from clp_package_utils.general import EXTRACT_FILE_CMD from clp_py_utils.clp_config import ClpConfig from pydantic import ValidationError from tests.utils.clp_mode_utils import compare_mode_signatures from tests.utils.config import PackageCompressionJob, PackageInstance, PackageTestConfig from tests.utils.docker_utils import list_running_services_in_compose_project -from tests.utils.utils import is_dir_tree_content_equal, load_yaml_to_dict, unlink +from tests.utils.utils import clear_directory, is_dir_tree_content_equal, load_yaml_to_dict logger = logging.getLogger(__name__) @@ -106,7 +107,7 @@ def _validate_running_mode_correct(package_instance: PackageInstance) -> None: pytest.fail("Mode mismatch: running configuration does not match intended configuration.") -def verify_compression( +def verify_package_compression( compression_job: PackageCompressionJob, package_test_config: PackageTestConfig, ) -> None: @@ -119,7 +120,7 @@ def verify_compression( """ mode = package_test_config.mode_config.mode_name - if mode in ("clp-json", "clp-presto"): + if mode == "clp-json": # TODO: Waiting for PR 1299 to be merged. assert True elif mode == "clp-text": @@ -128,11 +129,14 @@ def verify_compression( decompress_script_path = path_config.decompress_script_path decompression_dir = path_config.package_decompression_dir temp_config_file_path = package_test_config.temp_config_file_path + + clear_directory(decompression_dir) + decompress_cmd = [ str(decompress_script_path), "--config", str(temp_config_file_path), - "x", + EXTRACT_FILE_CMD, "--extraction-dir", str(decompression_dir), ] @@ -144,8 +148,9 @@ def verify_compression( path_to_dataset = compression_job.path_to_dataset output_path = decompression_dir / path_to_dataset.relative_to(path_to_dataset.anchor) - if not is_dir_tree_content_equal(path_to_dataset, output_path): - err_msg = f"Mismatch between clp input {path_to_dataset} and output {output_path}." - pytest.fail(err_msg) - - unlink(decompression_dir) + try: + if not is_dir_tree_content_equal(path_to_dataset, output_path): + err_msg = f"Mismatch between clp input {path_to_dataset} and output {output_path}." + pytest.fail(err_msg) + finally: + clear_directory(decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index c1dcaa1026..503847b0fb 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -15,7 +15,7 @@ ) from tests.utils.utils import ( - clean_directory, + clear_directory, unlink, validate_dir_exists, validate_file_exists, @@ -140,7 +140,7 @@ def decompress_script_path(self) -> Path: def clear_package_archives(self) -> None: """Removes the contents of `clp-package/var/data/archives`.""" archives_dir = self.clp_package_dir / "var" / "data" / "archives" - clean_directory(archives_dir) + clear_directory(archives_dir) @dataclass(frozen=True) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index ab447a945c..9108a31f02 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -10,7 +10,7 @@ import yaml -def clean_directory(directory: Path) -> None: +def clear_directory(directory: Path) -> None: """ Removes the contents of `directory` without removing `directory` itself. @@ -20,10 +20,7 @@ def clean_directory(directory: Path) -> None: return for item in directory.iterdir(): - if item.is_file() or item.is_symlink(): - item.unlink() - elif item.is_dir(): - shutil.rmtree(item) + remove_path(item) def get_binary_path(name: str) -> str: @@ -109,6 +106,23 @@ def load_yaml_to_dict(path: Path) -> dict[str, Any]: return target_dict +def remove_path(path_to_remove: Path) -> None: + """ + Remove a file, directory, or symlink at `path_to_remove` if it exists. + + :param path_to_remove: + :raise: Propagates `pathlib.Path.unlink`'s exceptions. + :raise: Propagates `shutil.rmtree`'s exceptions. + """ + if not path_to_remove.exists(): + return + + if path_to_remove.is_dir() and not path_to_remove.is_symlink(): + shutil.rmtree(path_to_remove) + else: + path_to_remove.unlink() + + def resolve_path_env_var(var_name: str) -> Path: """ :param var_name: Name of the environment variable holding a path. From 1ceaf8d2fffe9e6243179b9994739cb670d90d23 Mon Sep 17 00:00:00 2001 From: Quinn Date: Fri, 6 Feb 2026 18:06:48 +0000 Subject: [PATCH 13/13] Address Bill's comments. --- .../tests/fixtures/path_configs.py | 3 +++ .../package_tests/clp_json/test_clp_json.py | 23 +++++++---------- .../package_tests/clp_text/test_clp_text.py | 13 +++------- .../tests/utils/asserting_utils.py | 24 +++++++++++------- integration-tests/tests/utils/config.py | 25 +++++++++++++++---- .../tests/utils/package_utils.py | 7 ++---- 6 files changed, 52 insertions(+), 43 deletions(-) diff --git a/integration-tests/tests/fixtures/path_configs.py b/integration-tests/tests/fixtures/path_configs.py index bc6e270ac5..e041fabfa5 100644 --- a/integration-tests/tests/fixtures/path_configs.py +++ b/integration-tests/tests/fixtures/path_configs.py @@ -31,5 +31,8 @@ def fixt_package_path_config( """Provides paths for the clp-package directory and its contents.""" return PackagePathConfig( clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"), + package_test_scripts_dir=( + resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") / "tests" / "package_tests" + ), test_root_dir=integration_test_path_config.test_root_dir, ) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index df9e324574..bf5a471647 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -17,7 +17,6 @@ from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig from tests.utils.package_utils import run_package_compression_script -from tests.utils.utils import resolve_path_env_var logger = logging.getLogger(__name__) @@ -72,25 +71,21 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta # Compress a dataset. compression_job = PackageCompressionJob( - path_to_dataset=( - resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") - / "tests" - / "package_tests" - / "clp_json" - / "data" - / "json-multifile" - / "logs" + path_to_original_dataset=( + package_path_config.clp_json_test_data_path / "json-multifile" / "logs" ), - options={ - "--timestamp-key": "timestamp", - "--dataset": "json_multifile", - }, + options=[ + "--timestamp-key", + "timestamp", + "--dataset", + "json_multifile", + ], positional_args=None, ) run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. - verify_package_compression(compression_job, package_test_config) + verify_package_compression(compression_job.path_to_original_dataset, package_test_config) log_msg = "test_clp_json_compression_json_multifile was successful." logger.info(log_msg) diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index e6e9467e2d..a9ec962d6d 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -17,7 +17,6 @@ from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig from tests.utils.package_utils import run_package_compression_script -from tests.utils.utils import resolve_path_env_var logger = logging.getLogger(__name__) @@ -70,14 +69,8 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta # Compress a dataset. compression_job = PackageCompressionJob( - path_to_dataset=( - resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") - / "tests" - / "package_tests" - / "clp_text" - / "data" - / "text-multifile" - / "logs" + path_to_original_dataset=( + package_path_config.clp_text_test_data_path / "text-multifile" / "logs" ), options=None, positional_args=None, @@ -85,7 +78,7 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta run_package_compression_script(compression_job, package_test_config) # Check the correctness of compression. - verify_package_compression(compression_job, package_test_config) + verify_package_compression(compression_job.path_to_original_dataset, package_test_config) log_msg = "test_clp_text_compression_text_multifile was successful." logger.info(log_msg) diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index d85a4b2639..7cbc7c0e4a 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -3,6 +3,7 @@ import logging import shlex import subprocess +from pathlib import Path from typing import Any import pytest @@ -11,7 +12,7 @@ from pydantic import ValidationError from tests.utils.clp_mode_utils import compare_mode_signatures -from tests.utils.config import PackageCompressionJob, PackageInstance, PackageTestConfig +from tests.utils.config import PackageInstance, PackageTestConfig from tests.utils.docker_utils import list_running_services_in_compose_project from tests.utils.utils import clear_directory, is_dir_tree_content_equal, load_yaml_to_dict @@ -108,14 +109,15 @@ def _validate_running_mode_correct(package_instance: PackageInstance) -> None: def verify_package_compression( - compression_job: PackageCompressionJob, + path_to_original_dataset: Path, package_test_config: PackageTestConfig, ) -> None: """ - Verify that the compression_job has been executed correctly by decompressing the contents of - `clp-package/var/data/archives` and comparing the decompressed logs to the originals. + Verify that compression has been executed correctly by decompressing the contents of + `clp-package/var/data/archives` and comparing the decompressed logs to the originals stored at + `path_to_original_dataset`. - :param compression_job: + :param path_to_original_dataset: :param package_test_config: """ mode = package_test_config.mode_config.mode_name @@ -145,12 +147,16 @@ def verify_package_compression( run_and_assert(decompress_cmd) # Verify content equality. - path_to_dataset = compression_job.path_to_dataset - output_path = decompression_dir / path_to_dataset.relative_to(path_to_dataset.anchor) + output_path = decompression_dir / path_to_original_dataset.relative_to( + path_to_original_dataset.anchor + ) try: - if not is_dir_tree_content_equal(path_to_dataset, output_path): - err_msg = f"Mismatch between clp input {path_to_dataset} and output {output_path}." + if not is_dir_tree_content_equal(path_to_original_dataset, output_path): + err_msg = ( + f"Mismatch between clp input {path_to_original_dataset} and output" + f" {output_path}." + ) pytest.fail(err_msg) finally: clear_directory(decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 503847b0fb..993ed230bc 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -5,7 +5,6 @@ import re from dataclasses import dataclass, field, InitVar from pathlib import Path -from typing import Any import yaml from clp_py_utils.clp_config import ( @@ -72,6 +71,9 @@ class PackagePathConfig: #: Root directory containing all CLP package contents. clp_package_dir: Path + #: Root directory where all package test scripts and data are stored. + package_test_scripts_dir: Path + #: Root directory for package tests output. test_root_dir: InitVar[Path] @@ -99,7 +101,10 @@ def __post_init__(self, test_root_dir: Path) -> None: ) raise RuntimeError(err_msg) - # Initialize directory for package tests. + # Validate directory for package test scripts. + validate_dir_exists(self.package_test_scripts_dir) + + # Initialize directory for package test output. validate_dir_exists(test_root_dir) object.__setattr__(self, "temp_config_dir", test_root_dir / "temp_config_files") object.__setattr__( @@ -137,6 +142,16 @@ def decompress_script_path(self) -> Path: """:return: The absolute path to the package decompress script.""" return self.clp_package_dir / "sbin" / "decompress.sh" + @property + def clp_json_test_data_path(self) -> Path: + """:return: The absolute path to the data for clp-json tests.""" + return self.package_test_scripts_dir / "clp_json" / "data" + + @property + def clp_text_test_data_path(self) -> Path: + """:return: The absolute path to the data for clp-text tests.""" + return self.package_test_scripts_dir / "clp_text" / "data" + def clear_package_archives(self) -> None: """Removes the contents of `clp-package/var/data/archives`.""" archives_dir = self.clp_package_dir / "var" / "data" / "archives" @@ -148,10 +163,10 @@ class PackageCompressionJob: """A compression job for a package test.""" #: The absolute path to the dataset (either a file or directory). - path_to_dataset: Path + path_to_original_dataset: Path - #: Options to specify in the compression command, with their corresponding values. - options: dict[str, Any] | None + #: Options to specify in the compression command. + options: list[str] | None #: Positional arguments to specify in the compression command (do not put paths to compress) positional_args: list[str] | None diff --git a/integration-tests/tests/utils/package_utils.py b/integration-tests/tests/utils/package_utils.py index 9c39e514cd..01d67f4a07 100644 --- a/integration-tests/tests/utils/package_utils.py +++ b/integration-tests/tests/utils/package_utils.py @@ -70,15 +70,12 @@ def run_package_compression_script( ] if compression_job.options is not None: - for option, option_value in compression_job.options.items(): - compress_cmd.append(option) - if option_value is not None: - compress_cmd.append(str(option_value)) + compress_cmd.extend(compression_job.options) if compression_job.positional_args is not None: compress_cmd.extend(compression_job.positional_args) - compress_cmd.append(str(compression_job.path_to_dataset)) + compress_cmd.append(str(compression_job.path_to_original_dataset)) # Run compression command for this job and assert that it succeeds. run_and_assert(compress_cmd, timeout=DEFAULT_CMD_TIMEOUT_SECONDS)