diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 6124409099..e1cf224243 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -11,6 +11,7 @@ env = D:CLP_BUILD_DIR=../build D:CLP_CORE_BINS_DIR=../build/core D:CLP_PACKAGE_DIR=../build/clp-package + D:INTEGRATION_TESTS_PROJECT_ROOT=./ log_cli = True log_cli_date_format = %Y-%m-%d %H:%M:%S,%f log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s diff --git a/integration-tests/tests/fixtures/path_configs.py b/integration-tests/tests/fixtures/path_configs.py index bc6e270ac5..e041fabfa5 100644 --- a/integration-tests/tests/fixtures/path_configs.py +++ b/integration-tests/tests/fixtures/path_configs.py @@ -31,5 +31,8 @@ def fixt_package_path_config( """Provides paths for the clp-package directory and its contents.""" return PackagePathConfig( clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"), + package_test_scripts_dir=( + resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") / "tests" / "package_tests" + ), test_root_dir=integration_test_path_config.test_root_dir, ) diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md b/integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md new file mode 100644 index 0000000000..dd501c1ac4 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md @@ -0,0 +1,16 @@ +# Description of logs + +**Type** +JSON + +**Number of files** +5 + +**Number of events per file** +8 + +**Beginning timestamp (milliseconds)** +1310138944000 + +**End timestamp (milliseconds)** +1311208074120 diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl new file mode 100644 index 0000000000..1ba45d0ec1 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310138944000,"mission":"STS-135","mission_day_index":0,"event":"SRB_IGNITION_CONFIRMED","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters report stable ignition and rising thrust","line_index":0} +{"timestamp":1310139148373,"mission":"STS-135","mission_day_index":0,"event":"CLEAR_OF_TOWER","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle cleared tower, roll program initiated, tracking nominal trajectory","line_index":1} +{"timestamp":1310139352746,"mission":"STS-135","mission_day_index":0,"event":"ROLL_PROGRAM_COMPLETE","subsystem":"GUIDANCE","level":"INFO","detail":"Roll program complete, heads down attitude achieved for ascent","line_index":2} +{"timestamp":1310139557119,"mission":"STS-135","mission_day_index":0,"event":"MAX_Q_THROTTLE","subsystem":"PROP","level":"INFO","detail":"Main engines throttled to manage maximum dynamic pressure on stack","line_index":3} +{"timestamp":1310139761493,"mission":"STS-135","mission_day_index":0,"event":"SRB_SEPARATION","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters separation confirmed, separation motors firing nominally","line_index":4} +{"timestamp":1310139965866,"mission":"STS-135","mission_day_index":0,"event":"PRESS_TO_MECO","subsystem":"GUIDANCE","level":"INFO","detail":"Performance marks indicate press to main engine cutoff achieved","line_index":5} +{"timestamp":1310140170239,"mission":"STS-135","mission_day_index":0,"event":"MAIN_ENGINE_CUTOFF","subsystem":"PROP","level":"INFO","detail":"All three SSME engines shutdown, engine parameters within expected bands","line_index":6} +{"timestamp":1310140374613,"mission":"STS-135","mission_day_index":0,"event":"EXTERNAL_TANK_SEPARATION","subsystem":"STRUCTURES","level":"INFO","detail":"External tank separation command executed, umbilical disconnect nominal","line_index":7} diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl new file mode 100644 index 0000000000..89650afd46 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310169600000,"mission":"STS-135","mission_day_index":1,"event":"RNDZ_BURN","subsystem":"GUIDANCE","level":"INFO","detail":"Rendezvous correction burn complete, relative motion profile matches ISS plan","line_index":0} +{"timestamp":1310170175999,"mission":"STS-135","mission_day_index":1,"event":"RPM_MANEUVER","subsystem":"GUIDANCE","level":"INFO","detail":"R bar pitch maneuver complete for station imaging and TPS assessment","line_index":1} +{"timestamp":1310170751999,"mission":"STS-135","mission_day_index":1,"event":"FINAL_APPROACH","subsystem":"GUIDANCE","level":"INFO","detail":"Final approach corridor maintained, closing rate within docking constraints","line_index":2} +{"timestamp":1310171327999,"mission":"STS-135","mission_day_index":1,"event":"SOFT_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Soft capture confirmed at forward port, relative motion damped","line_index":3} +{"timestamp":1310171903999,"mission":"STS-135","mission_day_index":1,"event":"HARD_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Structural latches engaged, ISS and shuttle in hard dock configuration","line_index":4} +{"timestamp":1310172479999,"mission":"STS-135","mission_day_index":1,"event":"HATCH_OPEN","subsystem":"STRUCTURES","level":"INFO","detail":"Hatch open for ingress, pressure equalization verified within safe band","line_index":5} +{"timestamp":1310173055999,"mission":"STS-135","mission_day_index":1,"event":"MPLM_BERTHING","subsystem":"ROBOTICS","level":"INFO","detail":"Multipurpose logistics module berthed to Node, structural attach points verified","line_index":6} +{"timestamp":1310173631999,"mission":"STS-135","mission_day_index":1,"event":"CARGO_TRANSFER","subsystem":"PAYLOAD","level":"INFO","detail":"Cargo transfer underway according to integrated ISS and shuttle transfer list","line_index":7} diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl new file mode 100644 index 0000000000..cdbabf0945 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1310342400000,"mission":"STS-135","mission_day_index":3,"event":"JOINT_OPERATIONS_PLANNING","subsystem":"TIMELINE","level":"INFO","detail":"Daily planning conference complete, joint tasks synchronized","line_index":0} +{"timestamp":1310342975999,"mission":"STS-135","mission_day_index":3,"event":"EVA_PREP","subsystem":"EVA","level":"INFO","detail":"Extravehicular mobility units serviced, prebreathe protocols initiated","line_index":1} +{"timestamp":1310343551999,"mission":"STS-135","mission_day_index":3,"event":"EVA_TASK_STATUS","subsystem":"EVA","level":"INFO","detail":"Truss and payload bay tasks progressing, consumables within expected usage","line_index":2} +{"timestamp":1310344127999,"mission":"STS-135","mission_day_index":3,"event":"MPLM_UNBERTH","subsystem":"ROBOTICS","level":"INFO","detail":"Logistics module unberthed and reinstalled in payload bay","line_index":3} +{"timestamp":1310344703999,"mission":"STS-135","mission_day_index":3,"event":"ISS_SYSTEMS_HANDOFF","subsystem":"COMMS","level":"INFO","detail":"Command and telemetry handoff between shuttle and station completed","line_index":4} +{"timestamp":1310345279999,"mission":"STS-135","mission_day_index":3,"event":"MIDDECK_PAYLOAD_CHECK","subsystem":"PAYLOAD","level":"INFO","detail":"Middeck experiment racks powered and data recording verified","line_index":5} +{"timestamp":1310345855999,"mission":"STS-135","mission_day_index":3,"event":"ENVIRONMENTAL_MONITORING","subsystem":"ECLSS","level":"INFO","detail":"Atmospheric sampling shows stable CO2 and trace contaminant levels","line_index":6} +{"timestamp":1310346431999,"mission":"STS-135","mission_day_index":3,"event":"JOINT_CREW_MEETING","subsystem":"TIMELINE","level":"INFO","detail":"Joint crew status review completed, no blocking issues reported","line_index":7} diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl new file mode 100644 index 0000000000..c60ff4341a --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1311033600000,"mission":"STS-135","mission_day_index":11,"event":"UNDOCK_COMMAND","subsystem":"DOCKING","level":"INFO","detail":"Undock command executed, structural latches release verified","line_index":0} +{"timestamp":1311034175999,"mission":"STS-135","mission_day_index":11,"event":"SEPARATION_BURNS","subsystem":"GUIDANCE","level":"INFO","detail":"Separation maneuvers complete, increasing range rate from station","line_index":1} +{"timestamp":1311034751999,"mission":"STS-135","mission_day_index":11,"event":"TPS_INSPECTION","subsystem":"THERMAL","level":"INFO","detail":"Thermal protection system sensor data and imagery show no critical damage","line_index":2} +{"timestamp":1311035327999,"mission":"STS-135","mission_day_index":11,"event":"DEORBIT_BURN","subsystem":"PROP","level":"INFO","detail":"Deorbit burn complete, entry interface conditions within design envelope","line_index":3} +{"timestamp":1311035903999,"mission":"STS-135","mission_day_index":11,"event":"ENTRY_INTERFACE","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle at entry interface, guidance following nominal drag corridor","line_index":4} +{"timestamp":1311036479999,"mission":"STS-135","mission_day_index":11,"event":"COMM_BLACKOUT","subsystem":"COMMS","level":"INFO","detail":"Expected communications blackout region entered, tracking via ground radar only","line_index":5} +{"timestamp":1311037055999,"mission":"STS-135","mission_day_index":11,"event":"SUBSONIC_TRANSITION","subsystem":"GUIDANCE","level":"INFO","detail":"Shuttle transitions to subsonic flight, control surfaces active","line_index":6} +{"timestamp":1311037631999,"mission":"STS-135","mission_day_index":11,"event":"WHEELS_STOP","subsystem":"TIMELINE","level":"INFO","detail":"Vehicle stopped on runway, rollout distance within predicted range","line_index":7} diff --git a/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl new file mode 100644 index 0000000000..f2b7aa9ec8 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1311206400000,"mission":"STS-135","mission_day_index":13,"event":"PAYLOAD_UNLOAD","subsystem":"PAYLOAD","level":"INFO","detail":"Post flight payload processing under way, canisters removed from bay","line_index":0} +{"timestamp":1311206639160,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_SAFE","subsystem":"GROUND","level":"INFO","detail":"Hazardous systems safed, access teams cleared for orbiter entry","line_index":1} +{"timestamp":1311206878320,"mission":"STS-135","mission_day_index":13,"event":"DATA_ARCHIVE","subsystem":"GROUND","level":"INFO","detail":"Flight data downlinked and archived for post mission analysis","line_index":2} +{"timestamp":1311207117480,"mission":"STS-135","mission_day_index":13,"event":"CREW_DEBRIEF","subsystem":"TIMELINE","level":"INFO","detail":"Crew debrief with mission control completed without anomalies","line_index":3} +{"timestamp":1311207356640,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_TOW","subsystem":"GROUND","level":"INFO","detail":"Atlantis towed from runway to Orbiter Processing Facility","line_index":4} +{"timestamp":1311207595800,"mission":"STS-135","mission_day_index":13,"event":"RETIREMENT_CONFIG","subsystem":"GROUND","level":"INFO","detail":"Systems configured for extended safe power down and display conversion","line_index":5} +{"timestamp":1311207834960,"mission":"STS-135","mission_day_index":13,"event":"PUBLIC_EVENT","subsystem":"OUTREACH","level":"INFO","detail":"Crew participates in public event describing program closeout activities","line_index":6} +{"timestamp":1311208074120,"mission":"STS-135","mission_day_index":13,"event":"HARDWARE_REUSE_ASSESSMENT","subsystem":"ENGINEERING","level":"INFO","detail":"Component assessments identify candidates for museum display","line_index":7} diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index d3c48af72f..bf5a471647 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -12,9 +12,11 @@ from tests.utils.asserting_utils import ( validate_package_instance, + verify_package_compression, ) from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS -from tests.utils.config import PackageInstance, PackageModeConfig +from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig +from tests.utils.package_utils import run_package_compression_script logger = logging.getLogger(__name__) @@ -54,20 +56,43 @@ def test_clp_json_startup(fixt_package_instance: PackageInstance) -> None: @pytest.mark.compression -def test_clp_json_compression(fixt_package_instance: PackageInstance) -> None: +def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInstance) -> None: """ - Validate that the `clp-json` package successfully compresses some dataset. + Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. :param fixt_package_instance: """ validate_package_instance(fixt_package_instance) - # TODO: compress some dataset and check the correctness of compression. - assert True + # Clear archives before compressing. + package_test_config = fixt_package_instance.package_test_config + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() - log_msg = "test_clp_json_compression was successful." + # Compress a dataset. + compression_job = PackageCompressionJob( + path_to_original_dataset=( + package_path_config.clp_json_test_data_path / "json-multifile" / "logs" + ), + options=[ + "--timestamp-key", + "timestamp", + "--dataset", + "json_multifile", + ], + positional_args=None, + ) + run_package_compression_script(compression_job, package_test_config) + + # Check the correctness of compression. + verify_package_compression(compression_job.path_to_original_dataset, package_test_config) + + log_msg = "test_clp_json_compression_json_multifile was successful." logger.info(log_msg) + # Clear archives. + package_path_config.clear_package_archives() + @pytest.mark.search def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: @@ -78,7 +103,9 @@ def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: """ validate_package_instance(fixt_package_instance) - # TODO: compress some dataset and check the correctness of compression. + # TODO: compress a dataset + + # TODO: check the correctness of the compression # TODO: search through that dataset and check the correctness of the search results. @@ -86,3 +113,5 @@ def test_clp_json_search(fixt_package_instance: PackageInstance) -> None: log_msg = "test_clp_json_search was successful." logger.info(log_msg) + + # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md b/integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md new file mode 100644 index 0000000000..d3a910a18f --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md @@ -0,0 +1,16 @@ +# Description of logs + +**Type** +Unstructured text + +**Number of files** +5 + +**Number of events per file** +10 + +**Beginning timestamp (milliseconds)** +92572380000 + +**End timestamp (milliseconds)** +93614008000 diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day01.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day01.txt new file mode 100644 index 0000000000..6900c6d822 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day01.txt @@ -0,0 +1,10 @@ +1972-12-07T05:33:00.000 apollo-17 d01 e000 countdown sequencer resumes after hold and verifies launch commit criteria; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T05:42:35.999 apollo-17 d01 e001 terminal countdown proceeds at launch complex 39A with all stations reporting go; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T05:52:11.998 apollo-17 d01 e002 Saturn V first stage ignition and liftoff from Kennedy Space Center; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:01:47.997 apollo-17 d01 e003 tower cleared and roll program places vehicle on proper azimuth; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:11:23.996 apollo-17 d01 e004 pitch program guides ascent trajectory toward planned parking orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:20:59.995 apollo-17 d01 e005 first stage shutdown and staging event with clean separation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:30:35.994 apollo-17 d01 e006 second stage ignition with stable thrust and guidance lock; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:40:11.993 apollo-17 d01 e007 parking orbit achieved around Earth after third stage cutoff; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:49:47.992 apollo-17 d01 e008 systems checkout in Earth orbit confirms command and service module health; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-07T06:59:23.991 apollo-17 d01 e009 translunar injection burn ignition on third stage SIVB; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day04.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day04.txt new file mode 100644 index 0000000000..cfec6ff379 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day04.txt @@ -0,0 +1,10 @@ +1972-12-10T05:33:00.000 apollo-17 d04 e000 spacecraft enters lunar sphere of influence with gravity reference shifting; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T05:42:35.999 apollo-17 d04 e001 navigation update computes parameters for lunar orbit insertion burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T05:52:11.998 apollo-17 d04 e002 service propulsion system gimbals exercised in preparation for major burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:01:47.997 apollo-17 d04 e003 lunar orbit insertion burn ignition places spacecraft behind the Moon; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:11:23.996 apollo-17 d04 e004 loss of signal period begins as spacecraft passes behind lunar limb; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:20:59.995 apollo-17 d04 e005 lunar orbit insertion burn complete before acquisition of signal; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:30:35.994 apollo-17 d04 e006 acquisition of signal resumes with telemetry confirming stable lunar orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:40:11.993 apollo-17 d04 e007 initial lunar orbit tracking verifies altitude and period close to plan; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:49:47.992 apollo-17 d04 e008 crew performs visual inspection of lunar surface features from orbit; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-10T06:59:23.991 apollo-17 d04 e009 lunar module powered up for initial systems checkout in lunar orbit; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day07.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day07.txt new file mode 100644 index 0000000000..c30eb3129b --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day07.txt @@ -0,0 +1,10 @@ +1972-12-13T05:33:00.000 apollo-17 d07 e000 crew prepares for second surface excursion with suit and system checks; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T05:42:35.999 apollo-17 d07 e001 cabin depressurization and hatch opening procedures completed again; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T05:52:11.998 apollo-17 d07 e002 lunar rover driven toward designated geology stations away from landing site; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:01:47.997 apollo-17 d07 e003 sampling at Station Two documents dark mantle deposits and boulders; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:11:23.996 apollo-17 d07 e004 crew deploys additional experiment packages and marker flags along traverse; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:20:59.995 apollo-17 d07 e005 detailed core samples extracted for stratigraphic study of regolith; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:30:35.994 apollo-17 d07 e006 camera panoramas taken at multiple stations for context documentation; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:40:11.993 apollo-17 d07 e007 navigation data from rover used to refine local lunar topographic maps; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:49:47.992 apollo-17 d07 e008 communications remain stable during extended traverse distance from LM; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-13T06:59:23.991 apollo-17 d07 e009 crew notes color contrasts and layering in valley walls and boulders; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day10.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day10.txt new file mode 100644 index 0000000000..5255c9f241 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day10.txt @@ -0,0 +1,10 @@ +1972-12-16T05:33:00.000 apollo-17 d10 e000 service propulsion system ignition performs trans Earth injection burn; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T05:42:35.999 apollo-17 d10 e001 trajectory after burn shows proper corridor for atmospheric reentry; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T05:52:11.998 apollo-17 d10 e002 spacecraft exits lunar orbit and begins long coast back to Earth; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:01:47.997 apollo-17 d10 e003 navigation sightings compare well with ground based tracking solutions; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:11:23.996 apollo-17 d10 e004 crew conducts deep space science experiments during return flight; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:20:59.995 apollo-17 d10 e005 thermal control roll re-established for uniform heating on spacecraft; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:30:35.994 apollo-17 d10 e006 communications schedule alternates between worldwide tracking stations; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:40:11.993 apollo-17 d10 e007 crew performs housekeeping and stowage reconfiguration for entry phase; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:49:47.992 apollo-17 d10 e008 biomedical data indicates crew adapting well after lunar surface stay; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-16T06:59:23.991 apollo-17 d10 e009 fuel cell reactant status supports remaining mission with margin; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day13.txt b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day13.txt new file mode 100644 index 0000000000..994b0febdb --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/data/text-multifile/logs/apollo-17_day13.txt @@ -0,0 +1,10 @@ +1972-12-19T06:01:43.930 apollo-17 d13 e005 drogue parachute deployment sequence initiates to stabilize capsule; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:07:28.716 apollo-17 d13 e006 main parachutes deploy and show symmetrical inflation on visual checks; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:13:13.502 apollo-17 d13 e007 command module descends toward Pacific splashdown zone near recovery ship; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:18:58.288 apollo-17 d13 e008 impact with ocean surface recorded within predicted time and location; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:24:43.074 apollo-17 d13 e009 uprighting system inflates to place capsule in correct flotation attitude; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:30:27.860 apollo-17 d13 e010 recovery swimmers deploy from helicopters and secure lines to spacecraft; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:36:12.646 apollo-17 d13 e011 hatch opening occurs after cabin ventilation checks and safety verifications; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:41:57.432 apollo-17 d13 e012 crew exits command module and boards recovery helicopter for transport; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:47:42.218 apollo-17 d13 e013 command module hoisted onto recovery ship deck for post flight inspection; all primary systems nominal; communications lock; navigation solution within expected error +1972-12-19T06:53:27.004 apollo-17 d13 e014 mission debriefing begins with engineering and science team representatives; all primary systems nominal; communications lock; navigation solution within expected error diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index dbbc862f4c..a9ec962d6d 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -12,9 +12,11 @@ from tests.utils.asserting_utils import ( validate_package_instance, + verify_package_compression, ) from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS -from tests.utils.config import PackageInstance, PackageModeConfig +from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig +from tests.utils.package_utils import run_package_compression_script logger = logging.getLogger(__name__) @@ -49,3 +51,37 @@ def test_clp_text_startup(fixt_package_instance: PackageInstance) -> None: log_msg = "test_clp_text_startup was successful." logger.info(log_msg) + + +@pytest.mark.compression +def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInstance) -> None: + """ + Validate that the `clp-text` package successfully compresses the `text-multifile` dataset. + + :param fixt_package_instance: + """ + validate_package_instance(fixt_package_instance) + + # Clear archives before compressing. + package_test_config = fixt_package_instance.package_test_config + package_path_config = package_test_config.path_config + package_path_config.clear_package_archives() + + # Compress a dataset. + compression_job = PackageCompressionJob( + path_to_original_dataset=( + package_path_config.clp_text_test_data_path / "text-multifile" / "logs" + ), + options=None, + positional_args=None, + ) + run_package_compression_script(compression_job, package_test_config) + + # Check the correctness of compression. + verify_package_compression(compression_job.path_to_original_dataset, package_test_config) + + log_msg = "test_clp_text_compression_text_multifile was successful." + logger.info(log_msg) + + # Clear archives. + package_path_config.clear_package_archives() diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index 0a9b78e424..7cbc7c0e4a 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -3,16 +3,18 @@ import logging import shlex import subprocess +from pathlib import Path from typing import Any import pytest +from clp_package_utils.general import EXTRACT_FILE_CMD from clp_py_utils.clp_config import ClpConfig from pydantic import ValidationError from tests.utils.clp_mode_utils import compare_mode_signatures -from tests.utils.config import PackageInstance +from tests.utils.config import PackageInstance, PackageTestConfig from tests.utils.docker_utils import list_running_services_in_compose_project -from tests.utils.utils import load_yaml_to_dict +from tests.utils.utils import clear_directory, is_dir_tree_content_equal, load_yaml_to_dict logger = logging.getLogger(__name__) @@ -104,3 +106,57 @@ def _validate_running_mode_correct(package_instance: PackageInstance) -> None: if not compare_mode_signatures(intended_config, running_config): pytest.fail("Mode mismatch: running configuration does not match intended configuration.") + + +def verify_package_compression( + path_to_original_dataset: Path, + package_test_config: PackageTestConfig, +) -> None: + """ + Verify that compression has been executed correctly by decompressing the contents of + `clp-package/var/data/archives` and comparing the decompressed logs to the originals stored at + `path_to_original_dataset`. + + :param path_to_original_dataset: + :param package_test_config: + """ + mode = package_test_config.mode_config.mode_name + + if mode == "clp-json": + # TODO: Waiting for PR 1299 to be merged. + assert True + elif mode == "clp-text": + # Decompress the contents of `clp-package/var/data/archives`. + path_config = package_test_config.path_config + decompress_script_path = path_config.decompress_script_path + decompression_dir = path_config.package_decompression_dir + temp_config_file_path = package_test_config.temp_config_file_path + + clear_directory(decompression_dir) + + decompress_cmd = [ + str(decompress_script_path), + "--config", + str(temp_config_file_path), + EXTRACT_FILE_CMD, + "--extraction-dir", + str(decompression_dir), + ] + + # Run decompression command and assert that it succeeds. + run_and_assert(decompress_cmd) + + # Verify content equality. + output_path = decompression_dir / path_to_original_dataset.relative_to( + path_to_original_dataset.anchor + ) + + try: + if not is_dir_tree_content_equal(path_to_original_dataset, output_path): + err_msg = ( + f"Mismatch between clp input {path_to_original_dataset} and output" + f" {output_path}." + ) + pytest.fail(err_msg) + finally: + clear_directory(decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 76a8e2d43a..993ed230bc 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -14,6 +14,7 @@ ) from tests.utils.utils import ( + clear_directory, unlink, validate_dir_exists, validate_file_exists, @@ -70,12 +71,18 @@ class PackagePathConfig: #: Root directory containing all CLP package contents. clp_package_dir: Path + #: Root directory where all package test scripts and data are stored. + package_test_scripts_dir: Path + #: Root directory for package tests output. test_root_dir: InitVar[Path] #: Directory to store temporary package config files. temp_config_dir: Path = field(init=False, repr=True) + #: Directory where decompressed logs will be stored. + package_decompression_dir: Path = field(init=False, repr=True) + #: Directory where the CLP package writes logs. clp_log_dir: Path = field(init=False, repr=True) @@ -94,9 +101,15 @@ def __post_init__(self, test_root_dir: Path) -> None: ) raise RuntimeError(err_msg) - # Initialize directory for package tests. + # Validate directory for package test scripts. + validate_dir_exists(self.package_test_scripts_dir) + + # Initialize directory for package test output. validate_dir_exists(test_root_dir) object.__setattr__(self, "temp_config_dir", test_root_dir / "temp_config_files") + object.__setattr__( + self, "package_decompression_dir", test_root_dir / "package-decompressed-logs" + ) # Initialize log directory for the package. object.__setattr__( @@ -119,6 +132,45 @@ def stop_script_path(self) -> Path: """:return: The absolute path to the package stop script.""" return self.clp_package_dir / "sbin" / "stop-clp.sh" + @property + def compress_script_path(self) -> Path: + """:return: The absolute path to the package compress script.""" + return self.clp_package_dir / "sbin" / "compress.sh" + + @property + def decompress_script_path(self) -> Path: + """:return: The absolute path to the package decompress script.""" + return self.clp_package_dir / "sbin" / "decompress.sh" + + @property + def clp_json_test_data_path(self) -> Path: + """:return: The absolute path to the data for clp-json tests.""" + return self.package_test_scripts_dir / "clp_json" / "data" + + @property + def clp_text_test_data_path(self) -> Path: + """:return: The absolute path to the data for clp-text tests.""" + return self.package_test_scripts_dir / "clp_text" / "data" + + def clear_package_archives(self) -> None: + """Removes the contents of `clp-package/var/data/archives`.""" + archives_dir = self.clp_package_dir / "var" / "data" / "archives" + clear_directory(archives_dir) + + +@dataclass(frozen=True) +class PackageCompressionJob: + """A compression job for a package test.""" + + #: The absolute path to the dataset (either a file or directory). + path_to_original_dataset: Path + + #: Options to specify in the compression command. + options: list[str] | None + + #: Positional arguments to specify in the compression command (do not put paths to compress) + positional_args: list[str] | None + @dataclass(frozen=True) class PackageModeConfig: diff --git a/integration-tests/tests/utils/package_utils.py b/integration-tests/tests/utils/package_utils.py index 31254b8791..01d67f4a07 100644 --- a/integration-tests/tests/utils/package_utils.py +++ b/integration-tests/tests/utils/package_utils.py @@ -1,7 +1,10 @@ """Provides utility functions related to the CLP package used across `integration-tests`.""" from tests.utils.asserting_utils import run_and_assert -from tests.utils.config import PackageTestConfig +from tests.utils.config import ( + PackageCompressionJob, + PackageTestConfig, +) DEFAULT_CMD_TIMEOUT_SECONDS = 120.0 @@ -44,3 +47,35 @@ def stop_clp_package(package_test_config: PackageTestConfig) -> None: ] # fmt: on run_and_assert(stop_cmd, timeout=DEFAULT_CMD_TIMEOUT_SECONDS) + + +def run_package_compression_script( + compression_job: PackageCompressionJob, + package_test_config: PackageTestConfig, +) -> None: + """ + Constructs and runs a compression command on the CLP package. + + :param compression_job: + :param package_test_config: + """ + path_config = package_test_config.path_config + compress_script_path = path_config.compress_script_path + temp_config_file_path = package_test_config.temp_config_file_path + + compress_cmd = [ + str(compress_script_path), + "--config", + str(temp_config_file_path), + ] + + if compression_job.options is not None: + compress_cmd.extend(compression_job.options) + + if compression_job.positional_args is not None: + compress_cmd.extend(compression_job.positional_args) + + compress_cmd.append(str(compression_job.path_to_original_dataset)) + + # Run compression command for this job and assert that it succeeds. + run_and_assert(compress_cmd, timeout=DEFAULT_CMD_TIMEOUT_SECONDS) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index f58d3c70da..9108a31f02 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -10,6 +10,19 @@ import yaml +def clear_directory(directory: Path) -> None: + """ + Removes the contents of `directory` without removing `directory` itself. + + :param directory: + """ + if not directory.exists(): + return + + for item in directory.iterdir(): + remove_path(item) + + def get_binary_path(name: str) -> str: """ :param name: Name of the program binary to locate. @@ -93,6 +106,23 @@ def load_yaml_to_dict(path: Path) -> dict[str, Any]: return target_dict +def remove_path(path_to_remove: Path) -> None: + """ + Remove a file, directory, or symlink at `path_to_remove` if it exists. + + :param path_to_remove: + :raise: Propagates `pathlib.Path.unlink`'s exceptions. + :raise: Propagates `shutil.rmtree`'s exceptions. + """ + if not path_to_remove.exists(): + return + + if path_to_remove.is_dir() and not path_to_remove.is_symlink(): + shutil.rmtree(path_to_remove) + else: + path_to_remove.unlink() + + def resolve_path_env_var(var_name: str) -> Path: """ :param var_name: Name of the environment variable holding a path.