diff --git a/model_signing/serialization/conftest.py b/model_signing/serialization/conftest.py index 797515d7..addc9a6a 100644 --- a/model_signing/serialization/conftest.py +++ b/model_signing/serialization/conftest.py @@ -19,9 +19,20 @@ from model_signing.serialization import test_support +def pytest_addoption(parser): + """Adds a flag argument to update the goldens.""" + parser.addoption( + "--update_goldens", + action="store_true", + default=False, + help="update golden files", + ) + + # Note: Don't make fixtures with global scope as we are altering the models! @pytest.fixture def sample_model_file(tmp_path_factory): + """A model with just a single file.""" file = tmp_path_factory.mktemp("model") / "file" file.write_bytes(test_support.KNOWN_MODEL_TEXT) return file @@ -29,6 +40,7 @@ def sample_model_file(tmp_path_factory): @pytest.fixture def empty_model_file(tmp_path_factory): + """A model with just an empty file.""" file = tmp_path_factory.mktemp("model") / "file" file.write_bytes(b"") return file @@ -36,6 +48,7 @@ def empty_model_file(tmp_path_factory): @pytest.fixture def sample_model_folder(tmp_path_factory): + """A model with multiple files and directories.""" model_root = tmp_path_factory.mktemp("model") / "root" model_root.mkdir() @@ -55,13 +68,27 @@ def sample_model_folder(tmp_path_factory): @pytest.fixture def empty_model_folder(tmp_path_factory): + """A model with just an empty directory.""" + model_root = tmp_path_factory.mktemp("model") / "root" + model_root.mkdir() + return model_root + + +@pytest.fixture +def model_folder_with_empty_file(tmp_path_factory): + """A model with just an empty file, inside a directory.""" model_root = tmp_path_factory.mktemp("model") / "root" model_root.mkdir() + + empty_file = model_root / "empty_file" + empty_file.write_bytes(b"") + return model_root @pytest.fixture def deep_model_folder(tmp_path_factory): + """A model with a deep directory hierarchy.""" model_root = tmp_path_factory.mktemp("model") / "root" model_root.mkdir() diff --git a/model_signing/serialization/serialize_by_file_shard_test.py b/model_signing/serialization/serialize_by_file_shard_test.py index 90a1b0ca..59f94047 100644 --- a/model_signing/serialization/serialize_by_file_shard_test.py +++ b/model_signing/serialization/serialize_by_file_shard_test.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import dataclasses import pathlib +import pytest from model_signing.hashing import file from model_signing.hashing import memory @@ -21,6 +23,12 @@ from model_signing.serialization import test_support +# NOTE: This test uses a golden setup to compute digest of several test +# models. If the golden tests are failing, regenerate the golden files with +# +# pytest model_signing/serialization/ --update_goldens + + class TestShardedDFSSerializer: def _hasher_factory( @@ -37,17 +45,57 @@ def _hasher_factory_small_shards( path, memory.SHA256(), start=start, end=end, shard_size=2 ) - def test_known_file(self, sample_model_file): + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file_shard" + test_class_path = test_path / "TestShardedDFSSerializer" + golden_path = test_class_path / model_fixture_name + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) serializer = serialize_by_file_shard.ShardedDFSSerializer( self._hasher_factory, memory.SHA256() ) - expected = ( - "2ca48c47d5311a9b2f9305519cd5f927dcef09404fc32ef7886abe8f11450eff" + manifest = serializer.serialize(model) + + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + f.write(f"{manifest.digest.digest_hex}\n") + else: + with open(golden_path, "r", encoding="utf-8") as f: + expected_digest = f.read().strip() + + assert manifest.digest.digest_hex == expected_digest + + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models_small_shards(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file_shard" + test_class_path = test_path / "TestShardedDFSSerializer" + golden_path = test_class_path / f"{model_fixture_name}_small_shards" + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) + serializer = serialize_by_file_shard.ShardedDFSSerializer( + self._hasher_factory_small_shards, memory.SHA256() ) + manifest = serializer.serialize(model) - manifest = serializer.serialize(sample_model_file) + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + f.write(f"{manifest.digest.digest_hex}\n") + else: + with open(golden_path, "r", encoding="utf-8") as f: + expected_digest = f.read().strip() - assert manifest.digest.digest_hex == expected + assert manifest.digest.digest_hex == expected_digest def test_file_hash_is_not_same_as_hash_of_content(self, sample_model_file): serializer = serialize_by_file_shard.ShardedDFSSerializer( @@ -97,18 +145,6 @@ def test_directory_model_with_only_known_file(self, sample_model_file): assert manifest_file != manifest assert manifest.digest.digest_hex != content_digest.digest_hex - def test_known_folder(self, sample_model_folder): - serializer = serialize_by_file_shard.ShardedDFSSerializer( - self._hasher_factory, memory.SHA256() - ) - expected = ( - "d22e0441cfa5ac2bc09715ddd88c802a7f97e29c93dc50f5498bab2954958ebb" - ) - - manifest = serializer.serialize(sample_model_folder) - - assert manifest.digest.digest_hex == expected - def test_folder_model_hash_is_same_if_model_is_moved( self, sample_model_folder ): @@ -203,54 +239,6 @@ def test_folder_model_change_file(self, sample_model_folder): assert manifest != new_manifest - def test_deep_folder(self, deep_model_folder): - serializer = serialize_by_file_shard.ShardedDFSSerializer( - self._hasher_factory, memory.SHA256() - ) - expected = ( - "52fa3c459aec58bc5f9702c73cb3c6b8fd19e9342aa3e4db851e1bde69ab1727" - ) - - manifest = serializer.serialize(deep_model_folder) - - assert manifest.digest.digest_hex == expected - - def test_empty_file(self, empty_model_file): - serializer = serialize_by_file_shard.ShardedDFSSerializer( - self._hasher_factory, memory.SHA256() - ) - expected = ( - "5f2d126b0d3540c17481fdf724e31cf03b4436a2ebabaa1d2e94fe09831be64d" - ) - - manifest = serializer.serialize(empty_model_file) - - assert manifest.digest.digest_hex == expected - - def test_empty_folder(self, empty_model_folder): - serializer = serialize_by_file_shard.ShardedDFSSerializer( - self._hasher_factory, memory.SHA256() - ) - expected = ( - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ) - - manifest = serializer.serialize(empty_model_folder) - - assert manifest.digest.digest_hex == expected - - def test_directory_model_with_only_empty_file(self, empty_model_file): - serializer = serialize_by_file_shard.ShardedDFSSerializer( - self._hasher_factory, memory.SHA256() - ) - expected = ( - "74e81d0062f0a0674014c2f0e4b79985d5015f98a64089e7106a44d32e9ff11f" - ) - - manifest = serializer.serialize(empty_model_file.parent) - - assert manifest.digest.digest_hex == expected - def test_empty_folder_hashes_differently_than_empty_file( self, empty_model_file, empty_model_folder ): @@ -311,9 +299,18 @@ def test_shard_size_changes_digests(self, sample_model_folder): assert manifest1.digest.digest_value != manifest2.digest.digest_value +@dataclasses.dataclass(frozen=True) +class _Shard: + """A shard of a file from a manifest.""" + + path: str + start: int + end: int + + def _extract_shard_items_from_manifest( manifest: manifest.ShardLevelManifest, -) -> dict[tuple[str, int, int], str]: +) -> dict[_Shard, str]: """Builds a dictionary representation of the items in a manifest. Every item is mapped to its digest. @@ -322,11 +319,25 @@ def _extract_shard_items_from_manifest( """ return { # convert to file path (relative to model) string and endpoints - (str(shard[0]), shard[1], shard[2]): digest.digest_hex + _Shard(str(shard[0]), shard[1], shard[2]): digest.digest_hex for shard, digest in manifest._item_to_digest.items() } +def _parse_shard_and_digest(line: str) -> tuple[_Shard, str]: + """Reads a file shard and its digest from a line in the golden file. + + Args: + line: The line to parse. + + Returns: + The shard tuple and the digest corresponding to the line that was read. + """ + path, start, end, digest = line.strip().split(":") + shard = _Shard(path, int(start), int(end)) + return shard, digest + + class TestShardedFilesSerializer: def _hasher_factory( @@ -343,18 +354,71 @@ def _hasher_factory_small_shards( path, memory.SHA256(), start=start, end=end, shard_size=8 ) - def test_known_file(self, sample_model_file): + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file_shard" + test_class_path = test_path / "TestShardedFilesSerializer" + golden_path = test_class_path / model_fixture_name + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) serializer = serialize_by_file_shard.ShardedFilesSerializer( self._hasher_factory ) - expected = [ - "3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b" - ] + manifest = serializer.serialize(model) + items = _extract_shard_items_from_manifest(manifest) - manifest = serializer.serialize(sample_model_file) - digests = test_support.extract_digests_from_manifest(manifest) + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + for shard, digest in sorted(items.items()): + f.write( + f"{shard.path}:{shard.start}:{shard.end}:{digest}\n" + ) + else: + found_items: dict[_Shard, str] = {} + with open(golden_path, "r", encoding="utf-8") as f: + for line in f: + shard, digest = _parse_shard_and_digest(line) + found_items[shard] = digest + + assert items == found_items + + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models_small_shards(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file_shard" + test_class_path = test_path / "TestShardedFilesSerializer" + golden_path = test_class_path / f"{model_fixture_name}_small_shards" + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) + serializer = serialize_by_file_shard.ShardedFilesSerializer( + self._hasher_factory_small_shards + ) + manifest = serializer.serialize(model) + items = _extract_shard_items_from_manifest(manifest) - assert digests == expected + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + for shard, digest in sorted(items.items()): + f.write( + f"{shard.path}:{shard.start}:{shard.end}:{digest}\n" + ) + else: + found_items: dict[_Shard, str] = {} + with open(golden_path, "r", encoding="utf-8") as f: + for line in f: + shard, digest = _parse_shard_and_digest(line) + found_items[shard] = digest + + assert items == found_items def test_file_manifest_unchanged_when_model_moved(self, sample_model_file): serializer = serialize_by_file_shard.ShardedFilesSerializer( @@ -400,70 +464,6 @@ def test_directory_model_with_only_known_file(self, sample_model_file): assert manifest != manifest_file # different paths assert digests == digests_file - def test_known_folder(self, sample_model_folder): - serializer = serialize_by_file_shard.ShardedFilesSerializer( - self._hasher_factory - ) - # Long hashes, want to update easily, so pylint: disable=line-too-long - expected_items = { - ( - "f0", - 0, - 24, - ): "997b37cc51f1ca1c7a270466607e26847429cd7264c30148c1b9352e224083fc", - ( - "f1", - 0, - 24, - ): "c88a04d48353133fb065ba2c8ab369abab21395b9526aa20373ad828915fa7ae", - ( - "f2", - 0, - 24, - ): "700e3ba5065d8dd47e41fd928ea086670d628f891ba363be0ca3c31d20d7d719", - ( - "f3", - 0, - 24, - ): "912bcf5ebdf44dc7b4085b07940e0a81d157fba24b276e73fd911121d4544c4a", - ( - "d0/f00", - 0, - 23, - ): "fdd8925354242a7fd1515e79534317b800015607a609cd306e0b4dcfe6c92249", - ( - "d0/f01", - 0, - 23, - ): "e16940b5e44ce981150bda37c4ba95881a749a521b4a297c5cdf97bdcfe965e6", - ( - "d0/f02", - 0, - 23, - ): "407822246ea8f9e26380842c3f4cd10d7b23e78f1fe7c74c293608682886a426", - ( - "d1/f10", - 0, - 23, - ): "6a3b08b5df77c4d418ceee1ac136a9ad49fc7c41358b5e82c1176daccb21ff3f", - ( - "d1/f11", - 0, - 23, - ): "a484b3d8ea5e99b75f9f123f9a42c882388693edc7d85d82ccba54834712cadf", - ( - "d1/f12", - 0, - 23, - ): "8f577930f5f40c2c2133cb299d36f9527fde98c1608569017cae6b5bcd01abb3", - } - # Re-enable lint, so pylint: enable=line-too-long - - manifest = serializer.serialize(sample_model_folder) - items = _extract_shard_items_from_manifest(manifest) - - assert items == expected_items - def test_folder_model_hash_is_same_if_model_is_moved( self, sample_model_folder ): @@ -649,54 +649,6 @@ def test_folder_model_change_file(self, sample_model_folder): manifest, new_manifest, changed_entry ) - def test_deep_folder(self, deep_model_folder): - serializer = serialize_by_file_shard.ShardedFilesSerializer( - self._hasher_factory - ) - # Long hashes, want to update easily, so pylint: disable=line-too-long - expected_items = { - ( - "d0/d1/d2/d3/d4/f0", - 0, - 16, - ): "6efa14bb03544fcb76045c55f25b9315b6eb5be2d8a85f703193a76b7874c6ff", - ( - "d0/d1/d2/d3/d4/f1", - 0, - 16, - ): "a9bc149b70b9d325cd68d275d582cfdb98c0347d3ce54590aa6533368daed3d2", - ( - "d0/d1/d2/d3/d4/f2", - 0, - 16, - ): "5f597e6a92d1324d9adbed43d527926d11d0131487baf315e65ae1ef3b1ca3c0", - ( - "d0/d1/d2/d3/d4/f3", - 0, - 16, - ): "eaf677c35fec6b87889d9e4563d8bb65dcb9869ca0225697c9cc44cf49dca008", - } - # Re-enable lint, so pylint: enable=line-too-long - - manifest = serializer.serialize(deep_model_folder) - items = _extract_shard_items_from_manifest(manifest) - - assert items == expected_items - - def test_empty_file(self, empty_model_file): - serializer = serialize_by_file_shard.ShardedFilesSerializer( - self._hasher_factory - ) - manifest = serializer.serialize(empty_model_file) - assert not manifest._item_to_digest - - def test_empty_folder(self, empty_model_folder): - serializer = serialize_by_file_shard.ShardedFilesSerializer( - self._hasher_factory - ) - manifest = serializer.serialize(empty_model_folder) - assert not manifest._item_to_digest - def test_max_workers_does_not_change_digest(self, sample_model_folder): serializer1 = serialize_by_file_shard.ShardedFilesSerializer( self._hasher_factory @@ -714,167 +666,3 @@ def test_max_workers_does_not_change_digest(self, sample_model_folder): assert manifest1 == manifest2 assert manifest1 == manifest3 - - def test_known_folder_small_shards(self, sample_model_folder): - serializer = serialize_by_file_shard.ShardedFilesSerializer( - self._hasher_factory_small_shards - ) - # Long hashes, want to update easily, so pylint: disable=line-too-long - expected_items = { - ( - "f0", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "f0", - 8, - 16, - ): "6ceb6f182993c238d6ce291d3f72f0de743d81faff85f4b038f4dffcc0eea50b", - ( - "f0", - 16, - 24, - ): "df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581", - ( - "f1", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "f1", - 8, - 16, - ): "be0c7e3632df2704636ebb173240d67568b7736a5e4c86c3e3cdee24e765fe92", - ( - "f1", - 16, - 24, - ): "df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581", - ( - "f2", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "f2", - 8, - 16, - ): "336bbabbb79a106f9eecbdeb3f2f6c4949735fdb307c1703dc41a65fbba4faf5", - ( - "f2", - 16, - 24, - ): "df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581", - ( - "f3", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "f3", - 8, - 16, - ): "2a40ca276886e04559e3babe13a9baf15fb2e2820befa5db7a0105449a55e77d", - ( - "f3", - 16, - 24, - ): "df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581", - ( - "d0/f00", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d0/f00", - 8, - 16, - ): "03d5bb235fbf6771da18a650a7d01c1d6b85ada9de9cc62d27752a1c2a05548b", - ( - "d0/f00", - 16, - 23, - ): "003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a", - ( - "d0/f01", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d0/f01", - 8, - 16, - ): "819b976b5670c9c78a8bed6c2819757b5e2740db5e54e0a4da4b9bb1e5e80234", - ( - "d0/f01", - 16, - 23, - ): "003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a", - ( - "d0/f02", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d0/f02", - 8, - 16, - ): "635d7a05841e44a98e10ee9ac8346b8ed51653001a28d5c513a742c1408c7f33", - ( - "d0/f02", - 16, - 23, - ): "003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a", - ( - "d1/f10", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d1/f10", - 8, - 16, - ): "be0c2a2c271b113e22b02f495a1fa9b09bcc420b6ce969911827846519533811", - ( - "d1/f10", - 16, - 23, - ): "e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d", - ( - "d1/f11", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d1/f11", - 8, - 16, - ): "3a9436f3f057b24fd7f830c6a230a08922b7b5d6f8255ae987bec4773a2d6152", - ( - "d1/f11", - 16, - 23, - ): "e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d", - ( - "d1/f12", - 0, - 8, - ): "a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0", - ( - "d1/f12", - 8, - 16, - ): "93b0ae318b7ff95d0e9afdb975e1c6e7dca2d655d16a6801e0aa128bdfa65726", - ( - "d1/f12", - 16, - 23, - ): "e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d", - } - # Re-enable lint, so pylint: enable=line-too-long - - manifest = serializer.serialize(sample_model_folder) - items = _extract_shard_items_from_manifest(manifest) - - assert items == expected_items diff --git a/model_signing/serialization/serialize_by_file_test.py b/model_signing/serialization/serialize_by_file_test.py index e0a799b0..e4ec9e07 100644 --- a/model_signing/serialization/serialize_by_file_test.py +++ b/model_signing/serialization/serialize_by_file_test.py @@ -23,22 +23,42 @@ from model_signing.serialization import test_support +# NOTE: This test uses a goldens setup to compute digest of several test +# models. If the golden tests are failing, regenerate the golden files with +# +# pytest model_signing/serialization/ --update_goldens + + class TestDigestSerializer: - def test_known_file(self, sample_model_file): + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file" + test_class_path = test_path / "TestDigestSerializer" + golden_path = test_class_path / model_fixture_name + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) file_hasher = file.SimpleFileHasher( test_support.UNUSED_PATH, memory.SHA256() ) serializer = serialize_by_file.DigestSerializer( file_hasher, memory.SHA256 ) - expected = ( - "3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b" - ) + manifest = serializer.serialize(model) - manifest = serializer.serialize(sample_model_file) + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + f.write(f"{manifest.digest.digest_hex}\n") + else: + with open(golden_path, "r", encoding="utf-8") as f: + expected_digest = f.read().strip() - assert manifest.digest.digest_hex == expected + assert manifest.digest.digest_hex == expected_digest def test_file_hash_is_same_as_hash_of_content(self, sample_model_file): file_hasher = file.SimpleFileHasher( @@ -99,21 +119,6 @@ def test_directory_model_with_only_known_file(self, sample_model_file): assert manifest_file != manifest assert manifest.digest.digest_hex != content_digest.digest_hex - def test_known_folder(self, sample_model_folder): - file_hasher = file.SimpleFileHasher( - test_support.UNUSED_PATH, memory.SHA256() - ) - serializer = serialize_by_file.DigestSerializer( - file_hasher, memory.SHA256 - ) - expected = ( - "310af4fc4c52bf63cd1687c67076ed3e56bc5480a1b151539e6c550506ae0301" - ) - - manifest = serializer.serialize(sample_model_folder) - - assert manifest.digest.digest_hex == expected - def test_folder_model_hash_is_same_if_model_is_moved( self, sample_model_folder ): @@ -229,66 +234,6 @@ def test_folder_model_change_file(self, sample_model_folder): assert manifest != new_manifest - def test_deep_folder(self, deep_model_folder): - file_hasher = file.SimpleFileHasher( - test_support.UNUSED_PATH, memory.SHA256() - ) - serializer = serialize_by_file.DigestSerializer( - file_hasher, memory.SHA256 - ) - expected = ( - "36eed9389ebbbe15ac15d33c81dabb60ccb7c945ff641d78f59db9aa9dc47ac9" - ) - - manifest = serializer.serialize(deep_model_folder) - - assert manifest.digest.digest_hex == expected - - def test_empty_file(self, empty_model_file): - file_hasher = file.SimpleFileHasher( - test_support.UNUSED_PATH, memory.SHA256() - ) - serializer = serialize_by_file.DigestSerializer( - file_hasher, memory.SHA256 - ) - expected = ( - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ) - - manifest = serializer.serialize(empty_model_file) - - assert manifest.digest.digest_hex == expected - - def test_empty_folder(self, empty_model_folder): - file_hasher = file.SimpleFileHasher( - test_support.UNUSED_PATH, memory.SHA256() - ) - serializer = serialize_by_file.DigestSerializer( - file_hasher, memory.SHA256 - ) - expected = ( - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ) - - manifest = serializer.serialize(empty_model_folder) - - assert manifest.digest.digest_hex == expected - - def test_directory_model_with_only_empty_file(self, empty_model_file): - file_hasher = file.SimpleFileHasher( - test_support.UNUSED_PATH, memory.SHA256() - ) - serializer = serialize_by_file.DigestSerializer( - file_hasher, memory.SHA256 - ) - expected = ( - "8a587b2129fdecfbea38d5152b626299f5994d9b99d36b321aea356f69b38c61" - ) - - manifest = serializer.serialize(empty_model_file.parent) - - assert manifest.digest.digest_hex == expected - def test_empty_folder_hashes_differently_than_empty_file( self, empty_model_file, empty_model_folder ): @@ -334,16 +279,34 @@ class TestManifestSerializer: def _hasher_factory(self, path: pathlib.Path) -> file.FileHasher: return file.SimpleFileHasher(path, memory.SHA256()) - def test_known_file(self, sample_model_file): + @pytest.mark.parametrize("model_fixture_name", test_support.all_test_models) + def test_known_models(self, request, model_fixture_name): + # Set up variables (arrange) + testdata_path = request.path.parent / "testdata" + test_path = testdata_path / "serialize_by_file" + test_class_path = test_path / "TestManifestSerializer" + golden_path = test_class_path / model_fixture_name + should_update = request.config.getoption("update_goldens") + model = request.getfixturevalue(model_fixture_name) + + # Compute model manifest (act) serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) - expected = [ - "3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b" - ] + manifest = serializer.serialize(model) + items = test_support.extract_items_from_manifest(manifest) - manifest = serializer.serialize(sample_model_file) - digests = test_support.extract_digests_from_manifest(manifest) + # Compare with golden, or write to golden (approximately "assert") + if should_update: + with open(golden_path, "w", encoding="utf-8") as f: + for path, digest in sorted(items.items()): + f.write(f"{path}:{digest}\n") + else: + found_items: dict[str, str] = {} + with open(golden_path, "r", encoding="utf-8") as f: + for line in f: + path, digest = line.strip().split(":") + found_items[path] = digest - assert digests == expected + assert items == found_items def test_file_manifest_unchanged_when_model_moved(self, sample_model_file): serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) @@ -383,28 +346,6 @@ def test_directory_model_with_only_known_file(self, sample_model_file): assert manifest != manifest_file # different paths assert digests == digests_file - def test_known_folder(self, sample_model_folder): - serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) - # Long hashes, want to update easily, so pylint: disable=line-too-long - expected_items = { - "f0": "997b37cc51f1ca1c7a270466607e26847429cd7264c30148c1b9352e224083fc", - "f1": "c88a04d48353133fb065ba2c8ab369abab21395b9526aa20373ad828915fa7ae", - "f2": "700e3ba5065d8dd47e41fd928ea086670d628f891ba363be0ca3c31d20d7d719", - "f3": "912bcf5ebdf44dc7b4085b07940e0a81d157fba24b276e73fd911121d4544c4a", - "d0/f00": "fdd8925354242a7fd1515e79534317b800015607a609cd306e0b4dcfe6c92249", - "d0/f01": "e16940b5e44ce981150bda37c4ba95881a749a521b4a297c5cdf97bdcfe965e6", - "d0/f02": "407822246ea8f9e26380842c3f4cd10d7b23e78f1fe7c74c293608682886a426", - "d1/f10": "6a3b08b5df77c4d418ceee1ac136a9ad49fc7c41358b5e82c1176daccb21ff3f", - "d1/f11": "a484b3d8ea5e99b75f9f123f9a42c882388693edc7d85d82ccba54834712cadf", - "d1/f12": "8f577930f5f40c2c2133cb299d36f9527fde98c1608569017cae6b5bcd01abb3", - } - # Re-enable lint, so pylint: enable=line-too-long - - manifest = serializer.serialize(sample_model_folder) - items = test_support.extract_items_from_manifest(manifest) - - assert items == expected_items - def test_folder_model_hash_is_same_if_model_is_moved( self, sample_model_folder ): @@ -577,38 +518,6 @@ def test_folder_model_change_file(self, sample_model_folder): manifest, new_manifest, changed_entry ) - def test_deep_folder(self, deep_model_folder): - serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) - # Long hashes, want to update easily, so pylint: disable=line-too-long - expected_items = { - "d0/d1/d2/d3/d4/f0": "6efa14bb03544fcb76045c55f25b9315b6eb5be2d8a85f703193a76b7874c6ff", - "d0/d1/d2/d3/d4/f1": "a9bc149b70b9d325cd68d275d582cfdb98c0347d3ce54590aa6533368daed3d2", - "d0/d1/d2/d3/d4/f2": "5f597e6a92d1324d9adbed43d527926d11d0131487baf315e65ae1ef3b1ca3c0", - "d0/d1/d2/d3/d4/f3": "eaf677c35fec6b87889d9e4563d8bb65dcb9869ca0225697c9cc44cf49dca008", - } - # Re-enable lint, so pylint: enable=line-too-long - - manifest = serializer.serialize(deep_model_folder) - items = test_support.extract_items_from_manifest(manifest) - - assert items == expected_items - - def test_empty_file(self, empty_model_file): - serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) - expected = [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ] - - manifest = serializer.serialize(empty_model_file) - digests = test_support.extract_digests_from_manifest(manifest) - - assert digests == expected - - def test_empty_folder(self, empty_model_folder): - serializer = serialize_by_file.ManifestSerializer(self._hasher_factory) - manifest = serializer.serialize(empty_model_folder) - assert not manifest._item_to_digest - def test_max_workers_does_not_change_digest(self, sample_model_folder): serializer1 = serialize_by_file.ManifestSerializer(self._hasher_factory) serializer2 = serialize_by_file.ManifestSerializer( diff --git a/model_signing/serialization/test_support.py b/model_signing/serialization/test_support.py index bc7cb3f5..6d8aefaf 100644 --- a/model_signing/serialization/test_support.py +++ b/model_signing/serialization/test_support.py @@ -29,6 +29,19 @@ UNUSED_PATH = pathlib.Path("unused") +# All models to use in testing. Each of these is defined as a fixture in +# conftest.py. Here, we used them as strings, these names must match the +# function name used for the fixture. +all_test_models = [ + "sample_model_file", + "sample_model_folder", + "deep_model_folder", + "empty_model_file", + "empty_model_folder", + "model_folder_with_empty_file", +] + + def get_first_directory(path: pathlib.Path) -> pathlib.Path: """Returns the first directory that is a children of path. diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/deep_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/deep_model_folder new file mode 100644 index 00000000..fb544575 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/deep_model_folder @@ -0,0 +1 @@ +36eed9389ebbbe15ac15d33c81dabb60ccb7c945ff641d78f59db9aa9dc47ac9 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_file b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_file new file mode 100644 index 00000000..c3068040 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_file @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_folder new file mode 100644 index 00000000..c3068040 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/empty_model_folder @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/model_folder_with_empty_file b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/model_folder_with_empty_file new file mode 100644 index 00000000..1a4bcbef --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/model_folder_with_empty_file @@ -0,0 +1 @@ +68efd863f20e083173846a5e98ad11387a1979efe20ded426a7930bab8358a9c diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_file b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_file new file mode 100644 index 00000000..2ebc572a --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_file @@ -0,0 +1 @@ +3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b diff --git a/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_folder new file mode 100644 index 00000000..ed0f9979 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestDigestSerializer/sample_model_folder @@ -0,0 +1 @@ +310af4fc4c52bf63cd1687c67076ed3e56bc5480a1b151539e6c550506ae0301 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/deep_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/deep_model_folder new file mode 100644 index 00000000..44d8daa4 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/deep_model_folder @@ -0,0 +1,4 @@ +d0/d1/d2/d3/d4/f0:6efa14bb03544fcb76045c55f25b9315b6eb5be2d8a85f703193a76b7874c6ff +d0/d1/d2/d3/d4/f1:a9bc149b70b9d325cd68d275d582cfdb98c0347d3ce54590aa6533368daed3d2 +d0/d1/d2/d3/d4/f2:5f597e6a92d1324d9adbed43d527926d11d0131487baf315e65ae1ef3b1ca3c0 +d0/d1/d2/d3/d4/f3:eaf677c35fec6b87889d9e4563d8bb65dcb9869ca0225697c9cc44cf49dca008 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/empty_model_file b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/empty_model_file new file mode 100644 index 00000000..aff969ce --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/empty_model_file @@ -0,0 +1 @@ +.:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/empty_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/empty_model_folder new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/model_folder_with_empty_file b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/model_folder_with_empty_file new file mode 100644 index 00000000..2f7997e9 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/model_folder_with_empty_file @@ -0,0 +1 @@ +empty_file:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_file b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_file new file mode 100644 index 00000000..96d0bc82 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_file @@ -0,0 +1 @@ +.:3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b diff --git a/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_folder b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_folder new file mode 100644 index 00000000..4328868e --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file/TestManifestSerializer/sample_model_folder @@ -0,0 +1,10 @@ +d0/f00:fdd8925354242a7fd1515e79534317b800015607a609cd306e0b4dcfe6c92249 +d0/f01:e16940b5e44ce981150bda37c4ba95881a749a521b4a297c5cdf97bdcfe965e6 +d0/f02:407822246ea8f9e26380842c3f4cd10d7b23e78f1fe7c74c293608682886a426 +d1/f10:6a3b08b5df77c4d418ceee1ac136a9ad49fc7c41358b5e82c1176daccb21ff3f +d1/f11:a484b3d8ea5e99b75f9f123f9a42c882388693edc7d85d82ccba54834712cadf +d1/f12:8f577930f5f40c2c2133cb299d36f9527fde98c1608569017cae6b5bcd01abb3 +f0:997b37cc51f1ca1c7a270466607e26847429cd7264c30148c1b9352e224083fc +f1:c88a04d48353133fb065ba2c8ab369abab21395b9526aa20373ad828915fa7ae +f2:700e3ba5065d8dd47e41fd928ea086670d628f891ba363be0ca3c31d20d7d719 +f3:912bcf5ebdf44dc7b4085b07940e0a81d157fba24b276e73fd911121d4544c4a diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder new file mode 100644 index 00000000..528ab87c --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder @@ -0,0 +1 @@ +52fa3c459aec58bc5f9702c73cb3c6b8fd19e9342aa3e4db851e1bde69ab1727 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder_small_shards new file mode 100644 index 00000000..a4f2f81e --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/deep_model_folder_small_shards @@ -0,0 +1 @@ +abd66cd0d8a01f3f552ac5af717f49dc6e6575f0849ec3bfb3c9051962314ce6 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file new file mode 100644 index 00000000..9ac3ea65 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file @@ -0,0 +1 @@ +5f2d126b0d3540c17481fdf724e31cf03b4436a2ebabaa1d2e94fe09831be64d diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file_small_shards new file mode 100644 index 00000000..9ac3ea65 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_file_small_shards @@ -0,0 +1 @@ +5f2d126b0d3540c17481fdf724e31cf03b4436a2ebabaa1d2e94fe09831be64d diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder new file mode 100644 index 00000000..c3068040 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder_small_shards new file mode 100644 index 00000000..c3068040 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/empty_model_folder_small_shards @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file new file mode 100644 index 00000000..b6d24eaf --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file @@ -0,0 +1 @@ +230d217d5f4f388f5087ac4174dbc9b0ff358e3122a1267b0a56669a44f11ea1 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file_small_shards new file mode 100644 index 00000000..b6d24eaf --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/model_folder_with_empty_file_small_shards @@ -0,0 +1 @@ +230d217d5f4f388f5087ac4174dbc9b0ff358e3122a1267b0a56669a44f11ea1 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file new file mode 100644 index 00000000..a94a0fa0 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file @@ -0,0 +1 @@ +2ca48c47d5311a9b2f9305519cd5f927dcef09404fc32ef7886abe8f11450eff diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file_small_shards new file mode 100644 index 00000000..5b6697c8 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_file_small_shards @@ -0,0 +1 @@ +284b613e2e1576d87e5e1c912c82da8d87b6350276f36940516404b2a35f1a74 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder new file mode 100644 index 00000000..7fa49a73 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder @@ -0,0 +1 @@ +d22e0441cfa5ac2bc09715ddd88c802a7f97e29c93dc50f5498bab2954958ebb diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder_small_shards new file mode 100644 index 00000000..161cafdf --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedDFSSerializer/sample_model_folder_small_shards @@ -0,0 +1 @@ +82bb608d88cf741730c5bcb75a7630f560643acafdd8fa02ad24be20f51c1250 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder new file mode 100644 index 00000000..74057598 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder @@ -0,0 +1,4 @@ +d0/d1/d2/d3/d4/f0:0:16:6efa14bb03544fcb76045c55f25b9315b6eb5be2d8a85f703193a76b7874c6ff +d0/d1/d2/d3/d4/f1:0:16:a9bc149b70b9d325cd68d275d582cfdb98c0347d3ce54590aa6533368daed3d2 +d0/d1/d2/d3/d4/f2:0:16:5f597e6a92d1324d9adbed43d527926d11d0131487baf315e65ae1ef3b1ca3c0 +d0/d1/d2/d3/d4/f3:0:16:eaf677c35fec6b87889d9e4563d8bb65dcb9869ca0225697c9cc44cf49dca008 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder_small_shards new file mode 100644 index 00000000..d1069a28 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/deep_model_folder_small_shards @@ -0,0 +1,8 @@ +d0/d1/d2/d3/d4/f0:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/d1/d2/d3/d4/f0:8:16:a42438d3606f14e351888ad4be724eb98679674b9281a7dc532a57baace9439b +d0/d1/d2/d3/d4/f1:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/d1/d2/d3/d4/f1:8:16:9634ddd8e5388a5383bbb944456488549321f546896353395ce199dc852195a4 +d0/d1/d2/d3/d4/f2:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/d1/d2/d3/d4/f2:8:16:25ea3759cd0b40fdca74e4c35d3e9119ee222ac0284fa83f48f6da8a66dc9d8c +d0/d1/d2/d3/d4/f3:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/d1/d2/d3/d4/f3:8:16:4c6aead90eefc5202b6840120787fcee70cf9407b03209646d6f30472901ec86 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_file new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_file_small_shards new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_folder new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/empty_model_folder_small_shards new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/model_folder_with_empty_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/model_folder_with_empty_file new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/model_folder_with_empty_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/model_folder_with_empty_file_small_shards new file mode 100644 index 00000000..e69de29b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file new file mode 100644 index 00000000..6ef95249 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file @@ -0,0 +1 @@ +.:0:22:3aab065c7181a173b5dd9e9d32a9f79923440b413be1e1ffcdba26a7365f719b diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file_small_shards new file mode 100644 index 00000000..3fdc9b2f --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_file_small_shards @@ -0,0 +1,3 @@ +.:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +.:8:16:bd762002a3528a27fb9a8822f822b949d3c9ab7e860af33039c9aa70ebbbe682 +.:16:22:a791e1e893ea4260c77475725101fb4cc6ad85f6340f21f10b239184e318cd21 diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder new file mode 100644 index 00000000..eba78237 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder @@ -0,0 +1,10 @@ +d0/f00:0:23:fdd8925354242a7fd1515e79534317b800015607a609cd306e0b4dcfe6c92249 +d0/f01:0:23:e16940b5e44ce981150bda37c4ba95881a749a521b4a297c5cdf97bdcfe965e6 +d0/f02:0:23:407822246ea8f9e26380842c3f4cd10d7b23e78f1fe7c74c293608682886a426 +d1/f10:0:23:6a3b08b5df77c4d418ceee1ac136a9ad49fc7c41358b5e82c1176daccb21ff3f +d1/f11:0:23:a484b3d8ea5e99b75f9f123f9a42c882388693edc7d85d82ccba54834712cadf +d1/f12:0:23:8f577930f5f40c2c2133cb299d36f9527fde98c1608569017cae6b5bcd01abb3 +f0:0:24:997b37cc51f1ca1c7a270466607e26847429cd7264c30148c1b9352e224083fc +f1:0:24:c88a04d48353133fb065ba2c8ab369abab21395b9526aa20373ad828915fa7ae +f2:0:24:700e3ba5065d8dd47e41fd928ea086670d628f891ba363be0ca3c31d20d7d719 +f3:0:24:912bcf5ebdf44dc7b4085b07940e0a81d157fba24b276e73fd911121d4544c4a diff --git a/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder_small_shards b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder_small_shards new file mode 100644 index 00000000..3762e984 --- /dev/null +++ b/model_signing/serialization/testdata/serialize_by_file_shard/TestShardedFilesSerializer/sample_model_folder_small_shards @@ -0,0 +1,30 @@ +d0/f00:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/f00:8:16:03d5bb235fbf6771da18a650a7d01c1d6b85ada9de9cc62d27752a1c2a05548b +d0/f00:16:23:003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a +d0/f01:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/f01:8:16:819b976b5670c9c78a8bed6c2819757b5e2740db5e54e0a4da4b9bb1e5e80234 +d0/f01:16:23:003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a +d0/f02:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d0/f02:8:16:635d7a05841e44a98e10ee9ac8346b8ed51653001a28d5c513a742c1408c7f33 +d0/f02:16:23:003b089e217915b12ca9509b2f8a01be7cfe662ffadeb1cd3cf3f430c7b9773a +d1/f10:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d1/f10:8:16:be0c2a2c271b113e22b02f495a1fa9b09bcc420b6ce969911827846519533811 +d1/f10:16:23:e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d +d1/f11:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d1/f11:8:16:3a9436f3f057b24fd7f830c6a230a08922b7b5d6f8255ae987bec4773a2d6152 +d1/f11:16:23:e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d +d1/f12:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +d1/f12:8:16:93b0ae318b7ff95d0e9afdb975e1c6e7dca2d655d16a6801e0aa128bdfa65726 +d1/f12:16:23:e6607ae1f591bd75d53b4a52bcd86ecf08643270814d20950ce4e762a99f773d +f0:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +f0:8:16:6ceb6f182993c238d6ce291d3f72f0de743d81faff85f4b038f4dffcc0eea50b +f0:16:24:df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581 +f1:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +f1:8:16:be0c7e3632df2704636ebb173240d67568b7736a5e4c86c3e3cdee24e765fe92 +f1:16:24:df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581 +f2:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +f2:8:16:336bbabbb79a106f9eecbdeb3f2f6c4949735fdb307c1703dc41a65fbba4faf5 +f2:16:24:df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581 +f3:0:8:a37010c994067764d86540bf479d93b4d0c3bb3955de7b61f951caf2fd0301b0 +f3:8:16:2a40ca276886e04559e3babe13a9baf15fb2e2820befa5db7a0105449a55e77d +f3:16:24:df42fcbd1023b80c82e869872ea01e1e1b0bbd60ab4c68c4054e7343ff0ce581