From 86585ee27937dff6cd5b682c698a4bdbddc1bff6 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Thu, 22 Aug 2024 17:34:39 -0700 Subject: [PATCH] feat(47) read sha2 256 chunked (#50) * ManifestTest > testParseQuiltURI * testFromQuiltURI * FAIL testFromChunkedURI * SHA2_256_Chunked (underscores) * [0.1.4] - 2024-08-22 --- CHANGELOG.md | 5 ++ .../java/com/quiltdata/quiltcore/Entry.java | 20 +++++ .../com/quiltdata/quiltcore/Manifest.java | 89 ++++++++++++++++++- .../com/quiltdata/quiltcore/EntryTest.java | 43 +++++++++ .../com/quiltdata/quiltcore/ManifestTest.java | 51 +++++++++++ 5 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 lib/src/test/java/com/quiltdata/quiltcore/EntryTest.java create mode 100644 lib/src/test/java/com/quiltdata/quiltcore/ManifestTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 43ac03e..eff5016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG.md +## [0.1.4] - 2024-08-22 + +- Support chunked checksums via HasType.SHA2_256_CHUNKED +- Add high-level Manifest::fromURI function + ## [0.1.3] - 2024-08-21 - Fix failing tests diff --git a/lib/src/main/java/com/quiltdata/quiltcore/Entry.java b/lib/src/main/java/com/quiltdata/quiltcore/Entry.java index b737dae..282982a 100644 --- a/lib/src/main/java/com/quiltdata/quiltcore/Entry.java +++ b/lib/src/main/java/com/quiltdata/quiltcore/Entry.java @@ -43,6 +43,26 @@ public static enum HashType { * The SHA-256 hash algorithm. */ SHA256, + SHA2_256_Chunked; + + /** + * Returns the HashType corresponding to the given name. + * + * @param name the name of the hash type + * @return the corresponding HashType + * @throws IllegalArgumentException if the name does not correspond to any HashType + */ + public static HashType enumFor(String name) { + String nameWithoutHyphens = name.replace("-", "_"); + System.err.println("nameWithoutHyphens: " + nameWithoutHyphens); + for (HashType type : HashType.values()) { + System.err.println("type.name(): " + type.name()); + if (type.name().equalsIgnoreCase(nameWithoutHyphens)) { + return type; + } + } + throw new IllegalArgumentException("No enum constant " + HashType.class.getCanonicalName() + "." + name); + } } /** diff --git a/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java b/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java index 0ea4118..7654339 100644 --- a/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java +++ b/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java @@ -72,7 +72,90 @@ public class Manifest { TOP_HASH_MAPPER.registerModule(sm); TOP_HASH_MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); } + + /** + * Returns a map for a URI of the form + * "quilt+s3://bucket#package=package@hash&path=path" + * + * @param fragment + * @return Map + * @throws IllegalArgumentException + */ + + public static Map ParseQuiltURI(URI uri) throws IllegalArgumentException { + Map result = new TreeMap(); + String scheme = uri.getScheme(); + if (!scheme.equals("quilt+s3")) { + throw new IllegalArgumentException("Invalid scheme: " + scheme); + } + result.put("scheme", scheme); + result.put("bucket", uri.getHost()); + String fragment = uri.getFragment(); + if (fragment == null) { + throw new IllegalArgumentException("Missing fragment"); + } + String[] parts = fragment.split("&"); + for (String part : parts) { + String[] kv = part.split("="); + if (kv.length != 2) { + throw new IllegalArgumentException("Invalid fragment part: " + part); + } + String key = kv[0]; + String value = kv[1]; + if (key.equals("path")) { + result.put("path", value); + } else if (key.equals("package")) { + result.put("revision", "latest"); + if (value.contains("@")) { + String[] packageParts = value.split("@"); + if (packageParts.length > 2) { + throw new IllegalArgumentException("Invalid package part: " + value); + } + result.put("package", packageParts[0]); + if (packageParts.length == 2) { + result.put("hash", packageParts[1]); + } + } else { + result.put("package", value); + } + } else { + throw new IllegalArgumentException("Invalid fragment key: " + key); + } + } + System.out.println("ParseQuiltURI: " + uri + " -> " + result); + return result; + } + /** + * Returns a Manifest for a URI of the form "quilt+s3://bucket#package=package@hash&path=path" + * + * @param quiltURI The URI to create the manifest from. + * @return The created {@link Manifest} object. + * @throws IllegalArgumentException If the URI is invalid. + * + */ + public static Manifest FromQuiltURI(String quiltURI) throws URISyntaxException, IllegalArgumentException, IOException { + URI uri = new URI(quiltURI); + Map parts = ParseQuiltURI(uri); + + String s3_uri = "s3://" + parts.get("bucket") + "/"; + System.out.println("s3_uri: " + s3_uri); + URI s3_root = new URI(s3_uri); + System.out.println("s3_root: " + s3_root); + PhysicalKey p = PhysicalKey.fromUri(s3_root); + Registry r = new Registry(p); + String pkg_handle = parts.get("package"); + Namespace n = r.getNamespace(pkg_handle); + + String hash = parts.get("hash"); + if (hash == null) { + String revision = parts.get("revision"); + hash = n.getHash(revision); + } + Manifest m = n.getManifest(hash); + return m; + } + /** * Represents a builder for creating a {@link Manifest} object. */ @@ -145,9 +228,9 @@ public static Builder builder() { * @param path The path to the file to create the manifest from. * @return The created {@link Manifest} object. * @throws IOException If an I/O error occurs. - * @throws URISyntaxException If the URI is invalid. + * @throws IllegalArgumentException If the URI is invalid. */ - public static Manifest createFromFile(PhysicalKey path) throws IOException, URISyntaxException { + public static Manifest createFromFile(PhysicalKey path) throws IOException, IllegalArgumentException, URISyntaxException { Builder builder = builder(); ObjectMapper mapper = new ObjectMapper(); @@ -180,7 +263,7 @@ public static Manifest createFromFile(PhysicalKey path) throws IOException, URIS PhysicalKey physicalKey = PhysicalKey.fromUri(new URI(physicalKeyString)); long size = row.get("size").asLong(); JsonNode hashNode = row.get("hash"); - Entry.HashType hashType = Entry.HashType.valueOf(hashNode.get("type").asText()); + Entry.HashType hashType = Entry.HashType.enumFor(hashNode.get("type").asText()); String hashValue = hashNode.get("value").asText(); JsonNode meta = row.get("meta"); if (meta == null) { diff --git a/lib/src/test/java/com/quiltdata/quiltcore/EntryTest.java b/lib/src/test/java/com/quiltdata/quiltcore/EntryTest.java new file mode 100644 index 0000000..7d44f7b --- /dev/null +++ b/lib/src/test/java/com/quiltdata/quiltcore/EntryTest.java @@ -0,0 +1,43 @@ +package com.quiltdata.quiltcore; + +import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; + +public class EntryTest { + + @Test + void testEnumFor() { + // Arrange + String name = "SHA256"; + // Act + Entry.HashType result = Entry.HashType.enumFor(name); + // Assert + assertEquals(Entry.HashType.SHA256, result); + } + + @Test + void testEnumForChunked() { + // Arrange + String name = "sha2-256-chunked"; + // Act + Entry.HashType result = Entry.HashType.enumFor(name); + // Assert + assertEquals(Entry.HashType.SHA2_256_Chunked, result); + } + + @Test + void testEnumForInvalid() { + // Arrange + String name = "SHA-512"; + // Act + try { + Entry.HashType result = Entry.HashType.enumFor(name); + fail("Expected IllegalArgumentException"); + assert result != null; + } catch (IllegalArgumentException e) { + // Assert + assertEquals("No enum constant com.quiltdata.quiltcore.Entry.HashType.SHA-512", e.getMessage()); + } + } + +} diff --git a/lib/src/test/java/com/quiltdata/quiltcore/ManifestTest.java b/lib/src/test/java/com/quiltdata/quiltcore/ManifestTest.java new file mode 100644 index 0000000..3caadf2 --- /dev/null +++ b/lib/src/test/java/com/quiltdata/quiltcore/ManifestTest.java @@ -0,0 +1,51 @@ +package com.quiltdata.quiltcore; + +import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; +import java.net.URI; +import java.util.Map; + +public class ManifestTest { + + @Test + void testParseQuiltURI() { + // Arrange + String quiltURI = "quilt+s3://bkt#package=prefix/suffix@top_hash&path=file"; + try { + URI uri = new URI(quiltURI); + Map result = Manifest.ParseQuiltURI(uri); + assertEquals("bkt", result.get("bucket")); + assertEquals("prefix/suffix", result.get("package")); + assertEquals("top_hash", result.get("hash")); + assertEquals("latest", result.get("revision")); + assertEquals("file", result.get("path")); + } catch (Exception e) { + fail("Failed to create URI from quiltURI", e); + } + } + + @Test + void testFromQuiltURI() { + // Arrange + String quiltURI = "quilt+s3://quilt-example#package=examples/metadata"; + try { + Manifest manifest = Manifest.FromQuiltURI(quiltURI); + assert manifest != null; + } catch (Exception e) { + fail("Failed to create URI from quiltURI", e); + } + } + + @Test + void testFromChunkedURI() { + // Arrange + String quiltURI = "quilt+s3://udp-spec#package=nf-quilt/source"; + try { + Manifest manifest = Manifest.FromQuiltURI(quiltURI); + assert manifest != null; + } catch (Exception e) { + fail("Failed to create URI from quiltURI", e); + } + } + +}