Skip to content

Commit

Permalink
Merge pull request #35 from DataONEorg/feature-17-test-review
Browse files Browse the repository at this point in the history 
Feature-17: Enhanced Tests
  • Loading branch information
@doulikecookiedough
doulikecookiedough authored Jul 3, 2023
2 parents b76c82a + 5d87403 commit 44df73f
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 55 deletions.
11 changes: 0 additions & 11 deletions FileHashStore.properties
View file

This file was deleted.

82 changes: 40 additions & 42 deletions src/main/java/org/dataone/hashstore/filehashstore/FileHashStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ public FileHashStore(HashMap<String, Object> hashstoreProperties)
// Write configuration file 'hashstore.yaml' to store HashStore properties
Path hashstoreYaml = this.STORE_ROOT.resolve("hashstore.yaml");
if (!Files.exists(hashstoreYaml)) {
String hashstoreYamlContent = FileHashStore.buildHashStoreYamlString(this.STORE_ROOT, this.DIRECTORY_DEPTH,
String hashstoreYamlContent = this.buildHashStoreYamlString(this.STORE_ROOT, this.DIRECTORY_DEPTH,
this.DIRECTORY_WIDTH, this.OBJECT_STORE_ALGORITHM, this.METADATA_NAMESPACE);
this.putHashStoreYaml(hashstoreYamlContent);
logFileHashStore.info("FileHashStore - 'hashstore.yaml' written to storePath: " + hashstoreYaml);
Expand Down Expand Up @@ -314,7 +314,7 @@ protected void checkConfigurationEquality(String propertyName, Object suppliedVa
* @param storeMetadataNamespace default formatId of hashstore metadata
* @return String that representing the contents of 'hashstore.yaml'
*/
protected static String buildHashStoreYamlString(Path storePath, int storeDepth, int storeWidth,
protected String buildHashStoreYamlString(Path storePath, int storeDepth, int storeWidth,
String storeAlgorithm, String storeMetadataNamespace) {
return String.format(
"# Default configuration variables for HashStore\n\n" +
Expand Down Expand Up @@ -1229,6 +1229,38 @@ protected boolean writeToTmpMetadataFile(File tmpFile, InputStream metadataStrea
}
}

/**
* Get the absolute path of a HashStore object or metadata file
*
* @param pid Authority-based identifier
* @param entity "object" or "metadata"
* @param formatId Metadata namespace
* @return Actual path to object
* @throws IllegalArgumentException If entity is not object or metadata
* @throws NoSuchAlgorithmException If store algorithm is not supported
*/
protected Path getRealPath(String pid, String entity, String formatId)
throws IllegalArgumentException, NoSuchAlgorithmException {
Path realPath;
if (entity.equalsIgnoreCase("object")) {
String objectCid = this.getPidHexDigest(pid, this.OBJECT_STORE_ALGORITHM);
String objShardString = this.getHierarchicalPathString(this.DIRECTORY_DEPTH, this.DIRECTORY_WIDTH,
objectCid);
realPath = this.OBJECT_STORE_DIRECTORY.resolve(objShardString);

} else if (entity.equalsIgnoreCase("metadata")) {
String objectCid = this.getPidHexDigest(pid + formatId, this.OBJECT_STORE_ALGORITHM);
String objShardString = this.getHierarchicalPathString(this.DIRECTORY_DEPTH, this.DIRECTORY_WIDTH,
objectCid);
realPath = this.METADATA_STORE_DIRECTORY.resolve(objShardString);

} else {
throw new IllegalArgumentException("FileHashStore.getRealPath - entity must be 'object' or 'metadata'");

}
return realPath;
}

/**
* Deletes a given object and its parent directories if they are empty
*
Expand All @@ -1237,13 +1269,13 @@ protected boolean writeToTmpMetadataFile(File tmpFile, InputStream metadataStrea
* @param method Calling method
* @throws IOException I/O error when deleting object or accessing directories
*/
protected void deleteObjectAndParentDirectories(Path objectAbsPath, String pid, String method) throws IOException {
private void deleteObjectAndParentDirectories(Path objectAbsPath, String pid, String method) throws IOException {
// Delete file
Files.delete(objectAbsPath);

// Then delete any empty directories
Path parent = objectAbsPath.getParent();
while (parent != null && isDirectoryEmpty(parent)) {
while (parent != null && this.isDirectoryEmpty(parent)) {
if (parent.equals(this.METADATA_STORE_DIRECTORY)) {
// Do not delete the metadata store directory
break;
Expand All @@ -1267,7 +1299,7 @@ protected void deleteObjectAndParentDirectories(Path objectAbsPath, String pid,
* @return True if a file is found or the directory is empty, False otherwise
* @throws IOException If I/O occurs when accessing directory
*/
protected boolean isDirectoryEmpty(Path directory) throws IOException {
private boolean isDirectoryEmpty(Path directory) throws IOException {
try (Stream<Path> stream = Files.list(directory)) {
// The findFirst() method is called on the stream created from the given
// directory to retrieve the first element. If the stream is empty (i.e., the
Expand All @@ -1281,54 +1313,21 @@ protected boolean isDirectoryEmpty(Path directory) throws IOException {
}
}

/**
* Get the absolute path of a HashStore object or metadata file
*
* @param pid Authority-based identifier
* @param entity "object" or "metadata"
* @param formatId Metadata namespace
* @return Actual path to object
* @throws IllegalArgumentException If entity is not object or metadata
* @throws NoSuchAlgorithmException If store algorithm is not supported
*/
protected Path getRealPath(String pid, String entity, String formatId)
throws IllegalArgumentException, NoSuchAlgorithmException {
Path realPath;
if (entity.equalsIgnoreCase("object")) {
String objectCid = this.getPidHexDigest(pid, this.OBJECT_STORE_ALGORITHM);
String objShardString = this.getHierarchicalPathString(this.DIRECTORY_DEPTH, this.DIRECTORY_WIDTH,
objectCid);
realPath = this.OBJECT_STORE_DIRECTORY.resolve(objShardString);

} else if (entity.equalsIgnoreCase("metadata")) {
String objectCid = this.getPidHexDigest(pid + formatId, this.OBJECT_STORE_ALGORITHM);
String objShardString = this.getHierarchicalPathString(this.DIRECTORY_DEPTH, this.DIRECTORY_WIDTH,
objectCid);
realPath = this.METADATA_STORE_DIRECTORY.resolve(objShardString);

} else {
throw new IllegalArgumentException("FileHashStore.getRealPath - entity must be 'object' or 'metadata'");

}
return realPath;
}

/**
* Checks whether a given object is null and throws an exception if so
*
* @param object Object to check
* @param argument Value that is being checked
* @param method Calling method
*/
protected boolean isObjectNull(Object object, String argument, String method) {
private void isObjectNull(Object object, String argument, String method) {
if (object == null) {
String errMsg = "FileHashStore.isStringNullOrEmpty - Calling Method: " + method + "(): " + argument
+ " cannot be null.";
logFileHashStore.error(errMsg);
throw new NullPointerException(errMsg);

}
return false;
}

/**
Expand All @@ -1338,15 +1337,14 @@ protected boolean isObjectNull(Object object, String argument, String method) {
* @param argument Value that is being checked
* @param method Calling method
*/
protected boolean isStringEmpty(String string, String argument, String method) {
private void isStringEmpty(String string, String argument, String method) {
if (string.trim().isEmpty()) {
String errMsg = "FileHashStore.isStringNullOrEmpty - Calling Method: " + method + "(): " + argument
+ " cannot be empty.";
logFileHashStore.error(errMsg);
throw new IllegalArgumentException(errMsg);

}
return false;
}

/**
Expand All @@ -1359,7 +1357,7 @@ protected boolean isStringEmpty(String string, String argument, String method) {
* @throws IOException Error when calculating hex digest
* @throws NoSuchAlgorithmException Algorithm not supported
*/
protected String calculateHexDigest(Path objectPath, String algorithm)
private String calculateHexDigest(Path objectPath, String algorithm)
throws IOException, NoSuchAlgorithmException {
MessageDigest mdObject = MessageDigest.getInstance(algorithm);
try {
Expand Down
38 changes: 37 additions & 1 deletion src/test/java/org/dataone/hashstore/filehashstore/FileHashStoreInterfaceTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package org.dataone.hashstore.filehashstore;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
Expand All @@ -23,6 +26,7 @@
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TemporaryFolder;

import static org.junit.Assert.*;
Expand Down Expand Up @@ -100,7 +104,7 @@ public void storeObject_relPath() throws Exception {

// Check relative path
String objectCid = testData.pidData.get(pid).get("object_cid");
String objRelPath = fileHashStore.getHierarchicalPathString(3, 2, objectCid);
String objRelPath = this.fileHashStore.getHierarchicalPathString(3, 2, objectCid);
assertEquals(objRelPath, objInfo.getRelPath());
}
}
Expand Down Expand Up @@ -317,6 +321,38 @@ public void storeObject_duplicate() throws Exception {
}
}

/**
* Test that storeObject successfully stores a 1GB file
*
* Note, a 4GB successfully stored in approximately 1m30s
*/
@Test
public void storeObject_largeSparseFile() throws Exception {
long fileSize = 1L * 1024L * 1024L * 1024L; // 1GB
// Get tmp directory to initially store test file
Path storePath = (Path) this.fhsProperties.get("storePath");
Path testFilePath = storePath.resolve("random_file.bin");

// Generate a random file with the specified size
try (FileOutputStream fileOutputStream = new FileOutputStream(testFilePath.toString())) {
FileChannel fileChannel = fileOutputStream.getChannel();
FileLock lock = fileChannel.lock();
fileChannel.position(fileSize - 1);
fileChannel.write(java.nio.ByteBuffer.wrap(new byte[] { 0 }));
lock.release();
} catch (IOException ioe) {
ioe.printStackTrace();
throw ioe;
}

InputStream dataStream = Files.newInputStream(testFilePath);
String pid = "dou.sparsefile.1";
HashAddress sparseFileObjInfo = fileHashStore.storeObject(dataStream, pid, null, null, null);
Path testSparseFileAbsPath = sparseFileObjInfo.getAbsPath();
assertTrue(Files.exists(testSparseFileAbsPath));

}

/**
* Tests that the `storeObject` method can store an object successfully with
* multiple threads (5). This test uses five futures (threads) that run
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/dataone/hashstore/filehashstore/FileHashStoreProtectedTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ public void putObject_testHarness_relPath() throws Exception {

// Check relative path
String objAuthorityId = testData.pidData.get(pid).get("object_cid");
String objRelPath = fileHashStore.getHierarchicalPathString(3, 2, objAuthorityId);
String objRelPath = this.fileHashStore.getHierarchicalPathString(3, 2, objAuthorityId);
assertEquals(objRelPath, address.getRelPath());
}
}
Expand Down

0 comments on commit 44df73f

Please sign in to comment.