diff --git a/smart-action/src/main/java/org/smartdata/action/SmartAction.java b/smart-action/src/main/java/org/smartdata/action/SmartAction.java index d4870584097..4a68596fee4 100644 --- a/smart-action/src/main/java/org/smartdata/action/SmartAction.java +++ b/smart-action/src/main/java/org/smartdata/action/SmartAction.java @@ -85,9 +85,15 @@ public void init(Map args) { protected abstract void execute() throws Exception; + protected void preRun() throws Exception { + setStartTime(); + appendLog( + String.format("Action starts at %s", Utils.getFormatedCurrentTime())); + } + public final void run() { try { - setStartTime(); + preRun(); execute(); successful = true; } catch (Throwable t) { diff --git a/smart-common/src/main/java/org/smartdata/SmartConstants.java b/smart-common/src/main/java/org/smartdata/SmartConstants.java index 3de15ac3846..8dce87bbde9 100644 --- a/smart-common/src/main/java/org/smartdata/SmartConstants.java +++ b/smart-common/src/main/java/org/smartdata/SmartConstants.java @@ -17,22 +17,16 @@ */ package org.smartdata; -import java.util.HashMap; -import java.util.Map; - public class SmartConstants { public static final String SMART_HDFS_LAST_INOTIFY_TXID = - "smart_hadoop_last_inotify_txid"; + "smart_hadoop_last_inotify_txid"; public static final String SMART_CLIENT_PROTOCOL_NAME = - "org.smartdata.protocol.SmartClientProtocol"; - - public static final String SMART_ADMIN_PROTOCOL_NAME = - "org.smartdata.protocol.SmartAdminProtocol"; + "org.smartdata.protocol.SmartClientProtocol"; public static final String SMART_CLIENT_DISABLED_ID_FILE = - "/tmp/SMART_CLIENT_DISABLED_ID_FILE"; + "/tmp/SMART_CLIENT_DISABLED_ID_FILE"; public static final String NUMBER_OF_SMART_AGENT = "number_of_smart_agent_in_agents_file"; @@ -47,39 +41,6 @@ public class SmartConstants { public static final String AGENT_CMDLET_SERVICE_NAME = "AgentCmdletService"; - public static final byte STORAGE_POLICY_UNDEF_ID = 0; - public static final String STORAGE_POLICY_UNDEF_NAME = "UNDEF"; - - public static final byte STORAGE_POLICY_COLD_ID = 2; - public static final String STORAGE_POLICY_COLD_NAME = "COLD"; - - public static final byte STORAGE_POLICY_WARM_ID = 5; - public static final String STORAGE_POLICY_WARM_NAME = "WARM"; - - public static final byte STORAGE_POLICY_HOT_ID = 7; - public static final String STORAGE_POLICY_HOT_NAME = "HOT"; - - public static final byte STORAGE_POLICY_ONE_SSD_ID = 10; - public static final String STORAGE_POLICY_ONE_SSD_NAME = "ONE_SSD"; - - public static final byte STORAGE_POLICY_ALL_SSD_ID = 12; - public static final String STORAGE_POLICY_ALL_SSD_NAME = "ALL_SSD"; - - public static final byte STORAGE_POLICY_LAZY_PERSIST_ID = 15; - public static final String STORAGE_POLICY_LAZY_PERSIST_NAME = "LAZY_PERSIST"; - - public static final Map STORAGE_POLICY_MAP = new HashMap<>(); - - static { - STORAGE_POLICY_MAP.put(STORAGE_POLICY_UNDEF_ID, STORAGE_POLICY_UNDEF_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_COLD_ID, STORAGE_POLICY_COLD_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_WARM_ID, STORAGE_POLICY_WARM_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_HOT_ID, STORAGE_POLICY_HOT_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_ONE_SSD_ID, STORAGE_POLICY_ONE_SSD_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_ALL_SSD_ID, STORAGE_POLICY_ALL_SSD_NAME); - STORAGE_POLICY_MAP.put(STORAGE_POLICY_LAZY_PERSIST_ID, STORAGE_POLICY_LAZY_PERSIST_NAME); - } - public static final String SMART_FILE_CHECKSUM_XATTR_NAME = "user.checksum"; public static final String FS_HDFS_IMPL = "fs.hdfs.impl"; diff --git a/smart-common/src/main/java/org/smartdata/utils/PathUtil.java b/smart-common/src/main/java/org/smartdata/utils/PathUtil.java index 117b423e3b8..ea1f82948dd 100644 --- a/smart-common/src/main/java/org/smartdata/utils/PathUtil.java +++ b/smart-common/src/main/java/org/smartdata/utils/PathUtil.java @@ -17,14 +17,18 @@ */ package org.smartdata.utils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.io.IOException; import java.net.URI; import java.util.Optional; +import static org.smartdata.utils.ConfigUtil.toRemoteClusterConfig; + public class PathUtil { private static final String DIR_SEP = "/"; - private static final String HDFS_SCHEME = "hdfs"; private static final String[] GLOBS = new String[] { "*", "?" }; @@ -66,6 +70,10 @@ public static boolean pathStartsWith(String path, String prefixToCheck) { .startsWith(addPathSeparator(prefixToCheck)); } + public static boolean isAbsoluteRemotePath(String path) { + return isAbsoluteRemotePath(new Path(path)); + } + // todo replace 'stringPath.startsWith("hdfs")' calls with this method public static boolean isAbsoluteRemotePath(Path path) { return Optional.ofNullable(path) @@ -77,7 +85,32 @@ public static boolean isAbsoluteRemotePath(Path path) { public static boolean isAbsoluteRemotePath(URI uri) { return Optional.ofNullable(uri) .map(URI::getScheme) - .filter(HDFS_SCHEME::equals) .isPresent(); } + + public static Optional getScheme(String path) { + return Optional.ofNullable(path) + .map(URI::create) + .map(URI::getScheme); + } + + public static Optional getScheme(Path path) { + return Optional.ofNullable(path) + .map(Path::toUri) + .map(URI::getScheme); + } + + public static FileSystem getRemoteFileSystem(String path) throws IOException { + return getRemoteFileSystem(new Path(path)); + } + + // todo we use default HDFS config, add mechanism to provide + // hdfs config paths for remote clusters + public static FileSystem getRemoteFileSystem(Path path) throws IOException { + return path.getFileSystem(toRemoteClusterConfig(new Configuration())); + } + + public static String getRawPath(Path path) { + return path.toUri().getPath(); + } } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java index 8cfe1ed5cd9..0fead3ceb95 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java @@ -25,8 +25,8 @@ import org.smartdata.action.ActionRegistry; import org.smartdata.action.SmartAction; import org.smartdata.hdfs.action.HdfsAction; -import org.smartdata.hdfs.client.CachingDfsClientProvider; -import org.smartdata.hdfs.client.DfsClientProvider; +import org.smartdata.hdfs.client.CachingLocalFileSystemProvider; +import org.smartdata.hdfs.client.LocalFileSystemProvider; import org.smartdata.model.LaunchAction; import org.smartdata.protocol.message.LaunchCmdlet; @@ -40,15 +40,15 @@ public class CmdletFactory implements Closeable { static final Logger LOG = LoggerFactory.getLogger(CmdletFactory.class); private final SmartContext smartContext; - private final DfsClientProvider dfsClientProvider; + private final LocalFileSystemProvider localFileSystemProvider; public CmdletFactory(SmartContext smartContext) { - this(smartContext, new CachingDfsClientProvider(smartContext.getConf())); + this(smartContext, new CachingLocalFileSystemProvider(smartContext.getConf())); } - public CmdletFactory(SmartContext smartContext, DfsClientProvider dfsClientProvider) { + public CmdletFactory(SmartContext smartContext, LocalFileSystemProvider localFileSystemProvider) { this.smartContext = smartContext; - this.dfsClientProvider = dfsClientProvider; + this.localFileSystemProvider = localFileSystemProvider; } public Cmdlet createCmdlet(LaunchCmdlet launchCmdlet) throws ActionException { @@ -73,15 +73,15 @@ public SmartAction createAction(long cmdletId, boolean isLastAction, LaunchActio smartAction.init(launchAction.getArgs()); smartAction.setActionId(launchAction.getActionId()); if (smartAction instanceof HdfsAction) { - setDfsClient((HdfsAction) smartAction); + setLocalFileSystem((HdfsAction) smartAction); } return smartAction; } - private void setDfsClient(HdfsAction action) throws ActionException { + private void setLocalFileSystem(HdfsAction action) throws ActionException { try { - action.setDfsClient( - dfsClientProvider.provide(smartContext.getConf(), action.dfsClientType()) + action.setLocalFileSystem( + localFileSystemProvider.provide(smartContext.getConf(), action.localFsType()) ); } catch (IOException exception) { LOG.error("smartAction aid={} setDfsClient error", action.getActionId(), exception); @@ -92,7 +92,7 @@ private void setDfsClient(HdfsAction action) throws ActionException { @Override public void close() { try { - dfsClientProvider.close(); + localFileSystemProvider.close(); } catch (IOException exception) { String errorMessage = "Error closing DFS client provider"; log.error(errorMessage, exception); diff --git a/smart-hadoop-support/smart-hadoop-3/pom.xml b/smart-hadoop-support/smart-hadoop-3/pom.xml index 159c55f2c40..18e63d1ce71 100644 --- a/smart-hadoop-support/smart-hadoop-3/pom.xml +++ b/smart-hadoop-support/smart-hadoop-3/pom.xml @@ -77,6 +77,10 @@ smart-hadoop-common 2.0.0-SNAPSHOT + + org.projectlombok + lombok + org.apache.hadoop hadoop-aws diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/AddErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/AddErasureCodingPolicy.java index 0b77c54e805..2cedf8dcaa1 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/AddErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/AddErasureCodingPolicy.java @@ -44,7 +44,7 @@ public class AddErasureCodingPolicy extends HdfsAction { public static final String DATA_UNITS_NUM = "-dataNum"; public static final String PARITY_UNITS_NUM = "-parityNum"; public static final String CELL_SIZE = "-cellSize"; - private String policyName; + private String codecName; private int numDataUnits; private int numParityUnits; @@ -54,7 +54,7 @@ public class AddErasureCodingPolicy extends HdfsAction { public void init(Map args) { super.init(args); if (args.get(POLICY_NAME) != null && !args.get(POLICY_NAME).isEmpty()) { - this.policyName = args.get(POLICY_NAME); + String policyName = args.get(POLICY_NAME); String[] policySchema = policyName.split("-"); if (policySchema.length != 4) { return; @@ -81,28 +81,33 @@ public void init(Map args) { @Override public void execute() throws Exception { - if (codecName == null || numDataUnits <= 0 || numParityUnits <= 0 || - cellSize <= 0 || cellSize % 1024 != 0) { - throw new ActionException("Illegal EC policy Schema! " + - "A valid codec name should be given, " + - "the dataNum, parityNum and cellSize should be positive and " + - "the cellSize should be divisible by 1024."); - } + validateArgs(); + ECSchema ecSchema = new ECSchema(codecName, numDataUnits, numParityUnits); ErasureCodingPolicy ecPolicy = new ErasureCodingPolicy(ecSchema, cellSize); AddErasureCodingPolicyResponse addEcResponse = - dfsClient.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy})[0]; - if (addEcResponse.isSucceed()) { - appendLog(String.format("EC policy named %s is added successfully!", - addEcResponse.getPolicy().getName())); - } else { - appendLog(String.format("Failed to add the given EC policy!")); + localFileSystem.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy})[0]; + + if (!addEcResponse.isSucceed()) { + appendLog("Failed to add the given EC policy!"); throw new ActionException(addEcResponse.getErrorMsg()); + } else { + appendLog("EC policy is added successfully: " + addEcResponse.getPolicy().getName()); } } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; + } + + private void validateArgs() throws ActionException { + if (codecName == null || numDataUnits <= 0 || numParityUnits <= 0 || + cellSize <= 0 || cellSize % 1024 != 0) { + throw new ActionException("Illegal EC policy Schema! " + + "A valid codec name should be given, " + + "the dataNum, parityNum and cellSize should be positive and " + + "the cellSize should be divisible by 1024."); + } } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java index 663d985f632..3e69b50a5b7 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java @@ -17,7 +17,7 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.smartdata.action.annotation.ActionSignature; @@ -35,21 +35,21 @@ public class CheckErasureCodingPolicy extends HdfsAction { public static final String RESULT_OF_NULL_EC_POLICY = "The EC policy is replication."; - private String srcPath; + + private Path srcPath; @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(HdfsAction.FILE_PATH); + this.srcPath = getPathArg(FILE_PATH); } @Override public void execute() throws Exception { - if (StringUtils.isBlank(srcPath)) { - throw new IllegalArgumentException("File parameter is missing! "); - } + validateNonEmptyArg(FILE_PATH); - String result = Optional.ofNullable(dfsClient.getErasureCodingPolicy(srcPath)) + String result = Optional.ofNullable( + localFileSystem.getErasureCodingPolicy(srcPath)) .map(ErasureCodingPolicy::toString) .orElse(RESULT_OF_NULL_EC_POLICY); @@ -57,7 +57,7 @@ public void execute() throws Exception { } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/DisableErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/DisableErasureCodingPolicy.java index 38ff48336c7..49ff53d5b2e 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/DisableErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/DisableErasureCodingPolicy.java @@ -31,6 +31,7 @@ ) public class DisableErasureCodingPolicy extends HdfsAction { public static final String EC_POLICY_NAME = "-policy"; + private String policyName; @Override @@ -41,12 +42,12 @@ public void init(Map args) { @Override public void execute() throws Exception { - dfsClient.disableErasureCodingPolicy(policyName); + localFileSystem.disableErasureCodingPolicy(policyName); appendLog(String.format("The EC policy named %s is disabled!", policyName)); } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/EnableErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/EnableErasureCodingPolicy.java index 0341ac1db83..ac60aaa8e26 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/EnableErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/EnableErasureCodingPolicy.java @@ -31,6 +31,7 @@ ) public class EnableErasureCodingPolicy extends HdfsAction { public static final String EC_POLICY_NAME = "-policy"; + private String policyName; @Override @@ -41,12 +42,12 @@ public void init(Map args) { @Override public void execute() throws Exception { - dfsClient.enableErasureCodingPolicy(policyName); + localFileSystem.enableErasureCodingPolicy(policyName); appendLog(String.format("The EC policy named %s is enabled!", policyName)); } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 03c7fab0eba..2f4b8cfcb45 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -17,23 +17,17 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Options; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.utils.StringUtil; import java.io.IOException; -import java.util.EnumSet; -import java.util.HashMap; import java.util.Map; /** @@ -43,102 +37,80 @@ @ActionSignature( actionId = "ec", displayName = "ec", - usage = HdfsAction.FILE_PATH + " $src " + ErasureCodingAction.EC_POLICY_NAME + " $policy" + - ErasureCodingBase.BUF_SIZE + " $bufSize" + usage = HdfsAction.FILE_PATH + " $src " + + ErasureCodingAction.EC_POLICY_NAME + " $policy" + + ErasureCodingBase.BUF_SIZE + " $bufSize" ) public class ErasureCodingAction extends ErasureCodingBase { - private static final Logger LOG = - LoggerFactory.getLogger(ErasureCodingAction.class); public static final String EC_POLICY_NAME = "-policy"; @Override public void init(Map args) { super.init(args); - this.conf = getContext().getConf(); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(EC_TMP)) { - // this is a temp file kept for converting a file to another with other ec policy. - this.ecTmpPath = args.get(EC_TMP); - } - if (args.containsKey(EC_POLICY_NAME) && !args.get(EC_POLICY_NAME).isEmpty()) { - this.ecPolicyName = args.get(EC_POLICY_NAME); - } else { - String defaultEcPolicy = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, - DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); - this.ecPolicyName = defaultEcPolicy; - } - if (args.containsKey(BUF_SIZE) && !args.get(BUF_SIZE).isEmpty()) { - this.bufferSize = (int) StringUtil.parseToByte(args.get(BUF_SIZE)); - } + this.srcPath = getPathArg(FILE_PATH); + this.ecTmpPath = getPathArg(EC_TMP); + this.ecPolicyName = isArgPresent(EC_POLICY_NAME) + ? args.get(EC_POLICY_NAME) + : getConf().getTrimmed( + DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, + DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); + this.bufferSize = isArgPresent(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUF_SIZE; this.progress = 0.0F; } @Override protected void execute() throws Exception { - final String MATCH_RESULT = - "The current EC policy is already matched with the target one."; - final String DIR_RESULT = - "The EC policy is set successfully for the given directory."; - final String CONVERT_RESULT = - "The file is converted successfully with the given or default EC policy."; + validateNonEmptyArgs(FILE_PATH); // keep attribute consistent - HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); - if (fileStatus == null) { - throw new ActionException("File doesn't exist!"); - } + HdfsFileStatus fileStatus = (HdfsFileStatus) localFileSystem.getFileStatus(srcPath); + validateEcPolicy(ecPolicyName); ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); // if the current ecPolicy is already the target one, no need to convert - if (srcEcPolicy != null) { - if (srcEcPolicy.getName().equals(ecPolicyName)) { - appendLog(MATCH_RESULT); - this.progress = 1.0F; - return; - } - } else { - // if ecPolicy is null, it means replication. - if (ecPolicyName.equals(REPLICATION_POLICY_NAME)) { - appendLog(MATCH_RESULT); - this.progress = 1.0F; - return; - } + if (srcEcPolicy != null && srcEcPolicy.getName().equals(ecPolicyName) + // if ecPolicy is null, it means replication. + || srcEcPolicy == null && ecPolicyName.equals(REPLICATION_POLICY_NAME)) { + appendLog(MATCH_RESULT); + this.progress = 1.0F; + return; } + if (fileStatus.isDir()) { - dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); + localFileSystem.setErasureCodingPolicy(srcPath, ecPolicyName); this.progress = 1.0F; appendLog(DIR_RESULT); return; } - HdfsDataOutputStream outputStream = null; + + FSDataOutputStream outputStream = null; try { // a file only with replication policy can be appended. if (srcEcPolicy == null) { // append the file to acquire the lock to avoid modifying, real appending wouldn't occur. - outputStream = - dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); + outputStream = localFileSystem.append(srcPath, bufferSize); } convert(fileStatus); - /** - * The append operation will change the modification time accordingly, - * so we use the FileStatus obtained before append to set ecTmp file's most attributes - */ - setAttributes(srcPath, fileStatus, ecTmpPath); - dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + // The append operation will change the modification time accordingly, + // so we use the FileStatus obtained before append to set ecTmp file's most attributes + setAttributes(fileStatus); + localFileSystem.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendLog(CONVERT_RESULT); if (srcEcPolicy == null) { appendLog("The previous EC policy is replication."); } else { - appendLog(String.format("The previous EC policy is %s.", srcEcPolicy.getName())); + appendLog("The previous EC policy is " + srcEcPolicy.getName()); } - appendLog(String.format("The current EC policy is %s.", ecPolicyName)); + appendLog("The current EC policy is " + ecPolicyName); } catch (ActionException ex) { try { - if (dfsClient.getFileInfo(ecTmpPath) != null) { - dfsClient.delete(ecTmpPath, false); + if (localFileSystem.exists(ecTmpPath)) { + localFileSystem.delete(ecTmpPath, false); } } catch (IOException e) { - LOG.error("Failed to delete tmp file created during the conversion!"); + appendLog("Failed to delete tmp file created during the conversion!" + ex.getMessage()); } throw new ActionException(ex); } finally { @@ -153,25 +125,23 @@ protected void execute() throws Exception { } public void validateEcPolicy(String ecPolicyName) throws Exception { - Map ecPolicyNameToState = new HashMap<>(); - for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { - ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); - } - if (!ecPolicyNameToState.keySet().contains(ecPolicyName) && !ecPolicyName.equals(REPLICATION_POLICY_NAME)) { + ErasureCodingPolicyState ecPolicyState = localFileSystem.getAllErasureCodingPolicies() + .stream() + .filter(policyInfo -> policyInfo.getPolicy().getName().equals(ecPolicyName)) + .map(ErasureCodingPolicyInfo::getState) + .findFirst() + .orElse(null); + + if (ecPolicyState == null && !ecPolicyName.equals(REPLICATION_POLICY_NAME)) { throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); - } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED - || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { + } else if (ecPolicyState == ErasureCodingPolicyState.DISABLED + || ecPolicyState == ErasureCodingPolicyState.REMOVED) { throw new ActionException("The EC policy " + ecPolicyName + " is disabled or removed!"); } } @Override - public float getProgress() { - return progress; - } - - @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 503f631b81f..c6cbdcd50a0 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -17,105 +17,100 @@ */ package org.smartdata.hdfs.action; +import lombok.Getter; +import lombok.Setter; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.action.ActionException; -import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.StreamCopyHandler; import java.io.IOException; import java.util.EnumSet; import java.util.Map; +import static org.smartdata.utils.PathUtil.getRawPath; + /** * An abstract base class for ErasureCodingAction & UnErasureCodingAction. */ abstract public class ErasureCodingBase extends HdfsAction { - private static final Logger LOG = - LoggerFactory.getLogger(ErasureCodingBase.class); public static final String BUF_SIZE = "-bufSize"; - protected String srcPath; - protected String ecTmpPath; - protected int bufferSize = 1024 * 1024; - protected float progress; - protected SmartConf conf; - protected String ecPolicyName; // The value for -ecTmp is assigned by ErasureCodingScheduler. public static final String EC_TMP = "-ecTmp"; public static final String REPLICATION_POLICY_NAME = SystemErasureCodingPolicies.getReplicationPolicy().getName(); - protected void convert(HdfsFileStatus srcFileStatus) throws ActionException { - DFSInputStream in = null; - DFSOutputStream out = null; - try { - long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, - DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); - in = dfsClient.open(srcPath, bufferSize, true); - short replication = (short) conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, - DFSConfigKeys.DFS_REPLICATION_DEFAULT); - // use the same FsPermission as srcPath - FsPermission permission = srcFileStatus.getPermission(); - out = dfsClient.create(ecTmpPath, permission, EnumSet.of(CreateFlag.CREATE), true, - replication, blockSize, null, bufferSize, null, null, ecPolicyName); - // Keep storage policy according with original file except UNDEF storage policy - String storagePolicyName = dfsClient.getStoragePolicy(srcPath).getName(); + protected final static String MATCH_RESULT = + "The current EC policy is already matched with the target one."; + protected final static String DIR_RESULT = + "The EC policy is set successfully for the given directory."; + protected final static String CONVERT_RESULT = + "The file is converted successfully with the given or default EC policy."; + + public static final int DEFAULT_BUF_SIZE = 1024 * 1024; + + protected Path srcPath; + protected Path ecTmpPath; + + @Setter + @Getter + protected float progress; + protected int bufferSize; + protected String ecPolicyName; + + protected void convert(HdfsFileStatus srcFileStatus) throws Exception { + + long blockSize = localFileSystem.getDefaultBlockSize(); + short replication = localFileSystem.getDefaultReplication(); + FsPermission permission = srcFileStatus.getPermission(); + + try (FSDataInputStream in = localFileSystem.open(srcPath, bufferSize); + DFSOutputStream out = getLocalDfsClient().create( + getRawPath(ecTmpPath), permission, EnumSet.of(CreateFlag.CREATE), true, + replication, blockSize, null, bufferSize, null, null, ecPolicyName)) { + + // Keep storage policy according to original file except UNDEF storage policy + String storagePolicyName = localFileSystem.getStoragePolicy(srcPath).getName(); if (!storagePolicyName.equals("UNDEF")) { - dfsClient.setStoragePolicy(ecTmpPath, storagePolicyName); - } - long bytesRemaining = srcFileStatus.getLen(); - byte[] buf = new byte[bufferSize]; - while (bytesRemaining > 0L) { - int bytesToRead = - (int) (bytesRemaining < (long) buf.length ? bytesRemaining : - (long) buf.length); - int bytesRead = in.read(buf, 0, bytesToRead); - if (bytesRead == -1) { - break; - } - out.write(buf, 0, bytesRead); - bytesRemaining -= (long) bytesRead; - this.progress = (float) (srcFileStatus.getLen() - bytesRemaining) / srcFileStatus.getLen(); - } - } catch (Exception ex) { - throw new ActionException(ex); - } finally { - try { - if (in != null) { - in.close(); - } - if (out != null) { - out.close(); - } - } catch (IOException ex) { - LOG.error("IOException occurred when closing DFSInputStream or DFSOutputStream!"); + localFileSystem.setStoragePolicy(ecTmpPath, storagePolicyName); } + + StreamCopyHandler.of(in, out) + .count(srcFileStatus.getLen()) + .progressConsumer(this::setProgress) + .closeStreams(false) + .build() + .runCopy(); } } // set attributes for dest to keep them consistent with their counterpart of src - protected void setAttributes(String src, HdfsFileStatus fileStatus, String dest) + protected void setAttributes(HdfsFileStatus fileStatus) throws IOException { - dfsClient.setOwner(dest, fileStatus.getOwner(), fileStatus.getGroup()); - dfsClient.setPermission(dest, fileStatus.getPermission()); - dfsClient.setStoragePolicy(dest, dfsClient.getStoragePolicy(src).getName()); - dfsClient.setTimes(dest, fileStatus.getModificationTime(), fileStatus.getAccessTime()); - boolean aclsEnabled = getContext().getConf().getBoolean( + localFileSystem.setOwner(ecTmpPath, + fileStatus.getOwner(), fileStatus.getGroup()); + localFileSystem.setPermission(ecTmpPath, + fileStatus.getPermission()); + localFileSystem.setStoragePolicy(ecTmpPath, + localFileSystem.getStoragePolicy(srcPath).getName()); + localFileSystem.setTimes(ecTmpPath, + fileStatus.getModificationTime(), fileStatus.getAccessTime()); + boolean aclsEnabled = getConf().getBoolean( DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT); if (aclsEnabled) { - dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); + localFileSystem.setAcl(ecTmpPath, + localFileSystem.getAclStatus(srcPath).getEntries()); } //TODO: check ec related record to avoid paradox - for (Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { - dfsClient.setXAttr(dest, entry.getKey(), entry.getValue(), + for (Map.Entry entry : localFileSystem.getXAttrs(srcPath).entrySet()) { + localFileSystem.setXAttr(ecTmpPath, entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java index dc3f9c41317..90c8859ac4f 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java @@ -39,13 +39,13 @@ public void init(Map args) { @Override public void execute() throws Exception { - for (ErasureCodingPolicyInfo policyInfo : dfsClient.getErasureCodingPolicies()) { + for (ErasureCodingPolicyInfo policyInfo : localFileSystem.getAllErasureCodingPolicies()) { appendResult("{" + policyInfo.toString() + "}"); } } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/RemoveErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/RemoveErasureCodingPolicy.java index a96961b493d..d53004a311a 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/RemoveErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/RemoveErasureCodingPolicy.java @@ -41,12 +41,12 @@ public void init(Map args) { @Override public void execute() throws Exception { - dfsClient.removeErasureCodingPolicy(policyName); - appendLog(String.format("The EC policy named %s is removed!", policyName)); + localFileSystem.removeErasureCodingPolicy(policyName); + appendLog("The EC policy named is removed: " + policyName); } @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index c40e26de827..71969d4c8b6 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -24,7 +24,6 @@ import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.utils.StringUtil; import java.io.IOException; import java.util.Map; @@ -45,32 +44,22 @@ public class UnErasureCodingAction extends ErasureCodingBase { @Override public void init(Map args) { super.init(args); - this.conf = getContext().getConf(); - this.srcPath = args.get(FILE_PATH); + this.srcPath = getPathArg(FILE_PATH); this.ecPolicyName = REPLICATION_POLICY_NAME; - if (args.containsKey(EC_TMP)) { - this.ecTmpPath = args.get(EC_TMP); - } - if (args.containsKey(BUF_SIZE) && !args.get(BUF_SIZE).isEmpty()) { - this.bufferSize = (int) StringUtil.parseToByte(args.get(BUF_SIZE)); - } + this.ecTmpPath = getPathArg(EC_TMP); + this.bufferSize = isArgPresent(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUF_SIZE; this.progress = 0.0F; } @Override protected void execute() throws Exception { - final String MATCH_RESULT = - "The current EC policy is replication already."; - final String DIR_RESULT = - "The replication EC policy is set successfully for the given directory."; - final String CONVERT_RESULT = - "The file is converted successfully with replication EC policy."; + validateNonEmptyArgs(FILE_PATH); - HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); - if (fileStatus == null) { - throw new ActionException("File doesn't exist!"); - } + HdfsFileStatus fileStatus = (HdfsFileStatus) localFileSystem.getFileStatus(srcPath); ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); + // if ecPolicy is null, it means replication. if (srcEcPolicy == null) { this.progress = 1.0F; @@ -78,22 +67,23 @@ protected void execute() throws Exception { return; } if (fileStatus.isDir()) { - dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); + localFileSystem.setErasureCodingPolicy(srcPath, ecPolicyName); progress = 1.0F; appendLog(DIR_RESULT); return; } + try { convert(fileStatus); - setAttributes(srcPath, fileStatus, ecTmpPath); - dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + setAttributes(fileStatus); + localFileSystem.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendLog(CONVERT_RESULT); appendLog(String.format("The previous EC policy is %s.", srcEcPolicy.getName())); appendLog(String.format("The current EC policy is %s.", REPLICATION_POLICY_NAME)); } catch (ActionException ex) { try { - if (dfsClient.getFileInfo(ecTmpPath) != null) { - dfsClient.delete(ecTmpPath, false); + if (localFileSystem.exists(ecTmpPath)) { + localFileSystem.delete(ecTmpPath, false); } } catch (IOException e) { LOG.error("Failed to delete tmp file created during the conversion!"); @@ -103,12 +93,7 @@ protected void execute() throws Exception { } @Override - public float getProgress() { - return progress; - } - - @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index 114091feb1c..a843d4b669f 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -45,7 +45,7 @@ public void testEcActionForFile() throws Exception { args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); - ecAction.setDfsClient(dfsClient); + ecAction.setLocalFileSystem(dfs); ecAction.init(args); ecAction.run(); assertTrue(ecAction.getExpectedAfterRun()); @@ -77,7 +77,7 @@ public void testEcActionForDir() throws Exception { Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcDirPath); args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); - ecAction.setDfsClient(dfsClient); + ecAction.setLocalFileSystem(dfs); ecAction.init(args); ecAction.run(); assertTrue(ecAction.getExpectedAfterRun()); diff --git a/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index a4d44663db9..09d46c34cc8 100644 --- a/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -46,7 +46,7 @@ public void testUnEcActionForFile() throws Exception { Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); - unEcAction.setDfsClient(dfsClient); + unEcAction.setLocalFileSystem(dfs); unEcAction.init(args); unEcAction.run(); assertTrue(unEcAction.getExpectedAfterRun()); @@ -76,7 +76,7 @@ public void testUnEcActionForDir() throws Exception { unEcAction.setContext(smartContext); Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, testDir); - unEcAction.setDfsClient(dfsClient); + unEcAction.setLocalFileSystem(dfs); unEcAction.init(args); unEcAction.run(); assertNull(dfs.getErasureCodingPolicy(new Path(testDir))); diff --git a/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java b/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java index 04fa606ea63..0a4e20816c9 100644 --- a/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java +++ b/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java @@ -87,11 +87,20 @@ public class SmartFileSystem extends DistributedFileSystem { private SmartDFSClient smartDFSClient; private boolean verifyChecksum = true; + public SmartFileSystem() { + } + + public SmartFileSystem(SmartDFSClient smartDFSClient) { + this.smartDFSClient = smartDFSClient; + } + @Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); - this.smartDFSClient = new SmartDFSClient(uri, conf); + if (smartDFSClient == null) { + this.smartDFSClient = new SmartDFSClient(uri, conf); + } } @Override diff --git a/smart-hadoop-support/smart-hadoop-client-3/src/test/java/org/smartdata/hdfs/TestSmartDFSClientReadECData.java b/smart-hadoop-support/smart-hadoop-client-3/src/test/java/org/smartdata/hdfs/TestSmartDFSClientReadECData.java index 6dd6c2f7317..0cb48aa37c8 100644 --- a/smart-hadoop-support/smart-hadoop-client-3/src/test/java/org/smartdata/hdfs/TestSmartDFSClientReadECData.java +++ b/smart-hadoop-support/smart-hadoop-client-3/src/test/java/org/smartdata/hdfs/TestSmartDFSClientReadECData.java @@ -88,7 +88,7 @@ public void testReadECDataCreatedBySSM() throws IOException { args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); - ecAction.setDfsClient(dfsClient); + ecAction.setLocalFileSystem(dfs); ecAction.init(args); ecAction.run(); diff --git a/smart-hadoop-support/smart-hadoop-common/pom.xml b/smart-hadoop-support/smart-hadoop-common/pom.xml index 355ab06455e..0d70c4a3110 100644 --- a/smart-hadoop-support/smart-hadoop-common/pom.xml +++ b/smart-hadoop-support/smart-hadoop-common/pom.xml @@ -48,6 +48,12 @@ org.apache.hadoop hadoop-hdfs + + + org.projectlombok + lombok + + org.apache.hadoop hadoop-hdfs-client diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java index 6959be44a63..433cf44398f 100644 --- a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.ipc.RemoteException; @@ -44,6 +45,9 @@ import java.nio.file.Paths; import java.security.PrivilegedExceptionAction; import java.util.Map; +import java.util.Optional; + +import static org.smartdata.utils.PathUtil.getRawPath; /** * Contain utils related to hadoop cluster. @@ -95,7 +99,7 @@ public static void loadHadoopConf(String ssmConfDir, URL nameNodeUrl, Configurat * @param hadoopConfPath directory that hadoop config files located. */ public static void loadHadoopConf(String hadoopConfPath, Configuration conf) - throws IOException { + throws IOException { HdfsConfiguration hadoopConf = getHadoopConf(hadoopConfPath); if (hadoopConf != null) { for (Map.Entry entry : hadoopConf) { @@ -141,7 +145,7 @@ public static void setSmartConfByHadoop(SmartConf conf) { URI nnUri = HadoopUtil.getNameNodeUri(conf); if (nnUri != null) { conf.set(SmartConfKeys.SMART_DFS_NAMENODE_RPCSERVER_KEY, - nnUri.toString()); + nnUri.toString()); } } } catch (IOException ex) { @@ -155,7 +159,7 @@ public static void setSmartConfByHadoop(SmartConf conf) { * @param hadoopConfPath directory that hadoop config files located. */ public static HdfsConfiguration getHadoopConf(String hadoopConfPath) - throws IOException { + throws IOException { if (hadoopConfPath == null || hadoopConfPath.isEmpty()) { LOG.warn("Hadoop configuration path is not set"); return null; @@ -221,7 +225,7 @@ public static URI getNameNodeUri(Configuration conf) throws IOException { String nnRpcAddr = null; - String[] rpcAddrKeys = { + String[] rpcAddrKeys = { SmartConfKeys.SMART_DFS_NAMENODE_RPCSERVER_KEY, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, @@ -258,80 +262,50 @@ public static URI getNameNodeUri(Configuration conf) public static FileInfo convertFileStatus(HdfsFileStatus status, String path) { return FileInfo.newBuilder() - .setPath(path) - .setFileId(status.getFileId()) - .setLength(status.getLen()) - .setIsdir(status.isDir()) - .setBlockReplication(status.getReplication()) - .setBlocksize(status.getBlockSize()) - .setModificationTime(status.getModificationTime()) - .setAccessTime(status.getAccessTime()) - .setPermission(status.getPermission().toShort()) - .setOwner(status.getOwner()) - .setGroup(status.getGroup()) - .setStoragePolicy(status.getStoragePolicy()) - .setErasureCodingPolicy(CompatibilityHelperLoader.getHelper().getErasureCodingPolicy(status)) - .build(); + .setPath(path) + .setFileId(status.getFileId()) + .setLength(status.getLen()) + .setIsdir(status.isDir()) + .setBlockReplication(status.getReplication()) + .setBlocksize(status.getBlockSize()) + .setModificationTime(status.getModificationTime()) + .setAccessTime(status.getAccessTime()) + .setPermission(status.getPermission().toShort()) + .setOwner(status.getOwner()) + .setGroup(status.getGroup()) + .setStoragePolicy(status.getStoragePolicy()) + .setErasureCodingPolicy(CompatibilityHelperLoader.getHelper().getErasureCodingPolicy(status)) + .build(); } - public static DFSClient getDFSClient(final URI nnUri, final Configuration conf) - throws IOException{ + public static T doAsCurrentUser(PrivilegedExceptionAction action) throws IOException { try { - return UserGroupInformation.getCurrentUser() - .doAs(new PrivilegedExceptionAction() { - @Override - public DFSClient run() throws Exception { - return new DFSClient(nnUri, conf); - } - }); + return UserGroupInformation.getCurrentUser().doAs(action); } catch (InterruptedException e) { - LOG.error("Fail to new DFSClient for : " + e.getMessage()); - throw new IOException("Fail to new DFSClient for : " + e.getMessage()); + throw new IOException("Privileged action interrupted", e); } } - public static String translateStoragePoliceId2Name(int id) { - switch (id){ - case 15: - return "Lazy_Persist"; - case 12: - return "All_SSD"; - case 10: - return "One_SSD"; - case 7: - return "Hot"; - case 5: - return "Warm"; - case 2: - return "Cold"; - } - return ""; + public static DFSClient getDFSClient(final URI nnUri, final Configuration conf) + throws IOException { + return doAsCurrentUser(() -> new DFSClient(nnUri, conf)); } - public static byte translateStoragePoliceName2Id(String name) { - switch (name){ - case "Lazy_Persist": - return 15; - case "All_SSD": - return 12; - case "One_SSD": - return 10; - case "Hot": - return 7; - case "Warm": - return 5; - case "Cold": - return 2; - } - return 0; + public static DistributedFileSystem getDistributedFileSystem( + final URI nnUri, final Configuration conf) throws IOException { + return doAsCurrentUser(() -> { + DistributedFileSystem fileSystem = new DistributedFileSystem(); + fileSystem.initialize(nnUri, conf); + return fileSystem; + }); } /** * Return FileState, like SmartDFSClient#getFileState(). * @param dfsClient * @param filePath - * @return a FileState - * @throws IOException + * @return a FileState + * @throws IOException */ public static FileState getFileState(DFSClient dfsClient, String filePath) throws IOException { @@ -346,4 +320,20 @@ public static FileState getFileState(DFSClient dfsClient, String filePath) } return new NormalFileState(filePath); } + + public static FileState getFileState( + DistributedFileSystem fileSystem, + org.apache.hadoop.fs.Path filePath) + throws IOException { + try { + byte[] fileState = fileSystem.getXAttr( + filePath, SmartConstants.SMART_FILE_STATE_XATTR_NAME); + return Optional.ofNullable(fileState) + .map(SerializationUtils::deserialize) + .map(FileState.class::cast) + .orElseGet(() -> new NormalFileState(getRawPath(filePath))); + } catch (RemoteException e) { + return new NormalFileState(getRawPath(filePath)); + } + } } diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/StreamCopyHandler.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/StreamCopyHandler.java new file mode 100644 index 00000000000..a7b110ffb19 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/StreamCopyHandler.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs; + + +import lombok.Builder.Default; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import org.apache.hadoop.io.IOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.function.Consumer; + +@RequiredArgsConstructor +@Builder +public class StreamCopyHandler { + private final InputStream inputStream; + + private final OutputStream outputStream; + + @Default + private final int bufferSize = 4096; + + @Default + private final long offset = 0; + + @Default + private final long count = -1; + + @Default + private final boolean closeStreams = false; + + @Default + private final Consumer progressConsumer = progress -> {}; + + public static Builder of(InputStream inputStream, OutputStream outputStream) { + return builder() + .inputStream(inputStream) + .outputStream(outputStream); + } + + public void runCopy() throws IOException { + byte[] buf = new byte[bufferSize]; + long bytesRemaining = count; + int bytesRead; + + try { + maybeSkip(); + + while (bytesRemaining > 0) { + int bytesToRead = (int) Math.min(bytesRemaining, bufferSize); + + bytesRead = inputStream.read(buf, 0, bytesToRead); + if (bytesRead == -1) { + break; + } + + outputStream.write(buf, 0, bytesRead); + bytesRemaining -= bytesRead; + + progressConsumer.accept(((float) (count - bytesRemaining)) / count); + } + } finally { + if (closeStreams) { + IOUtils.closeStream(outputStream); + IOUtils.closeStream(inputStream); + } + } + } + + private void maybeSkip() throws IOException { + if (offset != 0) { + inputStream.skip(offset); + } + } +} diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java index 1afb21e412e..b82f7a9da02 100644 --- a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java @@ -17,23 +17,57 @@ */ package org.smartdata.hdfs.action; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.smartdata.action.SmartAction; +import org.smartdata.conf.SmartConf; import org.smartdata.conf.SmartConfKeys; import org.smartdata.model.CmdletDescriptor; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Optional; + +import static org.smartdata.utils.PathUtil.getRemoteFileSystem; +import static org.smartdata.utils.PathUtil.isAbsoluteRemotePath; + + /** * Base class for all HDFS actions. */ +@Setter public abstract class HdfsAction extends SmartAction { public static final String FILE_PATH = CmdletDescriptor.HDFS_FILE_PATH; - // SmartDFSClient - protected DFSClient dfsClient = null; - public void setDfsClient(DFSClient dfsClient) { - this.dfsClient = dfsClient; + protected DistributedFileSystem localFileSystem; + + public enum FsType { + SMART, + DEFAULT_HDFS + } + + public FsType localFsType() { + return FsType.SMART; + } + + @Override + protected void preRun() throws Exception { + super.preRun(); + withDefaultFs(); + } + + protected DFSClient getLocalDfsClient() { + return Optional.ofNullable(localFileSystem) + .map(DistributedFileSystem::getClient) + .orElse(null); } protected void withDefaultFs() { @@ -42,12 +76,48 @@ protected void withDefaultFs() { conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, nameNodeURL); } - public DfsClientType dfsClientType() { - return DfsClientType.SMART; + protected void validateNonEmptyArgs(String... keys) { + for (String key : keys) { + validateNonEmptyArg(key); + } } - public enum DfsClientType { - SMART, - DEFAULT_HDFS + protected void validateNonEmptyArg(String key) { + if (StringUtils.isBlank(getArguments().get(key))) { + throw new IllegalArgumentException(key + " parameter is missing."); + } + } + + protected SmartConf getConf() { + return getContext().getConf(); + } + + protected Path getPathArg(String key) { + return Optional.ofNullable(getArguments().get(key)) + .filter(StringUtils::isNotBlank) + .map(Path::new) + .orElse(null); + } + + protected FileSystem getFileSystemFor(Path path) throws IOException { + return isAbsoluteRemotePath(path) + ? getRemoteFileSystem(path) + : localFileSystem; + } + + protected boolean isArgPresent(String key) { + return StringUtils.isNotBlank(getArguments().get(key)); + } + + protected Optional getFileStatus(FileSystem fileSystem, Path path) throws IOException { + try { + return Optional.of(fileSystem.getFileStatus(path)); + } catch (FileNotFoundException e) { + return Optional.empty(); + } + } + + protected Optional getHdfsFileStatus(FileSystem fileSystem, Path path) throws IOException { + return getFileStatus(fileSystem, path).map(HdfsFileStatus.class::cast); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllDiskFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllDiskFileAction.java index 30a3a0e80fe..aad1f5ee955 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllDiskFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllDiskFileAction.java @@ -23,9 +23,9 @@ * An action to do all-disk for a file. */ @ActionSignature( - actionId = "alldisk", - displayName = "alldisk", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "alldisk", + displayName = "alldisk", + usage = HdfsAction.FILE_PATH + " $file " ) public class AllDiskFileAction extends MoveFileAction { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllSsdFileAction.java index 37f2c7cb040..06d3017c848 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllSsdFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AllSsdFileAction.java @@ -17,20 +17,17 @@ */ package org.smartdata.hdfs.action; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.annotation.ActionSignature; /** * An action to do all-ssd for a file. */ @ActionSignature( - actionId = "allssd", - displayName = "allssd", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "allssd", + displayName = "allssd", + usage = HdfsAction.FILE_PATH + " $file " ) public class AllSsdFileAction extends MoveFileAction { - private static final Logger LOG = LoggerFactory.getLogger(AllSsdFileAction.class); @Override public String getStoragePolicy() { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AppendFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AppendFileAction.java index de3bd2267f2..a5e71b5130a 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AppendFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/AppendFileAction.java @@ -17,74 +17,72 @@ */ package org.smartdata.hdfs.action; -import java.util.Map; -import java.util.Random; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; +import java.util.Map; +import java.util.Random; + @ActionSignature( - actionId = "append", - displayName = "append", - usage = HdfsAction.FILE_PATH + " $src" + - AppendFileAction.LENGTH + " $length" + - AppendFileAction.BUF_SIZE + " $size" + actionId = "append", + displayName = "append", + usage = HdfsAction.FILE_PATH + " $src" + + AppendFileAction.LENGTH + " $length" + + AppendFileAction.BUF_SIZE + " $size" ) -public class AppendFileAction extends HdfsAction { +public class AppendFileAction extends HdfsActionWithRemoteClusterSupport { static final String BUF_SIZE = "-bufSize"; static final String LENGTH = "-length"; - private String srcPath; - private long length = 1024; - private int bufferSize = 64 * 1024; + + private static final int DEFAULT_BUFF_SIZE = 64 * 1024; + private static final long DEFAULT_LENGTH = 1024L; + + + private Path filePath; + private long length; + private int bufferSize; @Override public void init(Map args) { - withDefaultFs(); super.init(args); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(BUF_SIZE)) { - bufferSize = Integer.parseInt(args.get(BUF_SIZE)); - } - if (args.containsKey(LENGTH)) { - length = Long.parseLong(args.get(LENGTH)); - } + this.filePath = getPathArg(FILE_PATH); + + this.bufferSize = args.containsKey(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUFF_SIZE; + + this.length = args.containsKey(LENGTH) + ? Long.parseLong(args.get(LENGTH)) + : DEFAULT_LENGTH; } @Override - protected void execute() throws Exception { - if (srcPath != null && !srcPath.isEmpty()) { - Path path = new Path(srcPath); - FileSystem fileSystem = path.getFileSystem(getContext().getConf()); - appendLog( - String.format("Action starts at %s : Read %s", - Utils.getFormatedCurrentTime(), srcPath)); - if (!fileSystem.exists(path)) { - throw new ActionException("Append Action fails, file doesn't exist!"); - } - appendLog( - String.format("Append to %s", srcPath)); - Random random = new Random(); - FSDataOutputStream os = null; - try { - os = fileSystem.append(path, bufferSize); - long remaining = length; - while (remaining > 0) { - int toAppend = (int) Math.min(remaining, bufferSize); - byte[] bytes = new byte[toAppend]; - random.nextBytes(bytes); - os.write(bytes); - remaining -= toAppend; - } - } finally { - if (os != null) { - os.close(); - } + protected void preExecute() throws Exception { + validateNonEmptyArg(FILE_PATH); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (!fileSystem.exists(filePath)) { + throw new ActionException("Append Action fails, file doesn't exist!"); + } + appendLog( + String.format("Append to %s", filePath)); + + Random random = new Random(); + try (FSDataOutputStream os = fileSystem.append(filePath, bufferSize)) { + long remaining = length; + + while (remaining > 0) { + int toAppend = (int) Math.min(remaining, bufferSize); + byte[] bytes = new byte[toAppend]; + random.nextBytes(bytes); + os.write(bytes); + remaining -= toAppend; } - } else { - throw new ActionException("File parameter is missing."); } } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ArchiveFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ArchiveFileAction.java index cfd4d2c1c56..cab684edbd0 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ArchiveFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ArchiveFileAction.java @@ -17,20 +17,17 @@ */ package org.smartdata.hdfs.action; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.annotation.ActionSignature; /** * An action to archive a file. */ @ActionSignature( - actionId = "archive", - displayName = "archive", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "archive", + displayName = "archive", + usage = HdfsAction.FILE_PATH + " $file " ) public class ArchiveFileAction extends MoveFileAction { - private static final Logger LOG = LoggerFactory.getLogger(ArchiveFileAction.class); @Override public String getStoragePolicy() { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CacheFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CacheFileAction.java index 85c1efdce5d..b4c6dc28701 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CacheFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CacheFileAction.java @@ -17,93 +17,76 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.hdfs.scheduler.CacheScheduler; -import java.util.EnumSet; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.Optional; /** * Move to Cache Action */ @ActionSignature( - actionId = "cache", - displayName = "cache", - usage = HdfsAction.FILE_PATH + " $file " + CacheFileAction.REPLICA + " $replica " + actionId = "cache", + displayName = "cache", + usage = HdfsAction.FILE_PATH + " $file " + + CacheFileAction.REPLICA + " $replica " ) public class CacheFileAction extends HdfsAction { public static final String REPLICA = "-replica"; - private String fileName; - private LinkedBlockingQueue actionEvents; - private short replication = 0; - public CacheFileAction() { - super(); - this.actionEvents = new LinkedBlockingQueue<>(); - } + private Path filePath; + private short replication; @Override public void init(Map args) { super.init(args); - fileName = args.get(FILE_PATH); - if (args.containsKey("-replica")) { - replication = (short) Integer.parseInt(args.get(REPLICA)); - } + + filePath = getPathArg(FILE_PATH); + replication = Optional.ofNullable(args.get(REPLICA)) + .map(Short::parseShort) + .orElse((short) 0); } @Override protected void execute() throws Exception { - if (fileName == null) { - throw new IllegalArgumentException("File parameter is missing! "); - } + validateNonEmptyArg(FILE_PATH); + // set cache replication as the replication number of the file if not set if (replication == 0) { - HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName); - replication = fileStatus.isDir() ? 1 : fileStatus.getReplication(); + FileStatus fileStatus = localFileSystem.getFileStatus(filePath); + replication = fileStatus.isDirectory() ? 1 : fileStatus.getReplication(); } - addActionEvent(fileName); - executeCacheAction(fileName); + executeCacheAction(); } - public void addActionEvent(String fileName) throws Exception { - actionEvents.put(fileName); + boolean isFileCached() throws Exception { + CacheDirectiveInfo filter = new CacheDirectiveInfo.Builder() + .setPath(filePath) + .build(); + return localFileSystem.listCacheDirectives(filter).hasNext(); } - private void executeCacheAction(String fileName) throws Exception { - if (isCached(fileName)) { + private void executeCacheAction() throws Exception { + if (isFileCached()) { this.appendLog("The given file has already been cached, " + "so there is no need to execute this action."); return; } - this.appendLog( - String.format( - "Action starts at %s : cache -> %s", Utils.getFormatedCurrentTime(), fileName)); - addDirective(fileName); - } - public boolean isCached(String fileName) throws Exception { - CacheDirectiveInfo.Builder filterBuilder = new CacheDirectiveInfo.Builder(); - filterBuilder.setPath(new Path(fileName)); - CacheDirectiveInfo filter = filterBuilder.build(); - RemoteIterator directiveEntries = dfsClient.listCacheDirectives(filter); - return directiveEntries.hasNext(); + addDirective(); } - private void addDirective(String fileName) throws Exception { - CacheDirectiveInfo.Builder filterBuilder = new CacheDirectiveInfo.Builder(); - filterBuilder.setPath(new Path(fileName)) + private void addDirective() throws Exception { + CacheDirectiveInfo filter = new CacheDirectiveInfo.Builder() + .setPath(filePath) .setPool(CacheScheduler.SSM_POOL) - .setReplication(replication); - CacheDirectiveInfo filter = filterBuilder.build(); - EnumSet flags = EnumSet.noneOf(CacheFlag.class); - dfsClient.addCacheDirective(filter, flags); + .setReplication(replication) + .build(); + + localFileSystem.addCacheDirective(filter); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckCompressAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckCompressAction.java index 6cc6a16579f..85395ad4b58 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckCompressAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckCompressAction.java @@ -17,13 +17,12 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.Path; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.hdfs.HadoopUtil; import org.smartdata.model.CompressionFileState; import org.smartdata.model.FileState; -import java.io.IOException; import java.util.Map; /** @@ -36,26 +35,26 @@ + " $file " ) public class CheckCompressAction extends HdfsAction { - private String srcPath; + private Path srcPath; @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(HdfsAction.FILE_PATH); + this.srcPath = getPathArg(FILE_PATH); } @Override protected void execute() throws Exception { - if (StringUtils.isBlank(srcPath)) { - throw new IOException("File path is not given!"); - } + validateNonEmptyArg(FILE_PATH); + // Consider directory case. - if (dfsClient.getFileInfo(srcPath).isDir()) { + if (localFileSystem.getFileStatus(srcPath).isDirectory()) { appendResult("The given path is a directory, " + "not applicable to checking compression status."); return; } - FileState fileState = HadoopUtil.getFileState(dfsClient, srcPath); + + FileState fileState = HadoopUtil.getFileState(localFileSystem, srcPath); if (fileState instanceof CompressionFileState) { appendLog("The given file has already been compressed by SSM."); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckStorageAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckStorageAction.java index d5e25d7d277..830293fafef 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckStorageAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckStorageAction.java @@ -17,14 +17,14 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.smartdata.action.ActionException; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageType; import org.smartdata.action.annotation.ActionSignature; -import java.util.List; +import java.io.IOException; import java.util.Map; /** @@ -35,51 +35,60 @@ displayName = "checkstorage", usage = HdfsAction.FILE_PATH + " $file " ) -public class CheckStorageAction extends HdfsAction { - private String fileName; +public class CheckStorageAction extends HdfsActionWithRemoteClusterSupport { + private Path filePath; @Override public void init(Map args) { super.init(args); - fileName = args.get(FILE_PATH); + filePath = getPathArg(FILE_PATH); } @Override - protected void execute() throws Exception { - if (StringUtils.isBlank(fileName)) { - throw new IllegalArgumentException("File parameter is missing! "); - } - HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName); - if (fileStatus == null) { - throw new ActionException("File does not exist."); - } - if (fileStatus.isDir()) { + protected void preExecute() { + validateNonEmptyArg(FILE_PATH); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + FileStatus fileStatus = fileSystem.getFileStatus(filePath); + if (fileStatus.isDirectory()) { appendResult("This is a directory which has no storage result!"); return; } - long length = fileStatus.getLen(); - List locatedBlocks = - dfsClient.getLocatedBlocks(fileName, 0, length).getLocatedBlocks(); - - if (locatedBlocks.isEmpty()) { - appendResult("File '" + fileName + "' has no blocks."); + BlockLocation[] fileBlockLocations = + fileSystem.getFileBlockLocations(filePath, 0, fileStatus.getLen()); + if (fileBlockLocations.length == 0) { + appendResult("File '" + filePath + "' has no blocks."); return; } - for (LocatedBlock locatedBlock : locatedBlocks) { - StringBuilder blockInfo = new StringBuilder(); - blockInfo.append("File offset = ").append(locatedBlock.getStartOffset()).append(", "); - blockInfo.append("Block locations = {"); - for (DatanodeInfoWithStorage datanodeInfo : locatedBlock.getLocations()) { - blockInfo.append(datanodeInfo.getName()) - .append("[") - .append(datanodeInfo.getStorageType()) - .append("]") - .append(" "); - } - blockInfo.append("}"); - appendResult(blockInfo.toString()); + for (BlockLocation blockLocation : fileBlockLocations) { + appendResult(buildBlockInfo(blockLocation)); + } + } + + private String buildBlockInfo(BlockLocation blockLocation) throws IOException { + StringBuilder blockInfo = new StringBuilder(); + + String[] names = blockLocation.getNames(); + StorageType[] storageTypes = blockLocation.getStorageTypes(); + + blockInfo.append("File offset = ") + .append(blockLocation.getOffset()) + .append(", ") + .append("Block locations = {"); + + for (int i = 0; i < names.length; i++) { + blockInfo.append(names[i]) + .append("[") + .append(storageTypes[i]) + .append("]") + .append(" "); } + + return blockInfo.toString(); } -} + +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckSumAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckSumAction.java index 2a62a2289fb..1a4644fc6af 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckSumAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CheckSumAction.java @@ -17,13 +17,11 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; @@ -33,70 +31,68 @@ import java.io.IOException; import java.util.Map; +import static org.smartdata.utils.PathUtil.getRawPath; + @ActionSignature( actionId = "checksum", displayName = "checksum", usage = HdfsAction.FILE_PATH + " $src " ) -public class CheckSumAction extends HdfsAction { - private String fileName; +public class CheckSumAction extends HdfsActionWithRemoteClusterSupport { + private String fileRawPath; @Override public void init(Map args) { super.init(args); - withDefaultFs(); - fileName = args.get(FILE_PATH); + this.fileRawPath = args.get(FILE_PATH); } @Override - protected void execute() throws Exception { - if (StringUtils.isBlank(fileName)) { - throw new IllegalArgumentException("Please specify file path!"); - } + protected void preExecute() throws Exception { + validateNonEmptyArg(FILE_PATH); + } - if (fileName.charAt(fileName.length() - 1) == '*') { - String directoryPath = fileName.substring(0, fileName.length() - 1); - directoryContentChecksum(directoryPath); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (fileRawPath.charAt(fileRawPath.length() - 1) == '*') { + String directoryPath = fileRawPath.substring(0, fileRawPath.length() - 1); + directoryContentChecksum(fileSystem, new Path(directoryPath)); return; } - HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName); + Path filePath = new Path(fileRawPath); + FileStatus fileStatus = fileSystem.getFileStatus(filePath); - if (fileStatus == null) { - throw new ActionException("Provided file doesn't exist: " + fileName); - } - - if (fileStatus.isDir()) { + if (fileStatus.isDirectory()) { appendResult("This is a directory which has no checksum result!"); return; } - checksum(fileName, fileStatus.getLen()); + checksum(fileSystem, filePath, fileStatus.getLen()); } - private void directoryContentChecksum(String directory) throws Exception { - Path directoryPath = new Path(directory); - FileSystem fileSystem = directoryPath.getFileSystem(getContext().getConf()); - + private void directoryContentChecksum( + FileSystem fileSystem, Path directoryPath) throws Exception { RemoteIterator statusIter; try { statusIter = fileSystem.listStatusIterator(directoryPath); } catch (FileNotFoundException e) { - throw new ActionException("Provided directory doesn't exist: " + directory); + throw new ActionException("Provided directory doesn't exist: " + directoryPath); } while (statusIter.hasNext()) { try { FileStatus status = statusIter.next(); - checksum(status.getPath().toUri().getPath(), status.getLen()); + checksum(fileSystem, status.getPath(), status.getLen()); } catch (FileNotFoundException e) { // skip file if it was deleted between listing and checksum requests } } } - private void checksum(String path, long fileSize) throws IOException { - MD5MD5CRC32FileChecksum md5 = dfsClient.getFileChecksum(path, fileSize); + private void checksum(FileSystem fileSystem, + Path path, long fileSize) throws IOException { + FileChecksum md5 = fileSystem.getFileChecksum(path, fileSize); ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(byteStream); @@ -104,7 +100,7 @@ private void checksum(String path, long fileSize) throws IOException { byte[] bytes = byteStream.toByteArray(); appendResult( String.format("%s\t%s\t%s", - path, + getRawPath(path), md5.getAlgorithmName(), byteArray2HexString(bytes) )); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ClusterBalanceAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ClusterBalanceAction.java deleted file mode 100644 index 903648c3222..00000000000 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ClusterBalanceAction.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An action to do balance for a cluster. - */ -public class ClusterBalanceAction extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(ClusterBalanceAction.class); - - @Override - protected void execute() throws Exception { - - } -} diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CompressionAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CompressionAction.java index 08ec01d34b2..99618c1fd5a 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CompressionAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CompressionAction.java @@ -20,21 +20,16 @@ import com.google.gson.Gson; import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.mutable.MutableFloat; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.CompressionCodecFactory; -import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.SmartCompressorStream; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.SmartConstants; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConfKeys; -import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.model.CompressionFileInfo; import org.smartdata.model.CompressionFileState; import org.smartdata.utils.StringUtil; @@ -46,6 +41,9 @@ import java.util.Map; import java.util.Set; +import static org.smartdata.SmartConstants.SMART_FILE_STATE_XATTR_NAME; +import static org.smartdata.utils.PathUtil.getRawPath; + /** * This action is used to compress a file. */ @@ -61,16 +59,16 @@ + " $codec" ) public class CompressionAction extends HdfsAction { - private static final Logger LOG = - LoggerFactory.getLogger(CompressionAction.class); - + public static final String COMPRESS_TMP = "-compressTmp"; public static final String BUF_SIZE = "-bufSize"; public static final String CODEC = "-codec"; + private static final Set SUPPORTED_CODECS = CompressionCodecFactory.getInstance().getSupportedCodecs(); - private String filePath; - private Configuration conf; + private final Gson compressionInfoSerializer; + + private Path filePath; private MutableFloat progress; // bufferSize is also chunk size. @@ -81,145 +79,111 @@ public class CompressionAction extends HdfsAction { private String compressCodec; // Specified by user in action arg. private int userDefinedBufferSize; - public static final String XATTR_NAME = - SmartConstants.SMART_FILE_STATE_XATTR_NAME; - private CompressionFileInfo compressionFileInfo; private CompressionFileState compressionFileState; - private String compressTmpPath; - public static final String COMPRESS_TMP = "-compressTmp"; + private Path compressTmpPath; + + public CompressionAction() { + this.compressionInfoSerializer = new Gson(); + } @Override public void init(Map args) { super.init(args); - this.conf = getContext().getConf(); - this.compressCodec = conf.get( + this.compressCodec = getContext().getConf().get( SmartConfKeys.SMART_COMPRESSION_CODEC, SmartConfKeys.SMART_COMPRESSION_CODEC_DEFAULT); - this.maxSplit = conf.getInt( + this.maxSplit = getContext().getConf().getInt( SmartConfKeys.SMART_COMPRESSION_MAX_SPLIT, SmartConfKeys.SMART_COMPRESSION_MAX_SPLIT_DEFAULT); - this.filePath = args.get(FILE_PATH); + this.filePath = getPathArg(FILE_PATH); if (args.containsKey(BUF_SIZE) && !args.get(BUF_SIZE).isEmpty()) { this.userDefinedBufferSize = (int) StringUtil.parseToByte(args.get(BUF_SIZE)); } this.compressCodec = args.get(CODEC) != null ? args.get(CODEC) : compressCodec; // This is a temp path for compressing a file. - this.compressTmpPath = args.containsKey(COMPRESS_TMP) ? - args.get(COMPRESS_TMP) : compressTmpPath; + this.compressTmpPath = getPathArg(COMPRESS_TMP); this.progress = new MutableFloat(0.0F); } @Override protected void execute() throws Exception { - if (filePath == null) { - throw new IllegalArgumentException("File path is missing."); - } - if (compressTmpPath == null) { - throw new IllegalArgumentException("Compression tmp path is not specified!"); - } + validateNonEmptyArgs(FILE_PATH, COMPRESS_TMP); + if (!SUPPORTED_CODECS.contains(compressCodec)) { throw new ActionException( "Compression Action failed due to unsupported codec: " + compressCodec); } - appendLog( - String.format("Compression Action started at %s for %s", - Utils.getFormatedCurrentTime(), filePath)); - if (!dfsClient.exists(filePath)) { + if (!localFileSystem.exists(filePath)) { throw new ActionException( "Failed to execute Compression Action: the given file doesn't exist!"); } - HdfsFileStatus srcFileStatus = dfsClient.getFileInfo(filePath); + FileStatus srcFileStatus = localFileSystem.getFileStatus(filePath); // Consider directory case. - if (srcFileStatus.isDir()) { + if (srcFileStatus.isDirectory()) { appendLog("Compression is not applicable to a directory."); return; } // Generate compressed file - compressionFileState = new CompressionFileState(filePath, bufferSize, compressCodec); + compressionFileState = new CompressionFileState(getRawPath(filePath), bufferSize, compressCodec); compressionFileState.setOriginalLength(srcFileStatus.getLen()); - OutputStream appendOut = null; - DFSInputStream in = null; - OutputStream out = null; - try { - if (srcFileStatus.getLen() == 0) { - compressionFileInfo = new CompressionFileInfo(false, compressionFileState); - } else { - short replication = srcFileStatus.getReplication(); - long blockSize = srcFileStatus.getBlockSize(); - long fileSize = srcFileStatus.getLen(); - appendLog("File length: " + fileSize); - bufferSize = getActualBuffSize(fileSize); - - // SmartDFSClient will fail to open compressing file with PROCESSING FileStage - // set by Compression scheduler. But considering DfsClient may be used, we use - // append operation to lock the file to avoid any modification. - appendOut = CompatibilityHelperLoader.getHelper(). - getDFSClientAppend(dfsClient, filePath, bufferSize); - in = dfsClient.open(filePath); - out = dfsClient.create(compressTmpPath, - true, replication, blockSize); - - // Keep storage policy consistent. - // The below statement is not supported on Hadoop-2.7.3 or CDH-5.10.1 - // String storagePolicyName = dfsClient.getStoragePolicy(filePath).getName(); - byte storagePolicyId = srcFileStatus.getStoragePolicy(); - String storagePolicyName = SmartConstants.STORAGE_POLICY_MAP.get(storagePolicyId); + CompressionFileInfo compressionFileInfo; + if (srcFileStatus.getLen() == 0) { + compressionFileInfo = new CompressionFileInfo(false, compressionFileState); + } else { + try ( + // SmartDFSClient will fail to open compressing file with PROCESSING FileStage + // set by Compression scheduler. But considering DfsClient may be used, we use + // append operation to lock the file to avoid any modification. + OutputStream lockStream = localFileSystem.append(filePath, bufferSize); + + FSDataInputStream in = localFileSystem.open(filePath); + OutputStream out = localFileSystem.create(compressTmpPath, + true, + getLocalDfsClient().getConf().getIoBufferSize(), + srcFileStatus.getReplication(), + srcFileStatus.getBlockSize()) + ) { + + appendLog("File length: " + srcFileStatus.getLen()); + bufferSize = getActualBuffSize(srcFileStatus.getLen()); + + String storagePolicyName = localFileSystem.getStoragePolicy(filePath).getName(); if (!storagePolicyName.equals("UNDEF")) { - dfsClient.setStoragePolicy(compressTmpPath, storagePolicyName); + localFileSystem.setStoragePolicy(compressTmpPath, storagePolicyName); } compress(in, out); - HdfsFileStatus destFileStatus = dfsClient.getFileInfo(compressTmpPath); - dfsClient.setOwner(compressTmpPath, srcFileStatus.getOwner(), srcFileStatus.getGroup()); - dfsClient.setPermission(compressTmpPath, srcFileStatus.getPermission()); + FileStatus destFileStatus = localFileSystem.getFileStatus(compressTmpPath); + localFileSystem.setOwner(compressTmpPath, srcFileStatus.getOwner(), srcFileStatus.getGroup()); + localFileSystem.setPermission(compressTmpPath, srcFileStatus.getPermission()); compressionFileState.setCompressedLength(destFileStatus.getLen()); appendLog("Compressed file length: " + destFileStatus.getLen()); compressionFileInfo = - new CompressionFileInfo(true, compressTmpPath, compressionFileState); - } - compressionFileState.setBufferSize(bufferSize); - appendLog("Compression buffer size: " + bufferSize); - appendLog("Compression codec: " + compressCodec); - String compressionInfoJson = new Gson().toJson(compressionFileInfo); - appendResult(compressionInfoJson); - LOG.warn(compressionInfoJson); - if (compressionFileInfo.needReplace()) { - // Add to temp path - // Please make sure content write to Xatte is less than 64K - dfsClient.setXAttr(compressionFileInfo.getTempPath(), - XATTR_NAME, SerializationUtils.serialize(compressionFileState), - EnumSet.of(XAttrSetFlag.CREATE)); - // Rename operation is moved from CompressionScheduler. - // Thus, modification for original file will be avoided. - dfsClient.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE); - } else { - // Add to raw path - dfsClient.setXAttr(filePath, - XATTR_NAME, SerializationUtils.serialize(compressionFileState), - EnumSet.of(XAttrSetFlag.CREATE)); - } - } catch (IOException e) { - throw new IOException(e); - } finally { - if (appendOut != null) { - try { - appendOut.close(); - } catch (IOException e) { - // Hide the expected exception that the original file is missing. - } - } - if (in != null) { - in.close(); - } - if (out != null) { - out.close(); + new CompressionFileInfo(true, getRawPath(compressTmpPath), compressionFileState); } } + + compressionFileState.setBufferSize(bufferSize); + appendLog("Compression buffer size: " + bufferSize); + appendLog("Compression codec: " + compressCodec); + String compressionInfoJson = compressionInfoSerializer.toJson(compressionFileInfo); + appendResult(compressionInfoJson); + if (compressionFileInfo.needReplace()) { + // Add to temp path + // Please make sure content write to Xattr is less than 64K + setXAttr(compressTmpPath, compressionFileState); + // Rename operation is moved from CompressionScheduler. + // Thus, modification for original file will be avoided. + localFileSystem.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE); + } else { + // Add to raw path + setXAttr(filePath, compressionFileState); + } } private void compress(InputStream inputStream, OutputStream outputStream) throws IOException { @@ -233,23 +197,29 @@ private int getActualBuffSize(long fileSize) { // The capacity of originalPos and compressedPos is maxSplit (1000, by default) in database // Calculated by max number of splits. int calculatedBufferSize = (int) (fileSize / maxSplit); - LOG.debug("Calculated buffer size: " + calculatedBufferSize); - LOG.debug("MaxSplit: " + maxSplit); + appendLog("Calculated buffer size: " + calculatedBufferSize); + appendLog("MaxSplit: " + maxSplit); // Determine the actual buffer size if (userDefinedBufferSize < bufferSize || userDefinedBufferSize < calculatedBufferSize) { if (bufferSize <= calculatedBufferSize) { - LOG.debug("User defined buffer size is too small, use the calculated buffer size:" + - calculatedBufferSize); + appendLog("User defined buffer size is too small, use the calculated buffer size: " + + calculatedBufferSize); } else { - LOG.debug("User defined buffer size is too small, use the default buffer size:" + - bufferSize); + appendLog("User defined buffer size is too small, use the default buffer size: " + + bufferSize); } } return Math.max(Math.max(userDefinedBufferSize, calculatedBufferSize), bufferSize); } + private void setXAttr(Path path, CompressionFileState compressionFileState) throws IOException { + localFileSystem.setXAttr(path, SMART_FILE_STATE_XATTR_NAME, + SerializationUtils.serialize(compressionFileState), + EnumSet.of(XAttrSetFlag.CREATE)); + } + @Override public float getProgress() { - return (float) this.progress.getValue(); + return progress.getValue(); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ConcatFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ConcatFileAction.java index 7c9c0d5f62c..61990450389 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ConcatFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ConcatFileAction.java @@ -17,20 +17,17 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import java.io.IOException; -import java.net.URI; +import java.util.ArrayDeque; import java.util.Arrays; -import java.util.LinkedList; +import java.util.Deque; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * An action to merge a list of file, @@ -43,84 +40,42 @@ usage = HdfsAction.FILE_PATH + " $src " + ConcatFileAction.DEST_PATH + " $dest" ) public class ConcatFileAction extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(ConcatFileAction.class); public static final String DEST_PATH = "-dest"; - private LinkedList srcPathList; - private String targetPath; + + private Deque srcPaths; + private Path targetPath; @Override public void init(Map args) { super.init(args); - String inputSrcPath = args.get(FILE_PATH); - //init the linkedList - String[] srcArray = inputSrcPath.split(","); - srcPathList = new LinkedList<>(Arrays.asList(srcArray)); - if (args.containsKey(DEST_PATH)) { - this.targetPath = args.get(DEST_PATH); - } + this.srcPaths = Optional.ofNullable(args.get(FILE_PATH)) + .map(paths -> paths.split(",")) + .map(Arrays::stream) + .orElseGet(Stream::empty) + .map(Path::new) + .collect(Collectors.toCollection(ArrayDeque::new)); + this.targetPath = getPathArg(DEST_PATH); } @Override protected void execute() throws Exception { - if (srcPathList == null || srcPathList.size() == 0) { - throw new IllegalArgumentException("Dest File parameter is missing."); + validateNonEmptyArgs(FILE_PATH, DEST_PATH); + if (srcPaths.isEmpty()) { + throw new IllegalArgumentException("Source files not provided"); } - if (srcPathList.size() == 1) { + if (srcPaths.size() == 1) { throw new IllegalArgumentException("Don't accept only one source file"); } - if (targetPath == null) { - throw new IllegalArgumentException("File parameter is missing."); - } - - appendLog( - String.format("Action starts at %s : Concat %s to %s", - Utils.getFormatedCurrentTime(), srcPathList, targetPath)); - //Merge the files - concatFiles(srcPathList, targetPath); - } - - private boolean concatFiles(LinkedList allFiles, String target) throws IOException { - if (target.startsWith("hdfs")) { - //merge in remote cluster - //check if all of the source file - // TODO read conf from files - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(URI.create(target), conf); - for (String sourceFile : allFiles) { - if (!fs.isFile(new Path(sourceFile))) { - throw new IllegalArgumentException("File parameter is not file"); - } - } - Path firstFile = new Path(allFiles.removeFirst()); - Path[] restFile = new Path[allFiles.size()]; - - int index = -1; - for (String transFile : allFiles) { - index++; - restFile[index] = new Path(transFile); - } - fs.concat(firstFile, restFile); - if (fs.exists(new Path(target))) { - fs.delete(new Path(target), true); + for (Path sourcePath : srcPaths) { + if (localFileSystem.getFileStatus(sourcePath).isDirectory()) { + throw new IllegalArgumentException("File parameter is not file"); } - fs.rename(firstFile, new Path(target)); - return true; - } else { - for (String sourceFile : allFiles) { - if (dfsClient.getFileInfo(sourceFile).isDir()) { - throw new IllegalArgumentException("File parameter is not file"); - } - } - String firstFile = allFiles.removeFirst(); - String[] restFile = new String[allFiles.size()]; - allFiles.toArray(restFile); - dfsClient.concat(firstFile, restFile); - if (dfsClient.exists(target)) { - dfsClient.delete(target, true); - } - dfsClient.rename(firstFile, target, Options.Rename.NONE); - return true; } + + Path firstPath = srcPaths.removeFirst(); + Path[] restPaths = srcPaths.toArray(new Path[0]); + localFileSystem.concat(firstPath, restPaths); + localFileSystem.rename(firstPath, targetPath, Options.Rename.OVERWRITE); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Copy2S3Action.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Copy2S3Action.java index 2ac800ba7cb..243f18ff7b8 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Copy2S3Action.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Copy2S3Action.java @@ -17,22 +17,22 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.hdfs.CompatibilityHelperLoader; +import org.smartdata.hdfs.StreamCopyHandler; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.util.Map; -import java.util.EnumSet; +import java.util.Optional; + +import static org.smartdata.utils.PathUtil.getRawPath; + /** * An action to copy a single file from src to destination. * If dest doesn't contains "hdfs" prefix, then destination will be set to @@ -45,122 +45,78 @@ usage = HdfsAction.FILE_PATH + " $src " + Copy2S3Action.DEST + " $dest " + Copy2S3Action.BUF_SIZE + " $size" ) -public class Copy2S3Action extends HdfsAction { - private static final Logger LOG = - LoggerFactory.getLogger(CopyFileAction.class); +public class Copy2S3Action extends HdfsActionWithRemoteClusterSupport { public static final String BUF_SIZE = "-bufSize"; public static final String SRC = HdfsAction.FILE_PATH; public static final String DEST = "-dest"; - private String srcPath; - private String destPath; - private int bufferSize = 64 * 1024; + + public static final String S3_SCHEME_PREFIX = "s3"; + public static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + + private Path srcPath; + private Path destPath; + private int bufferSize; @Override public void init(Map args) { withDefaultFs(); super.init(args); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(DEST)) { - this.destPath = args.get(DEST); - } - if (args.containsKey(BUF_SIZE)) { - bufferSize = Integer.parseInt(args.get(BUF_SIZE)); - } + this.srcPath = getPathArg(FILE_PATH); + this.destPath = getPathArg(DEST); + this.bufferSize = isArgPresent(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUFFER_SIZE; } @Override - protected void execute() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("File parameter is missing."); - } - if (destPath == null) { - throw new IllegalArgumentException("Dest File parameter is missing."); + protected void preExecute() throws Exception { + validateNonEmptyArgs(FILE_PATH, DEST); + + boolean isS3Scheme = Optional.ofNullable(destPath.toUri()) + .map(URI::getScheme) + .filter(scheme -> scheme.startsWith(S3_SCHEME_PREFIX)) + .isPresent(); + + if (!isS3Scheme) { + throw new ActionException("Destination is not a s3:// path: " + destPath); } - appendLog( - String.format("Action starts at %s : Read %s", - Utils.getFormatedCurrentTime(), srcPath)); - if (!dfsClient.exists(srcPath)) { + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + validateNonEmptyArgs(FILE_PATH, DEST); + if (!fileSystem.exists(srcPath)) { throw new ActionException("CopyFile Action fails, file doesn't exist!"); } + appendLog( String.format("Copy from %s to %s", srcPath, destPath)); - copySingleFile(srcPath, destPath); + copySingleFile(fileSystem); appendLog("Copy Successfully!!"); - setXAttribute(srcPath, destPath); - appendLog("SetXattr Successfully!!"); -} - - private long getFileSize(String fileName) throws IOException { - if (fileName.startsWith("hdfs")) { - // Get InputStream from URL - FileSystem fs = FileSystem.get(URI.create(fileName), getContext().getConf()); - return fs.getFileStatus(new Path(fileName)).getLen(); - } else { - return dfsClient.getFileInfo(fileName).getLen(); - } + setXAttribute(fileSystem); } - private boolean setXAttribute(String src, String dest) throws IOException { - + private void setXAttribute(FileSystem fileSystem) throws IOException { String name = "user.coldloc"; - dfsClient.setXAttr(srcPath, name, dest.getBytes(), EnumSet.of(XAttrSetFlag.CREATE,XAttrSetFlag.REPLACE) ); - appendLog(" SetXattr feature is set - srcPath " + srcPath + "destination" + dest.getBytes() ); - return true; + fileSystem.setXAttr(srcPath, name, getRawPath(destPath).getBytes()); + appendLog(" SetXattr feature is set - srcPath " + srcPath + " destination: " + destPath); } - private boolean copySingleFile(String src, String dest) throws IOException { - //get The file size of source file - InputStream in = null; - OutputStream out = null; - - try { - in = getSrcInputStream(src); - out = CompatibilityHelperLoader - .getHelper().getS3outputStream(dest, getContext().getConf()); - byte[] buf = new byte[bufferSize]; - long bytesRemaining = getFileSize(src); - - while (bytesRemaining > 0L) { - int bytesToRead = - (int) (bytesRemaining < (long) buf.length ? bytesRemaining : - (long) buf.length); - int bytesRead = in.read(buf, 0, bytesToRead); - if (bytesRead == -1) { - break; - } - out.write(buf, 0, bytesRead); - bytesRemaining -= (long) bytesRead; - } - return true; - } finally { - if (out != null) { - out.close(); - } - if (in != null) { - in.close(); - } - } - } + private void copySingleFile(FileSystem fileSystem) throws IOException { + try (InputStream in = fileSystem.open(srcPath); + OutputStream out = getDestOutPutStream()) { - private InputStream getSrcInputStream(String src) throws IOException { - if (!src.startsWith("hdfs")) { - // Copy between different remote clusters - // Get InputStream from URL - FileSystem fs = FileSystem.get(URI.create(src), getContext().getConf()); - return fs.open(new Path(src)); - } else { - // Copy from primary HDFS - return dfsClient.open(src); + StreamCopyHandler.of(in, out) + .closeStreams(false) + .count(fileSystem.getFileStatus(srcPath).getLen()) + .bufferSize(bufferSize) + .build() + .runCopy(); } } - private OutputStream getDestOutPutStream(String dest) throws IOException { - // Copy to remote S3 - if (!dest.startsWith("s3")) { - throw new IOException(); - } - // Copy to s3 - FileSystem fs = FileSystem.get(URI.create(dest), getContext().getConf()); - return fs.create(new Path(dest), true); + private OutputStream getDestOutPutStream() throws IOException { + FileSystem destFileSystem = destPath.getFileSystem(new Configuration()); + return destFileSystem.create(destPath, true); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyDirectoryAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyDirectoryAction.java index 9014ccbe23a..a2af48b5595 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyDirectoryAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyDirectoryAction.java @@ -18,20 +18,18 @@ package org.smartdata.hdfs.action; import com.google.common.collect.Sets; -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.GROUP; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.OWNER; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.PERMISSIONS; -import static org.smartdata.utils.ConfigUtil.toRemoteClusterConfig; /** * An action to copy a directory without content @@ -46,8 +44,8 @@ public class CopyDirectoryAction extends CopyPreservedAttributesAction { public static final String DEST_PATH = "-dest"; - private String srcPath; - private String destPath; + private Path srcPath; + private Path destPath; public static final Set SUPPORTED_PRESERVE_ATTRIBUTES = Sets.newHashSet(OWNER, GROUP, PERMISSIONS); @@ -59,8 +57,8 @@ public CopyDirectoryAction() { @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(FILE_PATH); - this.destPath = args.get(DEST_PATH); + this.srcPath = getPathArg(FILE_PATH); + this.destPath = getPathArg(DEST_PATH); } @Override @@ -68,37 +66,28 @@ protected void execute() throws Exception { validateArgs(); Set preserveAttributes = parsePreserveAttributes(); - createTargetDirectory(); - copyFileAttributes(srcPath, destPath, preserveAttributes); + FileSystem destFileSystem = getFileSystemFor(destPath); + createTargetDirectory(destFileSystem); + + copyFileAttributes( + getFileSystemFor(srcPath).getFileStatus(srcPath), + destPath, + destFileSystem, + preserveAttributes); appendLog("Copy directory success!"); } private void validateArgs() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("Source directory path is missing."); - } - if (!dfsClient.exists(srcPath)) { - throw new ActionException("DirCopy fails, src directory doesn't exist:" + srcPath); - } - if (destPath == null) { - throw new IllegalArgumentException("Target directory path is missing."); + validateNonEmptyArgs(FILE_PATH, DEST_PATH); + if (!localFileSystem.exists(srcPath)) { + throw new ActionException("Src directory doesn't exist:" + srcPath); } } - private void createTargetDirectory() throws IOException { + private void createTargetDirectory(FileSystem destFileSystem) throws IOException { appendLog( String.format("Creating directory %s", destPath)); - - if (!destPath.startsWith("hdfs")) { - dfsClient.mkdirs(destPath, null, true); - return; - } - - Configuration remoteClusterConfig = toRemoteClusterConfig(getContext().getConf()); - FileSystem remoteFileSystem = FileSystem.get(URI.create(destPath), remoteClusterConfig); - remoteFileSystem.mkdirs(new Path(destPath)); + destFileSystem.mkdirs(destPath); } } - - diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyFileAction.java index c469200ea6a..c94bf690491 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyFileAction.java @@ -17,32 +17,25 @@ */ package org.smartdata.hdfs.action; - import com.google.common.collect.Sets; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.util.Map; -import java.util.Set; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.hdfs.CompatibilityHelper; -import org.smartdata.hdfs.CompatibilityHelperLoader; +import org.smartdata.hdfs.StreamCopyHandler; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; +import java.util.Optional; +import java.util.Set; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.GROUP; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.OWNER; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.PERMISSIONS; import static org.smartdata.hdfs.action.CopyPreservedAttributesAction.PreserveAttribute.REPLICATION_NUMBER; -import static org.smartdata.utils.ConfigUtil.toRemoteClusterConfig; /** * An action to copy a single file from src to destination. @@ -61,7 +54,6 @@ + CopyFileAction.PRESERVE + " $attributes" ) public class CopyFileAction extends CopyPreservedAttributesAction { - public static final String BUF_SIZE = "-bufSize"; public static final String DEST_PATH = "-dest"; public static final String OFFSET_INDEX = "-offset"; @@ -70,12 +62,12 @@ public class CopyFileAction extends CopyPreservedAttributesAction { public static final Set DEFAULT_PRESERVE_ATTRIBUTES = Sets.newHashSet(OWNER, GROUP, PERMISSIONS); - private String srcPath; - private String destPath; - private long offset = 0; - private long length = 0; - private int bufferSize = 64 * 1024; - private boolean copyContent = true; + private Path srcPath; + private Path destPath; + private long offset; + private long length; + private int bufferSize; + private boolean copyContent; private Set preserveAttributes; @@ -83,16 +75,17 @@ public class CopyFileAction extends CopyPreservedAttributesAction { public CopyFileAction() { super(DEFAULT_PRESERVE_ATTRIBUTES); + this.offset = 0; + this.length = 0; + this.bufferSize = 64 * 1024; + this.copyContent = true; } @Override public void init(Map args) { - withDefaultFs(); super.init(args); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(DEST_PATH)) { - this.destPath = args.get(DEST_PATH); - } + this.srcPath = getPathArg(FILE_PATH); + this.destPath = getPathArg(DEST_PATH); if (args.containsKey(BUF_SIZE)) { bufferSize = Integer.parseInt(args.get(BUF_SIZE)); } @@ -109,117 +102,91 @@ public void init(Map args) { @Override protected void execute() throws Exception { - validateArgs(); - preserveAttributes = parsePreserveAttributes(); + FileSystem srcFileSystem = getFileSystemFor(srcPath); + FileSystem destFileSystem = getFileSystemFor(destPath); - appendLog( - String.format("Action starts at %s : Copy from %s to %s", - Utils.getFormatedCurrentTime(), srcPath, destPath)); + validateArgs(srcFileSystem); - srcFileStatus = getFileStatus(srcPath); + preserveAttributes = parsePreserveAttributes(); + srcFileStatus = srcFileSystem.getFileStatus(srcPath); if (!copyContent) { appendLog("Src and dest files are equal, no need to copy content"); } else if (length != 0) { - copyWithOffset(srcPath, destPath, bufferSize, offset, length); + copyWithOffset(srcFileSystem, destFileSystem, bufferSize, offset, length); } else if (offset == 0) { - copySingleFile(srcPath, destPath); + copySingleFile(srcFileSystem, destFileSystem); } - copyFileAttributes(srcPath, destPath, preserveAttributes); + copyFileAttributes(srcFileStatus, destPath, destFileSystem, preserveAttributes); appendLog("Copy Successfully!!"); } - private void validateArgs() throws Exception { - if (StringUtils.isBlank(srcPath)) { - throw new IllegalArgumentException("File parameter is missing."); - } - if (StringUtils.isBlank(destPath)) { - throw new IllegalArgumentException("Dest File parameter is missing."); - } - if (!dfsClient.exists(srcPath)) { + private void validateArgs(FileSystem srcFileSystem) throws Exception { + validateNonEmptyArgs(FILE_PATH, DEST_PATH); + if (!srcFileSystem.exists(srcPath)) { throw new ActionException("Src file doesn't exist!"); } } - private void copySingleFile(String src, String dest) throws IOException { + private void copySingleFile( + FileSystem srcFileSystem, FileSystem destFileSystem) throws IOException { appendLog( String.format("Copy the whole file with length %s", srcFileStatus.getLen())); - copyWithOffset(src, dest, bufferSize, 0, srcFileStatus.getLen()); + copyWithOffset(srcFileSystem, destFileSystem, bufferSize, 0, srcFileStatus.getLen()); } - private void copyWithOffset(String src, String dest, int bufferSize, - long offset, long length) throws IOException { + private void copyWithOffset( + FileSystem srcFileSystem, + FileSystem destFileSystem, + int bufferSize, long offset, long length) throws IOException { appendLog( String.format("Copy with offset %s and length %s", offset, length)); - try (InputStream in = getSrcInputStream(src); - OutputStream out = getDestOutPutStream(dest, offset)) { - //skip offset - in.skip(offset); - byte[] buf = new byte[bufferSize]; - long bytesRemaining = length; - - while (bytesRemaining > 0L) { - int bytesToRead = (int) (Math.min(bytesRemaining, buf.length)); - int bytesRead = in.read(buf, 0, bytesToRead); - if (bytesRead == -1) { - break; - } - out.write(buf, 0, bytesRead); - bytesRemaining -= bytesRead; - } + try (InputStream in = srcFileSystem.open(srcPath); + OutputStream out = getOutputStream(destFileSystem, offset)) { + StreamCopyHandler.of(in, out) + .offset(offset) + .count(length) + .bufferSize(bufferSize) + .closeStreams(false) + .build() + .runCopy(); } } - private InputStream getSrcInputStream(String src) throws IOException { - if (src.startsWith("hdfs")) { - // Copy between different remote clusters - // Get InputStream from URL - FileSystem fs = FileSystem.get(URI.create(src), getContext().getConf()); - return fs.open(new Path(src)); - } - return dfsClient.open(src); - } + // TODO add action option to fully re-copy the file in case if fs + // doesn't support appends + private OutputStream getOutputStream( + FileSystem fileSystem, long offset) throws IOException { + Optional destFileStatus = getFileStatus(fileSystem, destPath) + .map(this::validateDestFile); - private OutputStream getDestOutPutStream(String dest, long offset) throws IOException { - if (dest.startsWith("s3")) { - // Copy to s3 - FileSystem fs = FileSystem.get(URI.create(dest), getContext().getConf()); - return fs.create(new Path(dest), true); + if (!destFileStatus.isPresent() || offset == 0) { + short replication = getReplication(fileSystem.getDefaultReplication(destPath)); + return fileSystem.create(destPath, replication); } - if (dest.startsWith("hdfs")) { - // Copy between different clusters - // Copy to remote HDFS - // Get OutPutStream from URL - Configuration remoteClusterConfig = toRemoteClusterConfig(getContext().getConf()); - FileSystem fs = FileSystem.get(URI.create(dest), remoteClusterConfig); - Path destHdfsPath = new Path(dest); - - if (fs.exists(destHdfsPath) && offset != 0) { - appendLog("Append to existing file " + dest); - return fs.append(destHdfsPath); - } - - short replication = getReplication(fs.getDefaultReplication(destHdfsPath)); - return fs.create( - destHdfsPath, - true, - remoteClusterConfig.getInt( - IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), - replication, - fs.getDefaultBlockSize(destHdfsPath)); + if (destFileStatus.get().getLen() != offset) { + appendLog("Truncating existing file " + destPath + " to the new length " + offset); + fileSystem.truncate(destPath, offset); } - CompatibilityHelper compatibilityHelper = CompatibilityHelperLoader.getHelper(); - if (preserveAttributes.contains(REPLICATION_NUMBER)) { - return compatibilityHelper - .getDFSClientAppend(dfsClient, dest, bufferSize, offset, srcFileStatus.getReplication()); + appendLog("Appending to existing file " + destPath); + return fileSystem.append(destPath); + } + + private FileStatus validateDestFile(FileStatus destFileStatus) { + if (destFileStatus.getLen() < offset) { + String errorMessage = String.format( + "Destination file %s is shorter than it should be " + + "- expected min length: %d, actual length: %d", + destFileStatus.getPath(), offset, destFileStatus.getLen()); + throw new IllegalStateException(errorMessage); } - return compatibilityHelper.getDFSClientAppend(dfsClient, dest, bufferSize, offset); + return destFileStatus; } public static void validatePreserveArg(String option) { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyPreservedAttributesAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyPreservedAttributesAction.java index bd512d93611..fbfd6a4bb86 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyPreservedAttributesAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/CopyPreservedAttributesAction.java @@ -18,19 +18,21 @@ package org.smartdata.hdfs.action; import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.smartdata.model.FileInfoDiff; + import java.io.IOException; -import java.net.URI; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.smartdata.model.FileInfoDiff; + +import static org.smartdata.utils.PathUtil.getRawPath; /** @@ -51,7 +53,7 @@ public CopyPreservedAttributesAction(Set defaultAttributes) { } public CopyPreservedAttributesAction(Set supportedAttributes, - Set defaultAttributes) { + Set defaultAttributes) { this.supportedAttributes = supportedAttributes; this.defaultAttributes = defaultAttributes; } @@ -62,8 +64,7 @@ public void init(Map args) { if (StringUtils.isNotBlank(args.get(PRESERVE))) { rawPreserveAttributes = Arrays.asList(args.get(PRESERVE).split(",")); } - updateMetadataSupport = new UpdateFileMetadataSupport( - getContext().getConf(), getLogPrintStream()); + updateMetadataSupport = new UpdateFileMetadataSupport(getLogPrintStream()); } protected Set parsePreserveAttributes() { @@ -77,36 +78,29 @@ protected Set parsePreserveAttributes() { : attributesFromOptions; } - protected void copyFileAttributes(String srcPath, String destPath, - Set preserveAttributes) throws IOException { + protected void copyFileAttributes( + FileStatus srcFileStatus, + Path destPath, + FileSystem destFileSystem, + Set preserveAttributes) throws IOException { if (preserveAttributes.isEmpty()) { return; } appendLog( - String.format("Copy attributes from %s to %s", srcPath, destPath)); + String.format("Copy attributes from %s to %s", srcFileStatus.getPath(), destPath)); - FileStatus srcFileStatus = getFileStatus(srcPath); - FileInfoDiff fileInfoDiff = new FileInfoDiff().setPath(destPath); + FileInfoDiff fileInfoDiff = new FileInfoDiff().setPath(getRawPath(destPath)); supportedAttributes .stream() .filter(preserveAttributes::contains) .forEach(attribute -> attribute.applyToDiff(fileInfoDiff, srcFileStatus)); - updateMetadataSupport.changeFileMetadata(fileInfoDiff); + updateMetadataSupport.changeFileMetadata(destFileSystem, fileInfoDiff); appendLog("Successfully transferred file attributes: " + preserveAttributes); } - protected FileStatus getFileStatus(String fileName) throws IOException { - if (fileName.startsWith("hdfs")) { - FileSystem fs = FileSystem.get(URI.create(fileName), getContext().getConf()); - // Get InputStream from URL - return fs.getFileStatus(new Path(fileName)); - } - return (FileStatus) dfsClient.getFileInfo(fileName); - } - public enum PreserveAttribute { OWNER("owner") { @Override diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DecompressionAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DecompressionAction.java index 54516a197f0..6d83f1d5a07 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DecompressionAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DecompressionAction.java @@ -17,22 +17,22 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.SmartConstants; +import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.hdfs.HadoopUtil; +import org.smartdata.hdfs.StreamCopyHandler; import org.smartdata.model.CompressionFileState; import org.smartdata.model.FileState; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Map; +import static org.smartdata.hdfs.action.CompressionAction.BUF_SIZE; + /** * This class is used to decompress file. */ @@ -41,109 +41,85 @@ displayName = "decompress", usage = HdfsAction.FILE_PATH + " $file " - + CompressionAction.BUF_SIZE + + BUF_SIZE + " $bufSize " ) public class DecompressionAction extends HdfsAction { - public static final Logger LOG = - LoggerFactory.getLogger(DecompressionAction.class); public static final String COMPRESS_TMP = "-compressTmp"; - private Configuration conf; + public static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + private float progress; - private String compressTmpPath; - private String filePath; - private int buffSize = 64 * 1024; + private int buffSize; + private Path compressTmpPath; + private Path filePath; @Override public void init(Map args) { super.init(args); - this.conf = getContext().getConf(); - this.filePath = args.get(FILE_PATH); - // This is a temp path for compressing a file. - this.compressTmpPath = args.containsKey(COMPRESS_TMP) ? - args.get(COMPRESS_TMP) : compressTmpPath; + this.filePath = getPathArg(FILE_PATH); + this.compressTmpPath = getPathArg(COMPRESS_TMP); this.progress = 0.0F; + this.buffSize = args.containsKey(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUFFER_SIZE; } - protected void execute() throws Exception { - if (filePath == null) { - throw new IllegalArgumentException("File path is missing."); - } - if (compressTmpPath == null) { - throw new IllegalArgumentException( - "Compression tmp path is not specified!"); - } + private void validate() throws Exception { + validateNonEmptyArgs(FILE_PATH, COMPRESS_TMP); - if (!dfsClient.exists(filePath)) { + if (!localFileSystem.exists(filePath)) { throw new ActionException( "Failed to execute Compression Action: the given file doesn't exist!"); } + // Consider directory case. - if (dfsClient.getFileInfo(filePath).isDir()) { - appendLog("Decompression is not applicable to a directory."); - return; + if (localFileSystem.getFileStatus(filePath).isDirectory()) { + throw new ActionException("Decompression is not applicable to a directory."); } + } + + protected void execute() throws Exception { + validate(); - FileState fileState = HadoopUtil.getFileState(dfsClient, filePath); + FileState fileState = HadoopUtil.getFileState(localFileSystem, filePath); if (!(fileState instanceof CompressionFileState)) { - appendLog("The file is already decompressed!"); - return; + throw new ActionException("File is not compressed: " + filePath); } - OutputStream out = null; - InputStream in = null; - try { - // No need to lock the file by append operation, - // since compressed file cannot be modified. - out = dfsClient.create(compressTmpPath, true); + + FileStatus compressedFileStatus = localFileSystem.getFileStatus(filePath); + + try (InputStream in = localFileSystem.open(filePath); + // No need to lock the file by append operation, + // since compressed file cannot be modified. + OutputStream out = localFileSystem.create(compressTmpPath, true)) { // Keep storage policy consistent. - // The below statement is not supported on Hadoop-2.7.3 or CDH-5.10.1 - // String storagePolicyName = dfsClient.getStoragePolicy(filePath).getName(); - byte storagePolicyId = dfsClient.getFileInfo(filePath).getStoragePolicy(); - String storagePolicyName = SmartConstants.STORAGE_POLICY_MAP.get(storagePolicyId); + String storagePolicyName = localFileSystem.getStoragePolicy(filePath).getName(); if (!storagePolicyName.equals("UNDEF")) { - dfsClient.setStoragePolicy(compressTmpPath, storagePolicyName); + localFileSystem.setStoragePolicy(compressTmpPath, storagePolicyName); } - in = dfsClient.open(filePath); - long length = dfsClient.getFileInfo(filePath).getLen(); - outputDecompressedData(in, out, length); + StreamCopyHandler.of(in, out) + .count(compressedFileStatus.getLen()) + .bufferSize(buffSize) + .closeStreams(false) + .progressConsumer(this::updateProgress) + .build() + .runCopy(); + // Overwrite the original file with decompressed data - dfsClient.setOwner(compressTmpPath, dfsClient.getFileInfo(filePath).getOwner(), dfsClient.getFileInfo(filePath).getGroup()); - dfsClient.setPermission(compressTmpPath, dfsClient.getFileInfo(filePath).getPermission()); - dfsClient.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE); + localFileSystem.setOwner(compressTmpPath, + compressedFileStatus.getOwner(), + compressedFileStatus.getGroup()); + localFileSystem.setPermission(compressTmpPath, compressedFileStatus.getPermission()); + localFileSystem.rename(compressTmpPath, filePath, Options.Rename.OVERWRITE); appendLog("The given file is successfully decompressed by codec: " + ((CompressionFileState) fileState).getCompressionImpl()); - - } catch (IOException e) { - throw new IOException(e); - } finally { - if (out != null) { - out.close(); - } - if (in != null) { - in.close(); - } } } - private void outputDecompressedData(InputStream in, OutputStream out, - long length) throws IOException { - byte[] buff = new byte[buffSize]; - long remainSize = length; - while (remainSize != 0) { - int copySize = remainSize < buffSize ? (int) remainSize : buffSize; - // readSize may be smaller than copySize. Here, readSize is the actual - // number of bytes read to buff. - int readSize = in.read(buff, 0, copySize); - if (readSize == -1) { - break; - } - // Use readSize instead of copySize. - out.write(buff, 0, readSize); - remainSize -= readSize; - this.progress = (float) (length - remainSize) / length; - } + private void updateProgress(float progress) { + this.progress = progress; } @Override diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DeleteFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DeleteFileAction.java index c6c1ca5eaa1..ffd7078643c 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DeleteFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DeleteFileAction.java @@ -17,16 +17,11 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import java.io.IOException; -import java.net.URI; import java.util.Map; /** @@ -41,49 +36,28 @@ usage = HdfsAction.FILE_PATH + " $file" ) -public class DeleteFileAction extends HdfsAction { - private String filePath; +public class DeleteFileAction extends HdfsActionWithRemoteClusterSupport { + private Path filePath; @Override public void init(Map args) { super.init(args); - this.filePath = args.get(FILE_PATH); + filePath = getPathArg(FILE_PATH); } @Override - protected void execute() throws Exception { - if (StringUtils.isBlank(filePath)) { - throw new IllegalArgumentException("File parameter is missing."); - } - appendLog( - String.format("Action starts at %s : Delete %s", - Utils.getFormatedCurrentTime(), filePath)); - //delete File - deleteFile(filePath); + protected void preExecute() { + validateNonEmptyArg(FILE_PATH); } - private void deleteFile( - String filePath) throws IOException, ActionException { - if (filePath.startsWith("hdfs")) { - //delete in remote cluster - // TODO read conf from file - Configuration conf = new Configuration(); - //get FileSystem object - FileSystem fs = FileSystem.get(URI.create(filePath), conf); - if (!fs.exists(new Path(filePath))) { - throw new ActionException( - "DeleteFile Action fails, file doesn't exist!"); - } - fs.delete(new Path(filePath), true); - } else { - //delete in local cluster - if (!dfsClient.exists(filePath)) { - throw new ActionException( - "DeleteFile Action fails, file doesn't exist!"); - } - appendLog(String.format("Delete %s", filePath)); - dfsClient.delete(filePath, true); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (!fileSystem.exists(filePath)) { + throw new ActionException( + "DeleteFile Action fails, file doesn't exist!"); } + + fileSystem.delete(filePath, true); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DistCpAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DistCpAction.java index 1f90f9640b1..e24c1b1bba0 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DistCpAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/DistCpAction.java @@ -17,21 +17,20 @@ */ package org.smartdata.hdfs.action; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.OptionsParser; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + @ActionSignature( actionId = "distcp", displayName = "DistCp", @@ -77,15 +76,11 @@ protected void execute() throws Exception { // we need to perform validation and option parsing here // because SSM doesn't correctly handle exceptions thrown in the init method DistCpOptions distCpOptions = buildDistCpOptions(); - DistCp distCp = new DistCp(getContext().getConf(), distCpOptions); - appendLog( - String.format("DistCp Action started at %s for options %s", - Utils.getFormatedCurrentTime(), distCpOptions)); - try (JobCloseableWrapper jobWrapper = new JobCloseableWrapper(distCp.execute())) { - distCp.waitForJobCompletion(jobWrapper.job); - appendLog(jobWrapper.job.getCounters().toString()); + try (Job job = distCp.execute()) { + distCp.waitForJobCompletion(job); + appendLog(job.getCounters().toString()); } } @@ -137,22 +132,4 @@ private boolean containsPreserveOption(Map args) { .stream() .anyMatch(option -> option.startsWith(PRESERVE_DISTCP_OPTION_PREFIX)); } - - /** Used to gracefully close MapReduce job (MR Job is not AutoCloseable inheritor in Hadoop 2.7) */ - private static class JobCloseableWrapper implements AutoCloseable { - - private final Job job; - - private JobCloseableWrapper(Job job) { - this.job = job; - } - - @Override - public void close() throws Exception { - Cluster cluster = job.getCluster(); - if (cluster != null) { - cluster.close(); - } - } - } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java index bb4d382af50..bbf11ee4e1f 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java @@ -50,7 +50,6 @@ public class HdfsActionFactory extends AbstractActionFactory { addAction(DecompressionAction.class); addAction(CheckCompressAction.class); addAction(TruncateAction.class); - addAction(Truncate0Action.class); addAction(SmallFileCompactAction.class); addAction(SmallFileUncompactAction.class); addAction(CheckSumAction.class); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionWithRemoteClusterSupport.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionWithRemoteClusterSupport.java new file mode 100644 index 00000000000..439744ed349 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionWithRemoteClusterSupport.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import static org.smartdata.utils.PathUtil.getRemoteFileSystem; +import static org.smartdata.utils.PathUtil.isAbsoluteRemotePath; + +public abstract class HdfsActionWithRemoteClusterSupport extends HdfsAction { + + @Override + protected void execute() throws Exception { + preExecute(); + + Path targetPath = getTargetFile(); + if (isRemoteMode()) { + preRemoteExecute(); + execute(getRemoteFileSystem(targetPath)); + } else { + preLocalExecute(); + execute(localFileSystem); + } + + postExecute(); + } + + protected void preExecute() throws Exception { + + } + + protected void postExecute() throws Exception { + + } + + protected void preLocalExecute() throws Exception { + + } + + protected void preRemoteExecute() throws Exception { + + } + + protected Path getTargetFile() { + return getPathArg(FILE_PATH); + } + + protected boolean isRemoteMode() { + return isAbsoluteRemotePath(getTargetFile()); + } + + protected abstract void execute(FileSystem fileSystem) throws Exception; +} diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java index 0d847803178..636e8cfac7d 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java @@ -17,13 +17,10 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import java.io.IOException; @@ -46,7 +43,7 @@ + ListFileAction.RECURSIVELY + ListFileAction.PRETTY_SIZES ) -public class ListFileAction extends HdfsAction { +public class ListFileAction extends HdfsActionWithRemoteClusterSupport { // Options public static final String RECURSIVELY = "-r"; public static final String PRETTY_SIZES = "-h"; @@ -59,46 +56,26 @@ public class ListFileAction extends HdfsAction { "", "K", "M", "G", "T" ); - private String srcPath; - private boolean recursively = false; - private boolean human = false; + private Path srcPath; + private boolean recursively; + private boolean human; @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(RECURSIVELY)) { - this.recursively = true; - } - if (args.containsKey(PRETTY_SIZES)) { - this.human = true; - } + this.srcPath = getPathArg(FILE_PATH); + this.recursively = args.containsKey(RECURSIVELY); + this.human = args.containsKey(PRETTY_SIZES); } @Override - protected void execute() throws Exception { - Path listingRoot = Optional.ofNullable(srcPath) - .filter(StringUtils::isNotBlank) - .map(Path::new) - .orElseThrow(() -> new IllegalArgumentException("File parameter is missing.")); - - appendLog( - String.format("Action starts at %s : List %s", Utils.getFormatedCurrentTime(), srcPath)); - - if (srcPath.startsWith("hdfs")) { - listDirectory(listingRoot.getFileSystem(new Configuration()), listingRoot); - } else { - withDefaultFs(); - listDirectory(listingRoot.getFileSystem(getContext().getConf()), listingRoot); - } + protected void preExecute() { + validateNonEmptyArg(FILE_PATH); } - private void listDirectory(FileSystem fs, Path root) throws IOException { - FileStatus rootStatus = fs.getFileStatus(root); - if (rootStatus == null) { - appendResult("File not found!"); - return; - } + @Override + protected void execute(FileSystem fs) throws IOException { + FileStatus rootStatus = fs.getFileStatus(srcPath); if (!rootStatus.isDirectory()) { appendResult(formatFileStatus(rootStatus)); @@ -106,11 +83,11 @@ private void listDirectory(FileSystem fs, Path root) throws IOException { } Queue fileQueue = new ArrayDeque<>(); - addFilesFromDir(fs, root, fileQueue); + addFilesFromDir(fs, srcPath, fileQueue); while (!fileQueue.isEmpty()) { FileStatus fileStatus = fileQueue.poll(); - appendResult(String.format("%s", formatFileStatus(fileStatus))); + appendResult(formatFileStatus(fileStatus)); if (recursively && fileStatus.isDirectory()) { addFilesFromDir(fs, fileStatus.getPath(), fileQueue); @@ -147,4 +124,4 @@ private String readableFileSize(long size) { + " " + SIZE_UNITS.get(digitGroups); } -} +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MergeFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MergeFileAction.java index 6e097539138..a51754d379f 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MergeFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MergeFileAction.java @@ -17,22 +17,19 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URI; import java.util.Arrays; -import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * action to Merge File @@ -43,89 +40,54 @@ usage = HdfsAction.FILE_PATH + " $src " + MergeFileAction.DEST_PATH + " $dest " + MergeFileAction.BUF_SIZE + " $size" ) -public class MergeFileAction extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(MergeFileAction.class); +public class MergeFileAction extends HdfsActionWithRemoteClusterSupport { public static final String DEST_PATH = "-dest"; public static final String BUF_SIZE = "-bufSize"; - private LinkedList srcPathList; - private int bufferSize = 64 * 1024; - private String target; + public static final int DEFAULT_BUF_SIZE = 64 * 1024; + + private List srcPaths; + private Path target; + private int bufferSize; @Override public void init(Map args) { super.init(args); - String allSrcPath = args.get(FILE_PATH); - - String[] allSrcPathArr = allSrcPath.split(","); - srcPathList = new LinkedList(Arrays.asList(allSrcPathArr)); + this.srcPaths = Optional.ofNullable(args.get(FILE_PATH)) + .map(paths -> paths.split(",")) + .map(Arrays::stream) + .orElseGet(Stream::empty) + .map(Path::new) + .collect(Collectors.toList()); - if (args.containsKey(DEST_PATH)) { - this.target = args.get(DEST_PATH); - } - if (args.containsKey(BUF_SIZE)) { - bufferSize = Integer.valueOf(args.get(BUF_SIZE)); - } + this.target = getPathArg(DEST_PATH); + this.bufferSize = isArgPresent(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUF_SIZE; } @Override - protected void execute() throws Exception { - if (srcPathList == null || srcPathList.size() == 0) { - throw new IllegalArgumentException("File parameter is missing."); - } - if (target == null) { - throw new IllegalArgumentException("Dest File parameter is missing."); - } - if (srcPathList.size() == 1) { + protected void preExecute() throws Exception { + validateNonEmptyArgs(FILE_PATH, DEST_PATH); + if (srcPaths.size() == 1) { throw new IllegalArgumentException("Don't accept only one source file"); } - - appendLog( - String.format("Action starts at %s : Merge %s to %s", - Utils.getFormatedCurrentTime(), srcPathList, target)); - - //Merge - mergeFiles(srcPathList,target); } - private boolean mergeFiles(LinkedList srcFiles, String dest) throws IOException { - InputStream srcInputStream = null; - OutputStream destInputStream = getTargetOutputStream(dest); - for (String srcEle : srcPathList) { - srcInputStream = getSourceInputStream(srcEle); - IOUtils.copyBytes(srcInputStream, destInputStream, bufferSize, false); - IOUtils.closeStream(srcInputStream); - } - IOUtils.closeStream(destInputStream); - return true; - } - - private InputStream getSourceInputStream(String src) throws IOException { - if (src.startsWith("hdfs")) { - //get stream of source - // TODO read conf from files - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(URI.create(src), conf); - return fs.open(new Path(src)); - } else { - return dfsClient.open(src); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + try (OutputStream destInputStream = fileSystem.create(target, true)) { + for (Path srcPath : srcPaths) { + try (InputStream srcInputStream = getFileSystemFor(srcPath).open(srcPath)) { + IOUtils.copyBytes(srcInputStream, + destInputStream, bufferSize, false); + } + } } } - private OutputStream getTargetOutputStream(String dest) throws IOException { - if (dest.startsWith("hdfs")) { - // TODO read conf from files - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(URI.create(dest), conf); - if (fs.exists(new Path(target))) { - fs.delete(new Path(target), true); - } - return fs.create(new Path(dest), true); - } else { - if (dfsClient.exists(target)) { - dfsClient.delete(target, true); - } - return dfsClient.create(dest, true); - } + @Override + protected Path getTargetFile() { + return target; } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java index 65b894cb1f4..97a97c86953 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java @@ -17,12 +17,13 @@ */ package org.smartdata.hdfs.action; -import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hadoop.fs.FileSystem; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.model.FileInfoDiff; +import java.util.Map; + /** * action to set MetaData of file */ @@ -34,7 +35,7 @@ MetaDataAction.PERMISSION + " $permission " + MetaDataAction.MTIME + " $mtime " + MetaDataAction.ATIME + " $atime" ) -public class MetaDataAction extends HdfsAction { +public class MetaDataAction extends HdfsActionWithRemoteClusterSupport { public static final String OWNER_NAME = "-owner"; public static final String GROUP_NAME = "-group"; public static final String BLOCK_REPLICATION = "-replication"; @@ -66,16 +67,16 @@ public void init(Map args) { fileInfoDiff.setPermission(Short.parseShort(args.get(PERMISSION))); } - delegate = new UpdateFileMetadataSupport( - getContext().getConf(), getLogPrintStream()); + delegate = new UpdateFileMetadataSupport(getLogPrintStream()); } @Override - protected void execute() throws Exception { - if (StringUtils.isBlank(fileInfoDiff.getPath())) { - throw new IllegalArgumentException("File src is missing."); - } + protected void preExecute() throws Exception { + validateNonEmptyArg(FILE_PATH); + } - delegate.changeFileMetadata(fileInfoDiff); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + delegate.changeFileMetadata(fileSystem, fileInfoDiff); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MoveFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MoveFileAction.java index 13195788732..b6d66a47e84 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MoveFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MoveFileAction.java @@ -18,9 +18,9 @@ package org.smartdata.hdfs.action; import com.google.gson.Gson; -import org.apache.commons.lang3.StringUtils; +import lombok.Getter; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.smartdata.action.Utils; import org.smartdata.hdfs.action.move.AbstractMoveFileAction; import org.smartdata.hdfs.action.move.MoverExecutor; import org.smartdata.hdfs.action.move.MoverStatus; @@ -28,76 +28,63 @@ import java.io.IOException; import java.util.Map; +import java.util.Optional; /** * An action to set and enforce storage policy for a file. */ public class MoveFileAction extends AbstractMoveFileAction { - private MoverStatus status; + private static final Gson MOVER_PLAN_DESERIALIZER = new Gson(); + + private final MoverStatus status; + private Path filePath; + @Getter private String storagePolicy; - private String fileName; private FileMovePlan movePlan; public MoveFileAction() { - super(); this.status = new MoverStatus(); } - public MoverStatus getStatus() { - return this.status; - } - @Override public void init(Map args) { super.init(args); - this.fileName = args.get(FILE_PATH); - this.storagePolicy = getStoragePolicy() != null ? - getStoragePolicy() : args.get(STORAGE_POLICY); - if (args.containsKey(MOVE_PLAN)) { - String plan = args.get(MOVE_PLAN); - if (plan != null) { - Gson gson = new Gson(); - movePlan = gson.fromJson(plan, FileMovePlan.class); - status.setTotalBlocks(movePlan.getBlockIds().size()); - } - } + this.filePath = getPathArg(FILE_PATH); + this.storagePolicy = args.get(STORAGE_POLICY); + + Optional.ofNullable(args.get(MOVE_PLAN)) + .map(plan -> MOVER_PLAN_DESERIALIZER.fromJson(plan, FileMovePlan.class)) + .ifPresent(movePlan -> { + this.movePlan = movePlan; + status.setTotalBlocks(movePlan.getBlockIds().size()); + }); } @Override protected void execute() throws Exception { - if (StringUtils.isBlank(fileName)) { - throw new IllegalArgumentException("File parameter is missing!"); - } - - if (movePlan == null) { - throw new IllegalArgumentException("File move plan not specified."); - } + validateNonEmptyArgs(FILE_PATH, MOVE_PLAN); if (movePlan.isDir()) { - dfsClient.setStoragePolicy(fileName, storagePolicy); + localFileSystem.setStoragePolicy(filePath, storagePolicy); appendLog("Directory moved successfully."); return; } int totalReplicas = movePlan.getBlockIds().size(); - this.appendLog( - String.format( - "Action starts at %s : %s -> %s with %d replicas to move in total.", - Utils.getFormatedCurrentTime(), fileName, storagePolicy, totalReplicas)); int numFailed = move(); - if (numFailed == 0) { - appendLog("All scheduled " + totalReplicas + " replicas moved successfully."); - if (movePlan.isBeingWritten() || recheckModification()) { - appendResult("UpdateStoragePolicy=false"); - appendLog("NOTE: File may be changed during executing this action. " - + "Will move the corresponding blocks later."); - } - } else { + if (numFailed != 0) { String res = numFailed + " of " + totalReplicas + " replicas movement failed."; appendLog(res); throw new IOException(res); } + + appendLog("All scheduled " + totalReplicas + " replicas moved successfully."); + if (movePlan.isBeingWritten() || recheckModification()) { + appendResult("UpdateStoragePolicy=false"); + appendLog("NOTE: File may be changed during executing this action. " + + "Will move the corresponding blocks later."); + } } private int move() throws Exception { @@ -110,19 +97,15 @@ private int move() throws Exception { private boolean recheckModification() { try { - HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName); - if (fileStatus == null) { + Optional fileStatus = getHdfsFileStatus(localFileSystem, filePath); + if (!fileStatus.isPresent()) { return true; } - boolean closed = dfsClient.isFileClosed(fileName); - if (!closed - || (movePlan.getFileId() != 0 && fileStatus.getFileId() != movePlan.getFileId()) - || fileStatus.getLen() != movePlan.getFileLength() - || fileStatus.getModificationTime() != movePlan.getModificationTime()) { - return true; - } - return false; + return !localFileSystem.isFileClosed(filePath) + || (movePlan.getFileId() != 0 && fileStatus.get().getFileId() != movePlan.getFileId()) + || fileStatus.get().getLen() != movePlan.getFileLength() + || fileStatus.get().getModificationTime() != movePlan.getModificationTime(); } catch (Exception e) { return true; // check again for this case } @@ -133,12 +116,8 @@ public float getProgress() { return this.status.getPercentage(); } - public String getStoragePolicy() { - return storagePolicy; - } - @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneDiskFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneDiskFileAction.java index 8415b043de2..f79e16bcac5 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneDiskFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneDiskFileAction.java @@ -23,9 +23,9 @@ * An action to do one-disk for a file. */ @ActionSignature( - actionId = "onedisk", - displayName = "onedisk", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "onedisk", + displayName = "onedisk", + usage = HdfsAction.FILE_PATH + " $file " ) public class OneDiskFileAction extends MoveFileAction { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneSsdFileAction.java index dbacc23675c..5e4d3f184de 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneSsdFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/OneSsdFileAction.java @@ -17,20 +17,17 @@ */ package org.smartdata.hdfs.action; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.annotation.ActionSignature; /** * An action to do one-ssd for a file. */ @ActionSignature( - actionId = "onessd", - displayName = "onessd", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "onessd", + displayName = "onessd", + usage = HdfsAction.FILE_PATH + " $file " ) public class OneSsdFileAction extends MoveFileAction { - private static final Logger LOG = LoggerFactory.getLogger(OneSsdFileAction.class); @Override public String getStoragePolicy() { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RamDiskFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RamDiskFileAction.java index 47d999f2941..44bf6aed664 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RamDiskFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RamDiskFileAction.java @@ -23,9 +23,9 @@ * An action to do ram-disk for a file. */ @ActionSignature( - actionId = "ramdisk", - displayName = "ramdisk", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "ramdisk", + displayName = "ramdisk", + usage = HdfsAction.FILE_PATH + " $file " ) public class RamDiskFileAction extends MoveFileAction { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ReadFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ReadFileAction.java index 39ea1435a87..b1fa2a62637 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ReadFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ReadFileAction.java @@ -17,11 +17,12 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; +import java.io.InputStream; import java.util.Map; /** @@ -31,40 +32,43 @@ *

Arguments: file_path [buffer_size, default=64k] */ @ActionSignature( - actionId = "read", - displayName = "read", - usage = HdfsAction.FILE_PATH + " $file " + ReadFileAction.BUF_SIZE + " $size" + actionId = "read", + displayName = "read", + usage = HdfsAction.FILE_PATH + " $file " + + ReadFileAction.BUF_SIZE + " $size" ) -public class ReadFileAction extends HdfsAction { +public class ReadFileAction extends HdfsActionWithRemoteClusterSupport { public static final String BUF_SIZE = "-bufSize"; - private String filePath; - private int bufferSize = 64 * 1024; + public static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + + private Path filePath; + private int bufferSize; @Override public void init(Map args) { super.init(args); - this.filePath = args.get(FILE_PATH); - if (args.containsKey(BUF_SIZE)) { - bufferSize = Integer.parseInt(args.get(BUF_SIZE)); - } + this.filePath = getPathArg(FILE_PATH); + this.bufferSize = args.containsKey(BUF_SIZE) + ? Integer.parseInt(args.get(BUF_SIZE)) + : DEFAULT_BUFFER_SIZE; } @Override - protected void execute() throws Exception { - if (filePath == null) { - throw new IllegalArgumentException("File parameter is missing."); - } - appendLog( - String.format("Action starts at %s : Read %s", - Utils.getFormatedCurrentTime(), filePath)); - if (!dfsClient.exists(filePath)) { + protected void preExecute() { + validateNonEmptyArg(FILE_PATH); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (!fileSystem.exists(filePath)) { throw new ActionException("ReadFile Action fails, file " + filePath + " doesn't exist!"); } byte[] buffer = new byte[bufferSize]; - try (DFSInputStream inputStream = dfsClient.open(filePath)) { - while (inputStream.read(buffer, 0, bufferSize) != -1) {} + try (InputStream inputStream = fileSystem.open(filePath)) { + while (inputStream.read(buffer, 0, bufferSize) != -1) { + } } } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RenameFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RenameFileAction.java index 6f061b2f524..8da4f74140c 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RenameFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/RenameFileAction.java @@ -17,19 +17,16 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import java.io.IOException; -import java.net.URI; import java.util.Map; +import java.util.Optional; + +import static org.smartdata.utils.PathUtil.getScheme; /** * An action to rename a single file @@ -43,62 +40,58 @@ usage = HdfsAction.FILE_PATH + " $src " + RenameFileAction.DEST_PATH + " $dest" ) -public class RenameFileAction extends HdfsAction { - private static final Logger LOG = - LoggerFactory.getLogger(RenameFileAction.class); +public class RenameFileAction extends HdfsActionWithRemoteClusterSupport { public static final String DEST_PATH = "-dest"; - private String srcPath; - private String destPath; + + private Path srcPath; + private Path destPath; @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(FILE_PATH); - if (args.containsKey(DEST_PATH)) { - this.destPath = args.get(DEST_PATH); - } + this.srcPath = getPathArg(FILE_PATH); + this.destPath = getPathArg(DEST_PATH); + } + + @Override + protected boolean isRemoteMode() { + return getScheme(srcPath).isPresent() || getScheme(destPath).isPresent(); } @Override - protected void execute() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("File parameter is missing."); + protected void preExecute() { + validateNonEmptyArgs(FILE_PATH, DEST_PATH); + } + + @Override + protected void preRemoteExecute() throws Exception { + Optional srcScheme = getScheme(srcPath); + Optional destScheme = getScheme(destPath); + + // One of files is in local cluster and second is in remote + // TODO handle the case when absolute path's host is local cluster + if (!srcScheme.isPresent() || !destScheme.isPresent()) { + throw new ActionException("Paths are not in the same cluster"); } - if (destPath == null) { - throw new IllegalArgumentException("Dest File parameter is missing."); + + if (!srcScheme.get().equals(destScheme.get())) { + throw new ActionException("Paths have different schemes"); } - appendLog(String.format("Action starts at %s : Rename %s to %s", - Utils.getFormatedCurrentTime(), srcPath, destPath)); - if (!renameSingleFile(srcPath, destPath)) { - throw new IOException("Failed to rename " + srcPath + " -> " + destPath); + if (!destPath.toUri().getHost().equals(srcPath.toUri().getHost())) { + throw new ActionException("Paths are not in the same cluster"); } } - private boolean renameSingleFile(String src, - String dest) throws IOException, ActionException { - if (dest.startsWith("hdfs") && src.startsWith("hdfs")) { - //rename file in the same remote cluster - // TODO read conf from files - //check the file name - if (!URI.create(dest).getHost().equals(URI.create(src).getHost())) { - throw new ActionException("the file names are not in the same cluster"); - } - Configuration conf = new Configuration(); - //get FileSystem object - FileSystem fs = FileSystem.get(URI.create(dest), conf); - return fs.rename(new Path(src), new Path(dest)); - } else if (!dest.startsWith("hdfs") && !src.startsWith("hdfs")) { - //rename file in local cluster and overwrite - if (!dfsClient.exists(src)) { - throw new ActionException("the source file is not exist"); - } - dfsClient.rename(src, dest, Options.Rename.NONE); - return true; - } else { - // TODO handle the case when dest prefixed with the default hdfs uri - // while src not, the two path are in the same cluster - throw new ActionException("the file names are not in the same cluster"); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (!fileSystem.rename(srcPath, destPath)) { + throw new IOException("Failed to rename " + srcPath + " to " + destPath); } } + + @Override + protected void postExecute() { + appendLog("File " + srcPath + " was renamed to " + destPath); + } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SchemeHandlerRegistry.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SchemeHandlerRegistry.java new file mode 100644 index 00000000000..58284add00e --- /dev/null +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SchemeHandlerRegistry.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.smartdata.utils.PathUtil; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +import static org.smartdata.utils.PathUtil.getScheme; +import static org.smartdata.utils.PathUtil.isAbsoluteRemotePath; + +public class SchemeHandlerRegistry { + private final Map, ThrowingRunnable> schemeHandlers; + private ThrowingRunnable defaultHandler; + + public SchemeHandlerRegistry() { + this.schemeHandlers = new LinkedHashMap<>(); + } + + public SchemeHandlerRegistry onLocalPath(ThrowingRunnable handler) { + schemeHandlers.put(path -> !isAbsoluteRemotePath(path), handler); + return this; + } + + public SchemeHandlerRegistry onRemotePath(ThrowingRunnable handler) { + schemeHandlers.put(PathUtil::isAbsoluteRemotePath, handler); + return this; + } + + public SchemeHandlerRegistry onSchemes(ThrowingRunnable handler, String... schemes) { + schemeHandlers.put(SchemePredicate.forSchemes(schemes), handler); + return this; + } + + public SchemeHandlerRegistry onSchemesExcluding(ThrowingRunnable handler, String... excludedSchemes) { + schemeHandlers.put(SchemePredicate.forSchemesExcluding(excludedSchemes), handler); + return this; + } + + public SchemeHandlerRegistry defaultHandler(ThrowingRunnable handler) { + this.defaultHandler = handler; + return this; + } + + public void executeForPath(String path) throws Exception { + for (Map.Entry, ThrowingRunnable> entry : schemeHandlers.entrySet()) { + if (entry.getKey().test(path)) { + entry.getValue().run(); + break; + } + } + + if (defaultHandler == null) { + throw new IllegalArgumentException("Path is not supported: " + path); + } + defaultHandler.run(); + } + + private static class SchemePredicate implements Predicate { + + private final Set schemes; + private final boolean shouldBeOneOfSchemes; + + private SchemePredicate(boolean shouldBeOneOfSchemes, String... schemes) { + this.shouldBeOneOfSchemes = shouldBeOneOfSchemes; + this.schemes = new HashSet<>(Arrays.asList(schemes)); + } + + @Override + public boolean test(String path) { + return getScheme(path) + .filter(scheme -> schemes.contains(scheme) == shouldBeOneOfSchemes) + .isPresent(); + } + + private static SchemePredicate forSchemes(String... schemes) { + return new SchemePredicate(true, schemes); + } + + private static SchemePredicate forSchemesExcluding(String... schemes) { + return new SchemePredicate(false, schemes); + } + } + + public interface ThrowingRunnable { + void run() throws Exception; + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetStoragePolicyAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetStoragePolicyAction.java index 016ec9da0b0..8c8e0be9a42 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetStoragePolicyAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetStoragePolicyAction.java @@ -17,31 +17,39 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.smartdata.action.annotation.ActionSignature; import java.util.Map; /** Set storage policy */ @ActionSignature( - actionId = "setstoragepolicy", - displayName = "setstoragepolicy", - usage = HdfsAction.FILE_PATH + " $file " + SetStoragePolicyAction.STORAGE_POLICY + " $policy" + actionId = "setstoragepolicy", + displayName = "setstoragepolicy", + usage = HdfsAction.FILE_PATH + " $file " + + SetStoragePolicyAction.STORAGE_POLICY + " $policy" ) -public class SetStoragePolicyAction extends HdfsAction { +public class SetStoragePolicyAction extends HdfsActionWithRemoteClusterSupport { public static final String STORAGE_POLICY = "-storagePolicy"; - private String fileName; + private Path filePath; private String storagePolicy; @Override public void init(Map args) { super.init(args); - this.fileName = args.get(FILE_PATH); + this.filePath = getPathArg(FILE_PATH); this.storagePolicy = args.get(STORAGE_POLICY); } @Override - protected void execute() throws Exception { - dfsClient.setStoragePolicy(fileName, storagePolicy); + protected void preExecute() { + validateNonEmptyArg(FILE_PATH); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + fileSystem.setStoragePolicy(filePath, storagePolicy); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetXAttrAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetXAttrAction.java index 68d37ba4ae7..206c267d3cd 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetXAttrAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SetXAttrAction.java @@ -17,6 +17,8 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,52 +37,41 @@ usage = HdfsAction.FILE_PATH + " $src " + SetXAttrAction.ATT_NAME + " $name " + SetXAttrAction.ATT_VALUE + " $value" ) -public class SetXAttrAction extends HdfsAction { +public class SetXAttrAction extends HdfsActionWithRemoteClusterSupport { private static final Logger LOG = LoggerFactory.getLogger(SetXAttrAction.class); public static final String ATT_NAME = "-name"; public static final String ATT_VALUE = "-value"; - private String srcPath; + private Path srcPath; private String attName; private String attValue; - @Override public void init(Map args) { super.init(args); - if (args.containsKey(FILE_PATH)) { - this.srcPath = args.get(FILE_PATH); - } - if (args.containsKey(ATT_NAME)) { - this.attName = args.get(ATT_NAME); - } - if (args.containsKey(ATT_VALUE)) { - this.attValue = args.get(ATT_VALUE); - } + this.srcPath = getPathArg(FILE_PATH); + this.attName = args.get(ATT_NAME); + this.attValue = args.get(ATT_VALUE); } @Override - protected void execute() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("File parameter is missing."); - } - if (attName == null) { - throw new IllegalArgumentException("attName parameter is missing."); - } - if (attValue == null) { - throw new IllegalArgumentException("attValue parameter is missing."); - } - if (!dfsClient.exists(srcPath)) { + protected void preExecute() { + validateNonEmptyArgs(FILE_PATH, ATT_NAME, ATT_VALUE); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + if (!fileSystem.exists(srcPath)) { throw new ActionException("SetXAttr Action fails, file doesn't exist!"); } - LOG.debug("SetXattr path={} name={} value={}", srcPath, attName, attValue); - appendLog(String.format("SetXattr path=%s name=%s value=%s", + LOG.debug("Setting XAttribute path={} name={} value={}", + srcPath, attName, attValue); + appendLog(String.format("Setting XAttribute path=%s name=%s value=%s", srcPath, attName, attValue)); - dfsClient.setXAttr(srcPath, attName, attValue.getBytes(), + + fileSystem.setXAttr(srcPath, attName, attValue.getBytes(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - appendLog("SetXattr Successfully!!"); + appendLog("Xattr was set successfully!"); } - - } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileCompactAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileCompactAction.java index 5faf91b4abf..05ed851121a 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileCompactAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileCompactAction.java @@ -19,28 +19,33 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import lombok.Getter; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.SerializationUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.io.IOUtils; import org.smartdata.SmartConstants; import org.smartdata.SmartFilePermission; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.model.CompactFileState; import org.smartdata.model.FileContainerInfo; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Map; +import static org.smartdata.utils.PathUtil.getRawPath; + /** * An action to compact small files to a big container file. */ @@ -54,12 +59,17 @@ public class SmallFileCompactAction extends HdfsAction { public static final String CONTAINER_FILE = "-containerFile"; public static final String CONTAINER_FILE_PERMISSION = "-containerFilePermission"; - private float status = 0f; - private String smallFiles = null; - private String containerFile = null; - private String containerFilePermission = null; - private String xAttrNameFileSate = null; - private String xAttrNameCheckSum = null; + private static final Type SMALL_FILE_LIST_TYPE = + new TypeToken>() { + }.getType(); + + @Getter + private float progress; + private String smallFiles; + private Path containerFile; + private String containerFilePermission; + private String xAttrNameFileSate; + private String xAttrNameCheckSum; @Override public void init(Map args) { @@ -67,112 +77,96 @@ public void init(Map args) { this.xAttrNameFileSate = SmartConstants.SMART_FILE_STATE_XATTR_NAME; this.xAttrNameCheckSum = SmartConstants.SMART_FILE_CHECKSUM_XATTR_NAME; this.smallFiles = args.get(FILE_PATH); - this.containerFile = args.get(CONTAINER_FILE); + this.containerFile = getPathArg(CONTAINER_FILE); this.containerFilePermission = args.get(CONTAINER_FILE_PERMISSION); + this.progress = 0.0f; } @Override protected void execute() throws Exception { - // Get small file list - if (smallFiles == null || smallFiles.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid small files: %s.", smallFiles)); - } - ArrayList smallFileList = new Gson().fromJson( - smallFiles, new TypeToken>() { - }.getType()); - if (smallFileList == null || smallFileList.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid small files: %s.", smallFiles)); - } + validateNonEmptyArgs(FILE_PATH, CONTAINER_FILE); - // Get container file path - if (containerFile == null || containerFile.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid container file: %s.", containerFile)); - } + List smallFileList = parseSmallFileList(smallFiles); - // Get container file permission - SmartFilePermission filePermission = null; - if (containerFilePermission != null && !containerFilePermission.isEmpty()) { - filePermission = new Gson().fromJson( - containerFilePermission, new TypeToken() { - }.getType()); - } - appendLog(String.format("Action starts at %s : compact small files to %s.", - Utils.getFormatedCurrentTime(), containerFile)); // Get initial offset and output stream // Create container file and set permission if not exists - long offset; - OutputStream out; - boolean isContainerFileExist = dfsClient.exists(containerFile); - if (isContainerFileExist) { - offset = dfsClient.getFileInfo(containerFile).getLen(); - out = CompatibilityHelperLoader.getHelper() - .getDFSClientAppend(dfsClient, containerFile, 64 * 1024, offset); - } else { - out = dfsClient.create(containerFile, true); - if (filePermission != null) { - dfsClient.setOwner( - containerFile, filePermission.getOwner(), filePermission.getGroup()); - dfsClient.setPermission( - containerFile, new FsPermission(filePermission.getPermission())); - } - offset = 0L; - } - List compactFileStates = new ArrayList<>(); - - for (String smallFile : smallFileList) { - if ((smallFile != null) && !smallFile.isEmpty() && dfsClient.exists(smallFile)) { - HdfsDataOutputStream append = - (HdfsDataOutputStream) CompatibilityHelperLoader.getHelper().getDFSClientAppend(dfsClient, smallFile, 1024); - long fileLen = dfsClient.getFileInfo(smallFile).getLen(); - if (fileLen > 0) { - try (InputStream in = dfsClient.open(smallFile)) { - // Copy bytes of small file to container file - IOUtils.copyBytes(in, out, 4096); - - // Truncate small file, add file container info to XAttr - CompactFileState compactFileState = new CompactFileState( - smallFile, new FileContainerInfo(containerFile, offset, fileLen)); - append.close(); - truncateAndSetXAttr(smallFile, compactFileState); - - // Update compact file state map, offset, status, and log - compactFileStates.add(compactFileState); - offset += fileLen; - this.status = (smallFileList.indexOf(smallFile) + 1.0f) - / smallFileList.size(); - appendLog(String.format( - "Compact %s to %s successfully.", smallFile, containerFile)); - } catch (IOException e) { - // Close append, output streams and put compact file state map into action result - if (append != null) { - append.close(); - } - if (out != null) { - out.close(); - appendResult(new Gson().toJson(compactFileStates)); - } - if (!isContainerFileExist && compactFileStates.isEmpty()) { - dfsClient.delete(containerFile, false); - } - throw e; + boolean containerFileExists = localFileSystem.exists(containerFile); + long offset = containerFileExists + ? localFileSystem.getFileStatus(containerFile).getLen() + : 0; + + try (OutputStream out = getContainerOutputStream(containerFileExists)) { + List compactFileStates = new ArrayList<>(); + + for (int i = 0; i < smallFileList.size(); ++i) { + Path smallFile = new Path(smallFileList.get(i)); + + long fileLen = getFileStatus(localFileSystem, smallFile) + .map(FileStatus::getLen) + .orElse(0L); + + if (fileLen == 0) { + continue; + } + + try (InputStream in = localFileSystem.open(smallFile); + FSDataOutputStream append = localFileSystem.append(smallFile, 1024)) { + // Copy bytes of small file to container file + IOUtils.copyBytes(in, out, 4096); + + // Truncate small file, add file container info to XAttr + CompactFileState compactFileState = new CompactFileState( + smallFileList.get(i), new FileContainerInfo(getRawPath(containerFile), offset, fileLen)); + append.close(); + truncateAndSetXAttr(smallFile, compactFileState); + + // Update compact file state map, offset, status, and log + compactFileStates.add(compactFileState); + offset += fileLen; + this.progress = (i + 1.0f) / smallFileList.size(); + appendLog(String.format( + "Compact %s to %s successfully.", smallFile, containerFile)); + } catch (IOException e) { + if (out != null) { + out.close(); + appendResult(new Gson().toJson(compactFileStates)); + } + if (!containerFileExists && compactFileStates.isEmpty()) { + localFileSystem.delete(containerFile, false); } + throw e; } } + + appendResult(new Gson().toJson(compactFileStates)); + if (!containerFileExists && compactFileStates.isEmpty()) { + localFileSystem.delete(containerFile, false); + } + + appendLog(String.format( + "Compact all the small files to %s successfully.", containerFile)); } + } - appendResult(new Gson().toJson(compactFileStates)); - if (out != null) { - out.close(); + private OutputStream getContainerOutputStream(boolean containerFileExists) throws IOException { + if (containerFileExists) { + return localFileSystem.append(containerFile, 64 * 1024); } - if (!isContainerFileExist && compactFileStates.isEmpty()) { - dfsClient.delete(containerFile, false); + + OutputStream out = localFileSystem.create(containerFile, true); + + if (StringUtils.isNotBlank(containerFilePermission)) { + SmartFilePermission filePermission = new Gson().fromJson( + containerFilePermission, SmartFilePermission.class); + + localFileSystem.setOwner( + containerFile, filePermission.getOwner(), filePermission.getGroup()); + localFileSystem.setPermission( + containerFile, new FsPermission(filePermission.getPermission())); } - appendLog(String.format( - "Compact all the small files to %s successfully.", containerFile)); + + return out; } /** @@ -180,55 +174,60 @@ protected void execute() throws Exception { * To truncate the file length to zero, we delete the original file, then * create a new empty file with a different fid. */ - private void truncateAndSetXAttr(String path, CompactFileState compactFileState) + private void truncateAndSetXAttr(Path path, CompactFileState compactFileState) throws IOException { // Save original metadata of small file - HdfsFileStatus fileStatus = dfsClient.getFileInfo(path); - Map xAttr = dfsClient.getXAttrs(path); + FileStatus fileStatus = localFileSystem.getFileStatus(path); + Map xAttr = localFileSystem.getXAttrs(path); byte[] checksumBytes = getCheckSumByteArray(path, fileStatus.getLen()); // Delete file - dfsClient.delete(path, false); + localFileSystem.delete(path, false); // Create file with empty content. - OutputStream out = dfsClient.create(path, true); - if (out != null) { - out.close(); + try (OutputStream ignored = localFileSystem.create(path, true)) { } // Set metadata - dfsClient.setOwner(path, fileStatus.getOwner(), fileStatus.getGroup()); - dfsClient.setPermission(path, fileStatus.getPermission()); - dfsClient.setReplication(path, fileStatus.getReplication()); - dfsClient.setStoragePolicy(path, "Cold"); - dfsClient.setTimes(path, fileStatus.getModificationTime(), + localFileSystem.setOwner(path, fileStatus.getOwner(), fileStatus.getGroup()); + localFileSystem.setPermission(path, fileStatus.getPermission()); + localFileSystem.setReplication(path, fileStatus.getReplication()); + localFileSystem.setStoragePolicy(path, "Cold"); + localFileSystem.setTimes(path, fileStatus.getModificationTime(), fileStatus.getAccessTime()); - for(Map.Entry entry : xAttr.entrySet()) { - dfsClient.setXAttr(path, entry.getKey(), entry.getValue(), + for (Map.Entry entry : xAttr.entrySet()) { + localFileSystem.setXAttr(path, entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } // Set file container info into XAttr - dfsClient.setXAttr(path, + localFileSystem.setXAttr(path, xAttrNameFileSate, SerializationUtils.serialize(compactFileState), EnumSet.of(XAttrSetFlag.CREATE)); - dfsClient.setXAttr(path, xAttrNameCheckSum, + localFileSystem.setXAttr(path, xAttrNameCheckSum, checksumBytes, EnumSet.of(XAttrSetFlag.CREATE)); } - private byte[] getCheckSumByteArray(String path, long length) + private byte[] getCheckSumByteArray(Path path, long length) throws IOException { - return dfsClient.getFileChecksum(path, length).getBytes(); + return localFileSystem.getFileChecksum(path, length).getBytes(); } @Override - public float getProgress() { - return this.status; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } - @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + static List parseSmallFileList(String rawFiles) { + List smallFileList = new Gson() + .fromJson(rawFiles, SMALL_FILE_LIST_TYPE); + if (CollectionUtils.isEmpty(smallFileList)) { + throw new IllegalArgumentException( + String.format("Invalid small files: %s.", rawFiles)); + } + + return smallFileList; } + } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileUncompactAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileUncompactAction.java index ab7edc7bcdf..da1e8b2a471 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileUncompactAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/SmallFileUncompactAction.java @@ -17,23 +17,23 @@ */ package org.smartdata.hdfs.action; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; +import lombok.Getter; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.DFSInputStream; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.io.IOUtils; import org.smartdata.SmartConstants; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; import java.util.EnumSet; +import java.util.List; import java.util.Map; +import static org.smartdata.hdfs.action.SmallFileCompactAction.parseSmallFileList; + /** * An action to recovery contents of compacted ssm small files. */ @@ -46,86 +46,57 @@ public class SmallFileUncompactAction extends HdfsAction { public static final String CONTAINER_FILE = SmallFileCompactAction.CONTAINER_FILE; - private float status = 0f; - private String smallFiles = null; - private String xAttrNameFileState = null; - private String xAttrNameCheckSum = null; - private String containerFile = null; - private DFSClient smartDFSClient = null; + @Getter + private float progress; + private String smallFiles; + private String xAttrNameFileState; + private String xAttrNameCheckSum; + private Path containerFile; @Override public void init(Map args) { super.init(args); - this.smartDFSClient = dfsClient; this.xAttrNameFileState = SmartConstants.SMART_FILE_STATE_XATTR_NAME; this.xAttrNameCheckSum = SmartConstants.SMART_FILE_CHECKSUM_XATTR_NAME; this.smallFiles = args.get(FILE_PATH); - this.containerFile = args.get(CONTAINER_FILE); + this.containerFile = getPathArg(CONTAINER_FILE); + this.progress = 0.0f; } @Override protected void execute() throws Exception { // Get small file list - if (smallFiles == null || smallFiles.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid small files: %s.", smallFiles)); - } - ArrayList smallFileList = new Gson().fromJson( - smallFiles, new TypeToken>() { - }.getType()); - if (smallFileList == null || smallFileList.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid small files: %s.", smallFiles)); - } + validateNonEmptyArgs(FILE_PATH, CONTAINER_FILE); - // Get container file path - if (containerFile == null || containerFile.isEmpty()) { - throw new IllegalArgumentException( - String.format("Invalid container file: %s.", containerFile)); - } - appendLog(String.format( - "Action starts at %s : uncompact small files.", - Utils.getFormatedCurrentTime())); - - for (String smallFile : smallFileList) { - if ((smallFile != null) && !smallFile.isEmpty() - && dfsClient.exists(smallFile)) { - DFSInputStream in = null; - OutputStream out = null; - try { - // Get compact input stream - in = smartDFSClient.open(smallFile); + List smallFileList = parseSmallFileList(smallFiles); + for (int i = 0; i < smallFileList.size(); i++) { + Path smallFile = new Path(smallFileList.get(i)); - // Save original metadata of small file and delete original small file - HdfsFileStatus fileStatus = dfsClient.getFileInfo(smallFile); - Map xAttr = dfsClient.getXAttrs(smallFile); - dfsClient.delete(smallFile, false); + if (localFileSystem.exists(smallFile)) { + // Get compact input stream + try (FSDataInputStream in = localFileSystem.open(smallFile); + // Create new small file + OutputStream out = localFileSystem.create(smallFile, true)) { - // Create new small file - out = dfsClient.create(smallFile, true); + // Save original metadata of small file and delete original small file + FileStatus fileStatus = localFileSystem.getFileStatus(smallFile); + Map xAttr = localFileSystem.getXAttrs(smallFile); + localFileSystem.delete(smallFile, false); // Copy contents to original small file IOUtils.copyBytes(in, out, 4096); // Reset file meta data - resetFileMeta(smallFile, fileStatus, xAttr); + resetFileMeta(fileStatus, xAttr); // Set status and update log - this.status = (smallFileList.indexOf(smallFile) + 1.0f) - / smallFileList.size(); - appendLog(String.format("Uncompact %s successfully.", smallFile)); - } finally { - if (in != null) { - in.close(); - } - if (out != null) { - out.close(); - } + this.progress = (i + 1.0f) / smallFileList.size(); + appendLog("Uncompact successfully: " + smallFile); } } } - dfsClient.delete(containerFile, false); + localFileSystem.delete(containerFile, false); appendLog(String.format("Uncompact all the small files of %s successfully.", containerFile)); } @@ -133,27 +104,22 @@ protected void execute() throws Exception { * Reset meta data of small file. We should exclude the setting for * xAttrNameFileState or xAttrNameCheckSum. */ - private void resetFileMeta(String path, HdfsFileStatus fileStatus, + private void resetFileMeta(FileStatus fileStatus, Map xAttr) throws IOException { - dfsClient.setOwner(path, fileStatus.getOwner(), fileStatus.getGroup()); - dfsClient.setPermission(path, fileStatus.getPermission()); + localFileSystem.setOwner(fileStatus.getPath(), fileStatus.getOwner(), fileStatus.getGroup()); + localFileSystem.setPermission(fileStatus.getPath(), fileStatus.getPermission()); - for(Map.Entry entry : xAttr.entrySet()) { + for (Map.Entry entry : xAttr.entrySet()) { if (!entry.getKey().equals(xAttrNameFileState) && !entry.getKey().equals(xAttrNameCheckSum)) { - dfsClient.setXAttr(path, entry.getKey(), entry.getValue(), + localFileSystem.setXAttr(fileStatus.getPath(), entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } } } @Override - public float getProgress() { - return this.status; - } - - @Override - public DfsClientType dfsClientType() { - return DfsClientType.DEFAULT_HDFS; + public FsType localFsType() { + return FsType.DEFAULT_HDFS; } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Truncate0Action.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Truncate0Action.java deleted file mode 100644 index b53407e622a..00000000000 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/Truncate0Action.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.XAttrSetFlag; -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.smartdata.action.annotation.ActionSignature; - -import java.io.IOException; -import java.net.URI; -import java.util.EnumSet; -import java.util.Map; - -/** - * action to set file length to zero - */ -@ActionSignature( - actionId = "truncate0", - displayName = "truncate0", - usage = HdfsAction.FILE_PATH + " $src " -) -public class Truncate0Action extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(TruncateAction.class); - private String srcPath; - - @Override - public void init(Map args) { - super.init(args); - srcPath = args.get(FILE_PATH); - } - - @Override - protected void execute() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("File src is missing."); - } - setLen2Zero(srcPath); - } - - private boolean setLen2Zero(String srcPath) throws IOException { - if (srcPath.startsWith("hdfs")) { - // TODO read conf from files - Configuration conf = new Configuration(); - DistributedFileSystem fs = new DistributedFileSystem(); - fs.initialize(URI.create(srcPath), conf); - - return setLen2Zero(fs, srcPath); - } else { - return setLen2Zero(dfsClient, srcPath); - } - } - - private boolean setLen2Zero(DFSClient client, String src) throws IOException { - // return client.truncate(src, 0); - // Delete file and create file - // Save the metadata - HdfsFileStatus fileStatus = client.getFileInfo(src); - // AclStatus aclStatus = client.getAclStatus(src); - Map XAttr = client.getXAttrs(src); - // Delete file - client.delete(src, true); - // Create file - client.create(src, true); - // Set metadata - client.setOwner(src, fileStatus.getOwner(), fileStatus.getGroup()); - client.setPermission(src, fileStatus.getPermission()); - client.setReplication(src, fileStatus.getReplication()); - client.setStoragePolicy(src, "Cold"); - client.setTimes(src, fileStatus.getAccessTime(), - client.getFileInfo(src).getModificationTime()); - // client.setAcl(src, aclStatus.getEntries()); - for(Map.Entry entry : XAttr.entrySet()){ - client.setXAttr(src, entry.getKey(), entry.getValue(), - EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - } - return true; - } - - private boolean setLen2Zero(DistributedFileSystem fileSystem, String src) throws IOException { - // return fileSystem.truncate(new Path(src), 0); - // Delete file and create file - // Save the metadata - FileStatus fileStatus = fileSystem.getFileStatus(new Path(src)); - // AclStatus aclStatus = fileSystem.getAclStatus(new Path(src)); - Map XAttr = fileSystem.getXAttrs(new Path(src)); - // Delete file - fileSystem.delete(new Path(src), true); - // Create file - fileSystem.create(new Path(src), true); - // Set metadata - fileSystem.setOwner(new Path(src), fileStatus.getOwner(), fileStatus.getGroup()); - fileSystem.setPermission(new Path(src), fileStatus.getPermission()); - fileSystem.setReplication(new Path(src), fileStatus.getReplication()); - fileSystem.setStoragePolicy(new Path(src), "Cold"); - fileSystem.setTimes(new Path(src), fileStatus.getAccessTime(), - fileSystem.getFileStatus(new Path(src)).getModificationTime()); - // fileSystem.setAcl(new Path(src), aclStatus.getEntries()); - for(Map.Entry entry : XAttr.entrySet()){ - fileSystem.setXAttr(new Path(src), entry.getKey(), entry.getValue(), - EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - } - return true; - } -} diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/TruncateAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/TruncateAction.java index 4984d48dd77..1dfc73853b9 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/TruncateAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/TruncateAction.java @@ -17,18 +17,12 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.smartdata.action.annotation.ActionSignature; -import org.smartdata.hdfs.CompatibilityHelperLoader; -import java.io.IOException; -import java.net.URI; import java.util.Map; +import java.util.Optional; /** * action to truncate file @@ -36,61 +30,41 @@ @ActionSignature( actionId = "truncate", displayName = "truncate", - usage = HdfsAction.FILE_PATH + " $src " + TruncateAction.LENGTH + " $length" + usage = HdfsAction.FILE_PATH + " $src " + + TruncateAction.LENGTH + " $length" ) -public class TruncateAction extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(TruncateAction.class); +public class TruncateAction extends HdfsActionWithRemoteClusterSupport { public static final String LENGTH = "-length"; - private String srcPath; + private Path srcPath; private long length; @Override public void init(Map args) { super.init(args); - srcPath = args.get(FILE_PATH); - this.length = -1; - - if (args.containsKey(LENGTH)) { - this.length = Long.parseLong(args.get(LENGTH)); - } + this.srcPath = getPathArg(FILE_PATH); + this.length = Optional.ofNullable(args.get(LENGTH)) + .map(Long::parseLong) + .orElse(-1L); } @Override - protected void execute() throws Exception { - if (srcPath == null) { - throw new IllegalArgumentException("File src is missing."); - } + protected void preExecute() { + validateNonEmptyArgs(FILE_PATH, LENGTH); - if (length == -1) { - throw new IllegalArgumentException("Length is missing"); + if (length < 0) { + throw new IllegalArgumentException("Length should be non negative number"); } - - truncateClusterFile(srcPath, length); } - private boolean truncateClusterFile(String srcFile, long length) throws IOException { - if (srcFile.startsWith("hdfs")) { - // TODO read conf from files - Configuration conf = new Configuration(); - DistributedFileSystem fs = new DistributedFileSystem(); - fs.initialize(URI.create(srcFile), conf); - - //check the length - long oldLength = fs.getFileStatus(new Path(srcFile)).getLen(); - - if (length > oldLength) { - throw new IllegalArgumentException("Length is illegal"); - } - return fs.truncate(new Path(srcPath), length); - } else { - long oldLength = dfsClient.getFileInfo(srcFile).getLen(); + @Override + protected void execute(FileSystem fileSystem) throws Exception { + long oldLength = fileSystem.getFileStatus(srcPath).getLen(); - if (length > oldLength) { - throw new IllegalArgumentException("Length is illegal"); - } - return dfsClient.truncate(srcFile, length); + if (length > oldLength) { + throw new IllegalArgumentException("Length is illegal"); } + fileSystem.truncate(srcPath, length); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UncacheFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UncacheFileAction.java index 7dc4c05e76a..38dc2e5228d 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UncacheFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UncacheFileAction.java @@ -17,12 +17,10 @@ */ package org.smartdata.hdfs.action; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import java.util.Map; @@ -31,49 +29,44 @@ * An action to un-cache a file. */ @ActionSignature( - actionId = "uncache", - displayName = "uncache", - usage = HdfsAction.FILE_PATH + " $file " + actionId = "uncache", + displayName = "uncache", + usage = HdfsAction.FILE_PATH + " $file " ) public class UncacheFileAction extends HdfsAction { - private String fileName; + private Path filePath; @Override public void init(Map args) { super.init(args); - fileName = args.get(FILE_PATH); + filePath = getPathArg(FILE_PATH); } @Override protected void execute() throws Exception { - if (fileName == null) { - throw new IllegalArgumentException("File parameter is missing! "); + validateNonEmptyArg(FILE_PATH); + + Long id = getCacheId(); + if (id == null) { + this.appendLog(String.format("File %s is not in cache. " + + "So there is no need to execute this action.", filePath)); + return; } - this.appendLog( - String.format( - "Action starts at %s : %s -> uncache", Utils.getFormatedCurrentTime(), fileName)); - removeDirective(fileName); + localFileSystem.removeCacheDirective(id); } - @VisibleForTesting - Long getCacheId(String fileName) throws Exception { - CacheDirectiveInfo.Builder filterBuilder = new CacheDirectiveInfo.Builder(); - filterBuilder.setPath(new Path(fileName)); - CacheDirectiveInfo filter = filterBuilder.build(); - RemoteIterator directiveEntries = dfsClient.listCacheDirectives(filter); + private Long getCacheId() throws Exception { + CacheDirectiveInfo filter = new CacheDirectiveInfo.Builder() + .setPath(filePath) + .build(); + + RemoteIterator directiveEntries = + localFileSystem.listCacheDirectives(filter); if (!directiveEntries.hasNext()) { return null; } - return directiveEntries.next().getInfo().getId(); - } - - private void removeDirective(String fileName) throws Exception { - Long id = getCacheId(fileName); - if (id == null) { - this.appendLog(String.format("File %s is not in cache. " + - "So there is no need to execute this action.", fileName)); - return; - } - dfsClient.removeCacheDirective(id); + return directiveEntries.next() + .getInfo() + .getId(); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UpdateFileMetadataSupport.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UpdateFileMetadataSupport.java index 971502c5b14..ae1ebcbc224 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UpdateFileMetadataSupport.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/UpdateFileMetadataSupport.java @@ -17,51 +17,36 @@ */ package org.smartdata.hdfs.action; -import static org.smartdata.utils.ConfigUtil.toRemoteClusterConfig; - -import java.io.IOException; -import java.io.PrintStream; -import java.net.URI; -import java.util.Optional; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.smartdata.hdfs.HadoopUtil; import org.smartdata.model.FileInfoDiff; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Optional; + public class UpdateFileMetadataSupport { - private final Configuration configuration; private final PrintStream logOutput; - public UpdateFileMetadataSupport(Configuration configuration, PrintStream logOutput) { - this.configuration = configuration; + public UpdateFileMetadataSupport(PrintStream logOutput) { this.logOutput = logOutput; } - public void changeFileMetadata(FileInfoDiff fileInfoDiff) throws IOException { - if (fileInfoDiff.getPath().startsWith("hdfs")) { - changeRemoteFileMetadata(fileInfoDiff); - } else { - changeLocalFileMetadata(fileInfoDiff); - } - } - - private void changeRemoteFileMetadata(FileInfoDiff fileInfoDiff) throws IOException { - FileSystem remoteFileSystem = FileSystem.get(URI.create(fileInfoDiff.getPath()), - toRemoteClusterConfig(configuration)); - changeFileMetadata(fileInfoDiff, remoteFileSystem); - } + public void changeFileMetadata(FileSystem destFileSystem, + FileInfoDiff fileInfoDiff) throws IOException { + Path srcPath = new Path(fileInfoDiff.getPath()); + FileStatus srcFileStatus = destFileSystem.getFileStatus(srcPath); - private void changeLocalFileMetadata(FileInfoDiff fileInfoDiff) throws IOException { - FileSystem localFileSystem = FileSystem.get( - HadoopUtil.getNameNodeUri(configuration), configuration); - changeFileMetadata(fileInfoDiff, localFileSystem); + maybeChangeOwnerAndGroup(destFileSystem, fileInfoDiff, srcFileStatus); + maybeChangeBlockReplication(destFileSystem, fileInfoDiff, srcFileStatus); + maybeChangePermissions(destFileSystem, fileInfoDiff, srcFileStatus); + maybeChangeTimes(destFileSystem, fileInfoDiff, srcFileStatus); } - private void maybeChangeOwnerAndGroup(FileSystem fileSystem, - FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) throws IOException { + private void maybeChangeOwnerAndGroup(FileSystem destFileSystem, + FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) { String owner = Optional.ofNullable(fileInfoDiff.getOwner()) .orElseGet(srcFileStatus::getOwner); @@ -74,36 +59,51 @@ private void maybeChangeOwnerAndGroup(FileSystem fileSystem, "and file's group from '%s' to '%s'%n", srcFileStatus.getOwner(), owner, srcFileStatus.getGroup(), group); - fileSystem.setOwner(srcFileStatus.getPath(), owner, group); + + try { + destFileSystem.setOwner(srcFileStatus.getPath(), owner, group); + } catch (IOException e) { + logOutput.println("Error changing owner and group: " + e.getMessage()); + } } } - private void maybeChangeBlockReplication(FileSystem fileSystem, - FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) throws IOException { + private void maybeChangeBlockReplication(FileSystem destFileSystem, + FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) { Short newBlockReplication = fileInfoDiff.getBlockReplication(); if (newBlockReplication != null && !newBlockReplication.equals(srcFileStatus.getReplication())) { logOutput.printf("Updating file's replication factor from '%s' to '%s'%n", srcFileStatus.getReplication(), newBlockReplication); - fileSystem.setReplication(srcFileStatus.getPath(), newBlockReplication); + + try { + destFileSystem.setReplication(srcFileStatus.getPath(), newBlockReplication); + } catch (IOException e) { + logOutput.println("Error changing replication: " + e.getMessage()); + } } } - private void maybeChangePermissions(FileSystem fileSystem, - FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) throws IOException { + private void maybeChangePermissions(FileSystem destFileSystem, + FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) { Short newPermission = fileInfoDiff.getPermission(); if (newPermission != null && !newPermission.equals(srcFileStatus.getPermission().toShort())) { logOutput.printf("Updating file's permissions from '%s' to '%s'%n", srcFileStatus.getPermission().toShort(), newPermission); - fileSystem.setPermission(srcFileStatus.getPath(), new FsPermission(newPermission)); + + try { + destFileSystem.setPermission(srcFileStatus.getPath(), new FsPermission(newPermission)); + } catch (IOException e) { + logOutput.println("Error changing permissions: " + e.getMessage()); + } } } - private void maybeChangeTimes(FileSystem fileSystem, - FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) throws IOException { + private void maybeChangeTimes(FileSystem destFileSystem, + FileInfoDiff fileInfoDiff, FileStatus srcFileStatus) { long modificationTime = Optional.ofNullable(fileInfoDiff.getModificationTime()) .orElseGet(srcFileStatus::getModificationTime); @@ -116,18 +116,12 @@ private void maybeChangeTimes(FileSystem fileSystem, "and file's modification time from '%s' to '%s'%n", srcFileStatus.getAccessTime(), accessTime, srcFileStatus.getModificationTime(), modificationTime); - fileSystem.setTimes(srcFileStatus.getPath(), modificationTime, accessTime); - } - } - private void changeFileMetadata( - FileInfoDiff fileInfoDiff, FileSystem fileSystem) throws IOException { - Path srcPath = new Path(fileInfoDiff.getPath()); - FileStatus srcFileStatus = fileSystem.getFileStatus(srcPath); - - maybeChangeOwnerAndGroup(fileSystem, fileInfoDiff, srcFileStatus); - maybeChangeBlockReplication(fileSystem, fileInfoDiff, srcFileStatus); - maybeChangePermissions(fileSystem, fileInfoDiff, srcFileStatus); - maybeChangeTimes(fileSystem, fileInfoDiff, srcFileStatus); + try { + destFileSystem.setTimes(srcFileStatus.getPath(), modificationTime, accessTime); + } catch (IOException e) { + logOutput.println("Error changing times: " + e.getMessage()); + } + } } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/WriteFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/WriteFileAction.java index dc2207cf321..18bfd8f63ab 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/WriteFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/WriteFileAction.java @@ -20,10 +20,10 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.smartdata.action.Utils; import org.smartdata.action.annotation.ActionSignature; import java.util.Map; +import java.util.Optional; import java.util.Random; /** @@ -33,62 +33,65 @@ *

Arguments: file_path length [buffer_size, default=64k] */ @ActionSignature( - actionId = "write", - displayName = "write", - usage = - HdfsAction.FILE_PATH - + " $file " - + WriteFileAction.LENGTH - + " $length " - + WriteFileAction.BUF_SIZE - + " $size" + actionId = "write", + displayName = "write", + usage = + HdfsAction.FILE_PATH + + " $file " + + WriteFileAction.LENGTH + + " $length " + + WriteFileAction.BUF_SIZE + + " $size" ) -public class WriteFileAction extends HdfsAction { +public class WriteFileAction extends HdfsActionWithRemoteClusterSupport { public static final String LENGTH = "-length"; public static final String BUF_SIZE = "-bufSize"; - private String filePath; - private long length = -1; - private int bufferSize = 64 * 1024; + + public static final long DEFAULT_LENGTH = 1024; + public static final int DEFAULT_BUF_SIZE = 64 * 1024; + + private Path filePath; + private long length; + private int bufferSize; @Override public void init(Map args) { - withDefaultFs(); super.init(args); - this.filePath = args.get(FILE_PATH); - if (args.containsKey(LENGTH)) { - length = Long.parseLong(args.get(LENGTH)); - } - if (args.containsKey(BUF_SIZE)) { - this.bufferSize = Integer.parseInt(args.get(BUF_SIZE)); - } + this.filePath = getPathArg(FILE_PATH); + this.length = Optional.ofNullable(args.get(LENGTH)) + .map(Long::parseLong) + .orElse(DEFAULT_LENGTH); + + this.bufferSize = Optional.ofNullable(args.get(BUF_SIZE)) + .map(Integer::parseInt) + .orElse(DEFAULT_BUF_SIZE); } @Override - protected void execute() throws Exception { - if (filePath == null) { - throw new IllegalArgumentException("File parameter is missing! "); - } + protected void preExecute() throws Exception { + validateNonEmptyArg(FILE_PATH); + if (length == -1) { throw new IllegalArgumentException("Write Action provides wrong length! "); } - appendLog( - String.format( - "Action starts at %s : Write %s with length %s", - Utils.getFormatedCurrentTime(), filePath, length)); + } + + @Override + protected void execute(FileSystem fileSystem) throws Exception { + short replication = fileSystem.getServerDefaults(filePath).getReplication(); + + try (FSDataOutputStream out = fileSystem.create(filePath, replication)) { + // generate random data with given length + byte[] buffer = new byte[bufferSize]; + new Random().nextBytes(buffer); - Path path = new Path(filePath); - FileSystem fileSystem = path.getFileSystem(getContext().getConf()); - int replication = fileSystem.getServerDefaults(new Path(filePath)).getReplication(); - final FSDataOutputStream out = fileSystem.create(path, true, replication); - // generate random data with given length - byte[] buffer = new byte[bufferSize]; - new Random().nextBytes(buffer); - appendLog(String.format("Generate random data with length %d", length)); - for (long pos = 0; pos < length; pos += bufferSize) { - long writeLength = pos + bufferSize < length ? bufferSize : length - pos; - out.write(buffer, 0, (int) writeLength); + appendLog("Generate random data with length: " + length); + for (long pos = 0; pos < length; pos += bufferSize) { + long writeLength = pos + bufferSize < length ? bufferSize : length - pos; + out.write(buffer, 0, (int) writeLength); + } } - out.close(); + appendLog("Write Successfully!"); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java index 8e60ddf5934..805d76fa24b 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java @@ -134,7 +134,7 @@ public int executeMove(FileMovePlan plan, PrintStream resultOs, PrintStream logO * @throws Exception */ public int doMove(PrintStream resultOs, PrintStream logOs) throws Exception { - for (int retryTimes = 0; retryTimes < maxRetryTimes; retryTimes ++) { + for (int retryTimes = 0; retryTimes < maxRetryTimes; retryTimes++) { final AtomicInteger running = new AtomicInteger(0); for (final ReplicaMove replicaMove : allMoves) { moveExecutor.execute(new Runnable() { @@ -218,7 +218,7 @@ private void parseSchedulePlan(FileMovePlan plan) throws IOException { List targetStorageTypes = plan.getTargetStorageTypes(); List blockIds = plan.getBlockIds(); - for (int planIndex = 0; planIndex < blockIds.size(); planIndex ++) { + for (int planIndex = 0; planIndex < blockIds.size(); planIndex++) { // build block DBlock block = sourceBlockMap.get(blockIds.get(planIndex)); // build source diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java index e37923c5180..f6a70d202d7 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java @@ -26,10 +26,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.protocol.ClientProtocol; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.balancer.KeyManager; -import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java index f6fd67b4fa8..274f5ac8923 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos; -import org.apache.hadoop.hdfs.protocolPB.PBHelper; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.server.balancer.KeyManager; @@ -63,7 +62,7 @@ class ReplicaMove { private Configuration conf; public ReplicaMove(DBlock block, StorageGroup source, StorageGroup target, NameNodeConnector nnc, - SaslDataTransferClient saslClient, Configuration conf) { + SaslDataTransferClient saslClient, Configuration conf) { this.nnc = nnc; this.saslClient = saslClient; this.block = @@ -151,27 +150,28 @@ private void receiveResponse(DataInputStream in) throws IOException { DataTransferProtos.BlockOpResponseProto.parseFrom(CompatibilityHelperLoader.getHelper().getVintPrefixed(in)); while (response.getStatus() == DataTransferProtos.Status.IN_PROGRESS) { // read intermediate responses - response = DataTransferProtos.BlockOpResponseProto.parseFrom(CompatibilityHelperLoader.getHelper().getVintPrefixed(in)); + response = + DataTransferProtos.BlockOpResponseProto.parseFrom(CompatibilityHelperLoader.getHelper().getVintPrefixed(in)); } String logInfo = "block move is failed"; checkBlockOpStatus(response, logInfo); } public static void checkBlockOpStatus( - DataTransferProtos.BlockOpResponseProto response, - String logInfo) throws IOException { + DataTransferProtos.BlockOpResponseProto response, + String logInfo) throws IOException { if (response.getStatus() != DataTransferProtos.Status.SUCCESS) { if (response.getStatus() == DataTransferProtos.Status.ERROR_ACCESS_TOKEN) { throw new InvalidBlockTokenException( - "Got access token error" - + ", status message " + response.getMessage() - + ", " + logInfo + "Got access token error" + + ", status message " + response.getMessage() + + ", " + logInfo ); } else { throw new IOException( - "Got error" - + ", status message " + response.getMessage() - + ", " + logInfo + "Got error" + + ", status message " + response.getMessage() + + ", " + logInfo ); } } @@ -189,7 +189,7 @@ public static int failedMoves(List allMoves) { public static boolean allMoveFinished(List allMoves) { for (ReplicaMove move : allMoves) { - if (!move.status.isFinished()){ + if (!move.status.isFinished()) { return false; } } @@ -220,7 +220,7 @@ public static void countStatus(List allMoves, int[] ret) { * @return number of remaining moves */ public static int refreshMoverList(List allMoves) { - for (Iterator it = allMoves.iterator(); it.hasNext();) { + for (Iterator it = allMoves.iterator(); it.hasNext(); ) { ReplicaMove replicaMove = it.next(); if (replicaMove.status.isSuccessful()) { it.remove(); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseDfsClientCache.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseFileSystemCache.java similarity index 75% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseDfsClientCache.java rename to smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseFileSystemCache.java index a79ec2436eb..45251285b1d 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseDfsClientCache.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/BaseFileSystemCache.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.smartdata.hdfs.HadoopUtil; import java.io.IOException; @@ -37,14 +37,14 @@ @Slf4j -public abstract class BaseDfsClientCache implements DfsClientCache { +public abstract class BaseFileSystemCache implements FileSystemCache { - private final Cache clientCache; + private final Cache fileSystemCache; private final ScheduledExecutorService evictionHandlerExecutor; - public BaseDfsClientCache(Duration keyTtl) { + public BaseFileSystemCache(Duration keyTtl) { this.evictionHandlerExecutor = Executors.newSingleThreadScheduledExecutor(); - this.clientCache = Caffeine.newBuilder() + this.fileSystemCache = Caffeine.newBuilder() .expireAfterAccess(keyTtl) .removalListener(this::onEntryRemoved) .scheduler(Scheduler.forScheduledExecutorService(evictionHandlerExecutor)) @@ -55,27 +55,27 @@ public BaseDfsClientCache(Duration keyTtl) { public T get(Configuration config, InetSocketAddress ssmMasterAddress) throws IOException { CacheKey cacheKey = new CacheKey(ssmMasterAddress, HadoopUtil.getNameNodeUri(config)); - return clientCache.get(cacheKey, key -> createDfsClient(config, key)); + return fileSystemCache.get(cacheKey, key -> createFileSystem(config, key)); } @Override public void close() throws IOException { - clientCache.invalidateAll(); - clientCache.cleanUp(); + fileSystemCache.invalidateAll(); + fileSystemCache.cleanUp(); evictionHandlerExecutor.shutdown(); } - protected abstract T createDfsClient(Configuration config, CacheKey cacheKey); + protected abstract T createFileSystem(Configuration config, CacheKey cacheKey); private void onEntryRemoved(CacheKey key, T value, RemovalCause removalCause) { - Optional.ofNullable(value).ifPresent(this::closeClient); + Optional.ofNullable(value).ifPresent(this::closeFileSystem); } - private void closeClient(T dfsClient) { + private void closeFileSystem(T fileSystem) { try { - dfsClient.close(); + fileSystem.close(); } catch (IOException exception) { - log.error("Error closing cached dfsClient after expiration", exception); + log.error("Error closing cached fileSystem after expiration", exception); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingDfsClientProvider.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingDfsClientProvider.java deleted file mode 100644 index 6676711c080..00000000000 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingDfsClientProvider.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.client; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSClient; -import org.smartdata.hdfs.HadoopUtil; -import org.smartdata.hdfs.action.HdfsAction; -import org.smartdata.utils.StringUtil; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.time.Duration; - -import static org.smartdata.conf.SmartConfKeys.SMART_ACTION_CLIENT_CACHE_TTL_DEFAULT; -import static org.smartdata.conf.SmartConfKeys.SMART_ACTION_CLIENT_CACHE_TTL_KEY; -import static org.smartdata.conf.SmartConfKeys.SMART_CLIENT_CONCURRENT_REPORT_ENABLED; -import static org.smartdata.utils.ConfigUtil.getSsmMasterRpcAddress; - -@Slf4j -@RequiredArgsConstructor -public class CachingDfsClientProvider implements DfsClientProvider { - - private final DfsClientCache smartClientCache; - private final DfsClientCache hdfsClientCache; - - public CachingDfsClientProvider(Configuration config) { - String cacheKeyTtl = config.get( - SMART_ACTION_CLIENT_CACHE_TTL_KEY, SMART_ACTION_CLIENT_CACHE_TTL_DEFAULT); - Duration cacheKeyTtlDuration = Duration.ofMillis( - StringUtil.parseTimeString(cacheKeyTtl) - ); - - this.smartClientCache = new SmartDfsClientCache(cacheKeyTtlDuration); - this.hdfsClientCache = new HdfsClientCache(cacheKeyTtlDuration); - } - - @Override - public DFSClient provide(Configuration config, HdfsAction.DfsClientType clientType) - throws IOException { - InetSocketAddress ssmMasterAddress = getSsmMasterRpcAddress(config); - - return clientType == HdfsAction.DfsClientType.SMART - ? smartClientCache.get(config, ssmMasterAddress) - // we don't rely on SSM in case of pure HDFS client - : hdfsClientCache.get(config, null); - } - - @Override - public void close() throws IOException { - smartClientCache.close(); - hdfsClientCache.close(); - } - - private static class SmartDfsClientCache extends BaseDfsClientCache { - - private SmartDfsClientCache(Duration keyTtl) { - super(keyTtl); - } - - @Override - protected SmartDFSClient createDfsClient(Configuration config, CacheKey cacheKey) { - try { - Configuration clientConfig = new Configuration(config); - // smart server always have only 1 address set - // in the "smart.server.rpc.address" option - clientConfig.setBoolean(SMART_CLIENT_CONCURRENT_REPORT_ENABLED, false); - return new SmartDFSClient( - cacheKey.getNameNodeUri(), config, cacheKey.getSsmMasterAddress()); - } catch (IOException exception) { - throw new RuntimeException("Error creating smart client", exception); - } - } - } - - private static class HdfsClientCache extends BaseDfsClientCache { - private HdfsClientCache(Duration keyTtl) { - super(keyTtl); - } - - @Override - protected DFSClient createDfsClient(Configuration config, CacheKey cacheKey) { - try { - return HadoopUtil.getDFSClient(cacheKey.getNameNodeUri(), config); - } catch (IOException exception) { - throw new RuntimeException("Error creating hdfs client", exception); - } - } - } -} diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingLocalFileSystemProvider.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingLocalFileSystemProvider.java new file mode 100644 index 00000000000..b171969d3f2 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/CachingLocalFileSystemProvider.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.client; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.smartdata.hadoop.filesystem.SmartFileSystem; +import org.smartdata.hdfs.HadoopUtil; +import org.smartdata.hdfs.action.HdfsAction; +import org.smartdata.utils.StringUtil; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.time.Duration; + +import static org.smartdata.conf.SmartConfKeys.SMART_ACTION_CLIENT_CACHE_TTL_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_ACTION_CLIENT_CACHE_TTL_KEY; +import static org.smartdata.conf.SmartConfKeys.SMART_CLIENT_CONCURRENT_REPORT_ENABLED; +import static org.smartdata.hdfs.HadoopUtil.doAsCurrentUser; +import static org.smartdata.utils.ConfigUtil.getSsmMasterRpcAddress; + +@Slf4j +@RequiredArgsConstructor +public class CachingLocalFileSystemProvider implements LocalFileSystemProvider { + + private final FileSystemCache smartFsCache; + private final FileSystemCache defaultFsCache; + + public CachingLocalFileSystemProvider(Configuration config) { + String cacheKeyTtl = config.get( + SMART_ACTION_CLIENT_CACHE_TTL_KEY, SMART_ACTION_CLIENT_CACHE_TTL_DEFAULT); + Duration cacheKeyTtlDuration = Duration.ofMillis( + StringUtil.parseTimeString(cacheKeyTtl)); + + this.smartFsCache = new SmartFileSystemCache(cacheKeyTtlDuration); + this.defaultFsCache = new DefaultFileSystemCache(cacheKeyTtlDuration); + } + + @Override + public DistributedFileSystem provide(Configuration config, HdfsAction.FsType fsType) + throws IOException { + InetSocketAddress ssmMasterAddress = getSsmMasterRpcAddress(config); + + return fsType == HdfsAction.FsType.SMART + ? smartFsCache.get(config, ssmMasterAddress) + // we don't rely on SSM in case of pure HDFS client + : defaultFsCache.get(config, null); + } + + @Override + public void close() throws IOException { + smartFsCache.close(); + defaultFsCache.close(); + } + + private static class SmartFileSystemCache extends BaseFileSystemCache { + + private SmartFileSystemCache(Duration keyTtl) { + super(keyTtl); + } + + @Override + protected SmartFileSystem createFileSystem(Configuration config, CacheKey cacheKey) { + try { + Configuration fsConfig = new Configuration(config); + // smart server always have only 1 address set + // in the "smart.server.rpc.address" option + fsConfig.setBoolean(SMART_CLIENT_CONCURRENT_REPORT_ENABLED, false); + return doAsCurrentUser(() -> createSmartFileSystem(config, cacheKey)); + } catch (IOException exception) { + throw new RuntimeException("Error creating smart file system", exception); + } + } + + private SmartFileSystem createSmartFileSystem( + Configuration config, CacheKey cacheKey) throws IOException { + SmartDFSClient smartDfsClient = new SmartDFSClient( + cacheKey.getNameNodeUri(), config, cacheKey.getSsmMasterAddress()); + SmartFileSystem fileSystem = new SmartFileSystem(smartDfsClient); + fileSystem.initialize(cacheKey.getNameNodeUri(), config); + return fileSystem; + } + } + + private static class DefaultFileSystemCache extends BaseFileSystemCache { + private DefaultFileSystemCache(Duration keyTtl) { + super(keyTtl); + } + + @Override + protected DistributedFileSystem createFileSystem(Configuration config, CacheKey cacheKey) { + try { + return HadoopUtil.getDistributedFileSystem(cacheKey.getNameNodeUri(), config); + } catch (IOException exception) { + throw new RuntimeException("Error creating default file system", exception); + } + } + } +} diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientCache.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/FileSystemCache.java similarity index 88% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientCache.java rename to smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/FileSystemCache.java index b6589a7886f..a918b9f4c15 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientCache.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/FileSystemCache.java @@ -18,12 +18,12 @@ package org.smartdata.hdfs.client; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; -public interface DfsClientCache extends Closeable { +public interface FileSystemCache extends Closeable { T get(Configuration config, InetSocketAddress ssmMasterAddress) throws IOException; } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientProvider.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/LocalFileSystemProvider.java similarity index 81% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientProvider.java rename to smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/LocalFileSystemProvider.java index e9d86fe8e8f..5a6a8ca431a 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/DfsClientProvider.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/client/LocalFileSystemProvider.java @@ -18,13 +18,13 @@ package org.smartdata.hdfs.client; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.smartdata.hdfs.action.HdfsAction; import java.io.Closeable; import java.io.IOException; -public interface DfsClientProvider extends Closeable { - DFSClient provide( - Configuration config, HdfsAction.DfsClientType clientType) throws IOException; +public interface LocalFileSystemProvider extends Closeable { + DistributedFileSystem provide( + Configuration config, HdfsAction.FsType fsType) throws IOException; } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestAppendFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestAppendFileAction.java index 1fe63ade969..399bd6c3f45 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestAppendFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestAppendFileAction.java @@ -35,7 +35,7 @@ private void appendFile(String src, long length) throws IOException { args.put(AppendFileAction.FILE_PATH, src); args.put(AppendFileAction.LENGTH, "" + length); AppendFileAction appendFileAction = new AppendFileAction(); - appendFileAction.setDfsClient(dfsClient); + appendFileAction.setLocalFileSystem(dfs); appendFileAction.setContext(smartContext); appendFileAction.init(args); appendFileAction.run(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCacheFile.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCacheFile.java index e1423706cc8..64044bfe474 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCacheFile.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCacheFile.java @@ -48,14 +48,14 @@ public void testCacheFile() throws IOException { CacheFileAction cacheAction = new CacheFileAction(); cacheAction.setContext(smartContext); - cacheAction.setDfsClient(dfsClient); - Map args = new HashMap(); + cacheAction.setLocalFileSystem(dfs); + Map args = new HashMap<>(); args.put(CacheFileAction.FILE_PATH, file); cacheAction.init(args); try { - Assert.assertFalse(cacheAction.isCached(file)); + Assert.assertFalse(cacheAction.isFileCached()); cacheAction.run(); - Assert.assertTrue(cacheAction.isCached(file)); + Assert.assertTrue(cacheAction.isFileCached()); Assert.assertTrue(cacheAction.getExpectedAfterRun()); } catch (Exception e) { throw new RuntimeException(e); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckStorageAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckStorageAction.java index ce323b229d3..ab464d7989b 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckStorageAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckStorageAction.java @@ -34,7 +34,7 @@ public class TestCheckStorageAction extends MiniClusterHarness { @Test public void testCheckStorageAction() throws IOException { CheckStorageAction checkStorageAction = new CheckStorageAction(); - checkStorageAction.setDfsClient(dfsClient); + checkStorageAction.setLocalFileSystem(dfs); checkStorageAction.setContext(smartContext); final String file = "/testPath/file1"; dfsClient.mkdirs("/testPath", null, true); @@ -55,7 +55,7 @@ public void testCheckStorageAction() throws IOException { @Test public void testCheckStorageActionWithWrongFileName() throws IOException { CheckStorageAction checkStorageAction = new CheckStorageAction(); - checkStorageAction.setDfsClient(dfsClient); + checkStorageAction.setLocalFileSystem(dfs); checkStorageAction.setContext(smartContext); final String file = "/testPath/wrongfile"; diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckSumAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckSumAction.java index 3667000320d..cc85d247544 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckSumAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCheckSumAction.java @@ -24,6 +24,7 @@ import org.smartdata.action.ActionException; import org.smartdata.hdfs.MiniClusterHarness; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -45,7 +46,7 @@ public class TestCheckSumAction extends MiniClusterHarness { @Before public void setUp() { action = new CheckSumAction(); - action.setDfsClient(dfsClient); + action.setLocalFileSystem(dfs); action.setContext(smartContext); } @@ -105,8 +106,8 @@ public void testThrowIfFileNotFound() { Throwable error = action.getActionStatus().getThrowable(); assertNotNull(error); - assertTrue(error instanceof ActionException); - assertEquals("Provided file doesn't exist: /unknownFile", error.getMessage()); + assertTrue(error instanceof FileNotFoundException); + assertEquals("File does not exist: /unknownFile", error.getMessage()); } @Test @@ -120,7 +121,7 @@ public void testThrowIfDirectoryNotFound() { Throwable error = action.getActionStatus().getThrowable(); assertNotNull(error); assertTrue(error instanceof ActionException); - assertEquals("Provided directory doesn't exist: /unknownDir/", error.getMessage()); + assertEquals("Provided directory doesn't exist: /unknownDir", error.getMessage()); } private List getChecksumFiles() { diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCompressionAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCompressionAction.java index 2c973cb07c0..12daab75539 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCompressionAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCompressionAction.java @@ -41,7 +41,7 @@ public void init() throws Exception { protected void compression(String filePath, String bufferSize) throws IOException { CompressionAction compressionAction = new CompressionAction(); - compressionAction.setDfsClient(dfsClient); + compressionAction.setLocalFileSystem(dfs); compressionAction.setContext(smartContext); Map args = new HashMap<>(); args.put(compressionAction.FILE_PATH, filePath); @@ -67,7 +67,7 @@ public void testInit() throws IOException { // args.put(CompressionAction.COMPRESS_IMPL,"Bzip2"); // args.put(CompressionAction.COMPRESS_IMPL,"Zlib"); CompressionAction compressionAction = new CompressionAction(); - compressionAction.setDfsClient(dfsClient); + compressionAction.setLocalFileSystem(dfs); compressionAction.setContext(smartContext); compressionAction.init(args); } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestConcatFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestConcatFileAction.java index 7dbb5096381..23b451312fc 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestConcatFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestConcatFileAction.java @@ -18,106 +18,76 @@ package org.smartdata.hdfs.action; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.smartdata.hdfs.MiniClusterHarness; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; /** * Test for concatFileAction */ +@RunWith(Parameterized.class) public class TestConcatFileAction extends MiniClusterHarness { - @Test - public void testRemoteFileConcat() throws IOException { - final String srcPath = "/testConcat"; - final String file1 = "file1"; - final String file2 = "file2"; - final String target = "/target"; - dfs.mkdirs(new Path(srcPath)); - dfs.mkdirs(new Path(target)); - //write to DISK - //write 50 Bytes to file1 and 50 Byte to file2. then concat them - FSDataOutputStream out1 = dfs.create(new Path(srcPath + "/" + file1)); - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - out1.writeByte(1); - } - out1.close(); - - out1 = dfs.create(new Path(srcPath + "/" + file2)); - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - out1.writeByte(2); - } - out1.close(); + @Parameterized.Parameter + public boolean useAbsolutePath; - ConcatFileAction concatFileAction = new ConcatFileAction(); - concatFileAction.setDfsClient(dfsClient); - concatFileAction.setContext(smartContext); - Map args = new HashMap<>(); - args.put(CopyFileAction.FILE_PATH, dfs.getUri() + srcPath + "/" + - file1 + "," + dfs.getUri() + srcPath + "/" + "file2"); - args.put(ConcatFileAction.DEST_PATH, dfs.getUri() + target); - concatFileAction.init(args); - concatFileAction.run(); - - Assert.assertTrue(concatFileAction.getExpectedAfterRun()); - Assert.assertTrue(dfsClient.exists(target)); - //read and check file - FSDataInputStream in = dfs.open(new Path(target),50); - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - Assert.assertTrue(in.readByte() == 1); - } - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - Assert.assertTrue(in.readByte() == 2); - } + @Parameterized.Parameters(name = "useAbsolutePath = {0}") + public static Object[] parameters() { + return new Object[]{true, false}; } @Test - public void testLocalFileConcat() throws IOException { - final String srcPath = "/testConcat"; - final String file1 = "file1"; - final String file2 = "file2"; - final String target = "/target"; + public void testFileConcat() throws IOException { + String srcPath = "/testConcat"; + Path file1 = new Path(srcPath, "file1"); + Path file2 = new Path(srcPath, "file2"); + String target = "/target"; - dfs.mkdirs(new Path(srcPath)); - dfs.mkdirs(new Path(target)); - //write to DISK - FSDataOutputStream out1 = dfs.create(new Path(srcPath + "/" + file1)); - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - out1.writeByte(1); + if (useAbsolutePath) { + file1 = dfs.makeQualified(file1); + file2 = dfs.makeQualified(file2); } - out1.close(); - out1 = dfs.create(new Path(srcPath + "/" + file2)); - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - out1.writeByte(2); - } - out1.close(); + dfs.mkdirs(new Path(srcPath)); + DFSTestUtil.writeFile(dfs, new Path(target), ""); + //write to DISK + //write 50 Bytes to file1 and 50 Byte to file2. then concat them + createFileWithContent(file1, (byte) 1); + createFileWithContent(file2, (byte) 2); ConcatFileAction concatFileAction = new ConcatFileAction(); - concatFileAction.setDfsClient(dfsClient); + concatFileAction.setLocalFileSystem(dfs); concatFileAction.setContext(smartContext); Map args = new HashMap<>(); - args.put(CopyFileAction.FILE_PATH, srcPath + "/" + - file1 + "," + srcPath + "/" + "file2"); - args.put(ConcatFileAction.DEST_PATH, target); + args.put(CopyFileAction.FILE_PATH, file1 + "," + file2); + args.put(ConcatFileAction.DEST_PATH, dfs.getUri() + target); concatFileAction.init(args); concatFileAction.run(); Assert.assertTrue(concatFileAction.getExpectedAfterRun()); Assert.assertTrue(dfsClient.exists(target)); //read and check file - FSDataInputStream in = dfs.open(new Path(target),50); + FSDataInputStream in = dfs.open(new Path(target), 50); for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - Assert.assertTrue(in.readByte() == 1); + Assert.assertEquals(1, in.readByte()); } for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - Assert.assertTrue(in.readByte() == 2); + Assert.assertEquals(2, in.readByte()); } } + + private void createFileWithContent(Path path, byte content) throws IOException { + byte[] contentBytes = new byte[DEFAULT_BLOCK_SIZE]; + Arrays.fill(contentBytes, content); + DFSTestUtil.writeFile(dfs, path, contentBytes); + } } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyDirectoryAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyDirectoryAction.java index 4113dc2af45..86339b91752 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyDirectoryAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyDirectoryAction.java @@ -41,7 +41,7 @@ public class TestCopyDirectoryAction extends MultiClusterHarness { @Before public void setupAction() { copyDirectoryAction = new CopyDirectoryAction(); - copyDirectoryAction.setDfsClient(dfsClient); + copyDirectoryAction.setLocalFileSystem(dfs); copyDirectoryAction.setContext(smartContext); } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyFileAction.java index 789daaa6319..e3f76d8d21a 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestCopyFileAction.java @@ -53,7 +53,7 @@ private void copyFile(Path src, Path dest, long length, private void copyFile(Path src, Path dest, long length, long offset, Set preserveAttributes) throws Exception { CopyFileAction copyFileAction = new CopyFileAction(); - copyFileAction.setDfsClient(dfsClient); + copyFileAction.setLocalFileSystem(dfs); copyFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(CopyFileAction.FILE_PATH, src.toUri().getPath()); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDeleteFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDeleteFileAction.java index 76c5270e0d2..4bfe7352a2c 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDeleteFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDeleteFileAction.java @@ -46,7 +46,7 @@ public void testLocalFileDelete() throws IOException { Assert.assertTrue(dfsClient.exists(srcPath + "/" + file1)); DeleteFileAction deleteFileAction = new DeleteFileAction(); - deleteFileAction.setDfsClient(dfsClient); + deleteFileAction.setLocalFileSystem(dfs); deleteFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(DeleteFileAction.FILE_PATH, srcPath + "/" + file1); @@ -70,7 +70,7 @@ public void testRemoteFileDelete() throws Exception { Assert.assertTrue(dfsClient.exists(srcPath + "/" + file1)); DeleteFileAction deleteFileAction = new DeleteFileAction(); - deleteFileAction.setDfsClient(dfsClient); + deleteFileAction.setLocalFileSystem(dfs); deleteFileAction.setContext(smartContext); Map args = new HashMap<>(); // Destination with "hdfs" prefix diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDistCpAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDistCpAction.java index bff69ce2e0c..f2c26dffbd7 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDistCpAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestDistCpAction.java @@ -55,7 +55,7 @@ public class TestDistCpAction extends MiniClusterHarness { public DistCpAction createAction(Map args) { DistCpAction distCpAction = new DistCpAction(); - distCpAction.setDfsClient(dfsClient); + distCpAction.setLocalFileSystem(dfs); distCpAction.setContext(smartContext); distCpAction.init(args); return distCpAction; diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestListFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestListFileAction.java index 0d8ae70bd7c..fb65d0ee606 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestListFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestListFileAction.java @@ -63,7 +63,7 @@ public void buildAction() { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); listFileAction = new ListFileAction(); - listFileAction.setDfsClient(dfsClient); + listFileAction.setLocalFileSystem(dfs); listFileAction.setContext(smartContext); } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMergeFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMergeFileAction.java index a9c4ad56269..f53544589c8 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMergeFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMergeFileAction.java @@ -20,10 +20,14 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.junit.Assert; import org.junit.Test; +import org.junit.runners.Parameterized; import org.smartdata.hdfs.MiniClusterHarness; +import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -32,80 +36,38 @@ */ public class TestMergeFileAction extends MiniClusterHarness { - @Test - public void testLocalFileMerge() throws Exception { - final String srcPath = "/testMerge"; - final String file1 = "file1"; - final String file2 = "file2"; - final String target = "/target"; - - dfs.mkdirs(new Path(srcPath)); - dfs.mkdirs(new Path(target)); - //write to DISK - //write 40 Bytes to file1 and 50 Byte to file2. then concat them - FSDataOutputStream out1 = dfs.create(new Path(srcPath + "/" + file1)); - for (int i = 0; i < 60; i++) { - out1.writeByte(1); - } - out1.close(); - - out1 = dfs.create(new Path(srcPath + "/" + file2)); - for (int i = 0; i < 50; i++) { - out1.writeByte(2); - } - out1.close(); - - MergeFileAction mergeFileAction = new MergeFileAction(); - mergeFileAction.setDfsClient(dfsClient); - mergeFileAction.setContext(smartContext); - Map args = new HashMap<>(); - args.put(MergeFileAction.FILE_PATH, srcPath + "/" + - file1 + "," + dfs.getUri() + srcPath + "/" + "file2"); - args.put(MergeFileAction.DEST_PATH, target); - mergeFileAction.init(args); - mergeFileAction.run(); + @Parameterized.Parameter + public boolean useAbsolutePath; - Assert.assertTrue(mergeFileAction.getExpectedAfterRun()); - Assert.assertTrue(dfsClient.exists(target)); - //read and check file - FSDataInputStream in = dfs.open(new Path(target),50); - for (int i = 0; i < 60; i++) { - Assert.assertTrue(in.readByte() == 1); - } - for (int i = 0; i < 50; i++) { - Assert.assertTrue(in.readByte() == 2); - } + @Parameterized.Parameters(name = "useAbsolutePath = {0}") + public static Object[] parameters() { + return new Object[]{true, false}; } @Test - public void testRemoteFileMerge() throws Exception { - final String srcPath = "/testMerge"; - final String file1 = "file1"; - final String file2 = "file2"; - final String target = "/target"; + public void testFileMerge() throws IOException { + String srcPath = "/testConcat"; + Path file1 = new Path(srcPath, "file1"); + Path file2 = new Path(srcPath, "file2"); + String target = "/target"; - dfs.mkdirs(new Path(srcPath)); - dfs.mkdirs(new Path(target)); - //write to DISK - //write 40 Bytes to file1 and 50 Byte to file2. then concat them - FSDataOutputStream out1 = dfs.create(new Path(srcPath + "/" + file1)); - for (int i = 0; i < 60; i++) { - out1.writeByte(1); + if (useAbsolutePath) { + file1 = dfs.makeQualified(file1); + file2 = dfs.makeQualified(file2); } - out1.close(); - out1 = dfs.create(new Path(srcPath + "/" + file2)); - for (int i = 0; i < 50; i++) { - out1.writeByte(2); - } - out1.close(); + dfs.mkdirs(new Path(srcPath)); + DFSTestUtil.writeFile(dfs, new Path(target), ""); + //write to DISK + //write 50 Bytes to file1 and 50 Byte to file2. then concat them + createFileWithContent(file1, (byte) 1); + createFileWithContent(file2, (byte) 2); MergeFileAction mergeFileAction = new MergeFileAction(); - mergeFileAction.setDfsClient(dfsClient); + mergeFileAction.setLocalFileSystem(dfs); mergeFileAction.setContext(smartContext); Map args = new HashMap<>(); - args.put(MergeFileAction.FILE_PATH, dfs.getUri() + srcPath + "/" + - file1 + "," + dfs.getUri() + srcPath + "/" + "file2"); + args.put(MergeFileAction.FILE_PATH, file1 + "," + file2); args.put(MergeFileAction.DEST_PATH, dfs.getUri() + target); mergeFileAction.init(args); mergeFileAction.run(); @@ -113,12 +75,18 @@ public void testRemoteFileMerge() throws Exception { Assert.assertTrue(mergeFileAction.getExpectedAfterRun()); Assert.assertTrue(dfsClient.exists(target)); //read and check file - FSDataInputStream in = dfs.open(new Path(target),50); - for (int i = 0; i < 60; i++) { - Assert.assertTrue(in.readByte() == 1); + FSDataInputStream in = dfs.open(new Path(target), 50); + for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { + Assert.assertEquals(1, in.readByte()); } - for (int i = 0; i < 50; i++) { - Assert.assertTrue(in.readByte() == 2); + for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { + Assert.assertEquals(2, in.readByte()); } } + + private void createFileWithContent(Path path, byte content) throws IOException { + byte[] contentBytes = new byte[DEFAULT_BLOCK_SIZE]; + Arrays.fill(contentBytes, content); + DFSTestUtil.writeFile(dfs, path, contentBytes); + } } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMetaDataAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMetaDataAction.java index 457349cd204..971daac2110 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMetaDataAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestMetaDataAction.java @@ -87,7 +87,7 @@ private FileStatus updateMetadata(Map args) throws IOException { private void runAction(Map args) throws UnsupportedEncodingException { MetaDataAction metaFileAction = new MetaDataAction(); - metaFileAction.setDfsClient(dfsClient); + metaFileAction.setLocalFileSystem(dfs); metaFileAction.setContext(smartContext); metaFileAction.init(args); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestReadFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestReadFileAction.java index c76392e26ec..c8993da07b8 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestReadFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestReadFileAction.java @@ -48,7 +48,7 @@ public void testExecute() throws IOException { int size = 66560; writeFile(filePath, size); ReadFileAction readFileAction = new ReadFileAction(); - readFileAction.setDfsClient(dfsClient); + readFileAction.setLocalFileSystem(dfs); readFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(ReadFileAction.FILE_PATH, filePath); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestRenameFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestRenameFileAction.java index 772869c8650..4be47ae9626 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestRenameFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestRenameFileAction.java @@ -48,7 +48,7 @@ public void testLocalFileRename() throws IOException { Assert.assertTrue(dfsClient.exists(srcPath + "/" + file1)); RenameFileAction renameFileAction = new RenameFileAction(); - renameFileAction.setDfsClient(dfsClient); + renameFileAction.setLocalFileSystem(dfs); renameFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(RenameFileAction.FILE_PATH, srcPath + "/" + file1); @@ -74,7 +74,7 @@ public void testRemoteFileRename() throws IOException { DFSTestUtil.writeFile(dfs, new Path(dfs.getUri() + srcPath + "/" + file1), "testCopy1"); RenameFileAction renameFileAction = new RenameFileAction(); - renameFileAction.setDfsClient(dfsClient); + renameFileAction.setLocalFileSystem(dfs); renameFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(RenameFileAction.FILE_PATH , dfs.getUri() + srcPath + "/" +file1); @@ -113,7 +113,7 @@ public void testFileRenameFailed() throws Exception { private void verifyRenameFailed(String src, String dest) throws Exception { RenameFileAction renameFileAction = new RenameFileAction(); - renameFileAction.setDfsClient(dfsClient); + renameFileAction.setLocalFileSystem(dfs); renameFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(RenameFileAction.FILE_PATH , src); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetStoragePolicyAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetStoragePolicyAction.java index fa954ca414b..3b984f729e0 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetStoragePolicyAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetStoragePolicyAction.java @@ -67,7 +67,7 @@ public void testDifferentPolicies() throws IOException { private byte setStoragePolicy(String file, String storagePolicy) throws IOException { SetStoragePolicyAction action = new SetStoragePolicyAction(); - action.setDfsClient(dfsClient); + action.setLocalFileSystem(dfs); action.setContext(smartContext); Map args = new HashMap(); args.put(SetStoragePolicyAction.FILE_PATH, file); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetXAttrAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetXAttrAction.java index 242191f0b48..643c59f7fec 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetXAttrAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSetXAttrAction.java @@ -36,7 +36,7 @@ protected void setXAttr(String srcPath, String attName, String attValue) throws args.put(SetXAttrAction.ATT_NAME, "" + attName); args.put(SetXAttrAction.ATT_VALUE, "" + attValue); SetXAttrAction setXAttrAction = new SetXAttrAction(); - setXAttrAction.setDfsClient(dfsClient); + setXAttrAction.setLocalFileSystem(dfs); setXAttrAction.setContext(smartContext); setXAttrAction.init(args); setXAttrAction.run(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSmallFileCompactAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSmallFileCompactAction.java index 914513ffdfb..8b3699bb427 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSmallFileCompactAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestSmallFileCompactAction.java @@ -63,7 +63,7 @@ public void testAction() throws Exception { Assert.assertTrue(dfsClient.exists("/test/small_files/file_2")); SmallFileCompactAction smallFileCompactAction = new SmallFileCompactAction(); - smallFileCompactAction.setDfsClient(dfsClient); + smallFileCompactAction.setLocalFileSystem(dfs); smallFileCompactAction.setContext(smartContext); Map args = new HashMap<>(); args.put(SmallFileCompactAction.FILE_PATH , new Gson().toJson(smallFileList)); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestTruncate0Action.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestTruncate0Action.java deleted file mode 100644 index b4a172f2846..00000000000 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestTruncate0Action.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.XAttrSetFlag; -import org.junit.Assert; -import org.junit.Test; -import org.smartdata.hdfs.MiniClusterHarness; - -import java.io.IOException; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; - -public class TestTruncate0Action extends MiniClusterHarness { - @Test - public void testSetFileLen() throws IOException, InterruptedException { - final String srcPath = "/test"; - final String file = "file"; - - dfs.mkdirs(new Path(srcPath)); - FSDataOutputStream out = dfs.create(new Path(srcPath + "/" + file)); - - for (int i = 0; i < 50; i++) { - out.writeByte(1); - } - - out.close(); - - dfs.setXAttr(new Path(srcPath + "/" + file), "user.coldloc", "test".getBytes(), - EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - FileStatus oldFileStatus = dfs.getFileStatus(new Path(srcPath + "/" + file)); - Map oldXAttrs = dfs.getXAttrs(new Path(srcPath + "/" + file)); - - Truncate0Action setLen2ZeroAction = new Truncate0Action(); - setLen2ZeroAction.setDfsClient(dfsClient); - setLen2ZeroAction.setContext(smartContext); - Map args = new HashMap<>(); - args.put(Truncate0Action.FILE_PATH, srcPath + "/" + file); - - setLen2ZeroAction.init(args); - setLen2ZeroAction.run(); - Assert.assertTrue(setLen2ZeroAction.getExpectedAfterRun()); - - FileStatus newFileStatus = dfs.getFileStatus(new Path(srcPath + "/" + file)); - Map newXAttrs = dfs.getXAttrs(new Path(srcPath + "/" + file)); - - Assert.assertTrue(newFileStatus.getLen()==0); - Assert.assertTrue(oldFileStatus.getOwner().equals(newFileStatus.getOwner())); - Assert.assertTrue(oldFileStatus.getGroup().equals(newFileStatus.getGroup())); - Assert.assertTrue(oldFileStatus.getPermission().equals(newFileStatus.getPermission())); - Assert.assertTrue(oldFileStatus.getReplication()==newFileStatus.getReplication()); - //Assert.assertTrue(oldFileStatus.getAccessTime()==newFileStatus.getAccessTime()); - //Assert.assertTrue(oldFileStatus.getModificationTime()==newFileStatus.getModificationTime()); - Assert.assertTrue(oldXAttrs.size()==newXAttrs.size()); - for(Map.Entry oldXAttr : oldXAttrs.entrySet()){ - Assert.assertTrue(Arrays.equals(oldXAttr.getValue(),newXAttrs.get(oldXAttr.getKey()))); - } - } -} diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestUncacheFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestUncacheFileAction.java index 46e632f470e..97ab64d34da 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestUncacheFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestUncacheFileAction.java @@ -46,27 +46,27 @@ public void testUncacheFile() throws Exception { CacheScheduler.createCachePool(dfsClient); CacheFileAction cacheFileAction = new CacheFileAction(); - cacheFileAction.setDfsClient(dfsClient); + cacheFileAction.setLocalFileSystem(dfs); cacheFileAction.setContext(smartContext); - Map argsCache = new HashMap(); + Map argsCache = new HashMap<>(); argsCache.put(CacheFileAction.FILE_PATH, file); cacheFileAction.init(argsCache); UncacheFileAction uncacheFileAction = new UncacheFileAction(); - uncacheFileAction.setDfsClient(dfsClient); + uncacheFileAction.setLocalFileSystem(dfs); uncacheFileAction.setContext(smartContext); - Map argsUncache = new HashMap(); + Map argsUncache = new HashMap<>(); argsUncache.put(UncacheFileAction.FILE_PATH, file); uncacheFileAction.init(argsUncache); cacheFileAction.run(); Assert.assertTrue(cacheFileAction.getExpectedAfterRun()); - Assert.assertTrue(cacheFileAction.isCached(file)); + Assert.assertTrue(cacheFileAction.isFileCached()); uncacheFileAction.run(); Assert.assertTrue(uncacheFileAction.getExpectedAfterRun()); - Assert.assertFalse(cacheFileAction.isCached(file)); + Assert.assertFalse(cacheFileAction.isFileCached()); } @Test @@ -84,18 +84,18 @@ public void testUncacheNoncachedFile() throws Exception { CacheScheduler.createCachePool(dfsClient); UncacheFileAction uncacheFileAction = new UncacheFileAction(); - uncacheFileAction.setDfsClient(dfsClient); + uncacheFileAction.setLocalFileSystem(dfs); uncacheFileAction.setContext(smartContext); - Map argsUncache = new HashMap(); + Map argsUncache = new HashMap<>(); argsUncache.put(UncacheFileAction.FILE_PATH, file); uncacheFileAction.init(argsUncache); uncacheFileAction.run(); Assert.assertTrue(uncacheFileAction.getExpectedAfterRun()); CacheFileAction cacheFileAction = new CacheFileAction(); - cacheFileAction.setDfsClient(dfsClient); + cacheFileAction.setLocalFileSystem(dfs); cacheFileAction.setContext(smartContext); - Assert.assertFalse(cacheFileAction.isCached(file)); + Assert.assertFalse(cacheFileAction.isFileCached()); } } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestWriteFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestWriteFileAction.java index e30ebb30bfb..869cdabe3a9 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestWriteFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestWriteFileAction.java @@ -29,7 +29,7 @@ public class TestWriteFileAction extends MiniClusterHarness { protected void writeFile(String filePath, long length) throws IOException { WriteFileAction writeFileAction = new WriteFileAction(); - writeFileAction.setDfsClient(dfsClient); + writeFileAction.setLocalFileSystem(dfs); writeFileAction.setContext(smartContext); Map args = new HashMap<>(); args.put(WriteFileAction.FILE_PATH, filePath); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java index 69812e728be..6136ad66cc1 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java @@ -63,7 +63,7 @@ // // // schedule move to SSD // AllSsdFileAction action = new AllSsdFileAction(); -// action.setDfsClient(dfsClient); +// action.setLocalFileSystem(dfs); // action.setContext(smartContext); // action.setStatusReporter(new MockActionStatusReporter()); // Map args = new HashMap(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java index d49f0cc74ab..7df767b7a32 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java @@ -63,7 +63,7 @@ // // // schedule move to Archive // ArchiveFileAction action = new ArchiveFileAction(); -// action.setDfsClient(dfsClient); +// action.setLocalFileSystem(dfs); // action.setContext(smartContext); // action.setStatusReporter(new MockActionStatusReporter()); // Map args = new HashMap(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java index 2309d6ca482..2f12cc0fbee 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java @@ -53,7 +53,7 @@ public class TestCopy2S3Action extends MiniClusterHarness { // // private void copy2S3File(String src, String dest) throws Exception { // Copy2S3Action copy2S3Action = new Copy2S3Action(); - // copy2S3Action.setDfsClient(dfsClient); + // copy2S3Action.setLocalFileSystem(dfs); // copy2S3Action.setContext(testContext); // copy2S3Action.setStatusReporter(new MockActionStatusReporter()); // Map args = new HashMap<>(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java index 2fab4459f60..4938049d2d4 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java @@ -70,7 +70,7 @@ public void testParallelMove() throws Exception { //move to SSD AllSsdFileAction moveFileAction1 = new AllSsdFileAction(); - moveFileAction1.setDfsClient(dfsClient); + moveFileAction1.setLocalFileSystem(dfs); moveFileAction1.setContext(smartContext); Map args1 = new HashMap(); args1.put(MoveFileAction.FILE_PATH, dir); @@ -78,7 +78,7 @@ public void testParallelMove() throws Exception { args1.put(MoveFileAction.MOVE_PLAN, plan1.toString()); AllSsdFileAction moveFileAction2 = new AllSsdFileAction(); - moveFileAction2.setDfsClient(dfsClient); + moveFileAction2.setLocalFileSystem(dfs); moveFileAction2.setContext(smartContext); Map args2 = new HashMap(); args2.put(MoveFileAction.FILE_PATH, dir); @@ -108,7 +108,7 @@ public void testMove() throws Exception{ //move to SSD MoveFileAction moveFileAction = new MoveFileAction(); - moveFileAction.setDfsClient(dfsClient); + moveFileAction.setLocalFileSystem(dfs); moveFileAction.setContext(smartContext); Map args = new HashMap(); args.put(MoveFileAction.FILE_PATH, dir); @@ -129,7 +129,7 @@ public void testMoveNonexitedFile() throws Exception { // schedule move to ALL_SSD MoveFileAction moveFileAction = new MoveFileAction(); - moveFileAction.setDfsClient(dfsClient); + moveFileAction.setLocalFileSystem(dfs); moveFileAction.setContext(smartContext); Map args = new HashMap(); @@ -168,7 +168,7 @@ public void testMoveMultiblockFile() throws Exception { private void moveFile(String file) throws Exception { // schedule move to SSD ArchiveFileAction action = new ArchiveFileAction(); - action.setDfsClient(dfsClient); + action.setLocalFileSystem(dfs); action.setContext(smartContext); Map args = new HashMap(); args.put(ArchiveFileAction.FILE_PATH, file); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java index 00a831a1ed8..0f7b05c237e 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java @@ -63,7 +63,7 @@ // // // schedule move to Archive // OneSsdFileAction action = new OneSsdFileAction(); -// action.setDfsClient(dfsClient); +// action.setLocalFileSystem(dfs); // action.setContext(smartContext); // action.setStatusReporter(new MockActionStatusReporter()); // Map args = new HashMap(); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java index e58d636d5e6..97f978a0389 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java @@ -47,7 +47,7 @@ public void testLocalTruncateFile() throws IOException, InterruptedException { out.close(); TruncateAction truncateAction = new TruncateAction(); - truncateAction.setDfsClient(dfsClient); + truncateAction.setLocalFileSystem(dfs); truncateAction.setContext(smartContext); Map args = new HashMap<>(); args.put(TruncateAction.FILE_PATH, srcPath + "/" + file); @@ -84,7 +84,7 @@ public void testRemoteTruncateFile() throws IOException, InterruptedException { out.close(); TruncateAction truncateAction = new TruncateAction(); - truncateAction.setDfsClient(dfsClient); + truncateAction.setLocalFileSystem(dfs); truncateAction.setContext(smartContext); Map args = new HashMap<>(); args.put(TruncateAction.FILE_PATH, dfs.getUri() + srcPath + "/" + file); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java index c25c86a6c3f..7af852a722e 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java @@ -73,9 +73,11 @@ public void init() throws Exception { dfsClient = dfs.getClient(); smartContext = new SmartContext(conf); cachedListFetcher = new CachedListFetcher(conf, dfsClient, metaStore); + conf.set(SmartConfKeys.SMART_DFS_NAMENODE_RPCSERVER_KEY, + "hdfs://" + cluster.getNameNode().getNameNodeAddressHostPortString()); } - static void initConf(Configuration conf) { + private void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); @@ -130,7 +132,7 @@ public void testFetcher() throws Exception { fileInfos.add(createFileStatus(pathPrefix + fids[i])); cacheAction.setContext(smartContext); - cacheAction.setDfsClient(dfsClient); + cacheAction.setLocalFileSystem(dfs); Map args = new HashMap(); args.put(CacheFileAction.FILE_PATH, path); cacheAction.init(args); @@ -151,7 +153,7 @@ public void testFetcher() throws Exception { String path = pathPrefix + fids[i]; fileInfos.add(createFileStatus("fileTest/cache/" + fids[i])); uncacheFileAction.setContext(smartContext); - uncacheFileAction.setDfsClient(dfsClient); + uncacheFileAction.setLocalFileSystem(dfs); Map args = new HashMap(); args.put(UncacheFileAction.FILE_PATH, path); uncacheFileAction.init(args); diff --git a/smart-server/src/test/java/org/smartdata/hadoop/filesystem/TestSmartFileSystem.java b/smart-server/src/test/java/org/smartdata/hadoop/filesystem/TestSmartFileSystem.java index 9c600767af6..dd92941b404 100644 --- a/smart-server/src/test/java/org/smartdata/hadoop/filesystem/TestSmartFileSystem.java +++ b/smart-server/src/test/java/org/smartdata/hadoop/filesystem/TestSmartFileSystem.java @@ -68,7 +68,7 @@ private void createSmallFiles() throws Exception { } SmallFileCompactAction smallFileCompactAction = new SmallFileCompactAction(); - smallFileCompactAction.setDfsClient(dfsClient); + smallFileCompactAction.setLocalFileSystem(dfs); smallFileCompactAction.setContext(smartContext); Map args = new HashMap<>(); List smallFileList = new ArrayList<>(); diff --git a/smart-server/src/test/java/org/smartdata/hdfs/client/TestSmartDFSClient.java b/smart-server/src/test/java/org/smartdata/hdfs/client/TestSmartDFSClient.java index 76123e4b7fa..3bb968f7d5b 100644 --- a/smart-server/src/test/java/org/smartdata/hdfs/client/TestSmartDFSClient.java +++ b/smart-server/src/test/java/org/smartdata/hdfs/client/TestSmartDFSClient.java @@ -51,7 +51,7 @@ private void createSmallFiles() throws Exception { } // Compact small files SmallFileCompactAction smallFileCompactAction = new SmallFileCompactAction(); - smallFileCompactAction.setDfsClient(dfsClient); + smallFileCompactAction.setLocalFileSystem(dfs); smallFileCompactAction.setContext(smartContext); Map args = new HashMap<>(); List smallFileList = new ArrayList<>(); diff --git a/smart-server/src/test/java/org/smartdata/server/engine/cmdlet/TestCmdlet.java b/smart-server/src/test/java/org/smartdata/server/engine/cmdlet/TestCmdlet.java index 0614e8d3682..f93d541779d 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/cmdlet/TestCmdlet.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/cmdlet/TestCmdlet.java @@ -78,7 +78,7 @@ private Cmdlet runHelper() throws IOException { // actions[1].setContext(new SmartContext(smartConf)); // actions[1].getDNStorageReports(new String[]{"/testMoveFile/file2", "COLD"}); actions[2] = new CacheFileAction(); - ((HdfsAction) actions[2]).setDfsClient(dfsClient); + ((HdfsAction) actions[2]).setLocalFileSystem(dfs); actions[2].setContext(smartContext); Map args = new HashMap(); args.put(CacheFileAction.FILE_PATH, "/testCacheFile"); diff --git a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestSmallFileRead.java b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestSmallFileRead.java index c97a23a8db0..c94a50fcc6b 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestSmallFileRead.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestSmallFileRead.java @@ -64,7 +64,7 @@ private void createTestFiles() throws Exception { } SmallFileCompactAction smallFileCompactAction = new SmallFileCompactAction(); - smallFileCompactAction.setDfsClient(dfsClient); + smallFileCompactAction.setLocalFileSystem(dfs); smallFileCompactAction.setContext(smartContext); Map args = new HashMap<>(); List smallFileList = new ArrayList<>();