Skip to content
This repository was archived by the owner on Aug 4, 2022. It is now read-only.

Commit 2ff8581

Browse files
dwaijamdwaijam
committed
Add IFileSystem api to check if uri is a directory
SKIP e2e tests address cr comments handle path not found expception cr comments fix address cr comments fix cr comments cr comment fixes fix validate children test
1 parent 2e8b79d commit 2ff8581

File tree

11 files changed

+198
-40
lines changed

11 files changed

+198
-40
lines changed

‎lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
using System.IO;
2121
using System.Linq;
2222
using System.Text;
23+
using System.Linq;
2324
using Microsoft.WindowsAzure.Storage;
2425
using Microsoft.WindowsAzure.Storage.Blob;
2526
using Org.Apache.REEF.IO.FileSystem;
@@ -166,9 +167,9 @@ public void TestGetChildBlobsInContainerE2E()
166167
[Fact(Skip = SkipMessage)]
167168
public void TestGetChildContainerInStorageAccountE2E()
168169
{
169-
// List containers in the storage account
170-
Uri rootUri = _fileSystem.CreateUriForPath(string.Empty);
171-
ValidateChildren(rootUri, new List<Uri> { _container.Uri });
170+
// List container uris in the storage account
171+
var containerUris = _client.ListContainers().Select(container => container.Uri);
172+
ValidateChildren(_client.BaseUri, containerUris);
172173
}
173174

174175
private void ValidateChildren(Uri storageBlobUri, IEnumerable<Uri> expectedChildBlobs)
@@ -269,6 +270,21 @@ public void TestCopyFromLocalE2E()
269270
container.DeleteIfExistsAsync().Wait();
270271
}
271272

273+
[Fact(Skip = SkipMessage)]
274+
public void TestIsDirectory()
275+
{
276+
const string Directory1 = "dir1";
277+
const string Directory2 = "dir1/dir2";
278+
const string Directory3 = "dir3";
279+
var blockBlobs1 = CreateTempBlobs(Directory1);
280+
CreateTempBlobs(Directory2);
281+
282+
Assert.True(_fileSystem.IsDirectory(PathToFile(Directory1)));
283+
Assert.True(_fileSystem.IsDirectory(PathToFile(Directory2)));
284+
Assert.False(_fileSystem.IsDirectory(PathToFile(Directory3)));
285+
Assert.False(_fileSystem.IsDirectory(blockBlobs1.First().Uri));
286+
}
287+
272288
[Fact(Skip = SkipMessage)]
273289
public void TestDeleteDirectoryAtContainerE2E()
274290
{
@@ -279,16 +295,8 @@ public void TestDeleteDirectoryAtContainerE2E()
279295
[Fact(Skip = SkipMessage)]
280296
public void TestDeleteDirectoryFirstLevelE2E()
281297
{
282-
const string Directory = "dir";
283-
var blockBlobs = new List<CloudBlockBlob>();
284-
for (var i = 0; i < 3; i++)
285-
{
286-
var filePath = Directory + '/' + i;
287-
var blockBlob = _container.GetBlockBlobReference(filePath);
288-
UploadFromString(blockBlob, "hello");
289-
Assert.True(CheckBlobExists(blockBlob));
290-
blockBlobs.Add(blockBlob);
291-
}
298+
const string Directory = "dir1";
299+
var blockBlobs = CreateTempBlobs(Directory);
292300

293301
_fileSystem.DeleteDirectory(PathToFile(Directory));
294302

@@ -304,24 +312,11 @@ public void TestDeleteDirectoryFirstLevelE2E()
304312
public void TestDeleteDirectorySecondLevelE2E()
305313
{
306314
const string Directory1 = "dir1";
307-
const string Directory2 = "dir2";
308-
var blockBlobs1 = new List<CloudBlockBlob>();
309-
var blockBlobs2 = new List<CloudBlockBlob>();
310-
for (var i = 0; i < 3; i++)
311-
{
312-
var filePath1 = Directory1 + '/' + i;
313-
var filePath2 = Directory1 + '/' + Directory2 + '/' + i;
314-
var blockBlob1 = _container.GetBlockBlobReference(filePath1);
315-
var blockBlob2 = _container.GetBlockBlobReference(filePath2);
316-
UploadFromString(blockBlob1, "hello");
317-
UploadFromString(blockBlob2, "hello");
318-
Assert.True(CheckBlobExists(blockBlob1));
319-
Assert.True(CheckBlobExists(blockBlob2));
320-
blockBlobs1.Add(blockBlob1);
321-
blockBlobs2.Add(blockBlob2);
322-
}
315+
const string Directory2 = "dir1/dir2";
316+
var blockBlobs1 = CreateTempBlobs(Directory1);
317+
var blockBlobs2 = CreateTempBlobs(Directory2);
323318

324-
_fileSystem.DeleteDirectory(PathToFile(Directory1 + '/' + Directory2));
319+
_fileSystem.DeleteDirectory(PathToFile(Directory2));
325320

326321
foreach (var blockBlob in blockBlobs2)
327322
{
@@ -336,6 +331,20 @@ public void TestDeleteDirectorySecondLevelE2E()
336331
Assert.True(CheckContainerExists(_container));
337332
}
338333

334+
private List<CloudBlockBlob> CreateTempBlobs(string directory, int fileCount = 3)
335+
{
336+
var blockBlobs = new List<CloudBlockBlob>();
337+
for (var i = 0; i < fileCount; i++)
338+
{
339+
var filePath = directory + '/' + i;
340+
var blockBlob = _container.GetBlockBlobReference(filePath);
341+
UploadFromString(blockBlob, "hello");
342+
Assert.True(CheckBlobExists(blockBlob));
343+
blockBlobs.Add(blockBlob);
344+
}
345+
return blockBlobs;
346+
}
347+
339348
private static void UploadFromString(ICloudBlob blob, string str)
340349
{
341350
var byteArray = Encoding.UTF8.GetBytes(str);

‎lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,20 @@ public void TestCreateDirectoryE2E()
194194
Assert.True(_adlsClient.CheckExists(dirName));
195195
}
196196

197+
[Fact(Skip = SkipMessage)]
198+
public void TestIsDirectoryE2E()
199+
{
200+
string dirName = $"/{_defaultFolderName}";
201+
string fakeDirName = $"/fakeDir";
202+
203+
_adlsClient.CreateDirectory(dirName);
204+
string fileName = UploadFromString(ContentsText);
205+
206+
Assert.True(_fileSystem.IsDirectory(PathToFile(dirName)));
207+
Assert.False(_fileSystem.IsDirectory(PathToFile(fakeDirName)));
208+
Assert.False(_fileSystem.IsDirectory(PathToFile(fileName)));
209+
}
210+
197211
[Fact(Skip = SkipMessage)]
198212
public void TestDeleteDirectoryE2E()
199213
{

‎lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ namespace Org.Apache.REEF.IO.Tests
3333
/// <see cref="HadoopFileSystem" />
3434
public sealed class TestHadoopFileSystem
3535
{
36+
private const string SkipMessage = "These tests need to be run in an environment with HDFS installed."; // Use null to run tests
3637
private HadoopFileSystem _fileSystem;
3738

3839
private Uri GetTempUri()
@@ -56,7 +57,7 @@ public TestHadoopFileSystem()
5657
/// <summary>
5758
/// Creates a temp file locally, uploads it to HDFS and downloads it again.
5859
/// </summary>
59-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
60+
[Fact(Skip = SkipMessage)]
6061
public void TestCopyFromLocalAndBack()
6162
{
6263
var localFile = FileSystemTestUtilities.MakeLocalTempFile();
@@ -77,7 +78,7 @@ public void TestCopyFromLocalAndBack()
7778
/// <summary>
7879
/// Tests whether .Exists() works.
7980
/// </summary>
80-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
81+
[Fact(Skip = SkipMessage)]
8182
public void TestExists()
8283
{
8384
var remoteUri = GetTempUri();
@@ -93,7 +94,7 @@ public void TestExists()
9394
/// <summary>
9495
/// Tests for .GetChildren().
9596
/// </summary>
96-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
97+
[Fact(Skip = SkipMessage)]
9798
public void TestGetChildren()
9899
{
99100
// Make a directory
@@ -129,14 +130,14 @@ public void TestGetChildren()
129130
_fileSystem.DeleteDirectory(remoteDirectory);
130131
}
131132

132-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
133+
[Fact(Skip = SkipMessage)]
133134
public void TestOpen()
134135
{
135136
// Open() is not supported by HadoopFileSystem. Use CopyToLocal and open the local file instead.
136137
Assert.Throws<NotImplementedException>(() => _fileSystem.Open(GetTempUri()));
137138
}
138139

139-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
140+
[Fact(Skip = SkipMessage)]
140141
public void TestCreate()
141142
{
142143
// Create() is not supported by HadoopFileSystem. Create a local file and use CopyFromLocal instead.
@@ -163,10 +164,38 @@ public void CreateUriForPathWithPrefix()
163164
Assert.Equal(new Uri(uriString), _fileSystem.CreateUriForPath(uriString));
164165
}
165166

167+
/// <summary>
168+
/// Tests whether .IsDirectory() works.
169+
/// </summary>
170+
[Fact(Skip = SkipMessage)]
171+
public void TestIsDirectory()
172+
{
173+
// Create directory
174+
var remoteDirUri = GetTempUri();
175+
_fileSystem.CreateDirectory(remoteDirUri);
176+
177+
// Create fake directory uri
178+
var remoteFakeuri = GetTempUri();
179+
180+
// Create file
181+
var remoteFileUri = GetTempUri();
182+
var localFile = FileSystemTestUtilities.MakeLocalTempFile();
183+
_fileSystem.CopyFromLocal(localFile, remoteFileUri);
184+
185+
Assert.True(_fileSystem.IsDirectory(remoteDirUri));
186+
Assert.False(_fileSystem.IsDirectory(remoteFakeuri));
187+
Assert.False(_fileSystem.IsDirectory(remoteFileUri));
188+
189+
// Clean up
190+
_fileSystem.DeleteDirectory(remoteDirUri);
191+
_fileSystem.Delete(remoteFileUri);
192+
File.Delete(localFile);
193+
}
194+
166195
/// <summary>
167196
/// This test is to make sure with the HadoopFileSystemConfiguration, HadoopFileSystem can be injected.
168197
/// </summary>
169-
[Fact(Skip = "These tests need to be run in an environment with HDFS installed.")]
198+
[Fact(Skip = SkipMessage)]
170199
public void TestHadoopFileSystemConfiguration()
171200
{
172201
var fileSystemTest = TangFactory.GetTang().NewInjector(HadoopFileSystemConfiguration.ConfigurationModule

‎lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,22 @@ public void TestCopy()
136136
Assert.False(fs.Exists(sourceUri));
137137
}
138138

139+
[Fact]
140+
public void TestIsDirectory()
141+
{
142+
var fs = GetFileSystem();
143+
var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName) + "/");
144+
var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir") + "/");
145+
fs.CreateDirectory(directoryUri);
146+
var fileUri = new Uri(directoryUri, "testfile");
147+
MakeRemoteTestFile(fs, fileUri);
148+
Assert.True(fs.IsDirectory(directoryUri));
149+
Assert.False(fs.IsDirectory(fakeDirectoryUri));
150+
Assert.False(fs.IsDirectory(fileUri));
151+
fs.Delete(fileUri);
152+
fs.DeleteDirectory(directoryUri);
153+
}
154+
139155
[Fact]
140156
public void TestGetChildren()
141157
{

‎lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,19 @@ public void CreateDirectory(Uri directoryUri)
125125
{
126126
}
127127

128+
/// <summary>
129+
/// Checks if uri is a directory uri.
130+
/// </summary>
131+
/// <param name="uri">uri of the directory/file</param>
132+
/// <returns>true if uri is for a directory else false</returns>
133+
public bool IsDirectory(Uri uri)
134+
{
135+
var path = uri.AbsolutePath.TrimStart('/');
136+
const int maxBlobResults = 1;
137+
var blobItems = _client.ListBlobsSegmented(path, false, BlobListingDetails.Metadata, maxBlobResults, null, null, null).Results;
138+
return blobItems.Any() && blobItems.First() is CloudBlobDirectory;
139+
}
140+
128141
/// <summary>
129142
/// Recursively deletes blobs under a specified "directory URI."
130143
/// If only the container is specified, the entire container is deleted.
@@ -197,7 +210,7 @@ public IEnumerable<Uri> GetChildren(Uri directoryUri)
197210

198211
do
199212
{
200-
BlobResultSegment listing = _client.ListBlobsSegmented(
213+
BlobResultSegment listing = _client.ListDirectoryBlobsSegmented(
201214
containerName,
202215
relativeAddress,
203216
useFlatListing: false,

‎lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,27 @@ public ICloudBlockBlob GetBlockBlobReference(Uri uri)
6969
}
7070

7171
public BlobResultSegment ListBlobsSegmented(
72+
string prefix,
73+
bool useFlatListing,
74+
BlobListingDetails blobListingDetails,
75+
int? maxResults,
76+
BlobContinuationToken continuationToken,
77+
BlobRequestOptions blobRequestOptions,
78+
OperationContext operationContext)
79+
{
80+
var task = _client.ListBlobsSegmentedAsync(
81+
prefix,
82+
useFlatListing,
83+
blobListingDetails,
84+
maxResults,
85+
continuationToken,
86+
blobRequestOptions,
87+
operationContext);
88+
task.Wait();
89+
return task.Result;
90+
}
91+
92+
public BlobResultSegment ListDirectoryBlobsSegmented(
7293
string containerName,
7394
string relativeAddress,
7495
bool useFlatListing,

‎lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,21 @@ internal interface ICloudBlobClient
5656
ICloudBlockBlob GetBlockBlobReference(Uri uri);
5757

5858
/// <summary>
59-
/// Paginates a blob listing with container and relative path.
59+
/// Paginates a blob listing with prefix.
6060
/// </summary>
6161
BlobResultSegment ListBlobsSegmented(
62+
string path,
63+
bool useFlatListing,
64+
BlobListingDetails blobListingDetails,
65+
int? maxResults,
66+
BlobContinuationToken continuationToken,
67+
BlobRequestOptions blobRequestOptions,
68+
OperationContext operationContext);
69+
70+
/// <summary>
71+
/// Paginates a blob listing with container and relative path.
72+
/// </summary>
73+
BlobResultSegment ListDirectoryBlobsSegmented(
6274
string containerName,
6375
string relativeAddress,
6476
bool useFlatListing,

‎lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using System.Collections.Generic;
2020
using System.IO;
2121
using System.Linq;
22+
using System.Net;
2223
using System.Text;
2324
using Microsoft.Azure.DataLake.Store;
2425
using Microsoft.Azure.DataLake.Store.FileTransfer;
@@ -178,14 +179,34 @@ public void CreateDirectory(Uri directoryUri)
178179
}
179180
}
180181

182+
/// <summary>
183+
/// Checks if uri is a directory uri.
184+
/// </summary>
185+
/// <param name="uri">uri of the directory/file</param>
186+
/// <returns>true if uri is for a directory else false</returns>
187+
public bool IsDirectory(Uri uri)
188+
{
189+
try
190+
{
191+
return _adlsClient.GetDirectoryEntry(uri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY;
192+
}
193+
catch (AdlsException e)
194+
{
195+
if (e.HttpStatus == HttpStatusCode.NotFound)
196+
{
197+
return false;
198+
}
199+
throw e;
200+
}
201+
}
202+
181203
/// <summary>
182204
/// Deletes a directory.
183205
/// </summary>
184206
/// <exception cref="IOException">If directory cannot be deleted</exception>
185207
public void DeleteDirectory(Uri directoryUri)
186208
{
187-
bool deleteStatus = Exists(directoryUri) &&
188-
_adlsClient.GetDirectoryEntry(directoryUri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY &&
209+
bool deleteStatus = IsDirectory(directoryUri) &&
189210
_adlsClient.DeleteRecursive(directoryUri.AbsolutePath);
190211
if (!deleteStatus)
191212
{

‎lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,17 @@ public void CreateDirectory(Uri directoryUri)
152152
_commandRunner.Run("dfs -mkdir " + directoryUri);
153153
}
154154

155+
public bool IsDirectory(Uri uri)
156+
{
157+
// TODO[JIRA REEF - 2039]: HadoopFileSystem .IsDirectory() check needs to work on linux machines.
158+
var stdOut = _commandRunner.Run("dfs -test -d " + uri + "& call echo %^errorlevel%").StdOut;
159+
if (stdOut.Any())
160+
{
161+
return "0".Equals(stdOut.First());
162+
}
163+
return false;
164+
}
165+
155166
public void DeleteDirectory(Uri directoryUri)
156167
{
157168
_commandRunner.Run("dfs -rmdir " + directoryUri);

0 commit comments

Comments
 (0)