23
23
import java .nio .file .Path ;
24
24
import java .nio .file .Paths ;
25
25
import java .nio .file .StandardCopyOption ;
26
- import java .util .ArrayList ;
27
26
import java .util .Arrays ;
28
27
import java .util .List ;
29
28
import java .util .stream .Collectors ;
30
29
31
30
public class DownloadHelper {
32
- private static String bucketName = "astra-vector" ;
31
+ private static final String bucketName = "astra-vector" ;
33
32
34
- private static S3AsyncClientBuilder getS3AsyncClientBuilder () {
35
- S3AsyncClientBuilder s3ClientBuilder = S3AsyncClient .builder ()
33
+ private static S3AsyncClientBuilder s3AsyncClientBuilder () {
34
+ return S3AsyncClient .builder ()
36
35
.region (Region .US_EAST_1 )
37
36
.httpClient (AwsCrtAsyncHttpClient .builder ()
38
37
.maxConcurrency (1 )
39
38
.build ())
40
39
.credentialsProvider (AnonymousCredentialsProvider .create ());
41
- return s3ClientBuilder ;
42
40
}
43
41
44
42
public static void maybeDownloadFvecs (List <String > files ) {
45
43
List <String > keys ;
46
44
if (null == files || files .isEmpty ()) {
47
- keys = Arrays .asList (new String [] {
48
- "wikipedia_squad/100k/ada_002_100000_base_vectors.fvec" ,
49
- "wikipedia_squad/100k/ada_002_100000_query_vectors_10000.fvec" ,
50
- "wikipedia_squad/100k/ada_002_100000_indices_query_10000.ivec" ,
51
- });
45
+ keys = List .of ("wikipedia_squad/100k/ada_002_100000_base_vectors.fvec" ,
46
+ "wikipedia_squad/100k/ada_002_100000_query_vectors_10000.fvec" ,
47
+ "wikipedia_squad/100k/ada_002_100000_indices_query_10000.ivec" );
52
48
} else {
53
49
keys = files ;
54
50
}
@@ -65,7 +61,7 @@ public static void maybeDownloadFvecs(List<String> files) {
65
61
}
66
62
}
67
63
68
- try (S3AsyncClient s3Client = getS3AsyncClientBuilder ().build ()) {
64
+ try (S3AsyncClient s3Client = s3AsyncClientBuilder ().build ()) {
69
65
S3TransferManager tm = S3TransferManager .builder ().s3Client (s3Client ).build ();
70
66
for (String key : keys ) {
71
67
Path path = Paths .get ("fvec" , key );
@@ -118,7 +114,7 @@ public static void maybeDownloadHdf5(String datasetName) {
118
114
var url = "https://ann-benchmarks.com/" + datasetName ;
119
115
System .out .println ("Downloading: " + url );
120
116
121
- HttpURLConnection connection = null ;
117
+ HttpURLConnection connection ;
122
118
while (true ) {
123
119
int responseCode ;
124
120
try {
@@ -144,15 +140,4 @@ public static void maybeDownloadHdf5(String datasetName) {
144
140
System .exit (1 );
145
141
}
146
142
}
147
-
148
- public static List <String > s3FileListing () {
149
- S3Client s3 = S3Client .builder ().region (Region .US_EAST_1 ).credentialsProvider (AnonymousCredentialsProvider .create ()).build ();
150
- ListObjectsV2Request req = ListObjectsV2Request .builder ().bucket (bucketName ).build ();
151
- ListObjectsV2Response res = s3 .listObjectsV2 (req );
152
- List <String > filenames = res .contents ().stream ().map (S3Object ::key ).collect (Collectors .toList ());
153
- /*for (String filename : filenames) {
154
- System.out.println(filename);
155
- }*/
156
- return filenames ;
157
- }
158
143
}
0 commit comments