fix: 3512 load files from jar

backend/molgenis-emx2-datamodels/src/main/java/org/molgenis/emx2/datamodels/DataModels.java

@@ -20,7 +20,8 @@ public enum Profile {
     FAIR_DATA_POINT("_profiles/FAIRDataPoint.yaml"),
     BEACON_V2("_profiles/BeaconV2.yaml"),
     GDI("_profiles/GDI.yaml"),
-    SHARED_STAGING("_profiles/SharedStaging.yaml");
+    SHARED_STAGING("_profiles/SharedStaging.yaml"),
+    IMAGE_TEST("_profiles/ImageTest.yaml");
 
     public static boolean hasProfile(String nameOther) {
       return Arrays.stream(values()).anyMatch(profile -> profile.name().equals(nameOther));

backend/molgenis-emx2-io/src/main/java/org/molgenis/emx2/io/readers/CsvTableReader.java

@@ -75,10 +75,10 @@ public boolean hasNext() {
             }
 
             public Row next() {
-              HashMap<String, String> next = (HashMap<String, String>) it.next();
+              HashMap<String, Object> next = (HashMap<String, Object>) it.next();
               boolean isEmpty = next.values().stream().allMatch(Objects::isNull);
               while (isEmpty && it.hasNext()) {
-                next = (HashMap<String, String>) it.next();
+                next = (HashMap<String, Object>) it.next();
                 isEmpty = next.values().stream().allMatch(Objects::isNull);
               }
               return new Row(next);

backend/molgenis-emx2-io/src/main/java/org/molgenis/emx2/io/tablestore/TableStoreForCsvFilesClasspath.java

@@ -1,17 +1,29 @@
 package org.molgenis.emx2.io.tablestore;
 
 import java.io.*;
+import java.net.URI;
 import java.net.URL;
+import java.net.URLConnection;
+import java.nio.file.*;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.molgenis.emx2.BinaryFileWrapper;
 import org.molgenis.emx2.MolgenisException;
 import org.molgenis.emx2.Row;
 import org.molgenis.emx2.io.readers.CsvTableReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TableStoreForCsvFilesClasspath implements TableAndFileStore {
   public static final String CSV_EXTENSION = ".csv";
+  private static final Logger log = LoggerFactory.getLogger(TableStoreForCsvFilesClasspath.class);
   private final String directoryPath;
   private final Character separator;
+  private static final ConcurrentMap<String, Object> locks = new ConcurrentHashMap<>();
+  private final List<String> storefiles;
 
   public TableStoreForCsvFilesClasspath(String directoryPath, Character separator) {
     if (!directoryPath.startsWith("/")) {
@@ -22,6 +34,11 @@ public TableStoreForCsvFilesClasspath(String directoryPath, Character separator)
       throw new MolgenisException(
           "Import failed: Directory " + directoryPath + " doesn't exist in classpath");
     this.separator = separator;
+    try {
+      storefiles = jarSaveListFiles(directoryPath + "/_files/");
+    } catch (Exception e) {
+      throw new MolgenisException("File listing failed: " + e.getMessage(), e);
+    }
   }
 
   public TableStoreForCsvFilesClasspath(String directoryPath) {
@@ -66,19 +83,69 @@ public Collection<String> getTableNames() {
 
   @Override
   public BinaryFileWrapper getBinaryFileWrapper(String name) {
-    URL url = getClass().getResource(directoryPath + "/_files");
-    String path = url.getPath();
-    List<File> result =
-        Arrays.stream(new File(path).listFiles())
-            .filter(f -> f.getName().toString().startsWith(name + "."))
-            .toList();
-    if (result.isEmpty()) {
-      throw new MolgenisException("File not found for id " + name);
-    } else if (result.size() == 1) {
-      return new BinaryFileWrapper(result.get(0));
+    try {
+      String fileName =
+          storefiles.stream().filter(f -> f.startsWith(name + ".")).findFirst().orElseThrow();
+      byte[] bytes;
+      try (InputStream stream =
+          getClass().getResourceAsStream(directoryPath + "/_files/" + fileName)) {
+        bytes = Objects.requireNonNull(stream).readAllBytes();
+      }
+      String mimetype = URLConnection.getFileNameMap().getContentTypeFor(fileName);
+      return new BinaryFileWrapper(mimetype, fileName, bytes);
+
+    } catch (Exception ioe) {
+      throw new MolgenisException("Import '" + name + "' failed: " + ioe.getMessage(), ioe);
+    }
+  }
+
+  /** List files in a directory. This method is safe to use in a jar file. */
+  public List<String> jarSaveListFiles(String path) throws Exception {
+
+    URL url = getClass().getResource(path);
+    if (url == null) {
+      // no files directory
+      return Collections.emptyList();
+    }
+
+    URI uri = url.toURI();
+    List<String> files;
+    if ("jar".equals(uri.getScheme())) {
+      files = safeWalkJar(path, uri);
     } else {
-      throw new MolgenisException(
-          "File cannot be retrieved for id " + name + ": name is not unique");
+      return Arrays.stream(Objects.requireNonNull(new File(url.getPath()).listFiles()))
+          .map(File::getName)
+          .toList();
+    }
+    return files;
+  }
+
+  private List<String> safeWalkJar(String path, URI uri) throws Exception {
+    synchronized (getLock(uri)) {
+      // this'll close the FileSystem object at the end
+      try (FileSystem fs = getFileSystem(uri);
+          Stream<Path> walk = Files.walk(fs.getPath(path)); ) {
+        return walk.map(Path::getFileName).map(Path::toString).collect(Collectors.toList());
+      }
+    }
+  }
+
+  private Object getLock(URI uri) {
+    String fileName = parseFileName(uri);
+    locks.computeIfAbsent(fileName, s -> new Object());
+    return locks.get(fileName);
+  }
+
+  private String parseFileName(URI uri) {
+    String schemeSpecificPart = uri.getSchemeSpecificPart();
+    return schemeSpecificPart.substring(0, schemeSpecificPart.indexOf("!"));
+  }
+
+  private FileSystem getFileSystem(URI uri) throws IOException {
+    try {
+      return FileSystems.getFileSystem(uri);
+    } catch (FileSystemNotFoundException e) {
+      return FileSystems.newFileSystem(uri, Collections.<String, String>emptyMap());
     }
   }
 }

backend/molgenis-emx2-io/src/test/java/org/molgenis/emx2/io/readers/CsvTableReaderTest.java

@@ -0,0 +1,28 @@
+package org.molgenis.emx2.io.readers;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.molgenis.emx2.Row;
+
+class CsvTableReaderTest {
+
+  @Test
+  void readImageMeta() throws IOException {
+    File file =
+        new File(getClass().getClassLoader().getResource("profile/imagetest.csv").getFile());
+    Iterable<Row> rowsIterable = CsvTableReader.read(file);
+    List<Row> rows = Lists.newArrayList(rowsIterable);
+    assertEquals(2, rows.size());
+    assertEquals("User", rows.get(0).getString("tableName"));
+    assertEquals("username", rows.get(0).getString("columnName"));
+    assertEquals("ImageTest", rows.get(0).getString("profiles"));
+    assertEquals("User", rows.get(1).getString("tableName"));
+    assertEquals("picture", rows.get(1).getString("columnName"));
+    assertEquals("ImageTest", rows.get(1).getString("profiles"));
+  }
+}

backend/molgenis-emx2-io/src/test/java/org/molgenis/emx2/io/tablestore/TableStoreForCsvFilesClasspathTest.java

@@ -0,0 +1,18 @@
+package org.molgenis.emx2.io.tablestore;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import org.junit.jupiter.api.Test;
+import org.molgenis.emx2.BinaryFileWrapper;
+
+class TableStoreForCsvFilesClasspathTest {
+
+  @Test
+  void getBinaryFileWrapper() {
+    String directoryPath = "_demodata/applications/imagetest";
+    String fileName = "e019a97346d1416f91ba56fe842f44c6";
+    TableStoreForCsvFilesClasspath store = new TableStoreForCsvFilesClasspath(directoryPath);
+    BinaryFileWrapper binaryFileWrapper = store.getBinaryFileWrapper(fileName);
+    assertNotNull(binaryFileWrapper);
+  }
+}

backend/molgenis-emx2-io/src/test/resources/profile/imagetest.csv

@@ -0,0 +1,3 @@
+tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description,profiles
+User,,username,,,1,true,,,,,,,ImageTest
+User,,picture,file,,,,,,,,,,ImageTest

data/_demodata/applications/imagetest/User.csv

@@ -0,0 +1,2 @@
+username,picture,picture_filename
+bofke,e019a97346d1416f91ba56fe842f44c6,8hlbnm.jpg

data/_models/specific/imagetest.csv

@@ -0,0 +1,3 @@
+tableName,tableExtends,columnName,columnType,key,required,refSchema,refTable,refLink,refBack,validation,semantics,description,profiles
+User,,username,,1,true,,,,,,,,ImageTest
+User,,picture,file,,,,,,,,,,ImageTest

data/_profiles/ImageTest.yaml

@@ -0,0 +1,9 @@
+---
+name: Image test
+description: "test image loading"
+
+profileTags: ImageTest
+
+demoData: _demodata/applications/imagetest
+
+settings: _settings/datacatalogue