From f7af7a1aa3032ad03b806dbe688cc3e931bc34b2 Mon Sep 17 00:00:00 2001 From: connoratrug Date: Mon, 11 Nov 2024 09:42:55 +0100 Subject: [PATCH 1/5] read py file --- apps/build.gradle | 5 +++++ apps/nuxt3-ssr/migrations/version1/migrate.py | 1 + apps/yarn.lock | 18 --------------- .../org/molgenis/emx2/RunMolgenisEmx2.java | 22 ++++++++++++++++++- .../molgenis/emx2/web/StaticFileMapper.java | 20 +++++++++++++++++ 5 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 apps/nuxt3-ssr/migrations/version1/migrate.py diff --git a/apps/build.gradle b/apps/build.gradle index a8e5319909..fb4679bfc1 100644 --- a/apps/build.gradle +++ b/apps/build.gradle @@ -43,6 +43,11 @@ subprojects { subproject -> with nodeSpec } + task movePyFiles(type: Copy) { + from "migrations" + into "${parent.buildDir}/generated/main/resources/" + project.name + "/migration" + } + task format(type: YarnTask, dependsOn: parent.yarn_install) { if (!isCI) { args = ['run', 'format'] diff --git a/apps/nuxt3-ssr/migrations/version1/migrate.py b/apps/nuxt3-ssr/migrations/version1/migrate.py new file mode 100644 index 0000000000..8ce7c684aa --- /dev/null +++ b/apps/nuxt3-ssr/migrations/version1/migrate.py @@ -0,0 +1 @@ +print("Hello python migration script, app: nuxt3-ssr, version: 1"); \ No newline at end of file diff --git a/apps/yarn.lock b/apps/yarn.lock index 5f58e0e6b5..0e1e35da5a 100644 --- a/apps/yarn.lock +++ b/apps/yarn.lock @@ -7034,15 +7034,6 @@ handlebars@^4.7.7, handlebars@^4.7.8: optionalDependencies: uglify-js "^3.1.4" -happy-dom@15.10.1: - version "15.10.1" - resolved "https://registry.yarnpkg.com/happy-dom/-/happy-dom-15.10.1.tgz#db49321a7af97d8c18c45622de2817b382ac7595" - integrity sha512-FuGnj/qIB4QnBL6fWmD7Wnh6STxevLgOVWB6+nopDGgWG1+t9CXkNB2ldZ+iqwD2UKxD2D0SU8el8A6AX6Q1+g== - dependencies: - entities "^4.5.0" - webidl-conversions "^7.0.0" - whatwg-mimetype "^3.0.0" - happy-dom@15.10.2: version "15.10.2" resolved "https://registry.yarnpkg.com/happy-dom/-/happy-dom-15.10.2.tgz#14ae6652d1a80d2611e3f5832cb61ab5e2d1b539" @@ -7052,15 +7043,6 @@ happy-dom@15.10.2: webidl-conversions "^7.0.0" whatwg-mimetype "^3.0.0" -happy-dom@15.7.4: - version "15.7.4" - resolved "https://registry.yarnpkg.com/happy-dom/-/happy-dom-15.7.4.tgz#05aade59c1d307336001b7004c76dfc6a829f220" - integrity sha512-r1vadDYGMtsHAAsqhDuk4IpPvr6N8MGKy5ntBo7tSdim+pWDxus2PNqOcOt8LuDZ4t3KJHE+gCuzupcx/GKnyQ== - dependencies: - entities "^4.5.0" - webidl-conversions "^7.0.0" - whatwg-mimetype "^3.0.0" - has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java index fed59f427a..2d0a0bd780 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java @@ -3,12 +3,16 @@ import static org.molgenis.emx2.ColumnType.BOOL; import static org.molgenis.emx2.ColumnType.INT; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import org.molgenis.emx2.datamodels.BiobankDirectoryLoader; import org.molgenis.emx2.datamodels.DataModels; import org.molgenis.emx2.datamodels.PetStoreLoader; import org.molgenis.emx2.sql.SqlDatabase; import org.molgenis.emx2.utils.EnvironmentProperty; import org.molgenis.emx2.web.MolgenisWebservice; +import org.molgenis.emx2.web.StaticFileMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +32,7 @@ public class RunMolgenisEmx2 { (Boolean) EnvironmentProperty.getParameter(Constants.MOLGENIS_EXCLUDE_PETSTORE_DEMO, false, BOOL); - public static void main(String[] args) { + public static void main(String[] args) throws IOException { logger.info("Starting MOLGENIS EMX2 Software Version=" + Version.getVersion()); Integer port = @@ -64,6 +68,22 @@ public static void main(String[] args) { }); // start + logger.info("Before start"); MolgenisWebservice.start(port); + + logger.info("After start"); + // read py file from classpath + try (InputStream is = + StaticFileMapper.class.getResourceAsStream("/public_html/apps/ui/index.html")) { + if (is != null) { + logger.info("Found 3migrate.py"); + String text = new String(is.readAllBytes(), StandardCharsets.UTF_8); + logger.info("3migrate.py: " + text); + } else { + logger.error("Could not find 2migrate.py"); + } + } catch (IOException e) { + logger.error("Could not find 3migrate.py"); + } } } diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java index 356bb5d5d2..6a7f1df368 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java @@ -3,9 +3,13 @@ import com.google.common.io.ByteStreams; import io.javalin.Javalin; import io.javalin.http.Context; +import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * to allow for nice urls, and make it easier for 'schema' app developers we include the schema in @@ -16,12 +20,27 @@ */ public class StaticFileMapper { + private static Logger logger = LoggerFactory.getLogger(StaticFileMapper.class); + private StaticFileMapper() { // hide constructor } public static void create(Javalin app) { + try (InputStream is = + StaticFileMapper.class.getResourceAsStream("/public_html/apps/ui/index.html")) { + if (is != null) { + logger.info("Found 2migrate.py"); + String text = new String(is.readAllBytes(), StandardCharsets.UTF_8); + logger.info("2migrate.py: " + text); + } else { + logger.error("Could not find 2migrate.py"); + } + } catch (IOException e) { + logger.error("Could not find 2migrate.py"); + } + app.get("/{schema}/{appname}/theme.css", BootstrapThemeService::getCss); app.get("*/docs/", StaticFileMapper::redirectDocs); @@ -88,6 +107,7 @@ public static void addFileToContext(Context ctx, String path, String mimeType) { ctx.status(404).result("File not found: " + ctx.path()); return; } + logger.info("Serving file: " + path); if (mimeType == null) { mimeType = Files.probeContentType(Path.of(path)); if (mimeType == null) { From e23e1a95e30bcfb1f55ae3cec7b1a42d43d669eb Mon Sep 17 00:00:00 2001 From: connoratrug Date: Fri, 15 Nov 2024 14:42:29 +0100 Subject: [PATCH 2/5] run migration direct from file --- .../org/molgenis/emx2/RunMolgenisEmx2.java | 43 +++++++++++++------ .../molgenis/emx2/web/StaticFileMapper.java | 15 ------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java index 2d0a0bd780..42bc30df13 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java @@ -3,16 +3,15 @@ import static org.molgenis.emx2.ColumnType.BOOL; import static org.molgenis.emx2.ColumnType.INT; +import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import java.io.InputStreamReader; import org.molgenis.emx2.datamodels.BiobankDirectoryLoader; import org.molgenis.emx2.datamodels.DataModels; import org.molgenis.emx2.datamodels.PetStoreLoader; import org.molgenis.emx2.sql.SqlDatabase; import org.molgenis.emx2.utils.EnvironmentProperty; import org.molgenis.emx2.web.MolgenisWebservice; -import org.molgenis.emx2.web.StaticFileMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,18 +71,34 @@ public static void main(String[] args) throws IOException { MolgenisWebservice.start(port); logger.info("After start"); - // read py file from classpath - try (InputStream is = - StaticFileMapper.class.getResourceAsStream("/public_html/apps/ui/index.html")) { - if (is != null) { - logger.info("Found 3migrate.py"); - String text = new String(is.readAllBytes(), StandardCharsets.UTF_8); - logger.info("3migrate.py: " + text); - } else { - logger.error("Could not find 2migrate.py"); + + String relativeMigrationFilePath = "./apps/nuxt3-ssr/migrations/version1/migrate.py"; + String migrationFilePathString = "/nuxt3-ssr/migration/version1/migrate.py"; + + String runScriptCommand = "python3 -u " + relativeMigrationFilePath; + // String runScriptCommand = "pwd"; + ProcessBuilder builder = new ProcessBuilder("bash", "-c", runScriptCommand); + builder.redirectErrorStream(true); + Process process = builder.start(); + + // Read the process output + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + // Log each line of output + logger.info("Process output: {}", line); } - } catch (IOException e) { - logger.error("Could not find 3migrate.py"); } + + // Wait for the process to complete + int exitCode = 0; + try { + exitCode = process.waitFor(); + } catch (InterruptedException e) { + logger.error("Error waiting for migration process to complete", e); + throw new RuntimeException(e); + } + logger.info("Migration process exited with code: {}", exitCode); } } diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java index 6a7f1df368..39cee26622 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java @@ -3,9 +3,7 @@ import com.google.common.io.ByteStreams; import io.javalin.Javalin; import io.javalin.http.Context; -import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import org.slf4j.Logger; @@ -28,19 +26,6 @@ private StaticFileMapper() { public static void create(Javalin app) { - try (InputStream is = - StaticFileMapper.class.getResourceAsStream("/public_html/apps/ui/index.html")) { - if (is != null) { - logger.info("Found 2migrate.py"); - String text = new String(is.readAllBytes(), StandardCharsets.UTF_8); - logger.info("2migrate.py: " + text); - } else { - logger.error("Could not find 2migrate.py"); - } - } catch (IOException e) { - logger.error("Could not find 2migrate.py"); - } - app.get("/{schema}/{appname}/theme.css", BootstrapThemeService::getCss); app.get("*/docs/", StaticFileMapper::redirectDocs); From 7be9532d39eeee3333550c431fb2235bcd657aaf Mon Sep 17 00:00:00 2001 From: connoratrug Date: Fri, 15 Nov 2024 15:40:02 +0100 Subject: [PATCH 3/5] run migration with py client setup --- apps/nuxt3-ssr/migrations/requirements.txt | 13 +++++++++++++ apps/nuxt3-ssr/migrations/version1/migrate.py | 12 +++++++++++- .../org/molgenis/emx2/RunMolgenisEmx2.java | 18 +++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 apps/nuxt3-ssr/migrations/requirements.txt diff --git a/apps/nuxt3-ssr/migrations/requirements.txt b/apps/nuxt3-ssr/migrations/requirements.txt new file mode 100644 index 0000000000..7b21e98416 --- /dev/null +++ b/apps/nuxt3-ssr/migrations/requirements.txt @@ -0,0 +1,13 @@ +certifi==2024.7.4 +charset-normalizer==2.1.1 +idna==3.7 +numpy==1.26.3 +pandas==2.2.2 +python-dateutil==2.8.2 +python-decouple==3.8 +pytz==2022.7.1 +requests==2.32.3 +setuptools==70.0.0 +six==1.16.0 +molgenis_emx2_pyclient +urllib3==1.26.19 diff --git a/apps/nuxt3-ssr/migrations/version1/migrate.py b/apps/nuxt3-ssr/migrations/version1/migrate.py index 8ce7c684aa..423c62740f 100644 --- a/apps/nuxt3-ssr/migrations/version1/migrate.py +++ b/apps/nuxt3-ssr/migrations/version1/migrate.py @@ -1 +1,11 @@ -print("Hello python migration script, app: nuxt3-ssr, version: 1"); \ No newline at end of file +from molgenis_emx2_pyclient import Client + +username = 'admin' +password = 'admin' + +# Initialize the client as a context manager +with Client(url='http://localhost:8080') as client: + # Apply the 'signin' method with the username and password + client.signin(username, password) + + # Perform other tasks diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java index 42bc30df13..d375778ce7 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/RunMolgenisEmx2.java @@ -73,11 +73,27 @@ public static void main(String[] args) throws IOException { logger.info("After start"); String relativeMigrationFilePath = "./apps/nuxt3-ssr/migrations/version1/migrate.py"; + String relativeRequirementsFilePath = "./apps/nuxt3-ssr/migrations/requirements.txt"; String migrationFilePathString = "/nuxt3-ssr/migration/version1/migrate.py"; + // define commands (given tempDir as working directory) + String createVenvCommand = "python3 -m venv venv"; + String activateCommand = "source venv/bin/activate"; + String pipUpgradeCommand = "pip3 install --upgrade pip"; + String installRequirementsCommand = "pip3 install -r " + relativeRequirementsFilePath; String runScriptCommand = "python3 -u " + relativeMigrationFilePath; + String command = + String.join( + " && ", + createVenvCommand, + activateCommand, + pipUpgradeCommand, + installRequirementsCommand, + runScriptCommand); + logger.debug("Running migration script with command: {}", command); + // String runScriptCommand = "pwd"; - ProcessBuilder builder = new ProcessBuilder("bash", "-c", runScriptCommand); + ProcessBuilder builder = new ProcessBuilder("bash", "-c", command); builder.redirectErrorStream(true); Process process = builder.start(); From 08cde1c3372d8c09a5fb3b2638a328b2e1a52a6d Mon Sep 17 00:00:00 2001 From: connoratrug Date: Fri, 15 Nov 2024 15:42:14 +0100 Subject: [PATCH 4/5] remove debug code --- .../main/java/org/molgenis/emx2/web/StaticFileMapper.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java index 39cee26622..95b0ce9b5d 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java @@ -6,8 +6,6 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * to allow for nice urls, and make it easier for 'schema' app developers we include the schema in @@ -18,8 +16,6 @@ */ public class StaticFileMapper { - private static Logger logger = LoggerFactory.getLogger(StaticFileMapper.class); - private StaticFileMapper() { // hide constructor } @@ -92,7 +88,7 @@ public static void addFileToContext(Context ctx, String path, String mimeType) { ctx.status(404).result("File not found: " + ctx.path()); return; } - logger.info("Serving file: " + path); + if (mimeType == null) { mimeType = Files.probeContentType(Path.of(path)); if (mimeType == null) { From c2d860dc2439dc0a39e98c7c52b2e33a26c977a6 Mon Sep 17 00:00:00 2001 From: connoratrug Date: Fri, 15 Nov 2024 15:43:07 +0100 Subject: [PATCH 5/5] remove debug code --- .../src/main/java/org/molgenis/emx2/web/StaticFileMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java index 95b0ce9b5d..356bb5d5d2 100644 --- a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java +++ b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java @@ -88,7 +88,6 @@ public static void addFileToContext(Context ctx, String path, String mimeType) { ctx.status(404).result("File not found: " + ctx.path()); return; } - if (mimeType == null) { mimeType = Files.probeContentType(Path.of(path)); if (mimeType == null) {