From 49bc89cb424b574440705162310417c2bd34fae9 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Mon, 19 Sep 2016 15:24:18 -0700 Subject: [PATCH] Issue #8 - alternate /tests/ setup for 100% maven + Deploys jetty9:testing (tagged) image in /jetty9/ module during integration-test phase + Adds /tests/ + Adds /tests/webapps/ + Adds /tests/gcloud-testing-core/ for common testing lib --- jetty9/pom.xml | 37 +++++ pom.xml | 64 +++++++- tests/gcloud-testing-core/pom.xml | 49 +++++++ .../runtime/jetty/testing/AppDeployment.java | 57 ++++++++ .../runtime/jetty/testing/HttpURLUtil.java | 57 ++++++++ .../runtime/jetty/testing/ProcessUtil.java | 50 +++++++ .../runtime/jetty/testing/RemoteLog.java | 137 +++++++++++++++++ .../runtime/jetty/testing/RemoteLogTest.java | 23 +++ tests/pom.xml | 55 +++++++ tests/run.sh | 41 ++++++ tests/webapps/pom.xml | 138 ++++++++++++++++++ tests/webapps/test-war-hello/pom.xml | 81 ++++++++++ .../src/main/appengine/app.yaml | 3 + .../test-war-hello/src/main/docker/Dockerfile | 2 + .../runtimes/jetty/tests/HelloServlet.java | 20 +++ .../src/main/webapp/WEB-INF/web.xml | 8 + .../jetty/tests/DeploymentITCase.java | 25 ++++ 17 files changed, 844 insertions(+), 3 deletions(-) create mode 100644 tests/gcloud-testing-core/pom.xml create mode 100644 tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/AppDeployment.java create mode 100644 tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/HttpURLUtil.java create mode 100644 tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/ProcessUtil.java create mode 100644 tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/RemoteLog.java create mode 100644 tests/gcloud-testing-core/src/test/java/com/google/cloud/runtime/jetty/testing/RemoteLogTest.java create mode 100644 tests/pom.xml create mode 100644 tests/run.sh create mode 100644 tests/webapps/pom.xml create mode 100644 tests/webapps/test-war-hello/pom.xml create mode 100644 tests/webapps/test-war-hello/src/main/appengine/app.yaml create mode 100644 tests/webapps/test-war-hello/src/main/docker/Dockerfile create mode 100644 tests/webapps/test-war-hello/src/main/java/com/google/cloud/runtimes/jetty/tests/HelloServlet.java create mode 100644 tests/webapps/test-war-hello/src/main/webapp/WEB-INF/web.xml create mode 100644 tests/webapps/test-war-hello/src/test/java/com/google/cloud/runtimes/jetty/tests/DeploymentITCase.java diff --git a/jetty9/pom.xml b/jetty9/pom.xml index ae1e0c46..ec4c9d96 100644 --- a/jetty9/pom.xml +++ b/jetty9/pom.xml @@ -41,7 +41,9 @@ + org.apache.maven.plugins maven-resources-plugin + 3.0.0 copy-resources @@ -145,6 +147,41 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + tag-and-deploy-integration-tests + integration-test + + run + + + + + + + + + + app.deploy.project=${app.deploy.project} + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index f8079c0f..9d9dc9c8 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ 9.3.8.v20160314 1.9.40 async- + ${project.build.directory}/gcloud-projectid.properties @@ -79,6 +80,7 @@ jetty9-base jetty9 + tests @@ -245,9 +247,9 @@ org.apache.maven.plugins maven-failsafe-plugin 2.19.1 - - false - + + false + @@ -293,5 +295,61 @@ + + get-gcloud-project-id + + + !app.deploy.project + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + find-app-deploy-project + validate + + run + + + + app.deploy.project= + + + + + + + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + read-gcloud-properties + initialize + + read-project-properties + + + + ${gcloud-projectId-file} + + + + + + + + diff --git a/tests/gcloud-testing-core/pom.xml b/tests/gcloud-testing-core/pom.xml new file mode 100644 index 00000000..2b5397df --- /dev/null +++ b/tests/gcloud-testing-core/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + com.google.cloud.runtimes.jetty.tests + tests-parent + 0.1.0-SNAPSHOT + + gcloud-testing-core + Jetty-Runtime :: Testing Core Lib + jar + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + + org.yaml + snakeyaml + 1.17 + + + + org.eclipse.jetty.toolchain + jetty-test-helper + 4.0 + compile + + + junit + junit + compile + + + diff --git a/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/AppDeployment.java b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/AppDeployment.java new file mode 100644 index 00000000..09eff88a --- /dev/null +++ b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/AppDeployment.java @@ -0,0 +1,57 @@ +package com.google.cloud.runtime.jetty.testing; + +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.yaml.snakeyaml.Yaml; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; + +public final class AppDeployment { + public static final String PROJECT_ID; + public static final String VERSION_ID; + public static final String MODULE_ID; + public static final URI SERVER_URI; + + static { + String projectId = System.getProperty("app.deploy.project"); + String version = System.getProperty("app.deploy.version"); + + Objects.requireNonNull(projectId, "app.deploy.project"); + Objects.requireNonNull(version, "app.deploy.version"); + + PROJECT_ID = projectId; + VERSION_ID = version; + + String moduleId = null; + + Path appYamlPath = MavenTestingUtils.getProjectFilePath("src/main/appengine/app.yaml"); + if (Files.exists(appYamlPath)) { + try (BufferedReader reader = Files.newBufferedReader(appYamlPath, StandardCharsets.UTF_8)) { + Yaml yaml = new Yaml(); + Map map = (Map) yaml.load(reader); + moduleId = (String) map.get("module"); + } catch (IOException e) { + throw new RuntimeException("Unable to parse app.yaml", e); + } + } + + MODULE_ID = moduleId; + + StringBuilder uri = new StringBuilder(); + uri.append("https://"); + uri.append(version).append("-dot-"); + if (moduleId != null) { + uri.append(moduleId).append("-dot-"); + } + uri.append(projectId); + uri.append(".appspot.com/"); + + SERVER_URI = URI.create(uri.toString()); + } +} diff --git a/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/HttpURLUtil.java b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/HttpURLUtil.java new file mode 100644 index 00000000..97a27076 --- /dev/null +++ b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/HttpURLUtil.java @@ -0,0 +1,57 @@ +package com.google.cloud.runtime.jetty.testing; + +import org.eclipse.jetty.toolchain.test.IO; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public final class HttpURLUtil { + /** + * Open a new {@link HttpURLConnection} to the provided URI. + * + *

+ * Note: will also set the 'User-Agent' to {@code jetty-runtime/gcloud-testing-core} + *

+ * + * @param uri the URI to open to + * @return the open HttpURLConnection + * @throws IOException if unable to open the connection + */ + public static HttpURLConnection openTo(URI uri) throws IOException { + HttpURLConnection http = (HttpURLConnection) uri.toURL().openConnection(); + http.setRequestProperty("User-Agent", "jetty-runtime/gcloud-testing-core"); + return http; + } + + /** + * Obtain the text (non-binary) response body from an {@link HttpURLConnection}, + * using the response provided charset. + * + *

+ * Note: Normal HttpURLConnection doesn't use the provided charset properly. + *

+ * + * @param http the {@link HttpURLConnection} to obtain the response body from + * @return the text of the response body + * @throws IOException if unable to get the text of the response body + */ + public static String getResponseBody(HttpURLConnection http) throws IOException { + Charset responseEncoding = StandardCharsets.UTF_8; + if (http.getContentEncoding() != null) { + responseEncoding = Charset.forName(http.getContentEncoding()); + } + + try (InputStream in = http.getInputStream(); + InputStreamReader reader = new InputStreamReader(in, responseEncoding); + StringWriter writer = new StringWriter()) { + IO.copy(reader, writer); + return writer.toString(); + } + } +} diff --git a/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/ProcessUtil.java b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/ProcessUtil.java new file mode 100644 index 00000000..60c6b436 --- /dev/null +++ b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/ProcessUtil.java @@ -0,0 +1,50 @@ +package com.google.cloud.runtime.jetty.testing; + +import org.eclipse.jetty.toolchain.test.IO; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * Utilities for executing and collecting output from command line processes. + */ +public final class ProcessUtil { + private static final ExecutorService executor = Executors.newFixedThreadPool(5); + + /** + * Execute a command line. + *

+ * Report output from command line to Writer + *

+ * + * @param output where to put the output from the execution of the command line + * @param args the command line arguments + * @return the exit code from the execution + */ + public static int exec(OutputStream output, String... args) + throws IOException, InterruptedException, ExecutionException { + System.out.printf("exec(%s)%n", Arrays.toString(args)); + Process process = Runtime.getRuntime().exec(args); + + InputStream in = process.getInputStream(); + + Future fut = executor.submit(new Callable() { + @Override + public Void call() throws Exception { + IO.copy(in, output); + return null; + } + }); + + fut.get(); + + return process.waitFor(); + } +} diff --git a/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/RemoteLog.java b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/RemoteLog.java new file mode 100644 index 00000000..12f07dcb --- /dev/null +++ b/tests/gcloud-testing-core/src/main/java/com/google/cloud/runtime/jetty/testing/RemoteLog.java @@ -0,0 +1,137 @@ +package com.google.cloud.runtime.jetty.testing; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.yaml.snakeyaml.Yaml; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +public class RemoteLog { + public static class Entry { + String logName; + String timeStamp; + String textPayload; + + public String getTextPayload() { + return textPayload; + } + + public String getLogName() { + return logName; + } + + public String getTimeStamp() { + return timeStamp; + } + } + + /** + * Get the remote logs for the associated module and version. + * + * @param moduleId the module name + * @param versionId the deployed version + * @return the list of log entries + * @throws IOException if unable to fetch remote log entries + */ + public static List getLogs(String moduleId, String versionId) throws IOException { + String filter = String + .format("resource.labels.module_id=\"%s\" AND resource.labels.version_id=\"%s\"", moduleId, + versionId); + + List entries = new ArrayList<>(); + Path logPath = MavenTestingUtils.getTargetPath("remote-module-version.log"); + try (OutputStream output = Files.newOutputStream(logPath, StandardOpenOption.CREATE)) { + int retval = ProcessUtil.exec(output, "gcloud", "beta", "logging", "read", filter); + + if (retval != 0) { + return entries; + } + } catch (InterruptedException | ExecutionException e) { + throw new IOException("Unable to process gcloud beta logging read", e); + } + + try (InputStream in = Files.newInputStream(logPath, StandardOpenOption.READ)) { + Yaml yaml = new Yaml(); + for (Object yamlObj : yaml.loadAll(in)) { + Map yamlMap = (Map) yamlObj; + Entry entry = new Entry(); + entry.logName = (String) yamlMap.get("logName"); + entry.textPayload = (String) yamlMap.get("textPayload"); + entry.timeStamp = (String) yamlMap.get("timestamp"); + entries.add(entry); + } + } + return entries; + } + + /** + * Ensure that the list of log entries has the expected entries. + * + *

+ * Each expected entry is tested against the list of log entries + * via a {@link String#contains(CharSequence)} check. + *

+ * + *

+ * Missing entries are reported back in the Assertion failure + *

+ * + * @param logs the list of log entries + * @param expectedEntries the expected entries + */ + public static void assertHasEntries(List logs, List expectedEntries) { + List expected = new ArrayList<>(expectedEntries); + for (Entry entry : logs) { + if (expected.isEmpty()) { + return; // all good! + } + ListIterator expectedIter = expected.listIterator(); + while (expectedIter.hasNext()) { + String expectedText = expectedIter.next(); + if (entry.textPayload.contains(expectedText)) { + expectedIter.remove(); + } + } + } + + if (expected.size() > 0) { + StringBuilder err = new StringBuilder(); + err.append("Missing ").append(expected.size()).append(" expected entries"); + for (String expectedEntry : expected) { + err.append(System.lineSeparator()); + err.append(expectedEntry); + } + assertThat(err.toString(), expected.size(), is(0)); + } + } + + /** + * Find a specific log entry conforming to the {@link String#contains(CharSequence)} of + * the provided text. + * + * @param logs the list of entries to check in + * @param text the text to search for + * @return the first occurrence of text, or null if not found + */ + public static Entry findEntry(List logs, String text) { + for (Entry entry : logs) { + if (entry.textPayload.contains(text)) { + return entry; + } + } + + return null; + } +} diff --git a/tests/gcloud-testing-core/src/test/java/com/google/cloud/runtime/jetty/testing/RemoteLogTest.java b/tests/gcloud-testing-core/src/test/java/com/google/cloud/runtime/jetty/testing/RemoteLogTest.java new file mode 100644 index 00000000..2cb82ce8 --- /dev/null +++ b/tests/gcloud-testing-core/src/test/java/com/google/cloud/runtime/jetty/testing/RemoteLogTest.java @@ -0,0 +1,23 @@ +package com.google.cloud.runtime.jetty.testing; + +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertThat; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +public class RemoteLogTest { + @Test + @Ignore("This is module, and version specific (only useful for limited local testing)") + public void testGetLogs() throws IOException { + List logs = RemoteLog.getLogs("javautillogging", "20160914-143225"); + assertThat("Log Entries", logs.size(), greaterThan(1000)); + + for (RemoteLog.Entry entry : logs) { + System.out.printf("%s: %s%n", entry.timeStamp, entry.textPayload); + } + } +} diff --git a/tests/pom.xml b/tests/pom.xml new file mode 100644 index 00000000..9d9c6a17 --- /dev/null +++ b/tests/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + com.google.cloud.runtimes + jetty-parent + 0.1.0-SNAPSHOT + + + com.google.cloud.runtimes.jetty.tests + tests-parent + Jetty-Runtime :: Tests Parent + pom + + + gcloud-testing-core + webapps + + + + + + org.eclipse.jetty.toolchain + jetty-test-helper + 4.0 + test + + + + + + + + src/main/docker + true + ${project.build.directory}/docker + + + + diff --git a/tests/run.sh b/tests/run.sh new file mode 100644 index 00000000..7e36edb3 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +function appengine_deploy() +{ + module="$1" + + pushd "${module}" + + mvn clean appengine:deploy verify \ + -Dapp.deploy.project="${gcloud_projectid}" \ + -Dapp.deploy.version="${test_runid}" \ + -Dapp.deploy.promote=false + + ret=$? + + popd + + return ${ret} +} + +gcloud info +gcloud auth list + +gcloud_projectid=$(gcloud info | sed -rn 's/Project: \[(.*)\]/\1/p') +test_runid=$(date +%Y%m%d-%H%M%S) + +echo "GCloud Project: $gcloud_projectid" + +# Push the as-built jetty9:latest to gcr.io/${gcloud_projectid}/jetty9:testing" + +docker tag "jetty9:latest" "gcr.io/${gcloud_projectid}/jetty9:testing" +gcloud docker push "gcr.io/${gcloud_projectid}/jetty9:testing" + +# Deploy and Execute the individual test-war-* tests + +for module in test-war-* +do + echo "module: $module" + appengine_deploy "${module}" +done + diff --git a/tests/webapps/pom.xml b/tests/webapps/pom.xml new file mode 100644 index 00000000..39064a40 --- /dev/null +++ b/tests/webapps/pom.xml @@ -0,0 +1,138 @@ + + + + 4.0.0 + + com.google.cloud.runtimes.jetty.tests + tests-parent + 0.1.0-SNAPSHOT + + + tests-webapps-parent + Jetty-Runtime :: Tests :: WebApps Parent + pom + + + test-war-hello + + + + + + org.eclipse.jetty.toolchain + jetty-test-helper + 4.0 + test + + + + + + + + src/main/docker + true + ${project.build.directory}/docker + + + + + maven-checkstyle-plugin + + + true + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + org.apache.maven.surefire + surefire-junit47 + 2.19.1 + + + + + ${app.deploy.project} + ${maven.build.timestamp} + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + cleanup-integration-test + post-integration-test + + run + + + + + + + + + + + + + + + + + + diff --git a/tests/webapps/test-war-hello/pom.xml b/tests/webapps/test-war-hello/pom.xml new file mode 100644 index 00000000..07575fb4 --- /dev/null +++ b/tests/webapps/test-war-hello/pom.xml @@ -0,0 +1,81 @@ + + 4.0.0 + + com.google.cloud.runtimes.jetty.tests + tests-webapps-parent + 0.1.0-SNAPSHOT + + test-war-hello + war + Jetty-Runtime :: Tests :: Hello (Servlet 3.1) WebApp + + + javax.servlet + javax.servlet-api + 3.1.0 + provided + + + com.google.cloud.runtimes.jetty.tests + gcloud-testing-core + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + + run + + package + + + + + + + + + + com.google.cloud.tools + appengine-maven-plugin + 0.1.2 + + ${project.build.directory}/docker + + + + deploy-integration-test + pre-integration-test + + deploy + + + ${app.deploy.project} + ${maven.build.timestamp} + false + + + + + + + diff --git a/tests/webapps/test-war-hello/src/main/appengine/app.yaml b/tests/webapps/test-war-hello/src/main/appengine/app.yaml new file mode 100644 index 00000000..51203b3c --- /dev/null +++ b/tests/webapps/test-war-hello/src/main/appengine/app.yaml @@ -0,0 +1,3 @@ +runtime: custom +vm: true +module: hello diff --git a/tests/webapps/test-war-hello/src/main/docker/Dockerfile b/tests/webapps/test-war-hello/src/main/docker/Dockerfile new file mode 100644 index 00000000..6e666f82 --- /dev/null +++ b/tests/webapps/test-war-hello/src/main/docker/Dockerfile @@ -0,0 +1,2 @@ +FROM gcr.io/${app.deploy.project}/jetty9:testing +ADD root /app diff --git a/tests/webapps/test-war-hello/src/main/java/com/google/cloud/runtimes/jetty/tests/HelloServlet.java b/tests/webapps/test-war-hello/src/main/java/com/google/cloud/runtimes/jetty/tests/HelloServlet.java new file mode 100644 index 00000000..e5c028c3 --- /dev/null +++ b/tests/webapps/test-war-hello/src/main/java/com/google/cloud/runtimes/jetty/tests/HelloServlet.java @@ -0,0 +1,20 @@ +package com.google.cloud.runtimes.jetty.tests; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet(urlPatterns = {"/hello/*"}) +public class HelloServlet extends HttpServlet +{ + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException + { + resp.setContentType("text/plain"); + resp.getWriter().println("Hello from Servlet 3.1"); + } +} diff --git a/tests/webapps/test-war-hello/src/main/webapp/WEB-INF/web.xml b/tests/webapps/test-war-hello/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 00000000..79716f38 --- /dev/null +++ b/tests/webapps/test-war-hello/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,8 @@ + + + Web Application for Servlet 3.1 + diff --git a/tests/webapps/test-war-hello/src/test/java/com/google/cloud/runtimes/jetty/tests/DeploymentITCase.java b/tests/webapps/test-war-hello/src/test/java/com/google/cloud/runtimes/jetty/tests/DeploymentITCase.java new file mode 100644 index 00000000..030a6d06 --- /dev/null +++ b/tests/webapps/test-war-hello/src/test/java/com/google/cloud/runtimes/jetty/tests/DeploymentITCase.java @@ -0,0 +1,25 @@ +package com.google.cloud.runtimes.jetty.tests; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.cloud.runtime.jetty.testing.AppDeployment; +import com.google.cloud.runtime.jetty.testing.HttpURLUtil; + +import org.junit.Test; + +import java.io.IOException; +import java.net.HttpURLConnection; + +public class DeploymentITCase +{ + @Test + public void testGet() throws IOException + { + HttpURLConnection http = HttpURLUtil.openTo(AppDeployment.SERVER_URI.resolve("/hello/")); + assertThat(http.getResponseCode(), is(200)); + String responseBody = HttpURLUtil.getResponseBody(http); + assertThat(responseBody, containsString("Hello from Servlet 3.1")); + } +}