[#11080] Initial fixes for V8 (#11228)

.github/ISSUE_TEMPLATE/help-request.md

@@ -10,7 +10,6 @@ about: Request for technical help in development
 - **Operating System:**
 - **JDK Version:**
 - **Node.js Version:**
-- **Google Cloud SDK Version:**
 - **Firefox / Chrome Version (if applicable):**
 
 **What I was trying to do / What I need help with**

.gitignore

@@ -34,6 +34,18 @@ test-output/*
 *.iml
 out/
 
+# VSCode
+.vscode/
+*.code-workspace
+.history/
+
+# Netbeans
+nbproject/
+nbbuild/
+nbdist/
+nbactions.xml
+nb-configuration.xml
+
 # JMeter
 jmeter.log
 src/lnp/resources/data/*

build.gradle

@@ -49,10 +49,10 @@ dependencies {
     annotationProcessor(objectify)
 
     implementation("com.google.auth:google-auth-library-oauth2-http:0.24.1")
-    implementation("com.google.cloud:google-cloud-datastore:1.106.2")
-    implementation("com.google.cloud:google-cloud-tasks:1.30.11")
-    implementation("com.google.cloud:google-cloud-logging:2.1.2")
-    implementation("com.google.cloud:google-cloud-storage:1.113.9")
+    implementation("com.google.cloud:google-cloud-datastore:1.107.0")
+    implementation("com.google.cloud:google-cloud-tasks:1.33.1")
+    implementation("com.google.cloud:google-cloud-logging:2.3.1")
+    implementation("com.google.cloud:google-cloud-storage:1.116.0")
     implementation("com.google.code.gson:gson:2.8.6")
     implementation("com.google.guava:guava:30.1.1-jre")
     implementation(objectify)

src/client/java/teammates/client/scripts/PopulateCourseSearchDocuments.java

@@ -0,0 +1,106 @@
+package teammates.client.scripts;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import com.googlecode.objectify.cmd.Query;
+
+import teammates.client.util.BackDoor;
+import teammates.common.datatransfer.DataBundle;
+import teammates.common.datatransfer.attributes.InstructorAttributes;
+import teammates.common.datatransfer.attributes.StudentAttributes;
+import teammates.logic.api.Logic;
+import teammates.storage.entity.Course;
+
+/**
+ * Script to populate search documents into the system back-end.
+ */
+public class PopulateCourseSearchDocuments extends DataMigrationEntitiesBaseScript<Course> {
+
+    private static final int STUDENT_SIZE_LIMIT = 300;
+    private final Logic logic = new Logic();
+
+    public PopulateCourseSearchDocuments() {
+        numberOfScannedKey.set(0L);
+        numberOfAffectedEntities.set(0L);
+        numberOfUpdatedEntities.set(0L);
+    }
+
+    public static void main(String[] args) throws IOException {
+        PopulateCourseSearchDocuments populator = new PopulateCourseSearchDocuments();
+        populator.doOperationRemotely();
+    }
+
+    @Override
+    protected Query<Course> getFilterQuery() {
+        Instant createdAtBound = Instant.now();
+        // To change the boundary of the createdAt timestamp, uncomment the next line and insert the appropriate timestamp.
+        // createdAtBound = Instant.ofEpochMilli(1618053102147L);
+        return ofy().load().type(Course.class)
+                .filter("createdAt <=", createdAtBound)
+                .order("-createdAt");
+    }
+
+    @Override
+    protected boolean isPreview() {
+        return false;
+    }
+
+    @Override
+    protected boolean isMigrationNeeded(Course course) throws Exception {
+        return true;
+    }
+
+    @Override
+    protected void migrateEntity(Course course) throws Exception {
+        List<StudentAttributes> students = logic.getStudentsForCourse(course.getUniqueId());
+        List<InstructorAttributes> instructors = logic.getInstructorsForCourse(course.getUniqueId());
+
+        int nLoop = students.size() / STUDENT_SIZE_LIMIT;
+
+        System.out.println("---------");
+        System.out.println("Going to populate search documents for students and instructors from course "
+                + course.getUniqueId());
+        System.out.println("Course is created at epoch " + course.getCreatedAt().toEpochMilli());
+        System.out.println();
+
+        for (int i = 0; i <= nLoop + 1; i++) {
+            DataBundle bundle = new DataBundle();
+            bundle.students = new HashMap<>();
+            bundle.instructors = new HashMap<>();
+            if (i == nLoop + 1) {
+                // For final loop, migrate instructors
+                if (instructors.isEmpty()) {
+                    System.out.println("No instructors to migrate");
+                    System.out.println();
+                    continue;
+                }
+                instructors.forEach(instructor -> bundle.instructors.put(instructor.getEmail(), instructor));
+            } else {
+                List<StudentAttributes> studentsSubList =
+                        students.subList(i * STUDENT_SIZE_LIMIT,
+                                Math.min(students.size(), (i + 1) * STUDENT_SIZE_LIMIT));
+                if (studentsSubList.isEmpty()) {
+                    System.out.println("No students to migrate");
+                    System.out.println();
+                    continue;
+                }
+                studentsSubList.forEach(student -> bundle.students.put(student.getEmail(), student));
+            }
+
+            long time = System.currentTimeMillis();
+            System.out.println("Total load: " + bundle.students.size() + " students, "
+                    + bundle.instructors.size() + " instructors");
+            String result = BackDoor.getInstance().putDocuments(bundle);
+            System.out.println("Operation result: " + result);
+            System.out.println("Time elapsed: " + (System.currentTimeMillis() - time) + "ms");
+            System.out.println();
+        }
+
+        System.out.println("Search document insertion completed");
+        System.out.println("---------");
+    }
+
+}

src/client/java/teammates/client/util/BackDoor.java

@@ -0,0 +1,35 @@
+package teammates.client.util;
+
+import teammates.test.AbstractBackDoor;
+
+/**
+ * Used to create API calls to the back-end without going through the UI.
+ */
+public final class BackDoor extends AbstractBackDoor {
+
+    private static BackDoor instance = new BackDoor();
+
+    private BackDoor() {
+        // Utility class
+    }
+
+    public static BackDoor getInstance() {
+        return instance;
+    }
+
+    @Override
+    protected String getAppUrl() {
+        return ClientProperties.API_URL;
+    }
+
+    @Override
+    protected String getBackdoorKey() {
+        return ClientProperties.BACKDOOR_KEY;
+    }
+
+    @Override
+    protected String getCsrfKey() {
+        return ClientProperties.CSRF_KEY;
+    }
+
+}

src/client/java/teammates/client/util/ClientProperties.java

@@ -14,6 +14,15 @@ public final class ClientProperties {
     /** The value of "client.target.url" in client.properties file. */
     public static final String TARGET_URL;
 
+    /** The value of "client.api.url" in client.properties file. */
+    public static final String API_URL;
+
+    /** The value of "client.backdoor.key" in client.properties file. */
+    public static final String BACKDOOR_KEY;
+
+    /** The value of "client.csrf.key" in client.properties file. */
+    public static final String CSRF_KEY;
+
     static {
         Properties prop = new Properties();
         try {
@@ -22,6 +31,9 @@ public final class ClientProperties {
             }
 
             TARGET_URL = prop.getProperty("client.target.url");
+            API_URL = prop.getProperty("client.api.url");
+            BACKDOOR_KEY = prop.getProperty("client.backdoor.key");
+            CSRF_KEY = prop.getProperty("client.csrf.key");
 
         } catch (IOException e) {
             throw new RuntimeException(e);

src/client/resources/client.template.properties

@@ -8,5 +8,14 @@
 # For staging server, the URL should be the URL of the application.
 # e.g. client.target.url=http\://localhost\:8484
 # e.g. client.target.url=https\://7-0-0-dot-teammates-john.appspot.com
-# Note: the '.' in the url has been replaced by -dot- to support https connection for the staging server.
 client.target.url=http\://localhost\:8484
+
+# This is the URL of the API back-end that the script will be run against.
+# Note that most of the time, this URL will not be used as scripts will use the database URL (defined previously) instead.
+# This URL will be used for other back-end operations that need connection to other services such as search service.
+# The backdoor key and CSRF key will be the keys that are used in the specified API back-end.
+# e.g. client.api.url=https\://localhost\:8080
+# e.g. client.api.url=https\://7-0-0-dot-teammates-john.appspot.com
+client.api.url=
+client.backdoor.key=
+client.csrf.key=

src/main/appengine/app.template.yaml

@@ -56,24 +56,24 @@ handlers:
     expiration: 1d
 
   # Crawler-related files
-  - url: /sitemap.xml
-    static_files: dist/sitemap.xml
-    upload: dist/sitemap.xml
-    expiration: 1d
-  - url: /robots.txt
-    static_files: dist/robots.txt
-    upload: dist/robots.txt
-    expiration: 1d
+  # - url: /sitemap.xml
+  #   static_files: dist/sitemap.xml
+  #   upload: dist/sitemap.xml
+  #   expiration: 1d
+  # - url: /robots.txt
+  #   static_files: dist/robots.txt
+  #   upload: dist/robots.txt
+  #   expiration: 1d
 
   # Webmaster files
-  - url: /BingSiteAuth.xml
-    static_files: dist/BingSiteAuth.xml
-    upload: dist/BingSiteAuth.xml
-    expiration: 1d
-  - url: /google8c7ef1e995031e09.html
-    static_files: dist/google8c7ef1e995031e09.html
-    upload: dist/google8c7ef1e995031e09.html
-    expiration: 1d
+  # - url: /BingSiteAuth.xml
+  #   static_files: dist/BingSiteAuth.xml
+  #   upload: dist/BingSiteAuth.xml
+  #   expiration: 1d
+  # - url: /google8c7ef1e995031e09.html
+  #   static_files: dist/google8c7ef1e995031e09.html
+  #   upload: dist/google8c7ef1e995031e09.html
+  #   expiration: 1d
 
   # All incoming requests will be redirected to HTTPS.
   - url: /.*

src/main/appengine/cron.yaml

@@ -18,9 +18,9 @@ cron:
   timezone: 'Asia/Singapore'
   description: 'Checks and sends out emails for feedback sessions which have just been published in the past hour.'
 - url: '/auto/datastoreBackup'
-  schedule: 'every monday 05:30'
+  schedule: '1st sunday of month 05:30'
   timezone: 'Asia/Singapore'
-  description: 'Weekly Backup'
+  description: 'Monthly Backup'
 - url: '/auto/compileLogs'
   schedule: 'every 5 minutes synchronized'
   timezone: 'Asia/Singapore'

src/main/java/teammates/common/util/Config.java

@@ -107,7 +107,7 @@ public final class Config {
         }
         APP_ID = properties.getProperty("app.id");
         APP_REGION = properties.getProperty("app.region");
-        APP_VERSION = properties.getProperty("app.version").replace("-", ".");
+        APP_VERSION = properties.getProperty("app.version");
         APP_FRONTENDDEV_URL = properties.getProperty("app.frontenddev.url");
         APP_LOCALDATASTORE_PORT = Integer.parseInt(properties.getProperty("app.localdatastore.port", "8484"));
         TASKQUEUE_ACTIVE = Boolean.parseBoolean(properties.getProperty("app.taskqueue.active", "true"));
@@ -161,7 +161,7 @@ public static boolean isDevServer() {
         }
 
         return !appName.endsWith(APP_ID)
-                || !APP_VERSION.equals(version.replace("-", "."))
+                || !APP_VERSION.equals(version)
                 || !"standard".equals(env);
     }
 

src/main/java/teammates/logic/core/GoogleCloudTasksService.java

@@ -5,6 +5,7 @@
 import java.time.Instant;
 
 import com.google.cloud.tasks.v2.AppEngineHttpRequest;
+import com.google.cloud.tasks.v2.AppEngineRouting;
 import com.google.cloud.tasks.v2.CloudTasksClient;
 import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.cloud.tasks.v2.QueueName;
@@ -34,6 +35,9 @@ public void addDeferredTask(TaskWrapper task, long countdownTime) {
 
             AppEngineHttpRequest.Builder requestBuilder =
                     AppEngineHttpRequest.newBuilder()
+                            .setAppEngineRouting(AppEngineRouting.newBuilder()
+                                    .setVersion(Config.APP_VERSION)
+                                    .build())
                             .setHttpMethod(HttpMethod.POST);
 
             if (task.getRequestBody() == null) {

src/main/java/teammates/storage/entity/FeedbackQuestion.java

@@ -128,7 +128,9 @@ public String getId() {
         // and the produced legacy URL-safe key has additional space character at the end of the string,
         // resulting in incompatibility with old data.
         // Last four characters (ogEA) of the base64 encoded string are trimmed for backward compatibility.
-        return untrimmedId.substring(0, untrimmedId.length() - 4);
+        // Additional changes are needed for production system case, possibly due to very old Datastore instance used.
+        return untrimmedId.replaceFirst("KIBAA$", "AogEA")
+                .substring(0, untrimmedId.length() - 4);
     }
 
     public void setFeedbackQuestionId(Long feedbackQuestionId) {

src/main/java/teammates/storage/search/SearchManager.java

@@ -161,6 +161,7 @@ public void resetCollections() {
 
         try {
             client.deleteByQuery(getCollectionName(), "*:*");
+            client.commit(getCollectionName());
         } catch (SolrServerException e) {
             log.severe(String.format(ERROR_RESET_COLLECTION, e.getRootCause())
                     + TeammatesException.toStringWithStackTrace(e));

src/main/java/teammates/ui/webapi/RequestTraceFilter.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -18,7 +19,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.http.HttpStatus;
 
 import teammates.common.exception.TeammatesException;
@@ -85,7 +86,10 @@ public void doFilter(ServletRequest req, ServletResponse resp, FilterChain chain
             String traceId;
             String spanId = null;
             if (requestId == null) {
-                traceId = RandomStringUtils.randomAlphanumeric(32);
+                // Generate random hexadecimal string of length 32
+                byte[] resBuf = new byte[16];
+                new Random().nextBytes(resBuf);
+                traceId = Hex.encodeHexString(resBuf);
             } else {
                 // X-Cloud-Trace-Context header is in form of TRACE_ID/SPAN_ID;o=TRACE_TRUE
                 String[] traceAndSpan = requestId.split("/", 2);