Updated validation documents to prevent old packets from going into d…

…atabase. More integration tests

.editorconfig

@@ -35,6 +35,8 @@ indent_size = 4
 [{*.js,*.css,*.ts,*.tsx,*.jsx,*.graphql,*.ns}]
 indent_style = space
 indent_size = 2
+[core/**/*.js]  # CouchDB design document JavaScript files get special treatment
+indent_size = 4
 
 [syslog-ng.conf]
 indent_style = tab

client/src/main/java/me/retrodaredevil/solarthing/program/couchdb/CouchDbSetupMain.java

@@ -104,7 +104,10 @@ private void addPacketsDesign(SolarThingDatabaseType databaseType, CouchDbDataba
 			out.println("This database will have the simpleAllDocs view");
 			design.addSimpleAllDocsView();
 		}
-		if (databaseType.needsReadonlyValidateFunction()) {
+		if (databaseType == SolarThingDatabaseType.OPEN) {
+			out.println("This database will not allow editing of documents after they are uploaded.");
+			design.setSolarThingOpenValidate();
+		} else if (databaseType.needsReadonlyValidateFunction()) {
 			out.println("This database will be readonly");
 			design.setReadonlyAuth();
 		}

client/src/test/java/me/retrodaredevil/solarthing/integration/DatabaseOpenUploadOnlyTest.java

@@ -0,0 +1,64 @@
+package me.retrodaredevil.solarthing.integration;
+
+import me.retrodaredevil.couchdbjava.CouchDbInstance;
+import me.retrodaredevil.couchdbjava.exception.CouchDbException;
+import me.retrodaredevil.couchdbjava.exception.CouchDbUnauthorizedException;
+import me.retrodaredevil.solarthing.SolarThingDatabaseType;
+import me.retrodaredevil.solarthing.database.SolarThingDatabase;
+import me.retrodaredevil.solarthing.database.UpdateToken;
+import me.retrodaredevil.solarthing.database.couchdb.CouchDbSolarThingDatabase;
+import me.retrodaredevil.solarthing.database.exception.SolarThingDatabaseException;
+import me.retrodaredevil.solarthing.misc.device.CelsiusCpuTemperaturePacket;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollection;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollectionIdGenerator;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollections;
+import me.retrodaredevil.solarthing.packets.instance.InstanceFragmentIndicatorPackets;
+import me.retrodaredevil.solarthing.packets.instance.InstanceSourcePackets;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+@Tag("integration")
+public class DatabaseOpenUploadOnlyTest {
+
+	private static PacketCollection createSimplePacketCollection() {
+		// Note that in a production environment, we would make sure that the packets in a given packet collection
+		//   actually belong in a particular database, but for testing we don't care. (A CpuTemperaturePacket shouldn't end up in the events database)
+		return PacketCollections.createFromPackets(
+				Instant.now(),
+				Arrays.asList(
+						new CelsiusCpuTemperaturePacket(null, 20.2f, null),
+						InstanceSourcePackets.create("default"),
+						InstanceFragmentIndicatorPackets.create(1)
+				),
+				PacketCollectionIdGenerator.Defaults.UNIQUE_GENERATOR,
+				ZoneId.systemDefault()
+		);
+	}
+
+
+	@ParameterizedTest
+	@MethodSource("me.retrodaredevil.solarthing.integration.DatabaseService#all")
+	void test(DatabaseService databaseService) throws CouchDbException, SolarThingDatabaseException {
+		IntegrationSetup.setup(IntegrationUtil.createCouchDbInstance(databaseService, IntegrationUtil.DEFAULT_ADMIN_AUTH));
+
+		// Uploader user
+		CouchDbInstance uploaderInstance = IntegrationUtil.createCouchDbInstance(databaseService, IntegrationUtil.getAuthFor(SolarThingDatabaseType.UserType.UPLOADER));
+		SolarThingDatabase database = CouchDbSolarThingDatabase.create(uploaderInstance);
+
+		PacketCollection packetCollection = createSimplePacketCollection();
+		UpdateToken updateToken = database.getOpenDatabase().uploadPacketCollection(packetCollection, null);
+		try {
+			database.getOpenDatabase().uploadPacketCollection(packetCollection, updateToken);
+		} catch (SolarThingDatabaseException solarThingDatabaseException) {
+			// expect that we are unauthorized to change the document
+			assertInstanceOf(CouchDbUnauthorizedException.class, solarThingDatabaseException.getCause());
+		}
+	}
+}

client/src/test/java/me/retrodaredevil/solarthing/integration/DatabaseRejectOldPacketsTest.java

@@ -0,0 +1,104 @@
+package me.retrodaredevil.solarthing.integration;
+
+import me.retrodaredevil.couchdbjava.CouchDbInstance;
+import me.retrodaredevil.couchdbjava.exception.CouchDbCodeException;
+import me.retrodaredevil.couchdbjava.exception.CouchDbException;
+import me.retrodaredevil.couchdbjava.response.ErrorResponse;
+import me.retrodaredevil.solarthing.SolarThingDatabaseType;
+import me.retrodaredevil.solarthing.database.SolarThingDatabase;
+import me.retrodaredevil.solarthing.database.couchdb.CouchDbSolarThingDatabase;
+import me.retrodaredevil.solarthing.database.exception.SolarThingDatabaseException;
+import me.retrodaredevil.solarthing.misc.device.CelsiusCpuTemperaturePacket;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollection;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollectionIdGenerator;
+import me.retrodaredevil.solarthing.packets.collection.PacketCollections;
+import me.retrodaredevil.solarthing.packets.instance.InstanceFragmentIndicatorPackets;
+import me.retrodaredevil.solarthing.packets.instance.InstanceSourcePackets;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@Tag("integration")
+public class DatabaseRejectOldPacketsTest {
+
+	private static PacketCollection createSimplePacketCollection(Instant instant) {
+		// Note that in a production environment, we would make sure that the packets in a given packet collection
+		//   actually belong in a particular database, but for testing we don't care. (A CpuTemperaturePacket shouldn't end up in the events database)
+		return PacketCollections.createFromPackets(
+				instant,
+				Arrays.asList(
+						new CelsiusCpuTemperaturePacket(null, 20.2f, null),
+						InstanceSourcePackets.create("default"),
+						InstanceFragmentIndicatorPackets.create(1)
+				),
+				PacketCollectionIdGenerator.Defaults.UNIQUE_GENERATOR,
+				ZoneId.systemDefault()
+		);
+	}
+
+
+	@ParameterizedTest
+	@MethodSource("me.retrodaredevil.solarthing.integration.DatabaseService#all")
+	void test(DatabaseService databaseService) throws CouchDbException, SolarThingDatabaseException {
+		IntegrationSetup.setup(IntegrationUtil.createCouchDbInstance(databaseService, IntegrationUtil.DEFAULT_ADMIN_AUTH));
+
+		// Uploader user
+		CouchDbInstance uploaderInstance = IntegrationUtil.createCouchDbInstance(databaseService, IntegrationUtil.getAuthFor(SolarThingDatabaseType.UserType.UPLOADER));
+		SolarThingDatabase database = CouchDbSolarThingDatabase.create(uploaderInstance);
+
+		Instant pastInstant = Instant.now().minus(Duration.ofMinutes(30));
+		Instant futureInstant = Instant.now().plus(Duration.ofMinutes(4));
+		Instant tinyBitInTheFutureInstant = Instant.now().plus(Duration.ofMinutes(1));
+		PacketCollection pastPacketCollection = createSimplePacketCollection(pastInstant);
+		try {
+			database.getStatusDatabase().uploadPacketCollection(pastPacketCollection, null);
+			fail("We expect this to fail");
+		} catch (SolarThingDatabaseException solarThingDatabaseException) {
+			expectForbiddenResponse(solarThingDatabaseException);
+		}
+		try {
+			database.getEventDatabase().uploadPacketCollection(pastPacketCollection, null);
+			fail("We expect this to fail");
+		} catch (SolarThingDatabaseException solarThingDatabaseException) {
+			expectForbiddenResponse(solarThingDatabaseException);
+		}
+
+		try {
+			database.getOpenDatabase().uploadPacketCollection(pastPacketCollection, null);
+			fail("We expect this to fail");
+		} catch (SolarThingDatabaseException solarThingDatabaseException) {
+			expectForbiddenResponse(solarThingDatabaseException);
+		}
+		try {
+			// future solarthing_open test
+			database.getOpenDatabase().uploadPacketCollection(createSimplePacketCollection(futureInstant), null);
+			fail("We expect this to fail");
+		} catch (SolarThingDatabaseException solarThingDatabaseException) {
+			expectForbiddenResponse(solarThingDatabaseException);
+		}
+		// We expect to be able to upload a tiny bit in the future to SolarThing open
+		database.getOpenDatabase().uploadPacketCollection(createSimplePacketCollection(tinyBitInTheFutureInstant), null);
+	}
+	private static void expectForbiddenResponse(SolarThingDatabaseException solarThingDatabaseException) {
+		// NOTE: The cause is an implementation detail of a SolarThingDatabaseException
+		//   Remember that we don't want to rely on implementation details in actual code,
+		//   but it's OK in a test like this.
+		assertInstanceOf(CouchDbCodeException.class, solarThingDatabaseException.getCause());
+
+		CouchDbCodeException e = (CouchDbCodeException) solarThingDatabaseException.getCause();
+
+		// We expect to have thrown a forbidden error as described here: https://docs.couchdb.org/en/stable/ddocs/ddocs.html#validate-document-update-functions
+		assertEquals(403, e.getCode());
+		ErrorResponse errorResponse = e.getErrorResponse();
+		assertNotNull(errorResponse);
+		assertEquals("forbidden", errorResponse.getError());
+		assertTrue(errorResponse.getReason().contains("dateMillis"));
+	}
+}

core/src/main/java/me/retrodaredevil/couchdb/design/DesignResource.java

@@ -6,10 +6,16 @@
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
+/**
+ * Design resources are files used to create design documents for CouchDB.
+ * <p>
+ * Some of these design documents are validate functions: <a href="https://docs.couchdb.org/en/stable/ddocs/ddocs.html#validate-document-update-functions">Validate Document Update Functions</a>.
+ *
+ */
 public enum DesignResource {
 	// Validate
 	VALIDATE_JAVASCRIPT_READONLY_AUTH("validation/readonly_auth.js"),
-	VALIDATE_JAVASCRIPT_UPLOAD_ONLY("validation/upload_only.js"),
+	VALIDATE_JAVASCRIPT_SOLARTHING_OPEN_VALIDATE("validation/solarthing_open_validation.js"),
 
 	// View
 	VIEW_JAVASCRIPT_MILLIS_NULL("view/millisNull.js"),

core/src/main/java/me/retrodaredevil/couchdb/design/MutablePacketsDesign.java

@@ -20,6 +20,10 @@ public MutablePacketsDesign setReadonlyAuth() {
 		this.validateDocUpdate = DesignResource.VALIDATE_JAVASCRIPT_READONLY_AUTH.getAsString();
 		return this;
 	}
+	public MutablePacketsDesign setSolarThingOpenValidate() {
+		this.validateDocUpdate = DesignResource.VALIDATE_JAVASCRIPT_SOLARTHING_OPEN_VALIDATE.getAsString();
+		return this;
+	}
 	public MutablePacketsDesign addLast24HoursFilter() {
 		filters.put("last24Hours", DesignResource.FILTER_JAVASCRIPT_RECENT_PACKETS.getAsString());
 		return this;

core/src/main/java/me/retrodaredevil/solarthing/SolarThingDatabaseType.java

@@ -90,7 +90,9 @@ public boolean isReadonlyByAll() {
 		return this == CLOSED;
 	}
 	public @NotNull Set<UserType> getUsersWithWritePermission() {
-		if (this == CLOSED) { // This database is readonly by all
+		if (this == CLOSED || this == OPEN) { // This database is readonly by all
+			// closed is readonly by all users
+			// open already has permissive upload ability, so we don't need any user to get admin access
 			return Collections.emptySet();
 		}
 		if (this == CACHE || this == ALTER) {

core/src/main/resources/me/retrodaredevil/couchdb/design/validation/readonly_auth.js

@@ -1,25 +1,49 @@
+// solarthing, solarthing_alter, solarthing_events, solarthing_closed
+
 // noinspection JSUnusedGlobalSymbols,ReservedWordAsName
 
+// Readonly auth is designed to allow databases to be made public to the world, but not allow edits.
+// This also makes sure that any document with a dateMillis property is not uploading old data.
+//   This will not affect documents without a dateMillis property, so this can be used for many databases.
+
 function(newDoc, oldDoc, userCtx, secObj) {
+    var is_server_or_database_admin = function(userCtx, secObj) {
+        // see if the user is a server admin
+        if(userCtx.roles.indexOf('_admin') !== -1) {
+            return true; // a server admin
+        }
 
-    secObj.admins = secObj.admins || {};
-    secObj.admins.names = secObj.admins.names || [];
-    secObj.admins.roles = secObj.admins.roles || [];
+        // see if the user a database admin specified by name
+        if(secObj && secObj.admins && secObj.admins.names) {
+            if(secObj.admins.names.indexOf(userCtx.name) !== -1) {
+                return true; // database admin
+            }
+        }
 
-    var isAdmin = false;
-    if(userCtx.roles.indexOf('_admin') !== -1) {
-        isAdmin = true;
-    }
-    if(secObj.admins.names.indexOf(userCtx.name) !== -1) {
-        isAdmin = true;
-    }
-    for(var i = 0; i < userCtx.roles.length; i++) {
-        if(secObj.admins.roles.indexOf(userCtx.roles[i]) !== -1) {
-            isAdmin = true;
+        // see if the user a database admin specified by role
+        if(secObj && secObj.admins && secObj.admins.roles) {
+            var db_roles = secObj.admins.roles;
+            for(var idx = 0; idx < userCtx.roles.length; idx++) {
+                var user_role = userCtx.roles[idx];
+                if(db_roles.indexOf(user_role) !== -1) {
+                    return true; // role matches!
+                }
+            }
         }
+
+        return false; // default to no admin
     }
 
-    if(!isAdmin) {
+    if(!is_server_or_database_admin(userCtx, secObj)) {
         throw {'unauthorized':'This is read only when unauthorized'};
     }
+
+    // Only check dateMillis if this is a new document and dateMillis is present on the packet
+    if (!oldDoc && newDoc.dateMillis !== undefined) {
+        var currentMillis = +new Date();
+        var millisInPast = currentMillis - newDoc.dateMillis;
+        if (millisInPast > 25 * 60 * 1000) {
+            throw {forbidden: "dateMillis field is too far in the past!"}
+        }
+    }
 }

core/src/main/resources/me/retrodaredevil/couchdb/design/validation/solarthing_open_validation.js

@@ -0,0 +1,50 @@
+// This validation function made specifically for solarthing_open
+
+function(newDoc, oldDoc, userCtx, secObj) {
+    var is_server_or_database_admin = function(userCtx, secObj) {
+        // see if the user is a server admin
+        if(userCtx.roles.indexOf('_admin') !== -1) {
+            return true; // a server admin
+        }
+
+        // see if the user a database admin specified by name
+        if(secObj && secObj.admins && secObj.admins.names) {
+            if(secObj.admins.names.indexOf(userCtx.name) !== -1) {
+                return true; // database admin
+            }
+        }
+
+        // see if the user a database admin specified by role
+        if(secObj && secObj.admins && secObj.admins.roles) {
+            var db_roles = secObj.admins.roles;
+            for(var idx = 0; idx < userCtx.roles.length; idx++) {
+                var user_role = userCtx.roles[idx];
+                if(db_roles.indexOf(user_role) !== -1) {
+                    return true; // role matches!
+                }
+            }
+        }
+
+        return false; // default to no admin
+    }
+
+    // Perform certain checks only if this is a non-admin user
+    if (!is_server_or_database_admin(userCtx, secObj)) {
+        if (oldDoc) { // If someone is trying to update a document, prevent them from doing that
+            throw {'unauthorized':'You are not authorized to change this document!'};
+        }
+        if (typeof newDoc.dateMillis !== "number") {
+            throw {forbidden: "dateMillis must be a number"}
+        }
+        var currentMillis = +new Date();
+        var millisInPast = currentMillis - newDoc.dateMillis;
+        // Not only do we not want people to upload data that's far in the past,
+        //   we have strict rules that disallow documents targeted for the future.
+        if (millisInPast > 25 * 60 * 1000) {
+            throw {forbidden: "dateMillis field is too far in the past!"}
+        }
+        if (-millisInPast > 3 * 60 * 1000) {
+            throw {forbidden: "dateMillis field is too far in the future!"}
+        }
+    }
+}