Merge pull request #76 from Crusader99/dev

Full encryption, file upload/download, allow reference renaming, bug fixes

README.md

@@ -91,6 +91,8 @@ Make sure the android sdk is installed and path is correct.
 sdk.dir=/opt/android
 ```
 
+Note when opening the full project with Android Studio, the `local.properties` file will be generated automatically.
+
 ### With gradle & docker-compose:
 
 Build all modules, including android:

android-app/src/main/java/de/hsaalen/cmt/MainActivity.kt

@@ -11,7 +11,6 @@ import androidx.appcompat.app.AppCompatActivity
 import androidx.webkit.WebViewAssetLoader
 import androidx.webkit.WebViewAssetLoader.AssetsPathHandler
 import androidx.webkit.WebViewClientCompat
-import de.crusader.extensions.toFullString
 
 /**
  * Main activity that provides a web view for providing the web content.
@@ -74,7 +73,7 @@ class MainActivity : AppCompatActivity() {
             webView.settings.javaScriptEnabled = true
             webView.loadUrl(endpointWebAsserts)
         } catch (ex: Throwable) {
-            Log.e("APP-DEBUG", ex.toFullString())
+            Log.e("APP-DEBUG", ex.stackTraceToString())
         }
     }
 

backend-database/build.gradle.kts

@@ -27,14 +27,7 @@ dependencies {
     implementation("org.litote.kmongo:kmongo-coroutine-serialization:4.2.8")
 
     // Import Amazon AWS S3 driver for accessing Minio file storage
-    implementation("software.amazon.awssdk:s3:2.17.4")
-
-    // Statistics & logging frameworks
-    // See https://github.com/MicroUtils/kotlin-logging
-    implementation("io.github.microutils:kotlin-logging-jvm:2.0.10")
-    implementation("ch.qos.logback:logback-classic:1.2.3") {
-        because("Ktor depends on this library and has issues when missing")
-    }
+    implementation("software.amazon.awssdk:s3:2.17.7")
 
     // JUnit test framework
     testImplementation(kotlin("test"))

backend-database/src/main/kotlin/de/hsaalen/cmt/DatabaseModules.kt

@@ -36,7 +36,7 @@ object DatabaseModules {
         single<ReferenceRepository> { ReferenceRepositoryImpl }
         single<LabelRepository> { LabelRepositoryImpl }
         single<DocumentRepository> { DocumentRepositoryImpl }
+        single<FileRepository> { FileRepositoryImpl }
     }
 
-
 }

backend-database/src/main/kotlin/de/hsaalen/cmt/repository/AuthenticationRepositoryImpl.kt

@@ -4,6 +4,7 @@ import de.hsaalen.cmt.crypto.hashSHA256
 import de.hsaalen.cmt.environment.DEFAULT_CREDENTIAL_VALUE
 import de.hsaalen.cmt.environment.PASSWORD_SALT
 import de.hsaalen.cmt.network.dto.server.ServerUserInfoDto
+import de.hsaalen.cmt.session.currentSession
 import de.hsaalen.cmt.sql.schema.UserDao
 import de.hsaalen.cmt.sql.schema.UserTable
 import de.hsaalen.cmt.utils.validateEmailAndThrow
@@ -86,14 +87,14 @@ internal object AuthenticationRepositoryImpl : AuthenticationRepository {
     }
 
     /**
-     * Request server to restore client session. Session can only restored when JWT cookie is still valid.
+     * Request server to restore client session. Session can only be restored when JWT cookie is still valid.
      *
      * @return Session instance when email of session still registered.
      * @throws SecurityException user email seem not to be registered anymore.
      */
-    override suspend fun restore(email: String) = getUserByMail(email)
+    override suspend fun restore() = getUserByMail(currentSession.userMail)
         ?.toServerUserInfoDto()
-        ?: throw SecurityException("User with email '$email' is not registered")
+        ?: throw SecurityException("User with email '${currentSession.userMail}' is not registered")
 
     /**
      * Search in SQL database for a specific user and return it when found.

backend-database/src/main/kotlin/de/hsaalen/cmt/repository/DocumentRepositoryImpl.kt

@@ -5,6 +5,7 @@ import de.hsaalen.cmt.events.GlobalEventDispatcher
 import de.hsaalen.cmt.events.server.UserDocumentChangeEvent
 import de.hsaalen.cmt.mongo.MongoDB
 import de.hsaalen.cmt.mongo.TextDocument
+import de.hsaalen.cmt.network.dto.objects.ContentType
 import de.hsaalen.cmt.network.dto.objects.LineChangeMode.*
 import de.hsaalen.cmt.network.dto.objects.UUID
 import de.hsaalen.cmt.network.dto.rsocket.DocumentChangeDto
@@ -51,7 +52,7 @@ internal object DocumentRepositoryImpl : DocumentRepository {
     /**
      * Download the content of a specific reference by uuid.
      */
-    override suspend fun downloadContent(uuid: UUID): String {
+    override suspend fun downloadDocument(uuid: UUID): String {
         // Ensure user has permissions to access this document
         checkAccess(currentSession.userMail, uuid)
 
@@ -67,6 +68,7 @@ internal object DocumentRepositoryImpl : DocumentRepository {
         newSuspendedTransaction {
             val ref = ReferenceDao.findById(reference.id) ?: error("Reference not found: $reference")
             check(ref.owner.email == userMail) { "No permissions to access document" }
+            check(ref.contentType == ContentType.TEXT) { "Type " + ref.contentType.name + " is no document" }
         }
     }
 

backend-database/src/main/kotlin/de/hsaalen/cmt/repository/FileRepositoryImpl.kt

@@ -2,8 +2,6 @@ package de.hsaalen.cmt.repository
 
 import de.hsaalen.cmt.network.dto.objects.UUID
 import de.hsaalen.cmt.storage.StorageS3
-import kotlinx.coroutines.channels.ReceiveChannel
-import kotlinx.coroutines.channels.SendChannel
 
 /**
  * Server implementation of the file repository to provide access to the AWS S3 file storage.
@@ -13,21 +11,17 @@ internal object FileRepositoryImpl : FileRepository {
     /**
      * Download the reference content by a specific [UUID].
      */
-    override suspend fun download(uuid: UUID, contentStream: SendChannel<ByteArray>) {
+    override suspend fun download(uuid: UUID): ByteArray {
         // TODO: Ensure user has edit permissions for that file
-        StorageS3.downloadFile(uuid).buffered().use { inputStream ->
-            // TODO: implement
-//            contentStream.send(inputStream.read)
-        }
+        return StorageS3.downloadFile(uuid).readBytes()
     }
 
     /**
      * Upload or overwrite the reference content by a specific [UUID].
      */
-    override suspend fun upload(uuid: UUID, contentStream: ReceiveChannel<ByteArray>, contentLength: Long) {
+    override suspend fun upload(uuid: UUID, content: ByteArray) {
         // TODO: Ensure user has edit permissions for that file
-        // TODO: implement download
-//        StorageS3.uploadFile(uuid)
+        StorageS3.uploadFile(uuid, content.inputStream(), content.size.toLong())
     }
 
 }

backend-database/src/main/kotlin/de/hsaalen/cmt/repository/ReferenceRepositoryImpl.kt

@@ -8,9 +8,10 @@ import de.hsaalen.cmt.network.dto.client.ClientReferenceQueryDto
 import de.hsaalen.cmt.network.dto.objects.ContentType
 import de.hsaalen.cmt.network.dto.objects.Reference
 import de.hsaalen.cmt.network.dto.objects.UUID
-import de.hsaalen.cmt.network.dto.server.ServerReferenceListDto
 import de.hsaalen.cmt.network.dto.rsocket.ReferenceUpdateAddDto
 import de.hsaalen.cmt.network.dto.rsocket.ReferenceUpdateRemoveDto
+import de.hsaalen.cmt.network.dto.rsocket.ReferenceUpdateRenameDto
+import de.hsaalen.cmt.network.dto.server.ServerReferenceListDto
 import de.hsaalen.cmt.session.currentSession
 import de.hsaalen.cmt.sql.schema.ReferenceDao
 import de.hsaalen.cmt.sql.schema.ReferenceTable
@@ -19,6 +20,7 @@ import de.hsaalen.cmt.sql.schema.UserDao
 import de.hsaalen.cmt.storage.StorageS3
 import de.hsaalen.cmt.utils.id
 import de.hsaalen.cmt.utils.toUUID
+import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
 import org.joda.time.DateTime
@@ -43,10 +45,10 @@ internal object ReferenceRepositoryImpl : ReferenceRepository {
             val creator = UserDao.findUserByEmail(userEmail)
             val now = DateTime.now()
             val reference = ReferenceDao.new {
-                this.accessCode = "ACCESS_CODE"
                 this.displayName = request.displayName
                 this.contentType = request.contentType
                 this.owner = creator
+                this.dateLastModified = DateTime.now()
             }
             val revision = RevisionDao.new {
                 this.item = reference
@@ -60,7 +62,6 @@ internal object ReferenceRepositoryImpl : ReferenceRepository {
 
             Reference(
                 uuid = reference.id.toUUID(),
-                accessCode = reference.accessCode,
                 displayName = reference.displayName,
                 contentType = reference.contentType,
                 dateCreation = revision.dateCreation.millis,
@@ -98,7 +99,9 @@ internal object ReferenceRepositoryImpl : ReferenceRepository {
     override suspend fun listReferences(query: ClientReferenceQueryDto): ServerReferenceListDto {
         val refs = newSuspendedTransaction {
             val creator = UserDao.findUserByEmail(userEmail)
-            ReferenceDao.find(ReferenceTable.owner eq creator.id).map { it.toReference() }
+            ReferenceDao.find(ReferenceTable.owner eq creator.id)
+                .orderBy(ReferenceTable.dateLastModified to SortOrder.DESC)
+                .map { it.toReference() }
         }
         return ServerReferenceListDto(refs)
     }
@@ -134,4 +137,20 @@ internal object ReferenceRepositoryImpl : ReferenceRepository {
         }
     }
 
+    /**
+     * Give a new title name to a reference.
+     */
+    override suspend fun rename(uuid: UUID, newTitle: String) {
+        newSuspendedTransaction {
+            val ref = ReferenceDao.findById(uuid.id) ?: error("No reference with uuid=$uuid found!")
+            if (ref.owner.email != userEmail) {
+                throw SecurityException("Can not rename references from different users!")
+            }
+            ref.displayName = newTitle
+        }
+
+        // Call event handlers
+        GlobalEventDispatcher.notify(ReferenceUpdateRenameDto(uuid, newTitle))
+    }
+
 }

backend-database/src/main/kotlin/de/hsaalen/cmt/sql/schema/ReferenceSchema.kt

@@ -8,16 +8,17 @@ import org.jetbrains.exposed.dao.UUIDEntityClass
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.dao.id.UUIDTable
 import org.jetbrains.exposed.sql.ReferenceOption
+import org.jetbrains.exposed.sql.jodatime.datetime
 import java.util.*
 
 /**
  * The postgresql table of the reference data.
  */
 object ReferenceTable : UUIDTable("reference") {
-    val accessCode = varchar("access_code", 32) //.uniqueIndex()
     val displayName = varchar("display_name", 512)
     val contentType = enumeration("content_type", ContentType::class)
     val owner = reference("owner", UserTable, onDelete = ReferenceOption.CASCADE)
+    val dateLastModified = datetime("date_last_modified")
 }
 
 /**
@@ -26,10 +27,10 @@ object ReferenceTable : UUIDTable("reference") {
 class ReferenceDao(id: EntityID<UUID>) : UUIDEntity(id) {
     companion object : UUIDEntityClass<ReferenceDao>(ReferenceTable)
 
-    var accessCode by ReferenceTable.accessCode
     var displayName by ReferenceTable.displayName
     var contentType by ReferenceTable.contentType
     var owner by UserDao referencedOn ReferenceTable.owner
+    var dateLastModified by ReferenceTable.dateLastModified
     var labels by LabelDao via LabelRefMappingTable
 
     /**
@@ -38,6 +39,6 @@ class ReferenceDao(id: EntityID<UUID>) : UUIDEntity(id) {
     fun toReference(): Reference {
         val now = System.currentTimeMillis()
         val labels = labels.map { it.labelName }.toMutableSet()
-        return Reference(id.toUUID(), accessCode, displayName, contentType, now, now, labels)
+        return Reference(id.toUUID(), displayName, contentType, now, now, labels)
     }
 }

backend-database/src/main/kotlin/de/hsaalen/cmt/sql/schema/UserSchema.kt

@@ -1,6 +1,6 @@
 package de.hsaalen.cmt.sql.schema
 
-import de.crusader.extensions.toHexStr
+import de.hsaalen.cmt.crypto.toBase64
 import de.hsaalen.cmt.network.dto.server.ServerUserInfoDto
 import org.jetbrains.exposed.dao.UUIDEntity
 import org.jetbrains.exposed.dao.UUIDEntityClass
@@ -46,6 +46,6 @@ class UserDao(id: EntityID<UUID>) : UUIDEntity(id) {
     /**
      * Convert to data transfer object.
      */
-    fun toServerUserInfoDto() = ServerUserInfoDto(fullName, email, personalEncryptedKey.toHexStr())
+    fun toServerUserInfoDto() = ServerUserInfoDto(fullName, email, personalEncryptedKey.toBase64())
 
 }

backend-database/src/main/kotlin/de/hsaalen/cmt/storage/StorageS3.kt

@@ -66,6 +66,7 @@ internal object StorageS3 {
      * Create and upload a file with content.
      */
     fun uploadFile(uuid: UUID, contentStream: InputStream, contentLength: Long) {
+        logger.info("Uploading file content for $uuid")
         client.putObject({
             it.bucket(S3_BUCKET)
             it.key(uuid.value)
@@ -76,6 +77,7 @@ internal object StorageS3 {
      * Download the file content from a specific reference by [UUID].
      */
     fun downloadFile(uuid: UUID): InputStream {
+        logger.info("Downloading file content for $uuid")
         return client.getObject {
             it.bucket(S3_BUCKET)
             it.key(uuid.value)
@@ -87,6 +89,7 @@ internal object StorageS3 {
      */
     fun deleteFile(uuid: UUID) {
         try {
+            logger.info("Deleting file content from $uuid")
             client.deleteObject {
                 it.bucket(S3_BUCKET)
                 it.key(uuid.value)

backend-database/src/test/kotlin/de/hsaalen/cmt/test/ReferenceInfrastructureTest.kt

@@ -44,7 +44,7 @@ class ReferenceInfrastructureTest {
 
             suspend fun validate() {
                 assertEquals(info.displayName, ref.displayName)
-                assertEquals("", docRepo.downloadContent(ref.uuid))
+                assertEquals("", docRepo.downloadDocument(ref.uuid))
                 assertEquals(info.labels, labelNames)
             }
 
@@ -66,22 +66,22 @@ class ReferenceInfrastructureTest {
             ref = refRepo.listReferences().references.single { it.uuid == ref.uuid }
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 0, "line-1", MODIFY))
-            assertEquals("line-1", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-1", docRepo.downloadDocument(ref.uuid))
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 1, "line-2", ADD))
-            assertEquals("line-1\nline-2", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-1\nline-2", docRepo.downloadDocument(ref.uuid))
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 0, "line-0", ADD))
-            assertEquals("line-0\nline-1\nline-2", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-0\nline-1\nline-2", docRepo.downloadDocument(ref.uuid))
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 1, "x", ADD))
-            assertEquals("line-0\nx\nline-1\nline-2", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-0\nx\nline-1\nline-2", docRepo.downloadDocument(ref.uuid))
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 1, "y", MODIFY))
-            assertEquals("line-0\ny\nline-1\nline-2", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-0\ny\nline-1\nline-2", docRepo.downloadDocument(ref.uuid))
 
             docRepo.modifyDocument(DocumentChangeDto(ref.uuid, 1, "", DELETE))
-            assertEquals("line-0\nline-1\nline-2", docRepo.downloadContent(ref.uuid))
+            assertEquals("line-0\nline-1\nline-2", docRepo.downloadDocument(ref.uuid))
         }
     }
 

backend-server/build.gradle.kts

@@ -23,15 +23,15 @@ dependencies {
     implementation(project(":common"))
 
     // Network framework
-    implementation("io.ktor:ktor-server-core:1.6.1")
-    implementation("io.ktor:ktor-server-cio:1.6.1") {
+    implementation("io.ktor:ktor-server-core:1.6.2")
+    implementation("io.ktor:ktor-server-cio:1.6.2") {
         because("Known issues with netty & jetty")
     }
-    implementation("io.ktor:ktor-serialization:1.6.1")
-    implementation("io.ktor:ktor-websockets:1.6.1")
-    implementation("io.ktor:ktor-metrics-micrometer:1.6.1")
-    implementation("io.ktor:ktor-auth:1.6.1")
-    implementation("io.ktor:ktor-auth-jwt:1.6.1")
+    implementation("io.ktor:ktor-serialization:1.6.2")
+    implementation("io.ktor:ktor-websockets:1.6.2")
+    implementation("io.ktor:ktor-metrics-micrometer:1.6.2")
+    implementation("io.ktor:ktor-auth:1.6.2")
+    implementation("io.ktor:ktor-auth-jwt:1.6.2")
 
     // Use RSocket as better alternative to plain websockets (https://rsocket.io/)
     implementation("io.rsocket.kotlin:rsocket-transport-ktor-server:0.13.1")
@@ -50,11 +50,11 @@ dependencies {
     testImplementation(kotlin("test"))
     testImplementation("io.insert-koin:koin-test:3.1.2")
     testImplementation("io.mockk:mockk:1.12.0")
-    testImplementation("io.ktor:ktor-server-test-host:1.6.1")
+    testImplementation("io.ktor:ktor-server-test-host:1.6.2")
     testImplementation("de.crusader:webscraper-selenium:3.1.0")
     testImplementation("de.crusader:webscraper-htmlunit:3.1.0")
-    testImplementation("io.ktor:ktor-client-core:1.6.1")
-    testImplementation("io.ktor:ktor-client-cio:1.6.1")
+    testImplementation("io.ktor:ktor-client-core:1.6.2")
+    testImplementation("io.ktor:ktor-client-cio:1.6.2")
     testImplementation("io.rsocket.kotlin:rsocket-transport-ktor-client:0.13.1")
 
 }
@@ -71,7 +71,7 @@ tasks.test {
 
 // Configure detekt code analyze tool to generate HTML report
 detekt {
-    ignoreFailures = true // Currently only print warning
+    ignoreFailures = true // Currently, only print warning
     reports {
         html.enabled = true
     }

backend-server/src/main/kotlin/de/hsaalen/cmt/rest/Routing.kt

@@ -16,7 +16,8 @@ fun Application.registerRoutes() = routing {
 
     routeAuthentication()
     routeReferences()
+    routeFiles()
     routeLabels()
     routeMetrics()
-    routeWebSockets()
+    routeRSocket()
 }

backend-server/src/main/kotlin/de/hsaalen/cmt/rest/routes/RouteAuthentication.kt

@@ -64,8 +64,7 @@ fun Routing.routeAuthentication() = route("/" + RestPaths.base) {
 
     // Check authorization cookie is valid and refresh JWT token when logged in
     getWithSession(apiPathAuthRestore) {
-        val payload = call.request.readJwtCookie()
-        val user = repo.restore(payload.email)
+        val user = repo.restore()
         call.response.updateJwtCookie(user.generateJwtToken())
         call.respond(user)
     }