Remove s3 assets (#179)

* serve static assets from the public dir instead of s3

* did not mean to commit these files

* correct order of controllers

* remove unused asset parameter and remove unnecessary try block

* add pandomain bucket name

* use ssm lookup for dist bucket

* remove unused grunt tasks

* change order of bundle making and deb generating to ensure all assets are picked up

* remove unused config files

* fix path to public assets

* remove unused import

.gitignore

@@ -17,6 +17,7 @@ logs
 ivy-sbt
 
 tmp
+public/story-packages
 target
 .target
 /*/bin/
@@ -42,3 +43,7 @@ screenshots
 
 # Scala test framework #
 *.local.conf
+
+.bsp
+.java-version
+importmap.json

Gruntfile.js

@@ -40,27 +40,7 @@ module.exports = function (grunt) {
     /**
      * Compile tasks
      */
-    grunt.registerTask('compile', function () {
-        grunt.task.run(['clean', 'shell:packages', 'replace', 'cacheBust']);
-    });
     grunt.registerTask('bundle', function () {
-        grunt.task.run(['compile', 'copy:static', 'copy:debian', 'copy:deploy', 'compress:riffraff']);
-    });
-    grunt.registerTask('upload', function () {
-        var done = this.async();
-        process.env.ARTEFACT_PATH = __dirname;
-        var riffraff = require('node-riffraff-artefact');
-        var path = require('path');
-        riffraff.settings.leadDir = path.join(__dirname, 'tmp');
-        riffraff.settings.manifestProjectName = 'cms-fronts::story-packages';
-        riffraff.s3Upload()
-        .then(function () {
-            grunt.log.writeln('Artifacts uploaded!');
-            done();
-        })
-        .catch(function () {
-            grunt.log.error('Error uploading artifacts.');
-            done(false);
-        });
+        grunt.task.run(['clean', 'shell:packages', 'replace', 'cacheBust']);
     });
 };

app/Components.scala

@@ -42,6 +42,7 @@ class AppComponents(context: Context) extends BuiltInComponentsFromContext(conte
   val storyPackages = new StoryPackagesController(config, controllerComponents, database, updatesStream, frontsApi, reindex, wsClient)
   val vanity = new VanityRedirects(config, controllerComponents, wsClient)
   val views = new ViewsController(config, controllerComponents, assetsManager, wsClient)
+  val publicAssets = new PublicAssets(assets, config, controllerComponents, wsClient)
 
   val customGzipFilter = new GzipFilter(shouldGzip = (header, _) => !Responses.isImage(header))
 
@@ -50,7 +51,5 @@ class AppComponents(context: Context) extends BuiltInComponentsFromContext(conte
     new CORSFilter
   )
 
-  override lazy val assets = new Assets(httpErrorHandler, assetsMetadata)
-
-  val router: Router = new Routes(httpErrorHandler, status, pandaAuth, views, faciaTool, defaults, storyPackages, faciaProxy, vanity, assets)
+  val router: Router = new Routes(httpErrorHandler, status, pandaAuth, views, faciaTool, defaults, storyPackages, faciaProxy, vanity, publicAssets)
 }

app/conf/Configuration.scala

@@ -94,10 +94,6 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val env
     }
   }
 
-  object cdn {
-    lazy val host = getString("cdn.host").getOrElse("")
-  }
-
   object contentApi {
     val contentApiLiveHost: String = getMandatoryString("content.api.host")
     val packagesLiveHost: String = getString("content.api.packages.host").getOrElse(contentApiLiveHost)

app/controllers/PublicAssets.scala

@@ -0,0 +1,24 @@
+package controllers
+
+import conf.ApplicationConfiguration
+import play.api.libs.ws.WSClient
+import story_packages.model.NoCache
+import play.api.mvc.{AnyContent, ControllerComponents}
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class PublicAssets(assets: Assets,
+                   config: ApplicationConfiguration,
+                   components: ControllerComponents,
+                   wsClient: WSClient)
+  extends StoryPackagesBaseController(config, components, wsClient) {
+
+  def at(file: String): NoCache[AnyContent] = NoCache {
+    if (file.startsWith("story-packages/bundles")) {
+      assets.at("/public", file)
+    } else {
+      assets.at("/public/src", file)
+    }
+  }
+
+}

app/story_packages/services/AssetsManager.scala

@@ -1,10 +1,10 @@
 package story_packages.services
 
-import java.io.FileInputStream
-
-import play.api.libs.json._
-import play.api.libs.json.Reads._
 import conf.ApplicationConfiguration
+import play.api.libs.json.Reads._
+import play.api.libs.json._
+
+import scala.io.Source
 
 class InvalidAssetsException(msg: String) extends RuntimeException(msg)
 
@@ -19,22 +19,21 @@ case class Bundles (packages: String)
 
 
 class AssetsManager(config: ApplicationConfiguration, isDev: Boolean) {
-  val filePath = "/etc/gu/story-packages.assets-map.json"
-  val assetsMap: Option[Bundles] = if (isDev) None else Some(readFromPath(filePath))
+  val resourcePath = "public/story-packages/bundles/assets-map.json"
+  val assetsMap = if (isDev) None else Some(readFromPath(resourcePath))
 
   private def readFromPath(path: String): Bundles = {
-    val stream = new FileInputStream(filePath)
-    val maybeJson = try { Json.parse(stream) } finally { stream.close() }
+    val assetsMapSource = Source.fromResource(path)
+    val maybeJson = Json.parse(assetsMapSource.mkString)
     maybeJson.validate[Bundles] match {
-      case e: JsError => throw new InvalidAssetsException(s"JSON in $filePath does not match a valid Bundles")
-      case json: JsSuccess[Bundles] => json.getOrElse(throw new InvalidAssetsException(s"Invalid JSON Bundle in $filePath"))
+      case e: JsError => throw new InvalidAssetsException(s"JSON in $resourcePath does not match a valid Bundles: $e")
+      case json: JsSuccess[Bundles] => json.getOrElse(throw new InvalidAssetsException(s"Invalid JSON Bundle in $resourcePath"))
     }
   }
 
   def pathForPackages: String = pathFor(assetsMap.map(_.packages).getOrElse(""))
 
   private def pathFor(hashedFileName: String): String = {
-    val stage = config.environment.stage.toUpperCase
-    s"${config.cdn.host}/cms-fronts-static-assets/$stage/static-story-packages/$hashedFileName"
+    s"/assets/story-packages/bundles/$hashedFileName"
   }
 }

build.sbt

@@ -11,7 +11,6 @@ packageDescription := "Guardian story packages editor"
 scalaVersion := "2.12.16"
 
 import sbt.Resolver
-import sbt.io.Path._
 
 debianPackageDependencies := Seq("openjdk-8-jre-headless")
 
@@ -21,11 +20,10 @@ riffRaffPackageType := (Debian / packageBin).value
 riffRaffUploadArtifactBucket := Option("riffraff-artifact")
 riffRaffUploadManifestBucket := Option("riffraff-builds")
 riffRaffArtifactResources := {
-    val jsBundlesDir = baseDirectory.value / "tmp" / "bundles"
     Seq(
         (Debian / packageBin).value -> s"${name.value}/${name.value}_${version.value}_all.deb",
         baseDirectory.value / "riff-raff.yaml" -> "riff-raff.yaml"
-    ) ++ ((jsBundlesDir * "*") pair rebase(jsBundlesDir, "static-story-packages"))
+    )
 }
 
 javacOptions := Seq("-g","-encoding", "utf8")

conf/application.conf

@@ -82,6 +82,7 @@ aws {
 pandomain {
   service: "packages"
   roleArn: "arn:aws:iam::753338109777:role/Fronts-panda-IAM-FaciaToolRole-NKNXCYEGL0F6"
+  bucketName: "pan-domain-auth-settings"
 }
 
 logging {

conf/routes

@@ -57,4 +57,4 @@ GET         /add-trail/*id                           controllers.VanityRedirects
 GET         /*path/                                  controllers.VanityRedirects.untrail(path: String)
 
 # static files
-GET         /assets/*file  controllers.Assets.at(path = "/public/src", file)
+GET         /assets/*file                            controllers.PublicAssets.at(file)

grunt-configs/cacheBust.js

@@ -2,7 +2,7 @@ module.exports = function() {
     return {
         static: {
             options: {
-                baseDir: 'tmp/bundles/',
+                baseDir: 'public/story-packages/bundles/',
                 assets: ['*.js'],
                 deleteOriginals: true,
                 jsonOutput: true,

grunt-configs/clean.js

@@ -1,5 +1,5 @@
 module.exports = function() {
     return {
-        static: ['tmp']
+        static: ['public/story-packages']
     };
 };

grunt-configs/replace.js

@@ -11,8 +11,8 @@ module.exports = function() {
             files: [{
                 expand: true,
                 src: '*.js',
-                cwd: 'tmp/bundles',
-                dest: 'tmp/bundles/'
+                cwd: 'public/story-packages/bundles',
+                dest: 'public/story-packages/bundles/'
             }]
         }
     };

grunt-configs/shell.js

@@ -14,7 +14,7 @@ module.exports = function() {
                 'story-packages/widgets/fetch-latest-packages',
                 'story-packages/widgets/display-alerts',
                 'story-packages/widgets/sparklines-trails'
-            ].join(' + ') + ' tmp/bundles/packages.js'
+            ].join(' + ') + ' public/story-packages/bundles/packages.js'
         }
     };
 };

package.json

@@ -3,6 +3,9 @@
   "version": "1.0.0",
   "private": true,
   "description": "Guardian story packages pages editor",
+  "scripts": {
+    "bundle": "grunt bundle"
+  },
   "dependencies": {
     "@babel/cli": "^7.19.3",
     "eslint-plugin-jasmine": "^1.8.1",

riff-raff.yaml

@@ -1,25 +1,11 @@
 stacks:
 - cms-fronts
 deployments:
-  static-story-packages:
-    type: aws-s3
-    parameters:
-      publicReadAcl: true
-      cacheControl:
-      - pattern: .*\.js$
-        value: public, max-age=315360000
-      - pattern: .*
-        value: public, max-age=60
-      mimeTypes:
-        js: application/javascript
-        json: application/json
-        map: text/plain
-      prefixStack: false
-      bucket: cms-fronts-static-assets
   story-packages:
     type: autoscaling
     parameters:
-      bucket: story-packages-dist
+      bucketSsmLookup: true
+      bucketSsmKey: /account/services/artifact.bucket.story-packages
     dependencies: [packages-ami]
   packages-ami:
     type: ami-cloudformation-parameter

scripts/ci.sh

@@ -17,6 +17,6 @@ jspm registry export github
 jspm install
 
 grunt --stack validate test
-sbt compile test assets
 grunt --stack bundle
+sbt compile test assets
 sbt riffRaffUpload