From ee441d55a7d6e57cf8356e00c0d601fdc0636c83 Mon Sep 17 00:00:00 2001 From: Alina Boghiu Date: Mon, 7 Aug 2023 10:31:55 +0100 Subject: [PATCH] Remove s3 assets (#179) * serve static assets from the public dir instead of s3 * did not mean to commit these files * correct order of controllers * remove unused asset parameter and remove unnecessary try block * add pandomain bucket name * use ssm lookup for dist bucket * remove unused grunt tasks * change order of bundle making and deb generating to ensure all assets are picked up * remove unused config files * fix path to public assets * remove unused import --- .gitignore | 5 ++++ Gruntfile.js | 22 +-------------- app/Components.scala | 5 ++-- app/conf/Configuration.scala | 4 --- app/controllers/PublicAssets.scala | 24 ++++++++++++++++ .../services/AssetsManager.scala | 23 ++++++++------- build.sbt | 4 +-- conf/application.conf | 1 + conf/routes | 2 +- grunt-configs/cacheBust.js | 2 +- grunt-configs/clean.js | 2 +- grunt-configs/compress.js | 12 -------- grunt-configs/copy.js | 28 ------------------- grunt-configs/replace.js | 4 +-- grunt-configs/shell.js | 2 +- package.json | 3 ++ riff-raff.yaml | 18 ++---------- scripts/ci.sh | 2 +- 18 files changed, 57 insertions(+), 106 deletions(-) create mode 100644 app/controllers/PublicAssets.scala delete mode 100644 grunt-configs/compress.js delete mode 100644 grunt-configs/copy.js diff --git a/.gitignore b/.gitignore index b63aa58..90b1ae4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ logs ivy-sbt tmp +public/story-packages target .target /*/bin/ @@ -42,3 +43,7 @@ screenshots # Scala test framework # *.local.conf + +.bsp +.java-version +importmap.json diff --git a/Gruntfile.js b/Gruntfile.js index 124a698..0d0a030 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -40,27 +40,7 @@ module.exports = function (grunt) { /** * Compile tasks */ - grunt.registerTask('compile', function () { - grunt.task.run(['clean', 'shell:packages', 'replace', 'cacheBust']); - }); grunt.registerTask('bundle', function () { - grunt.task.run(['compile', 'copy:static', 'copy:debian', 'copy:deploy', 'compress:riffraff']); - }); - grunt.registerTask('upload', function () { - var done = this.async(); - process.env.ARTEFACT_PATH = __dirname; - var riffraff = require('node-riffraff-artefact'); - var path = require('path'); - riffraff.settings.leadDir = path.join(__dirname, 'tmp'); - riffraff.settings.manifestProjectName = 'cms-fronts::story-packages'; - riffraff.s3Upload() - .then(function () { - grunt.log.writeln('Artifacts uploaded!'); - done(); - }) - .catch(function () { - grunt.log.error('Error uploading artifacts.'); - done(false); - }); + grunt.task.run(['clean', 'shell:packages', 'replace', 'cacheBust']); }); }; diff --git a/app/Components.scala b/app/Components.scala index 99e156d..02aef56 100644 --- a/app/Components.scala +++ b/app/Components.scala @@ -42,6 +42,7 @@ class AppComponents(context: Context) extends BuiltInComponentsFromContext(conte val storyPackages = new StoryPackagesController(config, controllerComponents, database, updatesStream, frontsApi, reindex, wsClient) val vanity = new VanityRedirects(config, controllerComponents, wsClient) val views = new ViewsController(config, controllerComponents, assetsManager, wsClient) + val publicAssets = new PublicAssets(assets, config, controllerComponents, wsClient) val customGzipFilter = new GzipFilter(shouldGzip = (header, _) => !Responses.isImage(header)) @@ -50,7 +51,5 @@ class AppComponents(context: Context) extends BuiltInComponentsFromContext(conte new CORSFilter ) - override lazy val assets = new Assets(httpErrorHandler, assetsMetadata) - - val router: Router = new Routes(httpErrorHandler, status, pandaAuth, views, faciaTool, defaults, storyPackages, faciaProxy, vanity, assets) + val router: Router = new Routes(httpErrorHandler, status, pandaAuth, views, faciaTool, defaults, storyPackages, faciaProxy, vanity, publicAssets) } diff --git a/app/conf/Configuration.scala b/app/conf/Configuration.scala index ebcfa76..d0e43cc 100644 --- a/app/conf/Configuration.scala +++ b/app/conf/Configuration.scala @@ -94,10 +94,6 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val env } } - object cdn { - lazy val host = getString("cdn.host").getOrElse("") - } - object contentApi { val contentApiLiveHost: String = getMandatoryString("content.api.host") val packagesLiveHost: String = getString("content.api.packages.host").getOrElse(contentApiLiveHost) diff --git a/app/controllers/PublicAssets.scala b/app/controllers/PublicAssets.scala new file mode 100644 index 0000000..a3ef9ea --- /dev/null +++ b/app/controllers/PublicAssets.scala @@ -0,0 +1,24 @@ +package controllers + +import conf.ApplicationConfiguration +import play.api.libs.ws.WSClient +import story_packages.model.NoCache +import play.api.mvc.{AnyContent, ControllerComponents} + +import scala.concurrent.ExecutionContext.Implicits.global + +class PublicAssets(assets: Assets, + config: ApplicationConfiguration, + components: ControllerComponents, + wsClient: WSClient) + extends StoryPackagesBaseController(config, components, wsClient) { + + def at(file: String): NoCache[AnyContent] = NoCache { + if (file.startsWith("story-packages/bundles")) { + assets.at("/public", file) + } else { + assets.at("/public/src", file) + } + } + +} diff --git a/app/story_packages/services/AssetsManager.scala b/app/story_packages/services/AssetsManager.scala index 4564928..cebe600 100644 --- a/app/story_packages/services/AssetsManager.scala +++ b/app/story_packages/services/AssetsManager.scala @@ -1,10 +1,10 @@ package story_packages.services -import java.io.FileInputStream - -import play.api.libs.json._ -import play.api.libs.json.Reads._ import conf.ApplicationConfiguration +import play.api.libs.json.Reads._ +import play.api.libs.json._ + +import scala.io.Source class InvalidAssetsException(msg: String) extends RuntimeException(msg) @@ -19,22 +19,21 @@ case class Bundles (packages: String) class AssetsManager(config: ApplicationConfiguration, isDev: Boolean) { - val filePath = "/etc/gu/story-packages.assets-map.json" - val assetsMap: Option[Bundles] = if (isDev) None else Some(readFromPath(filePath)) + val resourcePath = "public/story-packages/bundles/assets-map.json" + val assetsMap = if (isDev) None else Some(readFromPath(resourcePath)) private def readFromPath(path: String): Bundles = { - val stream = new FileInputStream(filePath) - val maybeJson = try { Json.parse(stream) } finally { stream.close() } + val assetsMapSource = Source.fromResource(path) + val maybeJson = Json.parse(assetsMapSource.mkString) maybeJson.validate[Bundles] match { - case e: JsError => throw new InvalidAssetsException(s"JSON in $filePath does not match a valid Bundles") - case json: JsSuccess[Bundles] => json.getOrElse(throw new InvalidAssetsException(s"Invalid JSON Bundle in $filePath")) + case e: JsError => throw new InvalidAssetsException(s"JSON in $resourcePath does not match a valid Bundles: $e") + case json: JsSuccess[Bundles] => json.getOrElse(throw new InvalidAssetsException(s"Invalid JSON Bundle in $resourcePath")) } } def pathForPackages: String = pathFor(assetsMap.map(_.packages).getOrElse("")) private def pathFor(hashedFileName: String): String = { - val stage = config.environment.stage.toUpperCase - s"${config.cdn.host}/cms-fronts-static-assets/$stage/static-story-packages/$hashedFileName" + s"/assets/story-packages/bundles/$hashedFileName" } } diff --git a/build.sbt b/build.sbt index bfef74b..5e5dc0b 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,6 @@ packageDescription := "Guardian story packages editor" scalaVersion := "2.12.16" import sbt.Resolver -import sbt.io.Path._ debianPackageDependencies := Seq("openjdk-8-jre-headless") @@ -21,11 +20,10 @@ riffRaffPackageType := (Debian / packageBin).value riffRaffUploadArtifactBucket := Option("riffraff-artifact") riffRaffUploadManifestBucket := Option("riffraff-builds") riffRaffArtifactResources := { - val jsBundlesDir = baseDirectory.value / "tmp" / "bundles" Seq( (Debian / packageBin).value -> s"${name.value}/${name.value}_${version.value}_all.deb", baseDirectory.value / "riff-raff.yaml" -> "riff-raff.yaml" - ) ++ ((jsBundlesDir * "*") pair rebase(jsBundlesDir, "static-story-packages")) + ) } javacOptions := Seq("-g","-encoding", "utf8") diff --git a/conf/application.conf b/conf/application.conf index fad0e80..9cc98e0 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -82,6 +82,7 @@ aws { pandomain { service: "packages" roleArn: "arn:aws:iam::753338109777:role/Fronts-panda-IAM-FaciaToolRole-NKNXCYEGL0F6" + bucketName: "pan-domain-auth-settings" } logging { diff --git a/conf/routes b/conf/routes index 6c52cc0..7075da7 100644 --- a/conf/routes +++ b/conf/routes @@ -57,4 +57,4 @@ GET /add-trail/*id controllers.VanityRedirects GET /*path/ controllers.VanityRedirects.untrail(path: String) # static files -GET /assets/*file controllers.Assets.at(path = "/public/src", file) +GET /assets/*file controllers.PublicAssets.at(file) diff --git a/grunt-configs/cacheBust.js b/grunt-configs/cacheBust.js index 42fc7be..9338ecc 100644 --- a/grunt-configs/cacheBust.js +++ b/grunt-configs/cacheBust.js @@ -2,7 +2,7 @@ module.exports = function() { return { static: { options: { - baseDir: 'tmp/bundles/', + baseDir: 'public/story-packages/bundles/', assets: ['*.js'], deleteOriginals: true, jsonOutput: true, diff --git a/grunt-configs/clean.js b/grunt-configs/clean.js index 9e8ead5..fb2fb71 100644 --- a/grunt-configs/clean.js +++ b/grunt-configs/clean.js @@ -1,5 +1,5 @@ module.exports = function() { return { - static: ['tmp'] + static: ['public/story-packages'] }; }; diff --git a/grunt-configs/compress.js b/grunt-configs/compress.js deleted file mode 100644 index 6a108b4..0000000 --- a/grunt-configs/compress.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = function() { - return { - riffraff: { - options: { - archive: 'tmp/artifacts.zip' - }, - files: [ - {cwd: 'tmp/riffraff', expand: true, src: ['**']} - ] - } - }; -}; diff --git a/grunt-configs/copy.js b/grunt-configs/copy.js deleted file mode 100644 index e84b4a0..0000000 --- a/grunt-configs/copy.js +++ /dev/null @@ -1,28 +0,0 @@ -module.exports = function() { - return { - static: { - files: [{ - expand: true, - src: ['**'], - cwd: 'tmp/bundles', - dest: 'tmp/riffraff/packages/static-story-packages/' - }] - }, - debian: { - files: [{ - expand: true, - src: ['story-packages*.deb'], - cwd: 'target/', - dest: 'tmp/riffraff/packages/story-packages/' - }] - }, - deploy: { - files: [{ - expand: true, - src: ['deploy.json'], - cwd: 'conf/', - dest: 'tmp/riffraff/' - }] - } - }; -}; diff --git a/grunt-configs/replace.js b/grunt-configs/replace.js index 7c9cebd..de8eb52 100644 --- a/grunt-configs/replace.js +++ b/grunt-configs/replace.js @@ -11,8 +11,8 @@ module.exports = function() { files: [{ expand: true, src: '*.js', - cwd: 'tmp/bundles', - dest: 'tmp/bundles/' + cwd: 'public/story-packages/bundles', + dest: 'public/story-packages/bundles/' }] } }; diff --git a/grunt-configs/shell.js b/grunt-configs/shell.js index 72aca6a..4b370c7 100644 --- a/grunt-configs/shell.js +++ b/grunt-configs/shell.js @@ -14,7 +14,7 @@ module.exports = function() { 'story-packages/widgets/fetch-latest-packages', 'story-packages/widgets/display-alerts', 'story-packages/widgets/sparklines-trails' - ].join(' + ') + ' tmp/bundles/packages.js' + ].join(' + ') + ' public/story-packages/bundles/packages.js' } }; }; diff --git a/package.json b/package.json index ee423c7..3e1900c 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "private": true, "description": "Guardian story packages pages editor", + "scripts": { + "bundle": "grunt bundle" + }, "dependencies": { "@babel/cli": "^7.19.3", "eslint-plugin-jasmine": "^1.8.1", diff --git a/riff-raff.yaml b/riff-raff.yaml index 0787dd2..bce7974 100644 --- a/riff-raff.yaml +++ b/riff-raff.yaml @@ -1,25 +1,11 @@ stacks: - cms-fronts deployments: - static-story-packages: - type: aws-s3 - parameters: - publicReadAcl: true - cacheControl: - - pattern: .*\.js$ - value: public, max-age=315360000 - - pattern: .* - value: public, max-age=60 - mimeTypes: - js: application/javascript - json: application/json - map: text/plain - prefixStack: false - bucket: cms-fronts-static-assets story-packages: type: autoscaling parameters: - bucket: story-packages-dist + bucketSsmLookup: true + bucketSsmKey: /account/services/artifact.bucket.story-packages dependencies: [packages-ami] packages-ami: type: ami-cloudformation-parameter diff --git a/scripts/ci.sh b/scripts/ci.sh index b63fc1a..b160549 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -17,6 +17,6 @@ jspm registry export github jspm install grunt --stack validate test -sbt compile test assets grunt --stack bundle +sbt compile test assets sbt riffRaffUpload