Merge pull request #613 from hmrc/extensions

Extensions Framework

.gitignore

@@ -1,5 +1,7 @@
 # ignore tracking usage data config - we want a different one for each user
 usage-data-config.json
+# ignore extension SCSS because it's dynamically generated
+lib/extensions/_extensions.scss
 .env
 .sass-cache
 .DS_Store

__tests__/spec/sanity-checks.js

@@ -5,6 +5,9 @@ var path = require('path')
 var fs = require('fs')
 var assert = require('assert')
 
+function readFile (pathFromRoot) {
+  return fs.readFileSync(path.join(__dirname, '../../' + pathFromRoot), 'utf8')
+}
 /**
  * Basic sanity checks on the dev server
  */
@@ -40,4 +43,66 @@ describe('The Prototype Kit', () => {
       expect(response.type).toBe('text/html')
     })
   })
+
+  describe('extensions', () => {
+    it('should allow known assets to be loaded from node_modules', (done) => {
+      request(app)
+        .get('/extension-assets/govuk-frontend/all.js')
+        .expect('Content-Type', /application\/javascript; charset=UTF-8/)
+        .expect(200)
+        .end(function (err, res) {
+          if (err) {
+            done(err)
+          } else {
+            assert.strictEqual('' + res.text, readFile('node_modules/govuk-frontend/all.js'))
+            done()
+          }
+        })
+    })
+
+    it('should allow known assets to be loaded from node_modules', (done) => {
+      request(app)
+        .get('/assets/images/favicon.ico')
+        .expect('Content-Type', /image\/x-icon/)
+        .expect(200)
+        .end(function (err, res) {
+          if (err) {
+            done(err)
+          } else {
+            assert.strictEqual('' + res.body, readFile('node_modules/govuk-frontend/assets/images/favicon.ico'))
+            done()
+          }
+        })
+    })
+
+    it('should not expose everything', function (done) {
+      request(app)
+        .get('/assets/common.js')
+        .expect(404)
+        .end(function (err, res) {
+          if (err) {
+            done(err)
+          } else {
+            done()
+          }
+        })
+    })
+
+    describe('misconfigured prototype kit - while upgrading kit developer did not copy over changes in /app folder', () => {
+      it('should still allow known assets to be loaded from node_modules', (done) => {
+        request(app)
+          .get('/node_modules/govuk-frontend/all.js')
+          .expect('Content-Type', /application\/javascript; charset=UTF-8/)
+          .expect(200)
+          .end(function (err, res) {
+            if (err) {
+              done(err)
+            } else {
+              assert.strictEqual('' + res.text, readFile('node_modules/govuk-frontend/all.js'))
+              done()
+            }
+          })
+      })
+    })
+  })
 })

app/assets/sass/application.scss

@@ -1,8 +1,8 @@
 // global styles for <a> and <p> tags
 $govuk-global-styles: true;
 
-// Import GOV.UK Frontend
-@import "node_modules/govuk-frontend/all";
+// Import GOV.UK Frontend and any extension styles if extensions have been configured
+@import "lib/extensions/extensions";
 
 // Patterns that aren't in Frontend
 @import "patterns/step-by-step-navigation";

app/views/includes/head.html

@@ -1,2 +1,6 @@
 <!--[if lte IE 8]><link href="/public/stylesheets/application-ie8.css" rel="stylesheet" type="text/css" /><![endif]-->
 <!--[if gt IE 8]><!--><link href="/public/stylesheets/application.css" media="all" rel="stylesheet" type="text/css" /><!--<![endif]-->
+
+{% for stylesheetUrl in extensionConfig.stylesheets %}
+  <link href="{{ stylesheetUrl }}" rel="stylesheet" type="text/css" />
+{% endfor %}

app/views/includes/scripts.html

@@ -1,6 +1,10 @@
 <!-- Javascript -->
 <script src="/public/javascripts/jquery-1.11.3.js"></script>
-<script src="/node_modules/govuk-frontend/all.js"></script>
+
+{% for scriptUrl in extensionConfig.scripts %}
+  <script src="{{scriptUrl}}"></script>
+{% endfor %}
+
 <script src="/public/javascripts/application.js"></script>
 
 {% if useAutoStoreData %}

docs/documentation/extension-system.md

@@ -0,0 +1,3 @@
+# Extension system
+
+The extension system information should go here before it's adopted into GOVUK.

docs/views/includes/scripts.html

@@ -1,6 +1,6 @@
 <!-- Javascript -->
 <script src="/public/javascripts/jquery-1.11.3.js"></script>
-<script src="/node_modules/govuk-frontend/all.js"></script>
+<script src="/extension-assets/govuk-frontend/all.js"></script>
 <script src="/public/javascripts/docs.js"></script>
 
 {% if useAutoStoreData %}

gulp/sass.js

@@ -8,9 +8,19 @@
 const gulp = require('gulp')
 const sass = require('gulp-sass')
 const sourcemaps = require('gulp-sourcemaps')
+const path = require('path')
+const fs = require('fs')
 
+const extensions = require('../lib/extensions/extensions')
 const config = require('./config.json')
 
+gulp.task('sass-extensions', function (done) {
+  const fileContents = '$govuk-extensions-url-context: "/extension-assets"; ' + extensions.getFileSystemPaths('sass')
+    .map(filePath => `@import "${filePath.split(path.sep).join('/')}";`)
+    .join('\n')
+  fs.writeFile(path.join(config.paths.lib + 'extensions', '_extensions.scss'), fileContents, done)
+})
+
 gulp.task('sass', function () {
   return gulp.src(config.paths.assets + '/sass/*.scss')
     .pipe(sourcemaps.init())

gulpfile.js

@@ -18,6 +18,7 @@ requireDir('./gulp', { recurse: true })
 // We'll keep our top-level tasks in this file so that they are defined at the end of the chain, after their dependencies.
 gulp.task('generate-assets', gulp.series(
   'clean',
+  'sass-extensions',
   gulp.parallel(
     'sass',
     'copy-assets',

lib/extensions/extensions.js

@@ -0,0 +1,174 @@
+/**
+ * Extensions.js (Use with caution)
+ *
+ *  Experimental feature which is likely to change.
+ *  This file returns helper methods to enable services to include
+ *  their own departmental frontend(Styles, Scripts, nunjucks etc)
+ *
+ * Module.exports
+ *    getPublicUrls:
+ *      Params: (type | string ) eg. 'scripts', 'stylesheets'
+ *      Description:
+ *        returns array of urls for a type (script, stylesheet, nunjucks etc).
+ *    getFileSystemPaths:
+ *      Params: (type | string ) eg. 'scripts', 'stylesheets'
+ *      Description:
+ *        returns array paths to the file in the filesystem for a type (script, stylesheet, nunjucks etc)
+ *    getPublicUrlAndFileSystemPaths:
+ *      Params: (type | string ) eg. 'scripts', 'stylesheets'
+ *      Description:
+ *        returns Array of objects, each object is an extension and each obj has the filesystem & public url for the given type
+ *    getAppConfig:
+ *      Params: (type | string ) eg. 'scripts', 'stylesheets'
+ *      Description:
+ *        Returns an object containing two keys(scripts & stylesheets), each item contains an array of full paths to specific files.
+ *        This is used in the views to output links and scripts each file.
+ *    getAppViews:
+ *      Params: (additionalViews | Array ) eg.extensions.getAppViews([path.join(__dirname, '/app/views/'),path.join(__dirname, '/lib/')])
+ *      Description:
+ *        Returns an array of paths to nunjucks templates which is used to configure nunjucks in server.js
+ *    setExtensionsByType
+ *      Params: N/A
+ *      Description: only used for test purposes to reset mocked extensions items to ensure they are up-to-date when the tests run
+ *
+ * *
+ */
+
+// Core dependencies
+const fs = require('fs')
+const path = require('path')
+
+// Local dependencies
+const appConfig = require('../../app/config')
+
+// Generic utilities
+const removeDuplicates = arr => [...new Set(arr)]
+const filterOutParentAndEmpty = part => part && part !== '..'
+const objectMap = (object, mapFn) => Object.keys(object).reduce((result, key) => {
+  result[key] = mapFn(object[key], key)
+  return result
+}, {})
+
+// File utilities
+const getPathFromProjectRoot = (...all) => {
+  return path.join.apply(null, [__dirname, '..', '..'].concat(all))
+}
+const pathToPackageConfigFile = packageName => getPathFromProjectRoot('node_modules', packageName, 'govuk-prototype-kit.config.json')
+
+const readJsonFile = (filePath) => {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'))
+}
+const getPackageConfig = packageName => {
+  if (fs.existsSync(pathToPackageConfigFile(packageName))) {
+    return readJsonFile(pathToPackageConfigFile(packageName))
+  } else {
+    return {}
+  }
+}
+
+// Handle errors to do with extension paths
+// Example of `subject`: { packageName: 'govuk-frontend', item: '/all.js' }
+const throwIfBadFilepath = subject => {
+  if (('' + subject.item).indexOf('\\') > -1) {
+    throw new Error(`Can't use backslashes in extension paths - "${subject.packageName}" used "${subject.item}".`)
+  }
+  if (!('' + subject.item).startsWith('/')) {
+    throw new Error(`All extension paths must start with a forward slash - "${subject.packageName}" used "${subject.item}".`)
+  }
+}
+
+// Check for `baseExtensions` in config.js. If it's not there, default to `govuk-frontend`
+const getBaseExtensions = () => appConfig.baseExtensions || ['govuk-frontend']
+
+// Get all npm dependencies
+// Get baseExtensions in the order defined in `baseExtensions` in config.js
+// Then place baseExtensions before npm dependencies (and remove duplicates)
+const getPackageNamesInOrder = () => {
+  const dependencies = readJsonFile(getPathFromProjectRoot('package.json')).dependencies || {}
+  const allNpmDependenciesInAlphabeticalOrder = Object.keys(dependencies).sort()
+  const installedBaseExtensions = getBaseExtensions()
+    .filter(packageName => allNpmDependenciesInAlphabeticalOrder.includes(packageName))
+
+  return removeDuplicates(installedBaseExtensions.concat(allNpmDependenciesInAlphabeticalOrder))
+}
+
+// Extensions provide items such as sass scripts, asset paths etc.
+// This function groups them by type in a format which can used by getList
+// Example of return
+//    {
+//     nunjucksPaths: [
+//      { packageName: 'govuk-frontend', item: '/' },
+//      { packageName: 'govuk-frontend', item: '/components'}
+//    ],
+//    scripts: [
+//      { packageName: 'govuk-frontend', item: '/all.js' }
+//    ]
+//    assets: [
+//      { packageName: 'govuk-frontend', item: '/assets' }
+//    ],
+//    sass: [
+//      { packageName: 'govuk-frontend', item: '/all.scss' }
+//    ]}
+const getExtensionsByType = () => {
+  return getPackageNamesInOrder()
+    .reduce((accum, packageName) => Object.assign({}, accum, objectMap(
+      getPackageConfig(packageName),
+      (listOfItemsForType, type) => (accum[type] || [])
+        .concat([].concat(listOfItemsForType).map(item => ({
+          packageName,
+          item
+        })))
+    )), {})
+}
+
+let extensionsByType
+
+const setExtensionsByType = () => {
+  extensionsByType = getExtensionsByType()
+}
+
+setExtensionsByType()
+
+// The hard-coded reference to govuk-frontend allows us to soft launch without a breaking change.  After a hard launch
+// govuk-frontend assets will be served on /extension-assets/govuk-frontend
+const getPublicUrl = config => {
+  if (config.item === '/assets' && config.packageName === 'govuk-frontend') {
+    return '/assets'
+  } else {
+    return ['', 'extension-assets', config.packageName]
+      .concat(config.item.split('/').filter(filterOutParentAndEmpty))
+      .map(encodeURIComponent)
+      .join('/')
+  }
+}
+
+const getFileSystemPath = config => {
+  throwIfBadFilepath(config)
+  return getPathFromProjectRoot('node_modules',
+    config.packageName,
+    config.item.split('/').filter(filterOutParentAndEmpty).join(path.sep))
+}
+
+const getPublicUrlAndFileSystemPath = config => ({
+  fileSystemPath: getFileSystemPath(config),
+  publicUrl: getPublicUrl(config)
+})
+
+const getList = type => extensionsByType[type] || []
+
+// Exports
+const self = module.exports = {
+  getPublicUrls: type => getList(type).map(getPublicUrl),
+  getFileSystemPaths: type => getList(type).map(getFileSystemPath),
+  getPublicUrlAndFileSystemPaths: type => getList(type).map(getPublicUrlAndFileSystemPath),
+  getAppConfig: _ => ({
+    scripts: self.getPublicUrls('scripts'),
+    stylesheets: self.getPublicUrls('stylesheets')
+  }),
+  getAppViews: additionalViews => self
+    .getFileSystemPaths('nunjucksPaths')
+    .reverse()
+    .concat(additionalViews || []),
+
+  setExtensionsByType // exposed only for testing purposes
+}