Adds robots.txt file to discourage crawling

lib/appFactory.js

@@ -2,6 +2,7 @@ const express = require('express');
 const path = require('path');
 const { loggingMiddleware, getLogger } = require('./logger');
 const indexRouter = require('./routes/index');
+const robots = require('robots.txt');
 const requestInviteRouter = require('./routes/request-invite');
 const webFingerRouter = require('./routes/webfinger');
 const oAuthRouter = require('./routes/oauth');
@@ -34,13 +35,15 @@ module.exports = async function ({ hostIdentity, jwtSecret, accountMgr, storeRou
   app.set('accountMgr', accountMgr);
 
   // web browsers ask for this way too often
-  app.use('/favicon.ico', (req, res, _next) => {
+  app.get('/favicon.ico', (req, res, _next) => {
     res.set('Cache-Control', 'public, max-age=31536000');
     res.status(404).end();
   });
 
   app.use(loggingMiddleware);
 
+  app.use(robots(path.join(__dirname, 'robots.txt')));
+
   const helmetStorage = helmet({
     contentSecurityPolicy: {
       directives: {

lib/robots.txt

@@ -0,0 +1,6 @@
+User-agent: *
+Disallow: /
+Allow: /$
+Allow: /signup$
+Allow: /assets/
+Allow: /storage/*/public/

package.json

@@ -41,6 +41,7 @@
     "mkdirp": "^1.0.4",
     "node-mocks-http": "^1.14.1",
     "proquint": "^0.0.1",
+    "robots.txt": "^1.1.0",
     "winston": "^3.11.0",
     "yaml": "^2.4.0"
   },

spec/modular/robots.txt.spec.js

@@ -0,0 +1,35 @@
+/* eslint-env mocha */
+
+const chai = require('chai');
+const expect = chai.expect;
+chai.use(require('chai-http'));
+const { mockAccountFactory } = require('../util/mockAccount');
+const appFactory = require('../../lib/appFactory');
+const { configureLogger } = require('../../lib/logger');
+
+describe('robots.txt', function () {
+  before(async function () {
+    configureLogger({ log_dir: './test-log', stdout: [], log_files: ['error'] });
+
+    const app = await appFactory({
+      hostIdentity: 'autotest.ch',
+      jwtSecret: 'swordfish',
+      accountMgr: mockAccountFactory('autotest.ch'),
+      storeRouter: (_req, _res, next) => next()
+    });
+    app.locals.title = 'Test Armadietto';
+    app.locals.host = 'localhost:xxxx';
+    app.locals.signup = false;
+    this.app = app;
+  });
+
+  it('should serve robots.txt at expected location', async function () {
+    const res = await chai.request(this.app).get('/robots.txt');
+    expect(res).to.have.status(200);
+    expect(res).to.have.header('Content-Type', /^text\/plain/);
+    expect(parseInt(res.get('Content-Length'))).to.be.greaterThan(0);
+
+    expect(res.text).to.match(/^User-agent: \*$/m);
+    expect(res.text).to.match(/^Disallow: \/$/m);
+  });
+});

spec/runner.js

@@ -17,6 +17,7 @@ require('./stores/file_tree_spec');
 
 require('./modular/m_root.spec');
 require('./modular/m_not_found.spec');
+require('./modular/robots.txt.spec');
 require('./modular/m_static.spec');
 require('./modular/request_invite.spec');
 require('./modular/account.spec');