lint: collect unused l10n tokens

lib/common/files.js

@@ -299,6 +299,9 @@ var FileManager = function(baseURI, relBaseURI, console, flow){
   this.warns = [];
 
   this.readInfo = [];
+
+  // helpers
+  this.abspath = abspath;
 };
 
 FileManager.prototype = {

lib/extract/js/l10n.js

@@ -36,6 +36,34 @@ module.exports = function(file, flow, defineHandler, globalScope){
 
           fconsole.log('[basis.l10n] basis.l10n.Dictionary#token.compute ' + id);
           token_.obj = resolveL10nToken(key + '.{?}', dictPath).jsToken.obj;
+        }),
+        token: at.createRunner(function(token_, this_, args, scope){
+          var tokenKey = scope.simpleExpression(args[0]);
+
+          if (tokenKey && tokenKey.type == 'Literal')
+          {
+            tokenKey = tokenKey.value;
+
+            var file = tokenDescriptor.dictionary.file;
+            var id = key + '.' + tokenKey + '@' + file.filename;
+            // collect through refs
+            var parts = tokenKey.split('.');
+
+            fconsole.log('[basis.l10n] basis.l10n.Token#token ' + id);
+            parts.reduce(function(prev, current){
+              var l10nToken = flow.l10n.getToken(key + '.' + prev + '@' + file.filename);
+
+              l10nToken.addRef(this.file, token_, 'through');
+
+              return prev + '.' + current;
+            }.bind(this));
+
+            // add explicit ref
+            var l10nToken = resolveL10nToken(key + '.' + tokenKey, file.filename);
+
+            l10nToken.addRef(this.file, token_, 'explicit');
+            token_.obj = l10nToken.jsToken.obj;
+          }
         })
       };
 
@@ -63,12 +91,25 @@ module.exports = function(file, flow, defineHandler, globalScope){
 
           if (key && key.type == 'Literal')
           {
-            var id = key.value + '@' + file.filename;
+            key = key.value;
+
+            // collect through refs
+            var parts = key.split('.');
+            var id = key + '@' + file.filename;
 
             fconsole.log('[basis.l10n] basis.l10n.Dictionary#token ' + id);
+            parts.reduce(function(prev, current){
+              var l10nToken = flow.l10n.getToken(prev + '@' + file.filename);
+
+              l10nToken.addRef(this.file, token_, 'through');
+
+              return prev + '.' + current;
+            }.bind(this));
+
+            // add explicit ref
+            var l10nToken = resolveL10nToken(key, file.filename);
 
-            var l10nToken = resolveL10nToken(key.value, file.filename);
-            l10nToken.addRef(this.file, token_);
+            l10nToken.addRef(this.file, token_, 'explicit');
             token_.obj = l10nToken.jsToken.obj;
           }
           else
@@ -161,7 +202,7 @@ module.exports = function(file, flow, defineHandler, globalScope){
     var filename = scope.simpleExpression(args[0]);
     if (filename && filename.type == 'Literal')
     {
-      //filename = this.file.resolve(filename[1]);
+      // filename = this.file.resolve(filename.value);
       filename = basisResolveURI
         ? basisResolveURI(filename.value, flow.indexFile.baseURI)
         : resolveToBase(flow, filename.value, flow.indexFile.baseURI);

lib/extract/l10n/Token.js

@@ -10,14 +10,17 @@ var Token = function(dictionary, name){
 Token.prototype.type = 'default';
 Token.prototype.comment = null;
 
-Token.prototype.addRef = function(file, refToken){
+Token.prototype.addRef = function(file, refToken, type){
   for (var i = 0, ref; ref = this.ref[i]; i++)
     if (ref.file === file && ref.refToken === refToken)
       return;
 
+  // explicit has more priority than through
+  this.usage = type === 'explicit' ? type : this.usage || type;
   this.ref.push({
     file: file,
-    refToken: refToken
+    refToken: refToken,
+    type: type
   });
 };
 

lib/extract/l10n/index.js

@@ -25,6 +25,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl;
           if (l10nPrefix.test(bindName))
           {
             var l10nTokenRef = bindName.substr(5);
+            var parts = l10nTokenRef.split('@');
+            var tokenName = parts[0];
+            var dictFilename = parts[1];
+            var tokenNameParts = tokenName.match(/^(.+?)\.{(.+?)}/);
+
+            if (tokenNameParts && tokenNameParts.length == 3) {
+              tokenName = tokenNameParts[1];
+            }
+
+            l10nTokenRef = tokenName + '@' + dictFilename;
+
             var l10nToken = flow.l10n.getToken(l10nTokenRef);
             var name = l10nToken.name;
             var dictionary = l10nToken.dictionary;
@@ -43,7 +54,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl;
             dictionary.file.jsRefCount++;
             dictionary.addRef(this.file);
             this.file.link(dictionary.file);
-            l10nToken.addRef(this.file, tmplRef);
+
+            // collect through refs
+            tokenName.split('.').reduce(function(prev, current){
+              var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename);
+
+              l10nToken.addRef(this.file, tmplRef, 'through');
+
+              return prev + '.' + current;
+            }.bind(this));
+            // add explicit ref
+            l10nToken.addRef(this.file, tmplRef, 'explicit');
 
             tmplRefs.push(tmplRef);
           }
@@ -55,6 +76,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl;
               if (l10nPrefix.test(bindName))
               {
                 var l10nTokenRef = bindName.substr(5);
+                var parts = l10nTokenRef.split('@');
+                var tokenName = parts[0];
+                var dictFilename = parts[1];
+                var tokenNameParts = tokenName.match(/^(.+?)\.{(.+?)}/);
+
+                if (tokenNameParts && tokenNameParts.length == 3) {
+                  tokenName = tokenNameParts[1];
+                }
+
+                l10nTokenRef = tokenName + '@' + dictFilename;
+
                 var l10nToken = flow.l10n.getToken(l10nTokenRef);
                 var name = l10nToken.name;
                 var dictionary = l10nToken.dictionary;
@@ -73,7 +105,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl;
                 dictionary.file.jsRefCount++;
                 dictionary.addRef(this.file);
                 this.file.link(dictionary.file);
-                l10nToken.addRef(this.file, tmplRef);
+
+                // collect through refs
+                tokenName.split('.').reduce(function(prev, current){
+                  var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename);
+
+                  l10nToken.addRef(this.file, tmplRef, 'through');
+
+                  return prev + '.' + current;
+                }.bind(this));
+                // add explicit ref
+                l10nToken.addRef(this.file, tmplRef, 'explicit');
 
                 tmplRefs.push(tmplRef);
               }

lib/lint/command.js

@@ -2,6 +2,7 @@ var path = require('path');
 var clap = require.main.require('clap');
 var common = require('../common/command');
 var isChildProcess = typeof process.send == 'function'; // child process has send method
+var handleUnusedL10n = require('./reporter/parallel-process-unused-l10n');
 
 function resolveCwd(value){
   return path.resolve(process.env.PWD || process.cwd(), value);
@@ -20,6 +21,7 @@ module.exports = clap.create('lint', '[fileOrPreset]')
   .option('--no-color', 'Suppress color output')
   .option('--silent', 'No any output')
 
+  .option('--warn-unused-l10n <dir>', 'Warn about unused l10n tokens for specified path. Avoid using with --js-cut-dev since it might cause to incorrect results')
   .option('--filter <filename>', 'Show warnings only for specified file', resolveCwd)
   .option('-r, --reporter <name>', 'Reporter console (default), checkstyle, junit',
     function(reporter){
@@ -53,6 +55,8 @@ module.exports.getParallelOptions = function(){
     return {
       silent: true,
       callback: function(res){
+        handleUnusedL10n(res);
+
         var reporter = require(require('./reporter')[command.values.reporter]);
         var data = require('./reporter/parallel-process-warns.js')(res);
         console.log(reporter(data));

lib/lint/index.js

@@ -7,6 +7,7 @@ var extract = require('../extract');
 var command = require('./command');
 var chalk = require('chalk');
 var isChildProcess = typeof process.send == 'function'; // child process has send method
+var unusedL10n = require('./unused/l10n');
 
 if (isChildProcess)
   process.on('uncaughtException', function(error){
@@ -91,6 +92,13 @@ function lint(config){
     l10nInfo: true
   }).concat([
     function(flow){
+      if (options.warnUnusedL10n)
+      {
+        flow.usedL10nTokens = unusedL10n.collectUsed(flow);
+        if (!isChildProcess)
+          unusedL10n.warn(flow);
+      }
+
       flow.result = require('./reporter/process-warns')(flow.warns, options.filter);
     }
   ]);
@@ -148,6 +156,7 @@ function lint(config){
         event: 'done',
         success: !flow.warns.length,
         warnings: flow.warns,
+        usedL10nTokens: flow.usedL10nTokens,
         result: flow.result
       });
     });

lib/lint/reporter/parallel-process-unused-l10n.js

@@ -0,0 +1,60 @@
+var path = require('path');
+
+module.exports = function handleUnusedL10n(tasks){
+  var usedL10nTokens = {};
+  var unusedL10nTokens = {};
+
+  // merge used l10n tokens from tasks
+  // and collect used and unused l10n tokens
+  tasks.forEach(function(task){
+    if (task.result.usedL10nTokens)
+      for (var dictFileName in task.result.usedL10nTokens.items)
+      {
+        var absDictFilename = task.result.usedL10nTokens.basePath + dictFileName;
+        var tokenUsageInfo = task.result.usedL10nTokens.items[dictFileName];
+
+        usedL10nTokens[absDictFilename] = usedL10nTokens[absDictFilename] || {};
+        unusedL10nTokens[absDictFilename] = unusedL10nTokens[absDictFilename] || {};
+
+        for (var tokenName in tokenUsageInfo)
+          if (tokenUsageInfo.hasOwnProperty(tokenName))
+          {
+            var used = tokenUsageInfo[tokenName];
+            var alreadyUsed = usedL10nTokens[absDictFilename].hasOwnProperty(tokenName);
+
+            if (alreadyUsed)
+              continue;
+
+            if (!used)
+              unusedL10nTokens[absDictFilename][tokenName] = true;
+            else
+            {
+              usedL10nTokens[absDictFilename][tokenName] = true;
+              delete unusedL10nTokens[absDictFilename][tokenName];
+            }
+          }
+
+        if (!Object.keys(unusedL10nTokens[absDictFilename]).length)
+          delete unusedL10nTokens[absDictFilename];
+      }
+  });
+
+  // warn about unused l10n tokens
+  if (Object.keys(unusedL10nTokens).length)
+  {
+    for (var dictFileName in unusedL10nTokens)
+    {
+      var tokenNames = unusedL10nTokens[dictFileName];
+
+      dictFileName = path.relative(process.cwd(), dictFileName);
+      tasks.push({
+        name: 'unused l10n tokens at ' + dictFileName,
+        result: {
+          warnings: Object.keys(tokenNames).map(function(name){
+            return { file: 'Path "' + name + '" defined but never used' };
+          })
+        }
+      });
+    }
+  }
+};

lib/lint/reporter/parallel-process-warns.js

@@ -17,7 +17,7 @@ module.exports = function(tasks){
 
         failures.push({
           loc: warn.loc,
-          message: warn.message + ' at ' + filename
+          message: warn.message ? warn.message + ' at ' + filename : filename
         });
       });
   });