chore($ngLocale): refactor i18n closure slurper logic and parse extended datetime symbols

3 files changed, 232 insertions, 102 deletions

diff --git a/i18n/src/closureI18nExtractor.js b/i18n/src/closureI18nExtractor.js
new file mode 100644
index 00000000..a26f89b7
--- /dev/null
+++ b/i18n/src/closureI18nExtractor.js
@@ -0,0 +1,175 @@
+'use strict';
+
+var converter = require('./converter.js');
+
+exports.extractNumberSymbols = extractNumberSymbols;
+exports.extractCurrencySymbols = extractCurrencySymbols;
+exports.extractDateTimeSymbols = extractDateTimeSymbols;
+exports.pluralExtractor = pluralExtractor;
+exports.outputLocale = outputLocale;
+exports.correctedLocaleId = correctedLocaleId;
+exports.findLocaleId = findLocaleId;
+
+var goog = { provide: function() {},
+  require: function() {},
+  i18n: {currency: {}, pluralRules: {}} };
+
+function findLocaleId(str, type) {
+  if (type === 'num') {
+    return (str.match(/^NumberFormatSymbols_(.+)$/) || [])[1];
+  }
+
+  if (type != 'datetime') { throw new Error('unknown type in findLocaleId: ' + type); }
+
+  return (str.match(/^DateTimeSymbols_(.+)$/) || [])[1];
+}
+
+
+function getInfoForLocale(localeInfo, localeID) {
+  if (!localeInfo[localeID]) {
+    localeInfo[localeID] = {};
+    //localeIds.push(localeID);
+  }
+  return localeInfo[localeID];
+}
+
+function extractNumberSymbols(content, localeInfo, currencySymbols) {
+  //eval script in the current context so that we get access to all the symbols
+  eval(content.toString());
+  for (var propName in goog.i18n) {
+    var localeID = findLocaleId(propName, 'num');
+    if (localeID) {
+      var info = getInfoForLocale(localeInfo, localeID);
+      info.NUMBER_FORMATS =
+          converter.convertNumberData(goog.i18n[propName], currencySymbols);
+    }
+  }
+}
+
+function extractCurrencySymbols(content) {
+  //eval script in the current context so that we get access to all the symbols
+  eval(content.toString());
+  var currencySymbols = goog.i18n.currency.CurrencyInfo;
+  currencySymbols.__proto__ = goog.i18n.currency.CurrencyInfoTier2;
+
+  return currencySymbols;
+}
+
+function extractDateTimeSymbols(content, localeInfo) {
+  //eval script in the current context so that we get access to all the symbols
+  eval(content.toString());
+  for (var propName in goog.i18n) {
+    var localeID = findLocaleId(propName, 'datetime');
+    if (localeID) {
+      var info = getInfoForLocale(localeInfo, localeID);
+      localeInfo[localeID].DATETIME_FORMATS =
+          converter.convertDatetimeData(goog.i18n[propName]);
+    }
+  }
+}
+
+function pluralExtractor(content, localeInfo) {
+  var contentText = content.toString();
+  var localeIds = Object.keys(localeInfo);
+  for (var i = 0; i < localeIds.length; i++) {
+    //We don't need to care about country ID because the plural rules in more specific id are
+    //always the same as those in its language ID.
+    // e.g. plural rules for en_SG is the same as those for en.
+    goog.LOCALE = localeIds[i].match(/[^_]+/)[0];
+    try {
+      eval(contentText);
+    } catch(e) {
+      console.log("Error in eval(contentText): " + e.stack);
+    }
+    if (!goog.i18n.pluralRules.select) {
+      console.log('No select for lang [' + goog.LOCALE + ']');
+      continue;
+    }
+    var temp = goog.i18n.pluralRules.select.toString().
+        replace(/goog.i18n.pluralRules.Keyword/g, 'PLURAL_CATEGORY').replace(/\n/g, '');
+
+    ///@@ is a crazy place holder to be replaced before writing to file
+    localeInfo[localeIds[i]].pluralCat = "@@" + temp + "@@";
+  }
+}
+
+function correctedLocaleId(localeID) {
+// e.g. from zh_CN to zh-CN, from en_US to en-US
+  return localeID.replace(/_/g, '-').toLowerCase();
+}
+
+function canonicalizeForJsonStringify(unused_key, object) {
+  // This function is intended to be called as the 2nd argument to
+  // JSON.stringify.  The goal here is to ensure that the generated JSON has
+  // objects with their keys in ascending order.  Without this, it's much
+  // harder to diff the generated files in src/ngLocale as the order isn't
+  // exactly consistent.  We've gotten lucky in the past.
+  //
+  // Iteration order, for string keys, ends up being the same as insertion
+  // order.  Refer :-
+  //    1. http://ejohn.org/blog/javascript-in-chrome/
+  //       (search for "for loop order").
+  //       Currently all major browsers loop over the properties of an object
+  //       in the order in which they were defined.
+  //         - John Resig
+  //    2. https://code.google.com/p/v8/issues/detail?id=164
+  //       ECMA-262 does not specify enumeration order. The de facto standard
+  //       is to match insertion order, which V8 also does ...
+  if (typeof object != "object") {
+    return object;
+  }
+  var result = {};
+  Object.keys(object).sort().forEach(function(key) {
+    result[key] = object[key];
+  });
+  return result;
+}
+
+function outputLocale(localeInfo, localeID) {
+  var fallBackID = localeID.match(/[A-Za-z]+/)[0],
+      localeObj = localeInfo[localeID],
+      fallBackObj = localeInfo[fallBackID];
+
+  // fallBack to language formats when country format is missing
+  // e.g. if NUMBER_FORMATS of en_xyz is not present, use the NUMBER_FORMATS of en instead
+  if (!localeObj.NUMBER_FORMATS) {
+    localeObj.NUMBER_FORMATS = fallBackObj.NUMBER_FORMATS;
+  }
+
+  // datetimesymbolsext.js provides more top level locales than the other
+  // files.  We process datetimesymbolsext.js because we want the country
+  // specific formats that are missing from datetimesymbols.js.  However, we
+  // don't want to write locale files that only have dateformat (i.e. missing
+  // number formats.)  So we skip them.
+  if (!localeObj.NUMBER_FORMATS) {
+    console.log("Skipping locale %j: Don't have any number formats", localeID);
+    return null;
+  }
+
+  if (!localeObj.DATETIME_FORMATS) {
+    localeObj.DATETIME_FORMATS = fallBackObj.DATETIME_FORMATS;
+  }
+  localeObj.id = correctedLocaleId(localeID);
+
+  var prefix =
+      'angular.module("ngLocale", [], ["$provide", function($provide) {\n' +
+          'var PLURAL_CATEGORY = {' +
+          'ZERO: "zero", ONE: "one", TWO: "two", FEW: "few", MANY: "many", OTHER: "other"' +
+          '};\n' +
+          '$provide.value("$locale", ';
+
+  var suffix = ');\n}]);';
+
+  localeObj = {
+    DATETIME_FORMATS: localeObj.DATETIME_FORMATS,
+    NUMBER_FORMATS: localeObj.NUMBER_FORMATS,
+    pluralCat: localeObj.pluralCat,
+    id: localeObj.id
+  };
+
+  var content = JSON.stringify(localeInfo[localeID], canonicalizeForJsonStringify, '  ')
+      .replace(/\¤/g, '\\u00A4')
+      .replace(/"@@|@@"/g, '');
+
+  return prefix + content + suffix;
+}
diff --git a/i18n/src/closureSlurper.js b/i18n/src/closureSlurper.js
index 2e78ef64..9f236cf2 100755
--- a/i18n/src/closureSlurper.js
+++ b/i18n/src/closureSlurper.js
@@ -1,130 +1,85 @@
 #!/usr/bin/env node
 'use strict';
 
-var Q  = require('qq'),
+var Q  = require('q'),
     qfs  = require('q-fs'),
     converter = require('./converter.js'),
     util = require('./util.js'),
+    closureI18nExtractor = require('./closureI18nExtractor.js'),
     localeInfo = {},
-    localeIds = [],
     currencySymbols,
     goog = { provide: function() {},
              require: function() {},
              i18n: {currency: {}, pluralRules: {}} };
 
-createFolder('../../src/ngLocale/').then(function() {
-  var promiseA = Q.defer(),
-      promiseB = Q.defer();
 
-  qfs.read(__dirname + '/../closure/currencySymbols.js', 'b').then(function(content) {
-    eval(content.toString());
-    currencySymbols = goog.i18n.currency.CurrencyInfo;
-    currencySymbols.__proto__ = goog.i18n.currency.CurrencyInfoTier2;
+var NG_LOCALE_DIR = '../src/ngLocale/';
 
-    qfs.read(__dirname + '/../closure/numberSymbols.js', 'b').then(function(content) {
-      //eval script in the current context so that we get access to all the symbols
-      eval(content.toString());
-      for (var propName in goog.i18n) {
-        var localeID = util.findLocaleId(propName, 'num');
-        if (localeID) {
-          if (!localeInfo[localeID]) {
-            localeInfo[localeID] = {};
-            localeIds.push(localeID);
-          }
-          var convertedData = converter.convertNumberData(goog.i18n[propName], currencySymbols);
-          localeInfo[localeID].NUMBER_FORMATS = convertedData;
-        }
-      }
 
-      promiseA.resolve();
-    });
-  });
-
-
-  qfs.read(__dirname + '/../closure/datetimeSymbols.js', 'b').then(function(content) {
-    eval(content.toString());
-    for (var propName in goog.i18n) {
-      var localeID = util.findLocaleId(propName, 'datetime');
-      if (localeID) {
-        if (!localeInfo[localeID]) {
-          localeInfo[localeID] = {};
-          localeIds.push(localeID);
-        }
-        var convertedData = converter.convertDatetimeData(goog.i18n[propName]);
-        localeInfo[localeID].DATETIME_FORMATS = convertedData;
-      }
-    }
-
-    promiseB.resolve();
-  });
+function readSymbols() {
+  console.log("Processing currency and number symbols ...");
+  var numericStagePromise = qfs.read(__dirname + '/../closure/currencySymbols.js', 'b')
+    .then(function(content) {
+      var currencySymbols = closureI18nExtractor.extractCurrencySymbols(content);
+      return qfs.read(__dirname + '/../closure/numberSymbols.js', 'b').then(function(content) {
+          closureI18nExtractor.extractNumberSymbols(content, localeInfo, currencySymbols);
+        });
+      });
 
-  return Q.join(promiseA.promise, promiseB.promise, noop);
-}).then(function() {
-  var promise = Q.defer();
+  console.log("Processing datetime symbols ...");
+  var datetimeStagePromise = qfs.read(__dirname + '/../closure/datetimeSymbols.js', 'b')
+      .then(function(content) {
+        closureI18nExtractor.extractDateTimeSymbols(content, localeInfo);
+        return qfs.read(__dirname + '/../closure/datetimeSymbolsExt.js', 'b').then(function(content) {
+            closureI18nExtractor.extractDateTimeSymbols(content, localeInfo);
+        });
+    });
 
-  qfs.read(__dirname + '/../closure/pluralRules.js').then(function(content) {
-    for(var i = 0; i < localeIds.length; i++) {
-      //We don't need to care about country ID because the plural rules in more specific id are
-      //always the same as those in its language ID.
-      // e.g. plural rules for en_SG is the same as those for en.
-      goog.LOCALE = localeIds[i].match(/[^_]+/)[0];
-      eval(content);
-      var temp = goog.i18n.pluralRules.select.toString().
-                     replace(/goog.i18n.pluralRules.Keyword/g, 'PLURAL_CATEGORY').replace(/\n/g, '');
+    return Q.all([numericStagePromise, datetimeStagePromise]);
+}
 
-      ///@@ is a crazy place holder to be replaced before writing to file
-      localeInfo[localeIds[i]].pluralCat = "@@" + temp + "@@";
-    }
-    promise.resolve();
+function extractPlurals() {
+  console.log('Extracting Plurals ...');
+  return qfs.read(__dirname + '/../closure/pluralRules.js').then(function(content) {
+    closureI18nExtractor.pluralExtractor(content, localeInfo);
   });
+}
 
-  return promise.promise;
-}).then(function() {
+function writeLocaleFiles() {
+  console.log('Final stage: Writing angular locale files to directory: %j', NG_LOCALE_DIR);
+  var writePromises = [];
+  var localeIds = Object.keys(localeInfo);
+  var num_files = 0;
   localeIds.forEach(function(localeID) {
-    var fallBackID = localeID.match(/[A-Za-z]+/)[0],
-        localeObj = localeInfo[localeID],
-        fallBackObj = localeInfo[fallBackID];
-
-    // fallBack to language formats when country format is missing
-    // e.g. if NUMBER_FORMATS of en_xyz is not present, use the NUMBER_FORMATS of en instead
-    if (!localeObj.NUMBER_FORMATS) {
-      localeObj.NUMBER_FORMATS = fallBackObj.NUMBER_FORMATS;
-    }
-
-    if (!localeObj.DATETIME_FORMATS) {
-       localeObj.DATETIME_FORMATS = fallBackObj.DATETIME_FORMATS;
-    }
-
-    // e.g. from zh_CN to zh-CN, from en_US to en-US
-    var correctedLocaleId = localeID.replace(/_/g, '-').toLowerCase();
-    localeObj.id = correctedLocaleId;
-
-    var prefix =
-      'angular.module("ngLocale", [], ["$provide", function($provide) {\n' +
-         'var PLURAL_CATEGORY = {' +
-           'ZERO: "zero", ONE: "one", TWO: "two", FEW: "few", MANY: "many", OTHER: "other"' +
-         '};\n' +
-         '$provide.value("$locale", ';
-
-    var suffix = ');\n}]);';
-
-    var content = JSON.stringify(localeInfo[localeID]).replace(/\¤/g,'\\u00A4').
-                      replace(/"@@|@@"/g, '');
-
-    var toWrite = prefix + content + suffix;
-    qfs.write(__dirname + '/../locale/' + 'angular-locale_' + correctedLocaleId + '.js', toWrite);
+    var content = closureI18nExtractor.outputLocale(localeInfo, localeID);
+    if (!content) return;
+    var correctedLocaleId = closureI18nExtractor.correctedLocaleId(localeID);
+    var filename = NG_LOCALE_DIR + 'angular-locale_' + correctedLocaleId + '.js'
+    writePromises.push(
+      qfs.write(filename, content)
+      .then(function () {
+        console.log('Wrote ' + filename);
+        ++num_files;
+        }));
+    console.log('Writing ' + filename);
   });
-  console.log('Generated ' + localeIds.length + ' locale files!');
-}).end();
-
-function noop() {};
+  console.log('Generated %j locale files.', localeIds.length);
+  return Q.all(writePromises).then(function() { return num_files });
+}
 
 /**
 * Make a folder under current directory.
 * @param folder {string} name of the folder to be made
 */
 function createFolder(folder) {
-  return qfs.isDirectory(__dirname + '/' + folder).then(function(isDir) {
-    if (!isDir) return qfs.makeDirectory(__dirname + '/' + folder);
+  return qfs.isDirectory(folder).then(function(isDir) {
+    if (!isDir) return qfs.makeDirectory(folder).then(function() {
+        console.log('Created directory %j', folder); });
   });
 }
+
+createFolder(NG_LOCALE_DIR)
+  .then(readSymbols)
+  .then(extractPlurals)
+  .then(writeLocaleFiles)
+  .done(function(num_files) { console.log("Wrote %j files.\nAll Done!", num_files); });
diff --git a/i18n/src/parser.js b/i18n/src/parser.js
index db31aeef..31dea510 100644
--- a/i18n/src/parser.js
+++ b/i18n/src/parser.js
@@ -45,8 +45,8 @@ function parsePattern(pattern) {
   }
 
   var groups = integer.split(GROUP_SEP);
-  p.gSize = groups[1].length;
-  p.lgSize = (groups[2] || groups[1]).length;
+  p.gSize = groups[1] ? groups[1].length : 0;
+  p.lgSize = (groups[2] || groups[1]) ? (groups[2] || groups[1]).length : 0;
 
   if (negative) {
     var trunkLen = positive.length - p.posPre.length - p.posSuf.length,