Initial commit

packager/react-packager/src/Packager/Package.js

@@ -0,0 +1,108 @@
+'use strict';
+
+var _ = require('underscore');
+var SourceMapGenerator = require('source-map').SourceMapGenerator;
+var base64VLQ = require('./base64-vlq');
+
+module.exports = Package;
+
+function Package(sourceMapUrl) {
+  this._modules = [];
+  this._sourceMapUrl = sourceMapUrl;
+}
+
+Package.prototype.setMainModuleId = function(moduleId) {
+  this._mainModuleId = moduleId;
+};
+
+Package.prototype.addModule = function(
+  transformedCode,
+  sourceCode,
+  sourcePath
+) {
+  this._modules.push({
+    transformedCode: transformedCode,
+    sourceCode: sourceCode,
+    sourcePath: sourcePath
+  });
+};
+
+Package.prototype.finalize = function(options) {
+  if (options.runMainModule) {
+    var runCode = ';require("' + this._mainModuleId + '");';
+    this.addModule(
+      runCode,
+      runCode,
+      'RunMainModule.js'
+    );
+  }
+
+  Object.freeze(this._modules);
+  Object.seal(this._modules);
+};
+
+Package.prototype.getSource = function() {
+  return _.pluck(this._modules, 'transformedCode').join('\n') + '\n' +
+    'RAW_SOURCE_MAP = ' + JSON.stringify(this.getSourceMap({excludeSource: true})) + ';\n' +
+    '\/\/@ sourceMappingURL=' + this._sourceMapUrl;
+};
+
+Package.prototype.getSourceMap = function(options) {
+  options = options || {};
+  var mappings = this._getMappings();
+  var map = {
+    file: 'bundle.js',
+    sources: _.pluck(this._modules, 'sourcePath'),
+    version: 3,
+    names: [],
+    mappings: mappings,
+    sourcesContent: options.excludeSource
+      ? [] : _.pluck(this._modules, 'sourceCode')
+  };
+  return map;
+};
+
+
+Package.prototype._getMappings = function() {
+  var modules = this._modules;
+
+  // The first line mapping in our package is basically the base64vlq code for
+  // zeros (A).
+  var firstLine = 'AAAA';
+
+  // Most other lines in our mappings are all zeros (for module, column etc)
+  // except for the lineno mappinp: curLineno - prevLineno = 1; Which is C.
+  var line = 'AACA';
+
+  var mappings = '';
+  for (var i = 0; i < modules.length; i++) {
+    var module = modules[i];
+    var transformedCode = module.transformedCode;
+    var lastCharNewLine  = false;
+    module.lines = 0;
+    for (var t = 0; t < transformedCode.length; t++) {
+      if (t === 0 && i === 0) {
+        mappings += firstLine;
+      } else if (t === 0) {
+        mappings += 'AC';
+
+        // This is the only place were we actually don't know the mapping ahead
+        // of time. When it's a new module (and not the first) the lineno
+        // mapping is 0 (current) - number of lines in prev module.
+        mappings += base64VLQ.encode(0 - modules[i - 1].lines);
+        mappings += 'A';
+      } else if (lastCharNewLine) {
+        module.lines++;
+        mappings += line;
+       }
+      lastCharNewLine = transformedCode[t] === '\n';
+      if (lastCharNewLine) {
+        mappings += ';';
+      }
+    }
+    if (i != modules.length - 1) {
+      mappings += ';';
+    }
+  }
+  return mappings;
+};

packager/react-packager/src/Packager/__mocks__/source-map.js

@@ -0,0 +1,5 @@
+var SourceMapGenerator = jest.genMockFn();
+SourceMapGenerator.prototype.addMapping = jest.genMockFn();
+SourceMapGenerator.prototype.setSourceContent = jest.genMockFn();
+SourceMapGenerator.prototype.toJSON = jest.genMockFn();
+exports.SourceMapGenerator = SourceMapGenerator;

packager/react-packager/src/Packager/__tests__/Package-test.js

@@ -0,0 +1,95 @@
+'use strict';
+
+jest
+  .dontMock('underscore')
+  .dontMock('../base64-vlq')
+  .dontMock('source-map')
+  .dontMock('../Package');
+
+var SourceMapGenerator = require('source-map').SourceMapGenerator;
+
+describe('Package', function() {
+  var Package;
+  var ppackage;
+
+  beforeEach(function() {
+    Package = require('../Package');
+    ppackage = new Package('test_url');
+    ppackage.getSourceMap = jest.genMockFn().mockImpl(function() {
+      return 'test-source-map';
+    });
+  });
+
+  describe('source package', function() {
+    it('should create a package and get the source', function() {
+      ppackage.addModule('transformed foo;', 'source foo', 'foo path');
+      ppackage.addModule('transformed bar;', 'source bar', 'bar path');
+      ppackage.finalize({});
+      expect(ppackage.getSource()).toBe([
+        'transformed foo;',
+        'transformed bar;',
+        'RAW_SOURCE_MAP = "test-source-map";',
+        '\/\/@ sourceMappingURL=test_url',
+      ].join('\n'));
+    });
+
+    it('should create a package and add run module code', function() {
+      ppackage.addModule('transformed foo;', 'source foo', 'foo path');
+      ppackage.addModule('transformed bar;', 'source bar', 'bar path');
+      ppackage.setMainModuleId('foo');
+      ppackage.finalize({runMainModule: true});
+      expect(ppackage.getSource()).toBe([
+        'transformed foo;',
+        'transformed bar;',
+        ';require("foo");',
+        'RAW_SOURCE_MAP = "test-source-map";',
+        '\/\/@ sourceMappingURL=test_url',
+      ].join('\n'));
+    });
+  });
+
+  describe('sourcemap package', function() {
+    it('should create sourcemap', function() {
+      var ppackage = new Package('test_url');
+      ppackage.addModule('transformed foo;\n', 'source foo', 'foo path');
+      ppackage.addModule('transformed bar;\n', 'source bar', 'bar path');
+      ppackage.setMainModuleId('foo');
+      ppackage.finalize({runMainModule: true});
+      var s = ppackage.getSourceMap();
+      expect(s).toEqual(genSourceMap(ppackage._modules));
+    });
+  });
+});
+
+ function genSourceMap(modules) {
+   var sourceMapGen = new SourceMapGenerator({file: 'bundle.js', version: 3});
+   var packageLineNo = 0;
+   for (var i = 0; i < modules.length; i++) {
+     var module = modules[i];
+     var transformedCode = module.transformedCode;
+     var sourcePath = module.sourcePath;
+     var sourceCode = module.sourceCode;
+     var transformedLineCount = 0;
+     var lastCharNewLine = false;
+     for (var t = 0; t < transformedCode.length; t++) {
+       if (t === 0 || lastCharNewLine) {
+         sourceMapGen.addMapping({
+           generated: {line: packageLineNo + 1, column: 0},
+           original: {line: transformedLineCount + 1, column: 0},
+           source: sourcePath
+         });
+       }
+       lastCharNewLine = transformedCode[t] === '\n';
+       if (lastCharNewLine) {
+         transformedLineCount++;
+         packageLineNo++;
+       }
+     }
+     packageLineNo++;
+     sourceMapGen.setSourceContent(
+       sourcePath,
+       sourceCode
+     );
+   }
+   return sourceMapGen.toJSON();
+};

packager/react-packager/src/Packager/__tests__/Packager-test.js

@@ -0,0 +1,83 @@
+'use strict';
+
+jest
+  .setMock('worker-farm', function() { return function() {};})
+  .dontMock('path')
+  .dontMock('q')
+  .dontMock('os')
+  .dontMock('underscore')
+  .dontMock('../');
+
+var q = require('q');
+
+describe('Packager', function() {
+  var getDependencies;
+  var wrapModule;
+  var Packager;
+
+  beforeEach(function() {
+    getDependencies = jest.genMockFn();
+    wrapModule = jest.genMockFn();
+    require('../../DependencyResolver').mockImpl(function() {
+      return {
+        getDependencies: getDependencies,
+        wrapModule: wrapModule,
+      };
+    });
+
+    Packager = require('../');
+  });
+
+  pit('create a package', function() {
+    require('fs').statSync.mockImpl(function() {
+      return {
+        isDirectory: function() {return true;}
+      };
+    });
+
+    var packager = new Packager({});
+    var modules = [
+      {id: 'foo', path: '/root/foo.js', dependencies: []},
+      {id: 'bar', path: '/root/bar.js', dependencies: []},
+    ];
+
+    getDependencies.mockImpl(function() {
+      return q({
+        mainModuleId: 'foo',
+        dependencies: modules
+      });
+    });
+
+    require('../../JSTransformer').prototype.loadFileAndTransform
+      .mockImpl(function(tsets, path) {
+        return q({
+          code: 'transformed ' + path,
+          sourceCode: 'source ' + path,
+          sourcePath: path
+        });
+      });
+
+    wrapModule.mockImpl(function(module, code) {
+      return 'lol ' + code + ' lol';
+    });
+
+    return packager.package('/root/foo.js', true, 'source_map_url')
+      .then(function(p) {
+        expect(p.addModule.mock.calls[0]).toEqual([
+          'lol transformed /root/foo.js lol',
+          'source /root/foo.js',
+          '/root/foo.js'
+        ]);
+        expect(p.addModule.mock.calls[1]).toEqual([
+          'lol transformed /root/bar.js lol',
+          'source /root/bar.js',
+          '/root/bar.js'
+        ]);
+
+        expect(p.finalize.mock.calls[0]).toEqual([
+          {runMainModule: true}
+        ]);
+      });
+  });
+
+});

packager/react-packager/src/Packager/base64-vlq.js

@@ -0,0 +1,168 @@
+/* -*- Mode: js; js-indent-level: 2; -*- */
+/*
+ * Copyright 2011 Mozilla Foundation and contributors
+ * Licensed under the New BSD license. See LICENSE or:
+ * http://opensource.org/licenses/BSD-3-Clause
+ *
+ * Based on the Base 64 VLQ implementation in Closure Compiler:
+ * https://code.google.com/p/closure-compiler/source/browse/trunk/src/com/google/debugging/sourcemap/Base64VLQ.java
+ *
+ * Copyright 2011 The Closure Compiler Authors. All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials provided
+ *    with the distribution.
+ *  * Neither the name of Google Inc. nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+var charToIntMap = {};
+var intToCharMap = {};
+  
+'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+  .split('')
+  .forEach(function (ch, index) {
+    charToIntMap[ch] = index;
+    intToCharMap[index] = ch;
+  });
+
+var base64 = {};
+/**
+ * Encode an integer in the range of 0 to 63 to a single base 64 digit.
+ */
+base64.encode = function base64_encode(aNumber) {
+  if (aNumber in intToCharMap) {
+    return intToCharMap[aNumber];
+  }
+  throw new TypeError("Must be between 0 and 63: " + aNumber);
+};
+  
+/**
+ * Decode a single base 64 digit to an integer.
+ */
+base64.decode = function base64_decode(aChar) {
+  if (aChar in charToIntMap) {
+    return charToIntMap[aChar];
+  }
+  throw new TypeError("Not a valid base 64 digit: " + aChar);
+};
+  
+
+
+// A single base 64 digit can contain 6 bits of data. For the base 64 variable
+// length quantities we use in the source map spec, the first bit is the sign,
+// the next four bits are the actual value, and the 6th bit is the
+// continuation bit. The continuation bit tells us whether there are more
+// digits in this value following this digit.
+//
+//   Continuation
+//   |    Sign
+//   |    |
+//   V    V
+//   101011
+
+var VLQ_BASE_SHIFT = 5;
+
+// binary: 100000
+var VLQ_BASE = 1 << VLQ_BASE_SHIFT;
+
+// binary: 011111
+var VLQ_BASE_MASK = VLQ_BASE - 1;
+
+// binary: 100000
+var VLQ_CONTINUATION_BIT = VLQ_BASE;
+
+/**
+ * Converts from a two-complement value to a value where the sign bit is
+ * placed in the least significant bit.  For example, as decimals:
+ *   1 becomes 2 (10 binary), -1 becomes 3 (11 binary)
+ *   2 becomes 4 (100 binary), -2 becomes 5 (101 binary)
+ */
+function toVLQSigned(aValue) {
+  return aValue < 0
+    ? ((-aValue) << 1) + 1
+    : (aValue << 1) + 0;
+}
+
+/**
+ * Converts to a two-complement value from a value where the sign bit is
+ * placed in the least significant bit.  For example, as decimals:
+ *   2 (10 binary) becomes 1, 3 (11 binary) becomes -1
+ *   4 (100 binary) becomes 2, 5 (101 binary) becomes -2
+ */
+function fromVLQSigned(aValue) {
+  var isNegative = (aValue & 1) === 1;
+  var shifted = aValue >> 1;
+  return isNegative
+    ? -shifted
+    : shifted;
+}
+
+/**
+ * Returns the base 64 VLQ encoded value.
+ */
+exports.encode = function base64VLQ_encode(aValue) {
+  var encoded = "";
+  var digit;
+
+  var vlq = toVLQSigned(aValue);
+
+  do {
+    digit = vlq & VLQ_BASE_MASK;
+    vlq >>>= VLQ_BASE_SHIFT;
+    if (vlq > 0) {
+      // There are still more digits in this value, so we must make sure the
+      // continuation bit is marked.
+      digit |= VLQ_CONTINUATION_BIT;
+    }
+    encoded += base64.encode(digit);
+  } while (vlq > 0);
+
+  return encoded;
+};
+
+/**
+ * Decodes the next base 64 VLQ value from the given string and returns the
+ * value and the rest of the string via the out parameter.
+ */
+exports.decode = function base64VLQ_decode(aStr, aOutParam) {
+  var i = 0;
+  var strLen = aStr.length;
+  var result = 0;
+  var shift = 0;
+  var continuation, digit;
+
+  do {
+    if (i >= strLen) {
+      throw new Error("Expected more digits in base 64 VLQ value.");
+    }
+    digit = base64.decode(aStr.charAt(i++));
+    continuation = !!(digit & VLQ_CONTINUATION_BIT);
+    digit &= VLQ_BASE_MASK;
+    result = result + (digit << shift);
+    shift += VLQ_BASE_SHIFT;
+  } while (continuation);
+
+  aOutParam.value = fromVLQSigned(result);
+  aOutParam.rest = aStr.slice(i);
+};
+

packager/react-packager/src/Packager/index.js

@@ -0,0 +1,115 @@
+'use strict';
+
+var assert = require('assert');
+var fs = require('fs');
+var path = require('path');
+var Promise = require('q').Promise;
+var Transformer = require('../JSTransformer');
+var DependencyResolver = require('../DependencyResolver');
+var _ = require('underscore');
+var Package = require('./Package');
+var Activity = require('../Activity');
+
+var DEFAULT_CONFIG = {
+  /**
+   * RegExp used to ignore paths when scanning the filesystem to calculate the
+   * dependency graph.
+   */
+  blacklistRE: null,
+
+  /**
+   * The kind of module system/transport wrapper to use for the modules bundled
+   * in the package.
+   */
+  moduleFormat: 'haste',
+
+  /**
+   * An ordered list of module names that should be considered as dependencies
+   * of every module in the system. The list is ordered because every item in
+   * the list will have an implicit dependency on all items before it.
+   *
+   * (This ordering is necessary to build, for example, polyfills that build on
+   *  each other)
+   */
+  polyfillModuleNames: [],
+
+  /**
+   * DEPRECATED
+   *
+   * A string of code to be appended to the top of a package.
+   *
+   * TODO: THIS RUINS SOURCE MAPS. THIS OPTION SHOULD BE REMOVED ONCE WE GET
+   *       config.polyfillModuleNames WORKING!
+   */
+  runtimeCode: ''
+};
+
+function Packager(projectConfig) {
+  // Verify that the root exists.
+  var root = projectConfig.projectRoot;
+  assert(fs.statSync(root).isDirectory(), 'Root has to be a valid directory');
+  this._rootPath = root;
+
+  this._config = Object.create(DEFAULT_CONFIG);
+  for (var key in projectConfig) {
+    this._config[key] = projectConfig[key];
+  }
+
+  this._resolver = new DependencyResolver(this._config);
+
+  this._transformer = new Transformer(projectConfig);
+}
+
+Packager.prototype.kill = function() {
+  return this._transformer.kill();
+};
+
+Packager.prototype.package = function(main, runModule, sourceMapUrl) {
+  var transformModule = this._transformModule.bind(this);
+  var ppackage = new Package(sourceMapUrl);
+
+  var findEventId = Activity.startEvent('find dependencies');
+  var transformEventId;
+
+  return this._resolver.getDependencies(main)
+    .then(function(result) {
+      Activity.endEvent(findEventId);
+      transformEventId = Activity.startEvent('transform');
+
+      ppackage.setMainModuleId(result.mainModuleId);
+      return Promise.all(
+        result.dependencies.map(transformModule)
+      );
+    })
+    .then(function(transformedModules) {
+      Activity.endEvent(transformEventId);
+
+      transformedModules.forEach(function(transformed) {
+        ppackage.addModule(
+          transformed.code,
+          transformed.sourceCode,
+          transformed.sourcePath
+        );
+      });
+
+      ppackage.finalize({ runMainModule: runModule });
+      return ppackage;
+    });
+};
+
+Packager.prototype._transformModule = function(module) {
+  var resolver = this._resolver;
+  return this._transformer.loadFileAndTransform(
+    ['es6'],
+    path.resolve(module.path),
+    this._config.transformer || {}
+  ).then(function(transformed) {
+    return _.extend(
+      {},
+      transformed,
+      {code: resolver.wrapModule(module, transformed.code)}
+    );
+  });
+};
+
+module.exports = Packager;