Sourcemaps support for RAM

Summary:This rev adds support for production sourcemaps on RAM.

When we inject a module into JSC we use the original `sourceURL` and specify the `startingLineNumber` of the module relative to a "regular" bundle. By doing so, when an error is thrown, JSC will include the provided `sourceURL` as the filename and will use the indicated `startingLineNumber` to figure out on which line the error actually occurred.

To make things a bit simpler and avoid having to deal with columns, we tweak the generated bundle so that each module starts on a new line. Since we cannot assure that each module's code will be on a single line as the minifier might break it on multiple (UglifyJS does so due to a bug on old versions of Chrome), we include on the index the line number that should be used when invoking `JSEvaluateScript`. Since the module length was not being used we replaced the placeholder we have there for the line number.

Reviewed By: javache

Differential Revision: D2997520

fb-gh-sync-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f
shipit-source-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -8,6 +8,7 @@
  */
 'use strict';
 
+const buildUnbundleSourcemap = require('./buildUnbundleSourcemap');
 const fs = require('fs');
 const Promise = require('promise');
 const writeSourceMap = require('./write-sourcemap');
@@ -26,23 +27,27 @@ function saveAsIndexedFile(bundle, options, log) {
   const {
     'bundle-output': bundleOutput,
     'bundle-encoding': encoding,
-    dev,
     'sourcemap-output': sourcemapOutput,
   } = options;
 
   log('start');
-  const {startupCode, modules} = bundle.getUnbundle({minify: !dev});
+  const {startupCode, modules} = bundle.getUnbundle();
   log('finish');
 
   log('Writing unbundle output to:', bundleOutput);
   const writeUnbundle = writeBuffers(
     fs.createWriteStream(bundleOutput),
     buildTableAndContents(startupCode, modules, encoding)
-  );
+  ).then(() => log('Done writing unbundle output'));
 
-  writeUnbundle.then(() => log('Done writing unbundle output'));
-
-  return Promise.all([writeUnbundle, writeSourceMap(sourcemapOutput, '', log)]);
+  return Promise.all([
+    writeUnbundle,
+    writeSourceMap(
+      sourcemapOutput,
+      buildUnbundleSourcemap(bundle),
+      log,
+    ),
+  ]);
 }
 
 /* global Buffer: true */
@@ -60,13 +65,16 @@ function writeBuffers(stream, buffers) {
   });
 }
 
-const moduleToBuffer = ({name, code}, encoding) => ({
-  name,
-  buffer: Buffer.concat([
-    Buffer(code, encoding),
-    nullByteBuffer // create \0-terminated strings
-  ])
-});
+function moduleToBuffer(name, code, encoding) {
+  return {
+    name,
+    linesCount: code.split('\n').length,
+    buffer: Buffer.concat([
+      Buffer(code, encoding),
+      nullByteBuffer // create \0-terminated strings
+    ])
+  };
+}
 
 function uInt32Buffer(n) {
   const buffer = Buffer(4);
@@ -82,23 +90,28 @@ function buildModuleTable(buffers) {
   // entry:
   //  - module_id: NUL terminated utf8 string
   //  - module_offset: uint_32 offset into the module string
-  //  - module_length: uint_32 length of the module string, including terminating NUL byte
+  //  - module_line: uint_32 line on which module starts on the bundle
 
   const numBuffers = buffers.length;
 
   const tableLengthBuffer = uInt32Buffer(0);
   let tableLength = 4; // the table length itself, 4 == tableLengthBuffer.length
   let currentOffset = 0;
+  let currentLine = 1;
 
   const offsetTable = [tableLengthBuffer];
   for (let i = 0; i < numBuffers; i++) {
-    const {name, buffer: {length}} = buffers[i];
+    const {name, linesCount, buffer: {length}} = buffers[i];
+
     const entry = Buffer.concat([
       Buffer(i === 0 ? MAGIC_STARTUP_MODULE_ID : name, 'utf8'),
       nullByteBuffer,
       uInt32Buffer(currentOffset),
-      uInt32Buffer(length)
+      uInt32Buffer(currentLine),
     ]);
+
+    currentLine += linesCount - 1;
+
     currentOffset += length;
     tableLength += entry.length;
     offsetTable.push(entry);
@@ -110,14 +123,22 @@ function buildModuleTable(buffers) {
 
 function buildModuleBuffers(startupCode, modules, encoding) {
   return (
-    [moduleToBuffer({name: '', code: startupCode}, encoding)]
-      .concat(modules.map(module => moduleToBuffer(module, encoding)))
+    [moduleToBuffer('', startupCode, encoding, true)].concat(
+      modules.map(module =>
+        moduleToBuffer(
+          module.name,
+          module.code + '\n', // each module starts on a newline
+          encoding,
+        )
+      )
+    )
   );
 }
 
 function buildTableAndContents(startupCode, modules, encoding) {
   const buffers = buildModuleBuffers(startupCode, modules, encoding);
   const table = buildModuleTable(buffers, encoding);
+
   return [fileHeader, table].concat(buffers.map(({buffer}) => buffer));
 }
 

local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const sourceMap  = require('source-map');
+const SourceMapConsumer = sourceMap.SourceMapConsumer;
+
+/**
+ * Builds the sourcemaps for any type of unbundle provided the Bundle that
+ * contains the modules reachable from the entry point.
+ *
+ * The generated sourcemaps correspond to a regular bundle on which each module
+ * starts on a new line. Depending on the type of unbundle you're using, you
+ * will have to pipe the line number to native and use it when injecting the
+ * module's code into JSC. This way, we'll trick JSC to believe all the code is
+ * on a single big regular bundle where as it could be on an indexed bundle or
+ * as sparsed assets.
+ */
+function buildUnbundleSourcemap(bundle) {
+  const generator = new sourceMap.SourceMapGenerator({});
+  const nonPolyfillModules = bundle.getModules().filter(module =>
+    !module.polyfill
+  );
+
+  let offset = 1;
+  nonPolyfillModules.forEach(module => {
+    if (module.map) { // assets have no sourcemap
+      const consumer = new SourceMapConsumer(module.map);
+      consumer.eachMapping(mapping => {
+        generator.addMapping({
+          original: {
+            line: mapping.originalLine,
+            column: mapping.originalColumn,
+          },
+          generated: {
+            line: mapping.generatedLine + offset,
+            column: mapping.generatedColumn,
+          },
+          source: module.sourcePath,
+        });
+      });
+
+      generator.setSourceContent(module.sourcePath, module.sourceCode);
+    }
+
+    // some modules span more than 1 line
+    offset += module.code.split('\n').length;
+  });
+
+  return generator.toString();
+}
+
+module.exports = buildUnbundleSourcemap;

local-cli/bundle/output/unbundle/write-sourcemap.js

@@ -16,7 +16,7 @@ function writeSourcemap(fileName, contents, log) {
     return Promise.resolve();
   }
   log('Writing sourcemap output to:', fileName);
-  const writeMap = writeFile(fileName, '', null);
+  const writeMap = writeFile(fileName, contents, null);
   writeMap.then(() => log('Done writing sourcemap output'));
   return writeMap;
 }