Create an OutputFn that can build indexed RAM bundles

Summary: Adds functionality to assemble an indexed source map to the new Buck integration. This implementation supports startup section optimisations. Hooking it up, and grouping optimisations will be in follow-ups.

Reviewed By: jeanlauliac

Differential Revision: D5106985

fbshipit-source-id: cc4c6ac8cfe4e718fc8bb2a8a93cb88914c92e0b

local-cli/bundle/output/unbundle/as-assets.js

@@ -69,6 +69,7 @@ function saveAsAssets(
       buildSourceMapWithMetaData({
         startupModules: startupModules.concat(),
         lazyModules: lazyModules.concat(),
+        fixWrapperOffset: true,
       }),
       sourcemapSourcesRoot
     );

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -19,8 +19,7 @@ const writeSourceMap = require('./write-sourcemap');
 
 const {joinModules} = require('./util');
 
-import type ModuleTransport from '../../../../packager/src//lib/ModuleTransport';
-import type {Bundle, ModuleGroups, OutputOptions} from '../../types.flow';
+import type {Bundle, ModuleGroups, ModuleTransportLike, OutputOptions} from '../../types.flow';
 
 const SIZEOF_UINT32 = 4;
 
@@ -62,6 +61,7 @@ function saveAsIndexedFile(
         startupModules: startupModules.concat(),
         lazyModules: lazyModules.concat(),
         moduleGroups,
+        fixWrapperOffset: true,
       }),
       sourcemapSourcesRoot
     );
@@ -104,7 +104,7 @@ function entryOffset(n) {
   return (2 + n * 2) * SIZEOF_UINT32;
 }
 
-function buildModuleTable(startupCode, buffers, moduleGroups) {
+function buildModuleTable(startupCode, moduleBuffers, moduleGroups) {
   // table format:
   // - num_entries:      uint_32  number of entries
   // - startup_code_len: uint_32  length of the startup section
@@ -127,7 +127,7 @@ function buildModuleTable(startupCode, buffers, moduleGroups) {
 
   // entries
   let codeOffset = startupCode.length;
-  buffers.forEach(({id, buffer}) => {
+  moduleBuffers.forEach(({id, buffer}) => {
     const group = moduleGroups.groups.get(id);
     const idsInGroup = group ? [id].concat(Array.from(group)) : [id];
 
@@ -170,7 +170,12 @@ function buildModuleBuffers(modules, moduleGroups, encoding) {
     ));
 }
 
-function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
+function buildTableAndContents(
+  startupCode: string,
+  modules: $ReadOnlyArray<ModuleTransportLike>,
+  moduleGroups: ModuleGroups,
+  encoding?: 'utf8' | 'utf16le' | 'ascii',
+) {
   // file contents layout:
   // - magic number      char[4]  0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
   // - offset table      table    see `buildModuleTables`
@@ -190,7 +195,7 @@ function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
 
 function createModuleGroups(
   groups: Map<number, Set<number>>,
-  modules: Array<ModuleTransport>,
+  modules: $ReadOnlyArray<ModuleTransportLike>,
 ): ModuleGroups {
   return {
     groups,
@@ -205,4 +210,6 @@ function * concat(iterators) {
   }
 }
 
-module.exports = saveAsIndexedFile;
+exports.save = saveAsIndexedFile;
+exports.buildTableAndContents = buildTableAndContents;
+exports.createModuleGroups = createModuleGroups;

local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js

@@ -10,25 +10,31 @@
  */
 'use strict';
 
-const {combineSourceMaps, joinModules} = require('./util');
+const {combineSourceMaps, combineSourceMapsAddingOffsets, joinModules} = require('./util');
 
 import type {ModuleGroups, ModuleTransportLike} from '../../types.flow';
 
 type Params = {
+  fixWrapperOffset: boolean,
   lazyModules: Array<ModuleTransportLike>,
   moduleGroups?: ModuleGroups,
   startupModules: Array<ModuleTransportLike>,
 };
 
-module.exports = ({startupModules, lazyModules, moduleGroups}: Params) => {
+module.exports = ({fixWrapperOffset, lazyModules, moduleGroups, startupModules}: Params) => {
+  const options = fixWrapperOffset ? {fixWrapperOffset: true} : undefined;
   const startupModule: ModuleTransportLike = {
     code: joinModules(startupModules),
     id:  Number.MIN_SAFE_INTEGER,
-    map: combineSourceMaps({modules: startupModules}),
+    map: combineSourceMaps(startupModules, undefined, options),
+    sourcePath: '',
   };
-  return combineSourceMaps({
-    modules: [startupModule].concat(lazyModules),
+
+  const map = combineSourceMapsAddingOffsets(
+    [startupModule].concat(lazyModules),
     moduleGroups,
-    withCustomOffsets: true,
-  });
+    options,
+  );
+  delete map.x_facebook_offsets[Number.MIN_SAFE_INTEGER];
+  return map;
 };

local-cli/bundle/output/unbundle/index.js

@@ -14,7 +14,7 @@
 const Server = require('../../../../packager/src/Server');
 
 const asAssets = require('./as-assets');
-const asIndexedFile = require('./as-indexed-file');
+const asIndexedFile = require('./as-indexed-file').save;
 
 import type Bundle from '../../../../packager/src//Bundler/Bundle';
 import type {OutputOptions, RequestOptions} from '../../types.flow';

local-cli/bundle/output/unbundle/util.js

@@ -12,7 +12,7 @@
 
 const invariant = require('fbjs/lib/invariant');
 
-import type {IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap';
+import type {FBIndexMap, IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap';
 import type {ModuleGroups, ModuleTransportLike} from '../../types.flow';
 
 const newline = /\r\n?|\n|\u2028|\u2029/g;
@@ -43,39 +43,45 @@ const Section =
   (line: number, column: number, map: SourceMap) =>
     ({map, offset: {line, column}});
 
-type CombineSourceMapsOptions = {
-  moduleGroups?: ModuleGroups,
-  modules: Array<ModuleTransportLike>,
-  withCustomOffsets?: boolean,
-};
+type CombineOptions = {fixWrapperOffset: boolean};
 
-function combineSourceMaps({
-  moduleGroups,
-  modules,
-  withCustomOffsets,
-}: CombineSourceMapsOptions): IndexMap {
-  const offsets = [];
+function combineSourceMaps(
+  modules: Array<ModuleTransportLike>,
+  moduleGroups?: ModuleGroups,
+  options?: ?CombineOptions,
+): IndexMap {
+  const sections = combineMaps(modules, null, moduleGroups, options);
+  return {sections, version: 3};
+}
+
+function combineSourceMapsAddingOffsets(
+  modules: Array<ModuleTransportLike>,
+  moduleGroups?: ModuleGroups,
+  options?: ?CombineOptions,
+): FBIndexMap {
+  const x_facebook_offsets = [];
+  const sections = combineMaps(modules, x_facebook_offsets, moduleGroups, options);
+  return {sections, version: 3, x_facebook_offsets};
+}
+
+function combineMaps(modules, offsets: ?Array<number>, moduleGroups, options) {
   const sections = [];
-  const sourceMap: IndexMap = withCustomOffsets
-    ? {sections, version: 3, x_facebook_offsets: offsets}
-    : {sections, version: 3};
 
   let line = 0;
   modules.forEach(moduleTransport => {
     const {code, id, name} = moduleTransport;
     let column = 0;
-    let hasOffset = false;
     let group;
     let groupLines = 0;
     let {map} = moduleTransport;
 
-    if (withCustomOffsets) {
-      if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
-        // this is a module appended to another module
-        return;
-      }
+    if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
+      // this is a module appended to another module
+      return;
+    }
 
 
+    if (offsets != null) {
       group = moduleGroups && moduleGroups.groups.get(id);
       if (group && moduleGroups) {
         const {modulesById} = moduleGroups;
@@ -86,13 +92,10 @@ function combineSourceMaps({
         otherModules.forEach(m => {
           groupLines += countLines(m.code);
         });
-        map = combineSourceMaps({
-          modules: [moduleTransport].concat(otherModules),
-        });
+        map = combineSourceMaps([moduleTransport].concat(otherModules));
       }
 
-      hasOffset = id != null;
-      column = wrapperEnd(code);
+      column = options && options.fixWrapperOffset ? wrapperEnd(code) : 0;
     }
 
     invariant(
@@ -100,7 +103,7 @@ function combineSourceMaps({
       'Random Access Bundle source maps cannot be built from raw mappings',
     );
     sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
-    if (hasOffset) {
+    if (offsets != null && id != null) {
       offsets[id] = line;
       for (const moduleId of group || []) {
         offsets[moduleId] = line;
@@ -109,7 +112,7 @@ function combineSourceMaps({
     line += countLines(code) + groupLines;
   });
 
-  return sourceMap;
+  return sections;
 }
 
 const joinModules =
@@ -117,8 +120,9 @@ const joinModules =
     modules.map(m => m.code).join('\n');
 
 module.exports = {
-  countLines,
-  lineToLineSourceMap,
   combineSourceMaps,
+  combineSourceMapsAddingOffsets,
+  countLines,
   joinModules,
+  lineToLineSourceMap,
 };

local-cli/bundle/types.flow.js

@@ -19,7 +19,7 @@ export type {Bundle, FBSourceMap, ModuleTransport, SourceMap, Unbundle};
 
 export type ModuleGroups = {|
   groups: Map<number, Set<number>>,
-  modulesById: Map<number, ModuleTransport>,
+  modulesById: Map<number, ModuleTransportLike>,
   modulesInGroups: Set<number>,
 |};
 
@@ -28,6 +28,7 @@ export type ModuleTransportLike = {
   id: number,
   map?: $PropertyType<ModuleTransport, 'map'>,
   +name?: string,
+  sourcePath: string,
 };
 
 export type OutputOptions = {