fork unbundle output format per platform

Summary:
We decided to use different storage formats for android and ios. This diff changes the output format to asset files if the platform is `'android'`.

public

Reviewed By: martinbigio

Differential Revision: D2764739

fb-gh-sync-id: 4a5ac13ba7978112e9424573643e90cef2a1b75f

local-cli/bundle/output/unbundle/as-assets.js

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const fs = require('fs');
+const mkdirp = require('mkdirp');
+const path = require('path');
+const Promise = require('promise');
+
+const writeFile = require('../writeFile');
+const writeSourceMap = require('./write-sourcemap');
+const MODULES_DIR = 'js-modules';
+
+/**
+ * Saves all JS modules of an app as single files
+ * The startup code (prelude, polyfills etc.) are written to the file
+ * designated by the `bundleOuput` option.
+ * All other modules go into a 'js-modules' folder that in the same parent
+ * directory as the startup file.
+ */
+function saveAsAssets(bundle, options, log) {
+  const {
+    'bundle-output': bundleOutput,
+    'bundle-encoding': encoding,
+    dev,
+    'sourcemap-output': sourcemapOutput,
+  } = options;
+
+  log('start');
+  const {startupCode, modules} = bundle.getUnbundle({minify: !dev});
+  log('finish');
+
+  log('Writing bundle output to:', bundleOutput);
+  const writeUnbundle =
+    Promise.all([
+      writeModules(path.dirname(bundleOutput), modules, encoding),
+      writeStartupFile(bundleOutput, startupCode, encoding)
+    ]);
+  writeUnbundle.then(() => log('Done writing unbundle output'));
+
+  return Promise.all([writeUnbundle, writeSourceMap(sourcemapOutput, '', log)]);
+}
+
+function createDir(dirName) {
+  return new Promise((resolve, reject) =>
+    mkdirp(dirName, error => error ? reject(error) : resolve()));
+}
+
+function createDirectoriesForModules(modulesDir, modules) {
+  const dirNames =
+    modules.map(name => {
+      // get all needed directory names
+      const dir = path.dirname(name);
+      return dir === '.' ? modulesDir : path.join(modulesDir, dir);
+    })
+    .filter(Boolean) // remove empty directories
+    .sort()
+    .filter((dir, i, dirs) => {
+      // remove parent directories. After sorting, parent directories are
+      // located before child directories
+      const next = dirs[i + 1];
+      return !next || next !== dir && !next.startsWith(dir + path.sep);
+    });
+
+  return dirNames.reduce(
+    (promise, dirName) =>
+      promise.then(() => createDir(dirName)), Promise.resolve());
+}
+
+function writeModuleFile(module, modulesDir, encoding) {
+  const {name, code} = module;
+  return writeFile(path.join(modulesDir, name + '.js'), code, encoding);
+}
+
+function writeModuleFiles(modules, modulesDir, encoding) {
+  const writeFiles =
+    modules.map(module => writeModuleFile(module, modulesDir, encoding));
+  return Promise.all(writeFiles);
+}
+
+function writeModules(assetsDest, modules, encoding) {
+  const modulesDir = path.join(assetsDest, MODULES_DIR);
+
+  return (
+    createDirectoriesForModules(modulesDir, modules.map(({name}) => name))
+      .then(() => writeModuleFiles(modules, modulesDir, encoding))
+  );
+}
+
+function writeStartupFile(outputFile, code, encoding) {
+  return new Promise((resolve, reject) => {
+    fs.createWriteStream(outputFile).
+      write(code, encoding, error => error ? reject(error) : resolve());
+  });
+}
+
+module.exports = saveAsAssets;

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const fs = require('fs');
+const Promise = require('promise');
+const writeSourceMap = require('./write-sourcemap');
+
+const MAGIC_UNBUNDLE_FILE_HEADER = 0xFB0BD1E5;
+const MAGIC_STARTUP_MODULE_ID = '';
+
+/**
+ * Saves all JS modules of an app as a single file, separated with null bytes.
+ * The file begins with an offset table that contains module ids and their
+ * lengths/offsets.
+ * The module id for the startup code (prelude, polyfills etc.) is the
+ * empty string.
+ */
+function saveAsIndexedFile(bundle, options, log) {
+  const {
+    'bundle-output': bundleOutput,
+    'bundle-encoding': encoding,
+    dev,
+    'sourcemap-output': sourcemapOutput,
+  } = options;
+
+  log('start');
+  const {startupCode, modules} = bundle.getUnbundle({minify: !dev});
+  log('finish');
+
+  log('Writing unbundle output to:', bundleOutput);
+  const writeUnbundle = writeBuffers(
+    fs.createWriteStream(bundleOutput),
+    buildTableAndContents(startupCode, modules, encoding)
+  );
+
+  writeUnbundle.then(() => log('Done writing unbundle output'));
+
+  return Promise.all([writeUnbundle, writeSourceMap(sourcemapOutput, '', log)]);
+}
+
+/* global Buffer: true */
+
+const fileHeader = Buffer(4);
+fileHeader.writeUInt32LE(MAGIC_UNBUNDLE_FILE_HEADER);
+const nullByteBuffer = Buffer(1).fill(0);
+
+function writeBuffers(stream, buffers) {
+  buffers.forEach(buffer => stream.write(buffer));
+  return new Promise((resolve, reject) => {
+    stream.on('error', reject);
+    stream.on('finish', () => resolve());
+    stream.end();
+  });
+}
+
+const moduleToBuffer = ({name, code}, encoding) => ({
+  name,
+  buffer: Buffer.concat([
+    Buffer(code, encoding),
+    nullByteBuffer // create \0-terminated strings
+  ])
+});
+
+function uInt32Buffer(n) {
+  const buffer = Buffer(4);
+  buffer.writeUInt32LE(n, 0);
+  return buffer;
+}
+
+function buildModuleTable(buffers) {
+  // table format:
+  //  - table_length: uint_32 length of all table entries in bytes
+  //  - entries: entry...
+  //
+  // entry:
+  //  - module_id: NUL terminated utf8 string
+  //  - module_offset: uint_32 offset into the module string
+  //  - module_length: uint_32 length of the module string, including terminating NUL byte
+
+  const numBuffers = buffers.length;
+
+  const tableLengthBuffer = uInt32Buffer(0);
+  let tableLength = 4; // the table length itself, 4 == tableLengthBuffer.length
+  let currentOffset = 0;
+
+  const offsetTable = [tableLengthBuffer];
+  for (let i = 0; i < numBuffers; i++) {
+    const {name, buffer: {length}} = buffers[i];
+    const entry = Buffer.concat([
+      Buffer(i === 0 ? MAGIC_STARTUP_MODULE_ID : name, 'utf8'),
+      nullByteBuffer,
+      uInt32Buffer(currentOffset),
+      uInt32Buffer(length)
+    ]);
+    currentOffset += length;
+    tableLength += entry.length;
+    offsetTable.push(entry);
+  }
+
+  tableLengthBuffer.writeUInt32LE(tableLength, 0);
+  return Buffer.concat(offsetTable);
+}
+
+function buildModuleBuffers(startupCode, modules, encoding) {
+  return (
+    [moduleToBuffer({name: '', code: startupCode}, encoding)]
+      .concat(modules.map(module => moduleToBuffer(module, encoding)))
+  );
+}
+
+function buildTableAndContents(startupCode, modules, encoding) {
+  const buffers = buildModuleBuffers(startupCode, modules, encoding);
+  const table = buildModuleTable(buffers, encoding);
+  return [fileHeader, table].concat(buffers.map(({buffer}) => buffer));
+}
+
+module.exports = saveAsIndexedFile;

local-cli/bundle/output/unbundle/index.js

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const asIndexedFile = require('./as-indexed-file');
+const asAssets = require('./as-assets');
+
+function buildBundle(packagerClient, requestOptions) {
+  return packagerClient.buildBundle({...requestOptions, unbundle: true});
+}
+
+function saveUnbundle(bundle, options, log) {
+  // we fork here depending on the platform:
+  // while android is pretty good at loading individual assets, ios has a large
+  // overhead when reading hundreds pf assets from disk
+  return options.platform === 'android' ?
+    asAssets(bundle, options, log) :
+    asIndexedFile(bundle, options, log);
+}
+
+exports.build = buildBundle;
+exports.save = saveUnbundle;
+exports.formatName = 'bundle';

local-cli/bundle/output/unbundle/write-sourcemap.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const Promise = require('promise');
+const writeFile = require('../writeFile');
+
+function writeSourcemap(fileName, contents, log) {
+  if (!fileName) {
+    return Promise.resolve();
+  }
+  log('Writing sourcemap output to:', fileName);
+  const writeMap = writeFile(fileName, '', null);
+  writeMap.then(() => log('Done writing sourcemap output'));
+  return writeMap;
+}
+
+module.exports = writeSourcemap;