packager: preprocess outdated dependencies

Summary:
Note: if this changeset causes some breakage, consider disabling rather than reverting. To disable, the call to `_preprocessPotentialDependencies` in `ResolutionRequest` can be removed.

It's a bit of an experiment. I couldn't see any particular regression caused by this, but I could see net improvement of the global cache performance, as it unlock much, much stronger batching: indeed, instead of discovering dependencies progressively, we synchronously figure out the list of all potential modules of a bundle, and kick-off cache fetching and/or transformations. So when it comes to fetching from the global cache, it'll do less requests, and each requests will ask for considerably more keys at a time.

Potential problem caused by this changeset: if a module's dependencies completely changed, then the first time we try to build the bundle it'll start transforming modules that we probably don't care at all anymore, spending precious CPU time for nothing. I've been thinking about it and I cannot see such a case happening much often. Even if it happens, it should not cause any bug or corruption, it would just take additional time.

Other potential problem: that this new code doesn't handle some types of edge cases. It's quite hard to figure out what could possibly break in the `ResolutionRequest` code (and I think it would benefit from a larger refactor). We do have a good test coverage for `DependencyGraph` and it seems to work smoothly, so I'm relatively confident we're not breaking edge cases.

Reviewed By: davidaurelio

Differential Revision: D4875467

fbshipit-source-id: 2dfcc755bec638d3d1c47862ec1de5220953e812

packager/src/node-haste/DependencyGraph/ResolutionRequest.js

@@ -24,6 +24,8 @@ const getAssetDataFromName = require('../lib/getAssetDataFromName');
 import type {HasteFS} from '../types';
 import type DependencyGraphHelpers from './DependencyGraphHelpers';
 import type ResolutionResponse from './ResolutionResponse';
+import type {Options as TransformWorkerOptions} from '../../JSTransformer/worker/worker';
+import type {ReadResult, CachedReadResult} from '../Module';
 
 type DirExistsFn = (filePath: string) => boolean;
 
@@ -45,6 +47,8 @@ type Moduleish = {
   +path: string,
   getPackage(): ?Packageish,
   hash(): string,
+  readCached(transformOptions: TransformWorkerOptions): CachedReadResult,
+  readFresh(transformOptions: TransformWorkerOptions): Promise<ReadResult>,
 };
 
 type ModuleishCache<TModule, TPackage> = {
@@ -163,8 +167,8 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
 
   resolveModuleDependencies(
     module: TModule,
-    dependencyNames: Array<string>,
-  ): [Array<string>, Array<TModule>] {
+    dependencyNames: $ReadOnlyArray<string>,
+  ): [$ReadOnlyArray<string>, $ReadOnlyArray<TModule>] {
     const dependencies = dependencyNames.map(name => this.resolveDependency(module, name));
     return [dependencyNames, dependencies];
   }
@@ -176,7 +180,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
     recursive = true,
   }: {
     response: ResolutionResponse<TModule>,
-    transformOptions: Object,
+    transformOptions: TransformWorkerOptions,
     onProgress?: ?(finishedModules: number, totalModules: number) => mixed,
     recursive: boolean,
   }) {
@@ -186,10 +190,23 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
     let totalModules = 1;
     let finishedModules = 0;
 
-    const resolveDependencies = module => Promise.resolve().then(() => {
-      const result = module.readCached(transformOptions);
-      if (result != null) {
-        return this.resolveModuleDependencies(module, result.dependencies);
+    let preprocessedModuleCount = 1;
+    if (recursive) {
+      this._preprocessPotentialDependencies(transformOptions, entry, count => {
+        if (count + 1 <= preprocessedModuleCount) {
+          return;
+        }
+        preprocessedModuleCount = count + 1;
+        if (onProgress != null) {
+          onProgress(finishedModules, preprocessedModuleCount);
+        }
+      });
+    }
+
+    const resolveDependencies = (module: TModule) => Promise.resolve().then(() => {
+      const cached = module.readCached(transformOptions);
+      if (cached.result != null) {
+        return this.resolveModuleDependencies(module, cached.result.dependencies);
       }
       return module.readFresh(transformOptions)
         .then(({dependencies}) => this.resolveModuleDependencies(module, dependencies));
@@ -221,7 +238,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
       if (onProgress) {
         finishedModules += 1;
         totalModules += newDependencies.length;
-        onProgress(finishedModules, totalModules);
+        onProgress(finishedModules, Math.max(totalModules, preprocessedModuleCount));
       }
 
       if (recursive) {
@@ -273,6 +290,71 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
     });
   }
 
+  /**
+   * This synchronously look at all the specified modules and recursively kicks off global cache
+   * fetching or transforming (via `readFresh`). This is a hack that workaround the current
+   * structure, because we could do better. First off, the algorithm that resolves dependencies
+   * recursively should be synchronous itself until it cannot progress anymore (and needs to
+   * call `readFresh`), so that this algo would be integrated into it.
+   */
+  _preprocessPotentialDependencies(
+    transformOptions: TransformWorkerOptions,
+    module: TModule,
+    onProgress: (moduleCount: number) => mixed,
+  ): void {
+    const visitedModulePaths = new Set();
+    const pendingBatches = [this.preprocessModule(transformOptions, module, visitedModulePaths)];
+    onProgress(visitedModulePaths.size);
+    while (pendingBatches.length > 0) {
+      const dependencyModules = pendingBatches.pop();
+      while (dependencyModules.length > 0) {
+        const dependencyModule = dependencyModules.pop();
+        const deps = this.preprocessModule(transformOptions, dependencyModule, visitedModulePaths);
+        pendingBatches.push(deps);
+        onProgress(visitedModulePaths.size);
+      }
+    }
+  }
+
+  preprocessModule(
+    transformOptions: TransformWorkerOptions,
+    module: TModule,
+    visitedModulePaths: Set<string>,
+  ): Array<TModule> {
+    const cached = module.readCached(transformOptions);
+    if (cached.result == null) {
+      module.readFresh(transformOptions).catch(error => {
+        /* ignore errors, they'll be handled later if the dependency is actually
+         * not obsolete, and required from somewhere */
+      });
+    }
+    const dependencies = cached.result != null
+      ? cached.result.dependencies : cached.outdatedDependencies;
+    return this.tryResolveModuleDependencies(module, dependencies, visitedModulePaths);
+  }
+
+  tryResolveModuleDependencies(
+    module: TModule,
+    dependencyNames: $ReadOnlyArray<string>,
+    visitedModulePaths: Set<string>,
+  ): Array<TModule> {
+    const result = [];
+    for (let i = 0; i < dependencyNames.length; ++i) {
+      try {
+        const depModule = this.resolveDependency(module, dependencyNames[i]);
+        if (!visitedModulePaths.has(depModule.path)) {
+          visitedModulePaths.add(depModule.path);
+          result.push(depModule);
+        }
+      } catch (error) {
+        if (!(error instanceof UnableToResolveError)) {
+          throw error;
+        }
+      }
+    }
+    return result;
+  }
+
   _resolveHasteDependency(fromModule: TModule, toModuleName: string): TModule {
     toModuleName = normalizePath(toModuleName);