chore(docs): refactor the docs app search for better bootup time

This commit refactors how the search index is built. The docsSearch service
is now defined by a provider, which returns a different implementation of
the service depending upon whether the current browser supports WebWorkers
or now.

* **WebWorker supported**: The index is then built and stored in a new worker.
The service posts and receives messages to and from this worker to make
queries on the search index.

* **WebWorker no supported**: The index is built locally but with a 500ms
delay so that the initial page can render before the browser is blocked as
the index is built.

Also the way that the current app is identified has been modified so we can
slim down the js data files (pages-data.js) to again improve startup time.

Closes #9204
Closes #9203

docs/app/assets/css/docs.css

@@ -316,10 +316,10 @@ iframe.example {
 }
 
 .search-results-group.col-group-api { width:30%; }
-.search-results-group.col-group-guide { width:30%; }
-.search-results-group.col-group-tutorial { width:25%; }
+.search-results-group.col-group-guide,
+.search-results-group.col-group-tutorial { width:20%; }
 .search-results-group.col-group-misc,
-.search-results-group.col-group-error { float:right; clear:both; width:15% }
+.search-results-group.col-group-error { width:15%; float: right; }
 
 
 .search-results-group.col-group-api .search-result {

docs/app/assets/js/search-worker.js

@@ -0,0 +1,44 @@
+"use strict";
+/* jshint browser: true */
+/* global importScripts, onmessage: true, postMessage, lunr */
+
+// Load up the lunr library
+importScripts('../components/lunr.js-0.4.2/lunr.min.js');
+
+// Create the lunr index - the docs should be an array of object, each object containing
+// the path and search terms for a page
+var index = lunr(function() {
+  this.ref('path');
+  this.field('titleWords', {boost: 50});
+  this.field('members', { boost: 40});
+  this.field('keywords', { boost : 20 });
+});
+
+// Retrieve the searchData which contains the information about each page to be indexed
+var searchData = {};
+var searchDataRequest = new XMLHttpRequest();
+searchDataRequest.onload = function() {
+
+  // Store the pages data to be used in mapping query results back to pages
+  searchData = JSON.parse(this.responseText);
+  // Add search terms from each page to the search index
+  searchData.forEach(function(page) {
+    index.add(page);
+  });
+  postMessage({ e: 'index-ready' });
+};
+searchDataRequest.open('GET', 'search-data.json');
+searchDataRequest.send();
+
+// The worker receives a message everytime the web app wants to query the index
+onmessage = function(oEvent) {
+  var q = oEvent.data.q;
+  var hits = index.search(q);
+  var results = [];
+  // Only return the array of paths to pages
+  hits.forEach(function(hit) {
+    results.push(hit.ref);
+  });
+  // The results of the query are sent back to the web app via a new message
+  postMessage({ e: 'query-ready', q: q, d: results });
+};

docs/app/e2e/app.scenario.js

@@ -77,10 +77,13 @@ describe('docs.angularjs.org', function () {
     });
 
 
-    it("should display an error if the page does not exist", function() {
-      browser.get('index-debug.html#!/api/does/not/exist');
-      expect(element(by.css('h1')).getText()).toBe('Oops!');
-    });
   });
 
 });
+
+describe('Error Handling', function() {
+  it("should display an error if the page does not exist", function() {
+    browser.get('index-debug.html#!/api/does/not/exist');
+    expect(element(by.css('h1')).getText()).toBe('Oops!');
+  });
+});

docs/app/src/app.js

@@ -6,6 +6,7 @@ angular.module('docsApp', [
   'DocsController',
   'versionsData',
   'pagesData',
+  'navData',
   'directives',
   'errors',
   'examples',

docs/app/src/docs.js

@@ -6,31 +6,10 @@ angular.module('DocsController', [])
   function($scope, $rootScope, $location, $window, $cookies, openPlunkr,
               NG_PAGES, NG_NAVIGATION, NG_VERSION) {
 
-
   $scope.openPlunkr = openPlunkr;
 
   $scope.docsVersion = NG_VERSION.isSnapshot ? 'snapshot' : NG_VERSION.version;
 
-  $scope.fold = function(url) {
-    if(url) {
-      $scope.docs_fold = '/notes/' + url;
-      if(/\/build/.test($window.location.href)) {
-        $scope.docs_fold = '/build/docs' + $scope.docs_fold;
-      }
-      window.scrollTo(0,0);
-    }
-    else {
-      $scope.docs_fold = null;
-    }
-  };
-  var OFFLINE_COOKIE_NAME = 'ng-offline',
-      INDEX_PATH = /^(\/|\/index[^\.]*.html)$/;
-
-
-  /**********************************
-   Publish methods
-   ***********************************/
-
   $scope.navClass = function(navItem) {
     return {
       active: navItem.href && this.currentPage && this.currentPage.path,
@@ -38,55 +17,28 @@ angular.module('DocsController', [])
     };
   };
 
-  $scope.afterPartialLoaded = function() {
+
+
+  $scope.$on('$includeContentLoaded', function() {
     var pagePath = $scope.currentPage ? $scope.currentPage.path : $location.path();
     $window._gaq.push(['_trackPageview', pagePath]);
-  };
+  });
 
-  /** stores a cookie that is used by apache to decide which manifest ot send */
-  $scope.enableOffline = function() {
-    //The cookie will be good for one year!
-    var date = new Date();
-    date.setTime(date.getTime()+(365*24*60*60*1000));
-    var expires = "; expires="+date.toGMTString();
-    var value = angular.version.full;
-    document.cookie = OFFLINE_COOKIE_NAME + "="+value+expires+"; path=" + $location.path;
-
-    //force the page to reload so server can serve new manifest file
-    window.location.reload(true);
-  };
-
-
-
-  /**********************************
-   Watches
-   ***********************************/
+  $scope.$on('$includeContentError', function() {
+    $scope.partialPath = 'Error404.html';
+  });
 
 
   $scope.$watch(function docsPathWatch() {return $location.path(); }, function docsPathWatchAction(path) {
 
-    var currentPage = $scope.currentPage = NG_PAGES[path];
-    if ( !currentPage && path.charAt(0)==='/' ) {
-      // Strip off leading slash
-      path = path.substr(1);
-    }
+    path = path.replace(/^\/?(.+?)(\/index)?\/?$/, '$1');
 
-    currentPage = $scope.currentPage = NG_PAGES[path];
-    if ( !currentPage && path.charAt(path.length-1) === '/' && path.length > 1 ) {
-      // Strip off trailing slash
-      path = path.substr(0, path.length-1);
-    }
-
-    currentPage = $scope.currentPage = NG_PAGES[path];
-    if ( !currentPage && /\/index$/.test(path) ) {
-      // Strip off index from the end
-      path = path.substr(0, path.length - 6);
-    }
+    $scope.partialPath = 'partials/' + path + '.html';
 
     currentPage = $scope.currentPage = NG_PAGES[path];
 
     if ( currentPage ) {
-      $scope.currentArea = currentPage && NG_NAVIGATION[currentPage.area];
+      $scope.currentArea = NG_NAVIGATION[currentPage.area];
       var pathParts = currentPage.path.split('/');
       var breadcrumb = $scope.breadcrumb = [];
       var breadcrumbPath = '';
@@ -107,24 +59,12 @@ angular.module('DocsController', [])
 
   $scope.versionNumber = angular.version.full;
   $scope.version = angular.version.full + "  " + angular.version.codeName;
-  $scope.subpage = false;
-  $scope.offlineEnabled = ($cookies[OFFLINE_COOKIE_NAME] == angular.version.full);
-  $scope.futurePartialTitle = null;
   $scope.loading = 0;
-  $scope.$cookies = $cookies;
 
-  $cookies.platformPreference = $cookies.platformPreference || 'gitUnix';
 
+  var INDEX_PATH = /^(\/|\/index[^\.]*.html)$/;
   if (!$location.path() || INDEX_PATH.test($location.path())) {
     $location.path('/api').replace();
   }
 
-  // bind escape to hash reset callback
-  angular.element(window).on('keydown', function(e) {
-    if (e.keyCode === 27) {
-      $scope.$apply(function() {
-        $scope.subpage = false;
-      });
-    }
-  });
 }]);

docs/app/src/navigationService.js

@@ -1,24 +0,0 @@
-angular.module('docsApp.navigationService', [])
-
-.factory('navigationService', function($window) {
-  var service = {
-    currentPage: null,
-    currentVersion: null,
-    changePage: function(newPage) {
-
-    },
-    changeVersion: function(newVersion) {
-
-      //TODO =========
-    //   var currentPagePath = '';
-
-    // // preserve URL path when switching between doc versions
-    // if (angular.isObject($rootScope.currentPage) && $rootScope.currentPage.section && $rootScope.currentPage.id) {
-    //   currentPagePath = '/' + $rootScope.currentPage.section + '/' + $rootScope.currentPage.id;
-    // }
-
-    // $window.location = version.url + currentPagePath;
-
-    }
-  };
-});

docs/app/src/search.js

@@ -10,22 +10,35 @@ angular.module('search', [])
   $scope.search = function(q) {
     var MIN_SEARCH_LENGTH = 2;
     if(q.length >= MIN_SEARCH_LENGTH) {
-      var results = docsSearch(q);
-      var totalAreas = 0;
-      for(var i in results) {
-        ++totalAreas;
-      }
-      if(totalAreas > 0) {
-        $scope.colClassName = 'cols-' + totalAreas;
-      }
-      $scope.hasResults = totalAreas > 0;
-      $scope.results = results;
+      docsSearch(q).then(function(hits) {
+        var results = {};
+        angular.forEach(hits, function(hit) {
+          var area = hit.area;
+
+          var limit = (area == 'api') ? 40 : 14;
+          results[area] = results[area] || [];
+          if(results[area].length < limit) {
+            results[area].push(hit);
+          }
+        });
+
+        var totalAreas = 0;
+        for(var i in results) {
+          ++totalAreas;
+        }
+        if(totalAreas > 0) {
+          $scope.colClassName = 'cols-' + totalAreas;
+        }
+        $scope.hasResults = totalAreas > 0;
+        $scope.results = results;
+      });
     }
     else {
       clearResults();
     }
     if(!$scope.$$phase) $scope.$apply();
   };
+
   $scope.submit = function() {
     var result;
     for(var i in $scope.results) {
@@ -39,78 +52,124 @@ angular.module('search', [])
       $scope.hideResults();
     }
   };
+
   $scope.hideResults = function() {
     clearResults();
     $scope.q = '';
   };
 }])
 
-.controller('Error404SearchCtrl', ['$scope', '$location', 'docsSearch', function($scope, $location, docsSearch) {
-  $scope.results = docsSearch($location.path().split(/[\/\.:]/).pop());
+
+.controller('Error404SearchCtrl', ['$scope', '$location', 'docsSearch',
+        function($scope, $location, docsSearch) {
+  docsSearch($location.path().split(/[\/\.:]/).pop()).then(function(results) {
+    $scope.results = {};
+    angular.forEach(results, function(result) {
+      var area = $scope.results[result.area] || [];
+      area.push(result);
+      $scope.results[result.area] = area;
+    });
+  });
 }])
 
-.factory('lunrSearch', function() {
-  return function(properties) {
-    if (window.RUNNING_IN_NG_TEST_RUNNER) return null;
 
-    var engine = lunr(properties);
-    return {
-      store : function(values) {
-        engine.add(values);
-      },
-      search : function(q) {
-        return engine.search(q);
-      }
-    };
-  };
-})
+.provider('docsSearch', function() {
 
-.factory('docsSearch', ['$rootScope','lunrSearch', 'NG_PAGES',
-    function($rootScope, lunrSearch, NG_PAGES) {
-  if (window.RUNNING_IN_NG_TEST_RUNNER) {
-    return null;
-  }
+  // This version of the service builds the index in the current thread,
+  // which blocks rendering and other browser activities.
+  // It should only be used where the browser does not support WebWorkers
+  function localSearchFactory($http, $timeout, NG_PAGES) {
 
-  var index = lunrSearch(function() {
-    this.ref('id');
-    this.field('title', {boost: 50});
-    this.field('members', { boost: 40});
-    this.field('keywords', { boost : 20 });
-  });
+    console.log('Using Local Search Index');
 
-  angular.forEach(NG_PAGES, function(page, key) {
-    if(page.searchTerms) {
-      index.store({
-        id : key,
-        title : page.searchTerms.titleWords,
-        keywords : page.searchTerms.keywords,
-        members : page.searchTerms.members
+    // Create the lunr index
+    var index = lunr(function() {
+      this.ref('path');
+      this.field('titleWords', {boost: 50});
+      this.field('members', { boost: 40});
+      this.field('keywords', { boost : 20 });
+    });
+
+    // Delay building the index by loading the data asynchronously
+    var indexReadyPromise = $http.get('js/search-data.json').then(function(response) {
+      var searchData = response.data;
+      // Delay building the index for 500ms to allow the page to render
+      return $timeout(function() {
+        // load the page data into the index
+        angular.forEach(searchData, function(page) {
+          index.add(page);
+        });
+      }, 500);
+    });
+
+    // The actual service is a function that takes a query string and
+    // returns a promise to the search results
+    // (In this case we just resolve the promise immediately as it is not
+    // inherently an async process)
+    return function(q) {
+      return indexReadyPromise.then(function() {
+        var hits = index.search(q);
+        var results = [];
+        angular.forEach(hits, function(hit) {
+          results.push(NG_PAGES[hit.ref]);
+        });
+        return results;
       });
     };
-  });
+  }
+  localSearchFactory.$inject = ['$http', '$timeout', 'NG_PAGES'];
 
-  return function(q) {
-    var results = {
-      api : [],
-      tutorial : [],
-      guide : [],
-      error : [],
-      misc : []
+  // This version of the service builds the index in a WebWorker,
+  // which does not block rendering and other browser activities.
+  // It should only be used where the browser does support WebWorkers
+  function webWorkerSearchFactory($q, $rootScope, NG_PAGES) {
+
+    console.log('Using WebWorker Search Index')
+
+    var searchIndex = $q.defer();
+    var results;
+
+    var worker = new Worker('js/search-worker.js');
+
+    // The worker will send us a message in two situations:
+    // - when the index has been built, ready to run a query
+    // - when it has completed a search query and the results are available
+    worker.onmessage = function(oEvent) {
+      $rootScope.$apply(function() {
+
+        switch(oEvent.data.e) {
+          case 'index-ready':
+            searchIndex.resolve();
+            break;
+          case 'query-ready':
+            var pages = oEvent.data.d.map(function(path) {
+              return NG_PAGES[path];
+            });
+            results.resolve(pages);
+            break;
+        }
+      });
     };
-    angular.forEach(index.search(q), function(result) {
-      var key = result.ref;
-      var item = NG_PAGES[key];
-      var area = item.area;
-      item.path = key;
 
-      var limit = area == 'api' ? 40 : 14;
-      if(results[area].length < limit) {
-        results[area].push(item);
-      }
-    });
-    return results;
+    // The actual service is a function that takes a query string and
+    // returns a promise to the search results
+    return function(q) {
+
+      // We only run the query once the index is ready
+      return searchIndex.promise.then(function() {
+
+        results = $q.defer();
+        worker.postMessage({ q: q });
+        return results.promise;
+      });
+    };
+  }
+  webWorkerSearchFactory.$inject = ['$q', '$rootScope', 'NG_PAGES'];
+
+  return {
+    $get: window.Worker ? webWorkerSearchFactory : localSearchFactory
   };
-}])
+})
 
 .directive('focused', function($timeout) {
   return function(scope, element, attrs) {

docs/app/test/docsSpec.js

@@ -19,7 +19,7 @@ describe("DocsController", function() {
     it("should update the Google Analytics with currentPage path if currentPage exists", inject(function($window) {
       $window._gaq = [];
       $scope.currentPage = { path: 'a/b/c' };
-      $scope.afterPartialLoaded();
+      $scope.$broadcast('$includeContentLoaded');
       expect($window._gaq.pop()).toEqual(['_trackPageview', 'a/b/c']);
     }));
 
@@ -27,7 +27,7 @@ describe("DocsController", function() {
     it("should update the Google Analytics with $location.path if currentPage is missing", inject(function($window, $location) {
       $window._gaq = [];
       spyOn($location, 'path').andReturn('x/y/z');
-      $scope.afterPartialLoaded();
+      $scope.$broadcast('$includeContentLoaded');
       expect($window._gaq.pop()).toEqual(['_trackPageview', 'x/y/z']);
     }));
   });

docs/config/index.js

@@ -92,10 +92,7 @@ module.exports = new Package('angularjs', [
       }
       return docPath;
     },
-    getOutputPath: function(doc) {
-      return 'partials/' + doc.path +
-          ( doc.fileInfo.baseName === 'index' ? '/index.html' : '.html');
-    }
+    outputPathTemplate: 'partials/${path}.html'
   });
 
   computePathsProcessor.pathTemplates.push({
@@ -110,6 +107,16 @@ module.exports = new Package('angularjs', [
     outputPathTemplate: '${id}.html'
   });
 
+  computePathsProcessor.pathTemplates.push({
+    docTypes: ['module' ],
+    pathTemplate: '${area}/${name}',
+    outputPathTemplate: 'partials/${area}/${name}.html'
+  });
+  computePathsProcessor.pathTemplates.push({
+    docTypes: ['componentGroup' ],
+    pathTemplate: '${area}/${moduleName}/${groupType}',
+    outputPathTemplate: 'partials/${area}/${moduleName}/${groupType}.html'
+  });
 
   computeIdsProcessor.idTemplates.push({
     docTypes: ['overview', 'tutorial', 'e2e-test', 'indexPage'],

docs/config/processors/pages-data.js

@@ -147,24 +147,18 @@ module.exports = function generatePagesDataProcessor(log) {
   };
 
   return {
-    $runAfter: ['paths-computed'],
+    $runAfter: ['paths-computed', 'generateKeywordsProcessor'],
     $runBefore: ['rendering-docs'],
     $process: function(docs) {
 
-      _(docs)
-      .filter(function(doc) { return doc.area === 'api' && doc.docType === 'module'; })
-      .forEach(function(doc) { if ( !doc.path ) {
-        log.warn('Missing path property for ', doc.id);
-      }})
-      .map(function(doc) { return _.pick(doc, ['id', 'module', 'docType', 'area']); })
-      .tap(function(docs) {
-        log.debug(docs);
+      // We are only interested in docs that are in an area
+      var pages = _.filter(docs, function(doc) {
+        return doc.area;
       });
 
-
-      // We are only interested in docs that are in an area and are not landing pages
-      var navPages = _.filter(docs, function(page) {
-        return page.area && page.docType != 'componentGroup';
+      // We are only interested in pages that are not landing pages
+      var navPages = _.filter(pages, function(page) {
+        return page.docType != 'componentGroup';
       });
 
       // Generate an object collection of pages that is grouped by area e.g.
@@ -198,28 +192,48 @@ module.exports = function generatePagesDataProcessor(log) {
           area.navGroups = navGroupMapper(pages, area);
         });
 
+      docs.push({
+        docType: 'nav-data',
+        id: 'nav-data',
+        template: 'nav-data.template.js',
+        outputPath: 'js/nav-data.js',
+        areas: areas
+      });
+
+
+
+      var searchData = _(pages)
+        .filter(function(page) {
+            return page.searchTerms;
+        })
+        .map(function(page) {
+          return _.extend({ path: page.path }, page.searchTerms);
+        })
+        .value();
+
+      docs.push({
+        docType: 'json-doc',
+        id: 'search-data-json',
+        template: 'json-doc.template.json',
+        outputPath: 'js/search-data.json',
+        data: searchData
+      });
+
       // Extract a list of basic page information for mapping paths to partials and for client side searching
-      var pages = _(docs)
+      var pageData = _(docs)
         .map(function(doc) {
-          var page = _.pick(doc, [
-            'docType', 'id', 'name', 'area', 'outputPath', 'path', 'searchTerms'
-          ]);
-          return page;
+          return _.pick(doc, ['name', 'area', 'path']);
         })
         .indexBy('path')
         .value();
 
-      var docData = {
+      docs.push({
         docType: 'pages-data',
         id: 'pages-data',
         template: 'pages-data.template.js',
         outputPath: 'js/pages-data.js',
-
-        areas: areas,
-        pages: pages
-      };
-
-      docs.push(docData);
+        pages: pageData
+      });
     }
-  }
+  };
 };

docs/config/services/deployments/debug.js

@@ -26,6 +26,7 @@ module.exports = function debugDeployment(getVersion) {
       'components/google-code-prettify-' + getVersion('google-code-prettify') + '/src/lang-css.js',
       'js/versions-data.js',
       'js/pages-data.js',
+      'js/nav-data.js',
       'js/docs.js'
     ],
     stylesheets: [

docs/config/services/deployments/default.js

@@ -26,6 +26,7 @@ module.exports = function defaultDeployment(getVersion) {
       'components/google-code-prettify-' + getVersion('google-code-prettify') + '/src/lang-css.js',
       'js/versions-data.js',
       'js/pages-data.js',
+      'js/nav-data.js',
       'js/docs.js'
     ],
     stylesheets: [

docs/config/services/deployments/jquery.js

@@ -30,6 +30,7 @@ module.exports = function jqueryDeployment(getVersion) {
       'components/google-code-prettify-' + getVersion('google-code-prettify') + '/src/lang-css.js',
       'js/versions-data.js',
       'js/pages-data.js',
+      'js/nav-data.js',
       'js/docs.js'
     ],
     stylesheets: [

docs/config/services/deployments/production.js

@@ -29,6 +29,7 @@ module.exports = function productionDeployment(getVersion) {
       'components/google-code-prettify-' + getVersion('google-code-prettify') + '/src/lang-css.js',
       'js/versions-data.js',
       'js/pages-data.js',
+      'js/nav-data.js',
       'js/docs.js'
     ],
     stylesheets: [

docs/config/templates/indexPage.template.html

@@ -56,15 +56,6 @@
       }
     })();
 
-
-    // force page reload when new update is available
-    window.applicationCache && window.applicationCache.addEventListener('updateready', function(e) {
-      if (window.applicationCache.status == window.applicationCache.UPDATEREADY) {
-        window.applicationCache.swapCache();
-        window.location.reload();
-      }
-    }, false);
-
     // GA asynchronous tracker
     var _gaq = _gaq || [];
     _gaq.push(['_setAccount', 'UA-8594346-3']);
@@ -219,7 +210,7 @@
         </div>
         <div class="grid-right">
           <div id="loading" ng-show="loading">Loading...</div>
-          <div ng-hide="loading" ng-include="currentPage.outputPath || 'Error404.html'" onload="afterPartialLoaded()"  autoscroll></div>
+          <div ng-hide="loading" ng-include="partialPath" autoscroll></div>
         </div>
       </div>
     </section>

docs/config/templates/json-doc.template.json

@@ -0,0 +1 @@
+{$ doc.data | json $}

docs/config/templates/nav-data.template.js

@@ -0,0 +1,3 @@
+// Meta data used by the AngularJS docs app
+angular.module('navData', [])
+  .value('NG_NAVIGATION', {$ doc.areas | json $});

docs/config/templates/pages-data.template.js

@@ -1,4 +1,3 @@
 // Meta data used by the AngularJS docs app
 angular.module('pagesData', [])
-  .value('NG_PAGES', {$ doc.pages | json $})
-  .value('NG_NAVIGATION', {$ doc.areas | json $});
+  .value('NG_PAGES', {$ doc.pages | json $});

docs/gulpfile.js

@@ -17,6 +17,8 @@ var path = require('canonical-path');
 var outputFolder = '../build/docs';
 var bowerFolder = 'bower_components';
 
+var src = 'app/src/**/*.js';
+var assets = 'app/assets/**/*';
 
 var copyComponent = function(component, pattern, sourceFolder, packageFile) {
   pattern = pattern || '/**/*';
@@ -40,14 +42,14 @@ gulp.task('bower', function() {
 });
 
 gulp.task('build-app', function() {
-  gulp.src('app/src/**/*.js')
+  gulp.src(src)
     .pipe(concat('docs.js'))
     .pipe(gulp.dest(outputFolder + '/js/'));
 });
 
 gulp.task('assets', ['bower'], function() {
   return merge(
-    gulp.src(['app/assets/**/*']).pipe(gulp.dest(outputFolder)),
+    gulp.src([assets]).pipe(gulp.dest(outputFolder)),
     copyComponent('bootstrap', '/dist/**/*'),
     copyComponent('open-sans-fontface'),
     copyComponent('lunr.js','/*.js'),
@@ -77,3 +79,6 @@ gulp.task('jshint', ['doc-gen'], function() {
 // The default task that will be run if no task is supplied
 gulp.task('default', ['assets', 'doc-gen', 'build-app', 'jshint']);
 
+gulp.task('watch', function() {
+  gulp.watch([src, assets], ['assets', 'build-app']);
+});