Support v1beta2 indexes API (#1014)

src/commands/firestore-indexes-list.js

@@ -1,43 +0,0 @@
-"use strict";
-
-var Command = require("../command");
-var firestoreIndexes = require("../firestore/indexes.js");
-var requirePermissions = require("../requirePermissions");
-var logger = require("../logger");
-var _ = require("lodash");
-
-var _prettyPrint = function(indexes) {
-  indexes.forEach(function(index) {
-    logger.info(firestoreIndexes.toPrettyString(index));
-  });
-};
-
-var _makeJsonSpec = function(indexes) {
-  return {
-    indexes: indexes.map(function(index) {
-      return _.pick(index, ["collectionId", "fields"]);
-    }),
-  };
-};
-
-module.exports = new Command("firestore:indexes")
-  .description("List indexes in your project's Cloud Firestore database.")
-  .option(
-    "--pretty",
-    "Pretty print. When not specified the indexes are printed in the " +
-      "JSON specification format."
-  )
-  .before(requirePermissions, ["datastore.indexes.list"])
-  .action(function(options) {
-    return firestoreIndexes.list(options.project).then(function(indexes) {
-      var jsonSpec = _makeJsonSpec(indexes);
-
-      if (options.pretty) {
-        _prettyPrint(indexes);
-      } else {
-        logger.info(JSON.stringify(jsonSpec, undefined, 2));
-      }
-
-      return jsonSpec;
-    });
-  });

src/commands/firestore-indexes-list.ts

@@ -0,0 +1,37 @@
+import * as Command from "../command";
+import * as clc from "cli-color";
+import * as fsi from "../firestore/indexes";
+import * as logger from "../logger";
+import * as requirePermissions from "../requirePermissions";
+
+module.exports = new Command("firestore:indexes")
+  .description("List indexes in your project's Cloud Firestore database.")
+  .option(
+    "--pretty",
+    "Pretty print. When not specified the indexes are printed in the " +
+      "JSON specification format."
+  )
+  .before(requirePermissions, ["datastore.indexes.list"])
+  .action(async (options: any) => {
+    const indexApi = new fsi.FirestoreIndexes();
+
+    const indexes = await indexApi.listIndexes(options.project);
+    const fieldOverrides = await indexApi.listFieldOverrides(options.project);
+
+    const indexSpec = indexApi.makeIndexSpec(indexes, fieldOverrides);
+
+    if (options.pretty) {
+      logger.info(clc.bold.white("Compound Indexes"));
+      indexApi.prettyPrintIndexes(indexes);
+
+      if (fieldOverrides) {
+        logger.info();
+        logger.info(clc.bold.white("Field Overrides"));
+        indexApi.printFieldOverrides(fieldOverrides);
+      }
+    } else {
+      logger.info(JSON.stringify(indexSpec, undefined, 2));
+    }
+
+    return indexSpec;
+  });

src/deploy/firestore/deploy.js

@@ -2,85 +2,12 @@
 
 var _ = require("lodash");
 var clc = require("cli-color");
-var firestoreIndexes = require("../../firestore/indexes");
+
+var fsi = require("../../firestore/indexes");
 var logger = require("../../logger");
 
 var utils = require("../../utils");
 
-/**
- * Create an index if it does not already exist on the specified project.
- *
- * @param {string} projectId the Firestore project Id.
- * @param {object} index a Firestore index speficiation.
- * @param {object[]} existingIndexes array of existing indexes on the project.
- * @return {Promise} a promise for index creation.
- */
-var _createIfMissing = function(projectId, index, existingIndexes) {
-  var exists = existingIndexes.some(function(x) {
-    return firestoreIndexes.equal(x, index);
-  });
-
-  if (exists) {
-    logger.debug("Skipping existing index: " + JSON.stringify(index));
-    return Promise.resolve();
-  }
-
-  logger.debug("Creating new index: " + JSON.stringify(index));
-  return firestoreIndexes.create(projectId, index);
-};
-
-/**
- * Create a number of indexes on a project, chaining the operations
- * in sequence.
- *
- * @param {string} projectId the Firestore project Id.
- * @param {object[]} indexes array of indexes to create.
- * @param {object[]} existingIndexes array of existing indexes on the project.
- * @return {Promise} a promise representing the entire creation chain.
- */
-var _createAllChained = function(projectId, indexes, existingIndexes) {
-  if (indexes.length === 0) {
-    return Promise.resolve();
-  }
-
-  var index = indexes.shift();
-  return _createIfMissing(projectId, index, existingIndexes).then(function() {
-    return _createAllChained(projectId, indexes, existingIndexes);
-  });
-};
-
-/**
- * Warn the user of any indexes that exist in the project but not
- * in their index file.
- *
- * @param {object[]} indexes array of Firestore indexes to be created.
- * @param {object[]} existingIndexes array of Firestore indexes that exit.
- */
-var _logAllMissing = function(indexes, existingIndexes) {
-  var hashes = {};
-
-  indexes.forEach(function(index) {
-    hashes[firestoreIndexes.hash(index)] = true;
-  });
-
-  var missingIndexes = existingIndexes.filter(function(index) {
-    return !hashes[firestoreIndexes.hash(index)];
-  });
-
-  if (missingIndexes.length > 0) {
-    logger.info();
-    logger.info(
-      clc.bold("NOTE: ") +
-        "The following indexes are already deployed but not present in the specified indexes file:"
-    );
-
-    missingIndexes.forEach(function(index) {
-      logger.info(firestoreIndexes.toPrettyString(index));
-    });
-    logger.info();
-  }
-};
-
 function _deployRules(context) {
   var rulesDeploy = _.get(context, "firestore.rulesDeploy");
   if (!context.firestoreRules || !rulesDeploy) {
@@ -94,6 +21,7 @@ function _deployIndexes(context, options) {
     return Promise.resolve();
   }
 
+  var indexesFileName = _.get(context, "firestore.indexes.name");
   var indexesSrc = _.get(context, "firestore.indexes.content");
   if (!indexesSrc) {
     logger.debug("No Firestore indexes present.");
@@ -105,10 +33,18 @@ function _deployIndexes(context, options) {
     return utils.reject('Index file must contain an "indexes" property.');
   }
 
-  return firestoreIndexes.list(options.project).then(function(existingIndexes) {
-    _logAllMissing(indexes, existingIndexes);
-    return _createAllChained(options.project, indexes, existingIndexes);
-  });
+  var fieldOverrides = indexesSrc.fieldOverrides || [];
+
+  return new fsi.FirestoreIndexes()
+    .deploy(options.project, indexes, fieldOverrides)
+    .then(function() {
+      utils.logSuccess(
+        clc.bold.green("firestore:") +
+          " deployed indexes in " +
+          clc.bold(indexesFileName) +
+          " successfully"
+      );
+    });
 }
 
 /**

src/deploy/firestore/prepare.js

@@ -1,9 +1,11 @@
 "use strict";
 
 var _ = require("lodash");
+var clc = require("cli-color");
 
-var firestoreIndexes = require("../../firestore/indexes");
+var loadCJSON = require("../../loadCJSON");
 var RulesDeploy = require("../../RulesDeploy");
+var utils = require("../../utils");
 
 function _prepareRules(context, options) {
   var prepare = Promise.resolve();
@@ -24,7 +26,19 @@ function _prepareIndexes(context, options) {
     return Promise.resolve();
   }
 
-  return firestoreIndexes.prepare(context, options);
+  var indexesFileName = options.config.get("firestore.indexes");
+  var indexesPath = options.config.path(indexesFileName);
+  var parsedSrc = loadCJSON(indexesPath);
+
+  utils.logBullet(
+    clc.bold.cyan("firestore:") + " reading indexes from " + clc.bold(indexesFileName) + "..."
+  );
+
+  context.firestore = context.firestore || {};
+  context.firestore.indexes = {
+    name: indexesFileName,
+    content: parsedSrc,
+  };
 }
 
 module.exports = function(context, options) {

src/firestore/indexes-api.ts

@@ -0,0 +1,69 @@
+/**
+ * The v1beta1 indexes API used a 'mode' field to represent the indexing mode.
+ * This information has now been split into the fields 'arrayConfig' and 'order'.
+ * We allow use of 'mode' (for now) so that the move to v1beta2/v1 is not
+ * breaking when we can understand the developer's intent.
+ */
+export enum Mode {
+  ASCENDING = "ASCENDING",
+  DESCENDING = "DESCENDING",
+  ARRAY_CONTAINS = "ARRAY_CONTAINS",
+}
+
+export enum QueryScope {
+  COLLECTION = "COLLECTION",
+  COLLECTION_GROUP = "COLLECTION_GROUP",
+}
+
+export enum Order {
+  ASCENDING = "ASCENDING",
+  DESCENDING = "DESCENDING",
+}
+
+export enum ArrayConfig {
+  CONTAINS = "CONTAINS",
+}
+
+export enum State {
+  CREATING = "CREATING",
+  READY = "READY",
+  NEEDS_REPAIR = "NEEDS_REPAIR",
+}
+
+/**
+ * An Index as it is represented in the Firestore v1beta2 indexes API.
+ */
+export interface Index {
+  name?: string;
+  queryScope: QueryScope;
+  fields: IndexField[];
+  state?: State;
+}
+
+/**
+ * A field in an index.
+ */
+export interface IndexField {
+  fieldPath: string;
+  order?: Order;
+  arrayConfig?: ArrayConfig;
+}
+
+/**
+ * Represents a single field in the database.
+ *
+ * If a field has an empty indexConfig, that means all
+ * default indexes are exempted.
+ */
+export interface Field {
+  name: string;
+  indexConfig: IndexConfig;
+}
+
+/**
+ * Index configuration overrides for a field.
+ */
+export interface IndexConfig {
+  ancestorField?: string;
+  indexes: Index[];
+}

src/firestore/indexes-spec.ts

@@ -0,0 +1,36 @@
+import * as API from "./indexes-api";
+
+/**
+ * An entry specifying a compound or other non-default index.
+ */
+export interface Index {
+  collectionGroup: string;
+  queryScope: API.QueryScope;
+  fields: API.IndexField[];
+}
+
+/**
+ * An entry specifying field index configuration override.
+ */
+export interface FieldOverride {
+  collectionGroup: string;
+  fieldPath: string;
+  indexes: FieldIndex[];
+}
+
+/**
+ * Entry specifying a single-field index.
+ */
+export interface FieldIndex {
+  queryScope: API.QueryScope;
+  order: API.Order | undefined;
+  arrayConfig: API.ArrayConfig | undefined;
+}
+
+/**
+ * Specification for the JSON file that is used for index deployment,
+ */
+export interface IndexFile {
+  indexes: Index[];
+  fieldOverrides: FieldOverride[] | undefined;
+}

src/firestore/indexes.js

@@ -1,243 +0,0 @@
-"use strict";
-
-var api = require("../api");
-var clc = require("cli-color");
-var FirebaseError = require("../error");
-var loadCJSON = require("../loadCJSON");
-
-var VALID_INDEX_MODES = ["ASCENDING", "DESCENDING", "ARRAY_CONTAINS"];
-
-/**
- * Validate an index is correctly formed, throws an exception for all
- * fatal errors.
- *
- * See:
- * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index
- *
- * @param {string} index.collectionId index collection Id.
- * @param {object[]} index.fields array of index field specifications.
- */
-var _validate = function(index) {
-  var indexString = clc.cyan(JSON.stringify(index));
-  if (!index.collectionId) {
-    throw new FirebaseError('Index must contain "collectionId": ' + indexString);
-  }
-
-  if (!index.fields) {
-    throw new FirebaseError('Index must contain "fields": ' + indexString);
-  }
-
-  for (var i = 0; i < index.fields.length; i++) {
-    var field = index.fields[i];
-
-    if (!field.fieldPath) {
-      throw new FirebaseError('All index fields must contain "fieldPath": ' + indexString);
-    }
-
-    if (!field.mode) {
-      throw new FirebaseError('All index fields must contain "mode": ' + indexString);
-    }
-
-    if (VALID_INDEX_MODES.indexOf(field.mode) < 0) {
-      throw new FirebaseError(
-        "Index field mode must be one of " + VALID_INDEX_MODES.join(", ") + ": " + indexString
-      );
-    }
-  }
-};
-
-/**
- * Create an index in the specified project.
- *
- * See:
- * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index
- *
- * @param {string} project the Firestore project Id.
- * @param {string} index.collectionId index collection Id.
- * @param {object[]} index.fields array of index field specifications.
- * @return {Promise} a promise for index creation.
- */
-var create = function(project, index) {
-  _validate(index);
-
-  var url = "projects/" + project + "/databases/(default)/indexes";
-  return api.request("POST", "/v1beta1/" + url, {
-    auth: true,
-    data: index,
-    origin: api.firestoreOrigin,
-  });
-};
-
-/**
- * List all indexes in the specified project.
- *
- * See:
- * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index
- *
- * @param {string} project the Firestore project Id.
- * @return {Promise<object[]>} a promise for an array of indexes.
- */
-var list = function(project) {
-  var url = "projects/" + project + "/databases/(default)/indexes";
-
-  return api
-    .request("GET", "/v1beta1/" + url, {
-      auth: true,
-      origin: api.firestoreOrigin,
-    })
-    .then(function(res) {
-      var indexes = res.body.indexes || [];
-      var result = [];
-
-      // Clean up the index metadata so that they appear in the same
-      // format as they would be specified in firestore.indexes.json
-      for (var i = 0; i < indexes.length; i++) {
-        var index = indexes[i];
-        var sanitized = {};
-
-        sanitized.collectionId = index.collectionId;
-        sanitized.state = index.state;
-
-        sanitized.fields = index.fields.filter(function(field) {
-          return field.fieldPath !== "__name__";
-        });
-
-        result.push(sanitized);
-      }
-
-      return result;
-    });
-};
-
-/**
- * Determines if two indexes are equal by comparing their collectionId
- * and field specifications. All other properties are ignored.
- *
- * @param {object} a a Firestore index.
- * @param {object} b a Firestore index.
- * @return {boolean} true if the indexes are equal, false otherwise.
- */
-var equal = function(a, b) {
-  if (a.collectionId !== b.collectionId) {
-    return false;
-  }
-
-  if (a.fields.length !== b.fields.length) {
-    return false;
-  }
-
-  for (var i = 0; i < a.fields.length; i++) {
-    var aField = a.fields[i];
-    var bField = b.fields[i];
-
-    if (aField.fieldPath !== bField.fieldPath) {
-      return false;
-    }
-
-    if (aField.mode !== bField.mode) {
-      return false;
-    }
-  }
-
-  return true;
-};
-
-/**
- * Create a unique hash for a Firestore index that can be used
- * for deduplication.
- *
- * @param {object} index a Firestore index.
- * @return {string} a unique hash.
- */
-var hash = function(index) {
-  var result = "";
-  result += index.collectionId;
-  result += "[";
-
-  for (var i = 0; i < index.fields.length; i++) {
-    var field = index.fields[i];
-
-    // Skip __name__ fields
-    if (field.fieldPath === "__name__") {
-      continue;
-    }
-
-    // Append the field description
-    result += "(";
-    result += field.fieldPath + "," + field.mode;
-    result += ")";
-  }
-
-  result += "]";
-
-  return result;
-};
-
-/**
- * Get a colored, pretty printed representation of an index.
- *
- * @param {object} index a Firestore index.
- * @return {string} string for logging.
- */
-var toPrettyString = function(index) {
-  var result = "";
-
-  if (index.state) {
-    var stateMsg = "[" + index.state + "] ";
-
-    if (index.state === "READY") {
-      result += clc.green(stateMsg);
-    } else if (index.state === "CREATING") {
-      result += clc.yellow(stateMsg);
-    } else {
-      result += clc.red(stateMsg);
-    }
-  }
-
-  result += clc.cyan("(" + index.collectionId + ")");
-  result += " -- ";
-
-  index.fields.forEach(function(field) {
-    if (field.fieldPath === "__name__") {
-      return;
-    }
-
-    result += "(" + field.fieldPath + "," + field.mode + ") ";
-  });
-
-  return result;
-};
-
-/**
- * Prepare indexes for deployment.
- */
-var prepare = function(context, options) {
-  var indexesFileName = options.config.get("firestore.indexes");
-  var indexesPath = options.config.path(indexesFileName);
-  var parsedSrc = loadCJSON(indexesPath);
-
-  if (!parsedSrc.indexes) {
-    throw new FirebaseError('Indexes file must contain "indexes" property: ' + indexesPath);
-  }
-
-  parsedSrc.indexes.forEach(function(index) {
-    _validate(index);
-  });
-
-  context.firestore = context.firestore || {};
-  context.firestore.indexes = {
-    name: indexesFileName,
-    content: parsedSrc,
-  };
-
-  return Promise.resolve();
-};
-
-module.exports = {
-  create: create,
-  list: list,
-  equal: equal,
-  hash: hash,
-  toPrettyString: toPrettyString,
-  prepare: prepare,
-};

src/firestore/indexes.ts

@@ -0,0 +1,564 @@
+import * as clc from "cli-color";
+
+import * as api from "../api";
+import * as FirebaseError from "../error";
+import * as logger from "../logger";
+import * as utils from "../utils";
+import * as validator from "./validator";
+
+import * as API from "./indexes-api";
+import * as Spec from "./indexes-spec";
+
+// projects/$PROJECT_ID/databases/(default)/collectionGroups/$COLLECTION_GROUP_ID/indexes/$INDEX_ID
+const INDEX_NAME_REGEX = /projects\/([^\/]+?)\/databases\/\(default\)\/collectionGroups\/([^\/]+?)\/indexes\/([^\/]*)/;
+
+// projects/$PROJECT_ID/databases/(default)/collectionGroups/$COLLECTION_GROUP_ID/fields/$FIELD_ID
+const FIELD_NAME_REGEX = /projects\/([^\/]+?)\/databases\/\(default\)\/collectionGroups\/([^\/]+?)\/fields\/([^\/]*)/;
+
+interface IndexName {
+  projectId: string;
+  collectionGroupId: string;
+  indexId: string;
+}
+
+interface FieldName {
+  projectId: string;
+  collectionGroupId: string;
+  fieldPath: string;
+}
+
+export class FirestoreIndexes {
+  /**
+   * Deploy an index specification to the specified project.
+   * @param project the Firebase project ID.
+   * @param indexes an array of objects, each will be validated and then converted
+   * to an {@link Spec.Index}.
+   * @param fieldOverrides an array of objects, each will be validated and then
+   * converted to an {@link Spec.FieldOverride}.
+   */
+  async deploy(project: string, indexes: any[], fieldOverrides: any[]): Promise<void> {
+    const spec = this.upgradeOldSpec({
+      indexes,
+      fieldOverrides,
+    });
+
+    this.validateSpec(spec);
+
+    // Now that the spec is validated we can safely assert these types.
+    const indexesToDeploy: Spec.Index[] = spec.indexes;
+    const fieldOverridesToDeploy: Spec.FieldOverride[] = spec.fieldOverrides;
+
+    const existingIndexes = await this.listIndexes(project);
+    const existingFieldOverrides = await this.listFieldOverrides(project);
+
+    if (existingIndexes.length > indexesToDeploy.length) {
+      utils.logBullet(
+        clc.bold.cyan("firestore:") +
+          " there are some indexes defined in your project that are not present in your " +
+          "firestore indexes file. Run firebase firestore:indexes and save the result to correct the discrepancy."
+      );
+    }
+
+    indexesToDeploy.forEach(async (index) => {
+      const exists = existingIndexes.some((x) => this.indexMatchesSpec(x, index));
+      if (exists) {
+        logger.debug(`Skipping existing index: ${JSON.stringify(index)}`);
+        return;
+      }
+
+      logger.debug(`Creating new index: ${JSON.stringify(index)}`);
+      await this.createIndex(project, index);
+    });
+
+    if (existingFieldOverrides.length > fieldOverridesToDeploy.length) {
+      utils.logBullet(
+        clc.bold.cyan("firestore:") +
+          " there are some field overrides defined in your project that are not present in your " +
+          "firestore indexes file. Run firebase firestore:indexes and save the result to correct the discrepancy."
+      );
+    }
+
+    fieldOverridesToDeploy.forEach(async (field) => {
+      const exists = existingFieldOverrides.some((x) => this.fieldMatchesSpec(x, field));
+      if (exists) {
+        logger.debug(`Skipping existing field override: ${JSON.stringify(field)}`);
+        return;
+      }
+
+      logger.debug(`Updating field override: ${JSON.stringify(field)}`);
+      await this.patchField(project, field);
+    });
+  }
+
+  /**
+   * List all indexes that exist on a given project.
+   * @param project the Firebase project id.
+   */
+  async listIndexes(project: string): Promise<API.Index[]> {
+    const url = `projects/${project}/databases/(default)/collectionGroups/-/indexes`;
+
+    const res = await api.request("GET", `/v1beta2/${url}`, {
+      auth: true,
+      origin: api.firestoreOrigin,
+    });
+
+    const indexes = res.body.indexes;
+    return indexes.map(
+      (index: any): API.Index => {
+        // Ignore any fields that point at the document ID, as those are implied
+        // in all indexes.
+        const fields = index.fields.filter((field: API.IndexField) => {
+          return field.fieldPath !== "__name__";
+        });
+
+        return {
+          name: index.name,
+          state: index.state,
+          queryScope: index.queryScope,
+          fields,
+        };
+      }
+    );
+  }
+
+  /**
+   * List all field configuration overrides defined on the given project.
+   * @param project the Firebase project.
+   */
+  async listFieldOverrides(project: string): Promise<API.Field[]> {
+    const parent = `projects/${project}/databases/(default)/collectionGroups/-`;
+    const url = `${parent}/fields?filter=indexConfig.usesAncestorConfig=false`;
+
+    const res = await api.request("GET", `/v1beta2/${url}`, {
+      auth: true,
+      origin: api.firestoreOrigin,
+    });
+
+    const fields = res.body.fields as API.Field[];
+
+    // Ignore the default config, only list other fields.
+    return fields.filter((field) => {
+      return field.name.indexOf("__default__") < 0;
+    });
+  }
+
+  /**
+   * Turn an array of indexes and field overrides into a {@link Spec.IndexFile} suitable for use
+   * in an indexes.json file.
+   */
+  makeIndexSpec(indexes: API.Index[], fields?: API.Field[]): Spec.IndexFile {
+    const indexesJson = indexes.map((index) => {
+      return {
+        collectionGroup: this.parseIndexName(index.name).collectionGroupId,
+        queryScope: index.queryScope,
+        fields: index.fields,
+      };
+    });
+
+    if (!fields) {
+      logger.debug("No field overrides specified, using [].");
+      fields = [];
+    }
+
+    const fieldsJson = fields.map((field) => {
+      const parsedName = this.parseFieldName(field.name);
+      return {
+        collectionGroup: parsedName.collectionGroupId,
+        fieldPath: parsedName.fieldPath,
+
+        indexes: field.indexConfig.indexes.map((index) => {
+          const firstField = index.fields[0];
+          return {
+            order: firstField.order,
+            arrayConfig: firstField.arrayConfig,
+            queryScope: index.queryScope,
+          };
+        }),
+      };
+    });
+
+    return {
+      indexes: indexesJson,
+      fieldOverrides: fieldsJson,
+    };
+  }
+
+  /**
+   * Print an array of indexes to the console.
+   * @param indexes the array of indexes.
+   */
+  prettyPrintIndexes(indexes: API.Index[]): void {
+    indexes.forEach((index) => {
+      logger.info(this.prettyIndexString(index));
+    });
+  }
+
+  /**
+   * Print an array of field overrides to the console.
+   * @param fields  the array of field overrides.
+   */
+  printFieldOverrides(fields: API.Field[]): void {
+    fields.forEach((field) => {
+      logger.info(this.prettyFieldString(field));
+    });
+  }
+
+  /**
+   * Validate that an object is a valid index specification.
+   * @param spec the object, normally parsed from JSON.
+   */
+  validateSpec(spec: any): void {
+    validator.assertHas(spec, "indexes");
+
+    spec.indexes.forEach((index: any) => {
+      this.validateIndex(index);
+    });
+
+    if (spec.fieldOverrides) {
+      spec.fieldOverrides.forEach((field: any) => {
+        this.validateField(field);
+      });
+    }
+  }
+
+  /**
+   * Validate that an arbitrary object is safe to use as an {@link API.Field}.
+   */
+  validateIndex(index: any): void {
+    validator.assertHas(index, "collectionGroup");
+    validator.assertHas(index, "queryScope");
+    validator.assertEnum(index, "queryScope", Object.keys(API.QueryScope));
+
+    validator.assertHas(index, "fields");
+
+    index.fields.forEach((field: any) => {
+      validator.assertHas(field, "fieldPath");
+      validator.assertHasOneOf(field, ["order", "arrayConfig"]);
+
+      if (field.order) {
+        validator.assertEnum(field, "order", Object.keys(API.Order));
+      }
+
+      if (field.arrayConfig) {
+        validator.assertEnum(field, "arrayConfig", Object.keys(API.ArrayConfig));
+      }
+    });
+  }
+
+  /**
+   * Validate that an arbitrary object is safe to use as an {@link Spec.FieldOverride}.
+   * @param field
+   */
+  validateField(field: any): void {
+    validator.assertHas(field, "collectionGroup");
+    validator.assertHas(field, "fieldPath");
+    validator.assertHas(field, "indexes");
+
+    field.indexes.forEach((index: any) => {
+      validator.assertHasOneOf(index, ["arrayConfig", "order"]);
+
+      if (index.arrayConfig) {
+        validator.assertEnum(index, "arrayConfig", Object.keys(API.ArrayConfig));
+      }
+
+      if (index.order) {
+        validator.assertEnum(index, "order", Object.keys(API.Order));
+      }
+
+      if (index.queryScope) {
+        validator.assertEnum(index, "queryScope", Object.keys(API.QueryScope));
+      }
+    });
+  }
+
+  /**
+   * Update the configuration of a field. Note that this kicks off a long-running
+   * operation for index creation/deletion so the update is complete when this
+   * method returns.
+   * @param project the Firebase project.
+   * @param spec the new field override specification.
+   */
+  async patchField(project: string, spec: Spec.FieldOverride): Promise<any> {
+    const url = `projects/${project}/databases/(default)/collectionGroups/${
+      spec.collectionGroup
+    }/fields/${spec.fieldPath}`;
+
+    const indexes = spec.indexes.map((index) => {
+      return {
+        queryScope: index.queryScope,
+        fields: [
+          {
+            fieldPath: spec.fieldPath,
+            arrayConfig: index.arrayConfig,
+            order: index.order,
+          },
+        ],
+      };
+    });
+
+    const data = {
+      indexConfig: {
+        indexes,
+      },
+    };
+
+    await api.request("PATCH", `/v1beta2/${url}`, {
+      auth: true,
+      origin: api.firestoreOrigin,
+      data,
+    });
+  }
+
+  /**
+   * Create a new index on the specified project.
+   */
+  createIndex(project: string, index: Spec.Index): Promise<any> {
+    const url = `projects/${project}/databases/(default)/collectionGroups/${
+      index.collectionGroup
+    }/indexes`;
+    return api.request("POST", "/v1beta2/" + url, {
+      auth: true,
+      data: {
+        fields: index.fields,
+        queryScope: index.queryScope,
+      },
+      origin: api.firestoreOrigin,
+    });
+  }
+
+  /**
+   * Determine if an API Index and a Spec Index are functionally equivalent.
+   */
+  indexMatchesSpec(index: API.Index, spec: Spec.Index): boolean {
+    const collection = this.parseIndexName(index.name).collectionGroupId;
+    if (collection !== spec.collectionGroup) {
+      return false;
+    }
+
+    if (index.queryScope !== spec.queryScope) {
+      return false;
+    }
+
+    if (index.fields.length !== spec.fields.length) {
+      return false;
+    }
+
+    let i = 0;
+    while (i < index.fields.length) {
+      const iField = index.fields[i];
+      const sField = spec.fields[i];
+
+      if (iField.fieldPath !== sField.fieldPath) {
+        return false;
+      }
+
+      if (iField.order !== sField.order) {
+        return false;
+      }
+
+      if (iField.arrayConfig !== sField.arrayConfig) {
+        return false;
+      }
+
+      i++;
+    }
+
+    return true;
+  }
+
+  /**
+   * Determine if an API Field and a Spec Field are functionally equivalent.
+   */
+  fieldMatchesSpec(field: API.Field, spec: Spec.FieldOverride): boolean {
+    const parsedName = this.parseFieldName(field.name);
+
+    if (parsedName.collectionGroupId !== spec.collectionGroup) {
+      return false;
+    }
+
+    if (parsedName.fieldPath !== spec.fieldPath) {
+      return false;
+    }
+
+    if (field.indexConfig.indexes.length !== spec.indexes.length) {
+      return false;
+    }
+
+    const fieldModes = field.indexConfig.indexes.map((index) => {
+      const firstField = index.fields[0];
+      return firstField.order || firstField.arrayConfig;
+    });
+
+    const specModes = spec.indexes.map((index) => {
+      return index.order || index.arrayConfig;
+    });
+
+    // Confirms that the two objects have the same set of enabled indexes without
+    // caring about specification order.
+    for (const mode of fieldModes) {
+      if (specModes.indexOf(mode) < 0) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Parse an Index name into useful pieces.
+   */
+  parseIndexName(name?: string): IndexName {
+    if (!name) {
+      throw new FirebaseError(`Cannot parse undefined index name.`);
+    }
+
+    const m = name.match(INDEX_NAME_REGEX);
+    if (!m || m.length < 4) {
+      throw new FirebaseError(`Error parsing index name: ${name}`);
+    }
+
+    return {
+      projectId: m[1],
+      collectionGroupId: m[2],
+      indexId: m[3],
+    };
+  }
+
+  /**
+   * Parse an Field name into useful pieces.
+   */
+  parseFieldName(name: string): FieldName {
+    const m = name.match(FIELD_NAME_REGEX);
+    if (!m || m.length < 4) {
+      throw new FirebaseError(`Error parsing field name: ${name}`);
+    }
+
+    return {
+      projectId: m[1],
+      collectionGroupId: m[2],
+      fieldPath: m[3],
+    };
+  }
+
+  /**
+   * Take a object that may represent an old v1beta1 indexes spec
+   * and convert it to the new v1beta2/v1 spec format.
+   *
+   * This function is meant to be run **before** validation and
+   * works on a purely best-effort basis.
+   */
+  upgradeOldSpec(spec: any): any {
+    const result = {
+      indexes: [],
+      fieldOverrides: spec.fieldOverrides || [],
+    };
+
+    if (!spec.indexes) {
+      return;
+    }
+
+    // Try to detect use of the old API, warn the users.
+    if (spec.indexes[0].collectionId) {
+      utils.logBullet(
+        clc.bold.cyan("firestore:") +
+          " your indexes indexes are specified in the v1beta1 API format. " +
+          "Please upgrade to the new index API format by running " +
+          clc.bold("firebase firestore:indexes") +
+          " again and saving the result."
+      );
+    }
+
+    result.indexes = spec.indexes.map((index: any) => {
+      const i = {
+        collectionGroup: index.collectionGroup || index.collectionId,
+        queryScope: index.queryScope || API.QueryScope.COLLECTION,
+        fields: [],
+      };
+
+      if (index.fields) {
+        i.fields = index.fields.map((field: any) => {
+          const f: any = {
+            fieldPath: field.fieldPath,
+          };
+
+          if (field.order) {
+            f.order = field.order;
+          } else if (field.arrayConfig) {
+            f.arrayConfig = field.arrayConfig;
+          } else if (field.mode === API.Mode.ARRAY_CONTAINS) {
+            f.arrayConfig = API.ArrayConfig.CONTAINS;
+          } else {
+            f.order = field.mode;
+          }
+
+          return f;
+        });
+      }
+
+      return i;
+    });
+
+    return result;
+  }
+
+  /**
+   * Get a colored, pretty-printed representation of an index.
+   */
+  private prettyIndexString(index: API.Index): string {
+    let result = "";
+
+    if (index.state) {
+      const stateMsg = `[${index.state}] `;
+
+      if (index.state === API.State.READY) {
+        result += clc.green(stateMsg);
+      } else if (index.state === API.State.CREATING) {
+        result += clc.yellow(stateMsg);
+      } else {
+        result += clc.red(stateMsg);
+      }
+    }
+
+    const nameInfo = this.parseIndexName(index.name);
+
+    result += clc.cyan(`(${nameInfo.collectionGroupId})`);
+    result += " -- ";
+
+    index.fields.forEach((field) => {
+      if (field.fieldPath === "__name__") {
+        return;
+      }
+
+      // Normal field indexes have an "order" while array indexes have an "arrayConfig",
+      // we want to display whichever one is present.
+      const orderOrArrayConfig = field.order ? field.order : field.arrayConfig;
+      result += `(${field.fieldPath},${orderOrArrayConfig}) `;
+    });
+
+    return result;
+  }
+
+  /**
+   * Get a colored, pretty-printed representation of a field
+   */
+  private prettyFieldString(field: API.Field): string {
+    let result = "";
+
+    const parsedName = this.parseFieldName(field.name);
+
+    result +=
+      "[" +
+      clc.cyan(parsedName.collectionGroupId) +
+      "." +
+      clc.yellow(parsedName.fieldPath) +
+      "] --";
+
+    field.indexConfig.indexes.forEach((index) => {
+      const firstField = index.fields[0];
+      const mode = firstField.order || firstField.arrayConfig;
+      result += " (" + mode + ")";
+    });
+
+    return result;
+  }
+}

src/firestore/validator.ts

@@ -0,0 +1,41 @@
+import * as clc from "cli-color";
+import * as FirebaseError from "../error";
+
+/**
+ * Throw an error if 'obj' does not have a value for the property 'prop'.
+ */
+export function assertHas(obj: any, prop: string): void {
+  const objString = clc.cyan(JSON.stringify(obj));
+  if (!obj[prop]) {
+    throw new FirebaseError(`Must contain "${prop}": ${objString}`);
+  }
+}
+
+/**
+ * throw an error if 'obj' does not have a value for exactly one of the
+ * properties in 'props'.
+ */
+export function assertHasOneOf(obj: any, props: string[]): void {
+  const objString = clc.cyan(JSON.stringify(obj));
+  let count = 0;
+  props.forEach((prop) => {
+    if (obj[prop]) {
+      count++;
+    }
+  });
+
+  if (count !== 1) {
+    throw new FirebaseError(`Must contain exactly one of "${props.join(",")}": ${objString}`);
+  }
+}
+
+/**
+ * Throw an error if the value of the property 'prop' on 'obj' is not one of
+ * the values in the the array 'valid'.
+ */
+export function assertEnum(obj: any, prop: string, valid: any[]): void {
+  const objString = clc.cyan(JSON.stringify(obj));
+  if (valid.indexOf(obj[prop]) < 0) {
+    throw new FirebaseError(`Field "${prop}" must be one of  ${valid.join(", ")}: ${objString}`);
+  }
+}

src/init/features/firestore.js

@@ -5,7 +5,7 @@ var fs = require("fs");
 
 var FirebaseError = require("../../error");
 var gcp = require("../../gcp");
-var indexes = require("../../firestore/indexes");
+var iv2 = require("../../firestore/indexes");
 var fsutils = require("../../fsutils");
 var prompt = require("../../prompt");
 var logger = require("../../logger");
@@ -13,6 +13,8 @@ var utils = require("../../utils");
 var requireAccess = require("../../requireAccess");
 var scopes = require("../../scopes");
 
+var indexes = new iv2.FirestoreIndexes();
+
 var RULES_TEMPLATE = fs.readFileSync(
   __dirname + "/../../../templates/init/firestore/firestore.rules",
   "utf8"
@@ -141,10 +143,12 @@ var _initIndexes = function(setup, config) {
 };
 
 var _getIndexesFromConsole = function(projectId) {
-  return indexes
-    .list(projectId)
-    .then(function(indexes) {
-      return JSON.stringify({ indexes: indexes }, null, 2);
+  var indexesPromise = indexes.listIndexes(projectId);
+  var fieldOverridesPromise = indexes.listFieldOverrides(projectId);
+
+  return Promise.all([indexesPromise, fieldOverridesPromise])
+    .then(function(res) {
+      return indexes.makeIndexSpec(res[0], res[1]);
     })
     .catch(function(e) {
       if (e.message.indexOf("is not a Cloud Firestore enabled project") >= 0) {

src/test/firestore/indexes.spec.ts

@@ -0,0 +1,222 @@
+import { expect } from "chai";
+import { FirestoreIndexes } from "../../firestore/indexes";
+import * as FirebaseError from "../../error";
+import * as API from "../../firestore/indexes-api";
+import * as Spec from "../../firestore/indexes-spec";
+
+const idx = new FirestoreIndexes();
+
+const VALID_SPEC = {
+  indexes: [
+    {
+      collectionGroup: "collection",
+      queryScope: "COLLECTION",
+      fields: [
+        { fieldPath: "foo", order: "ASCENDING" },
+        { fieldPath: "bar", order: "DESCENDING" },
+        { fieldPath: "baz", arrayConfig: "CONTAINS" },
+      ],
+    },
+  ],
+  fieldOverrides: [
+    {
+      collectionGroup: "collection",
+      fieldPath: "foo",
+      indexes: [
+        { order: "ASCENDING", scope: "COLLECTION" },
+        { arrayConfig: "CONTAINS", scope: "COLLECTION" },
+      ],
+    },
+  ],
+};
+
+describe("IndexValidation", () => {
+  it("should accept a valid v1beta2 index spec", () => {
+    idx.validateSpec(VALID_SPEC);
+  });
+
+  it("should not change a valid v1beta2 index spec after upgrade", () => {
+    const upgraded = idx.upgradeOldSpec(VALID_SPEC);
+    expect(upgraded).to.eql(VALID_SPEC);
+  });
+
+  it("should accept a valid v1beta1 index spec after upgrade", () => {
+    idx.validateSpec(
+      idx.upgradeOldSpec({
+        indexes: [
+          {
+            collectionId: "collection",
+            fields: [
+              { fieldPath: "foo", mode: "ASCENDING" },
+              { fieldPath: "bar", mode: "DESCENDING" },
+              { fieldPath: "baz", mode: "ARRAY_CONTAINS" },
+            ],
+          },
+        ],
+      })
+    );
+  });
+
+  it("should reject an incomplete index spec", () => {
+    expect(() => {
+      idx.validateSpec({
+        indexes: [
+          {
+            collectionGroup: "collection",
+            fields: [
+              { fieldPath: "foo", order: "ASCENDING" },
+              { fieldPath: "bar", order: "DESCENDING" },
+            ],
+          },
+        ],
+      });
+    }).to.throw(FirebaseError, /Must contain "queryScope"/);
+  });
+
+  it("should reject an overspecified index spec", () => {
+    expect(() => {
+      idx.validateSpec({
+        indexes: [
+          {
+            collectionGroup: "collection",
+            queryScope: "COLLECTION",
+            fields: [
+              { fieldPath: "foo", order: "ASCENDING", arrayConfig: "CONTAINES" },
+              { fieldPath: "bar", order: "DESCENDING" },
+            ],
+          },
+        ],
+      });
+    }).to.throw(FirebaseError, /Must contain exactly one of "order,arrayConfig"/);
+  });
+});
+
+describe("IndexNameParsing", () => {
+  it("should parse an index name correctly", () => {
+    const name =
+      "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123/";
+    expect(idx.parseIndexName(name)).to.eql({
+      projectId: "myproject",
+      collectionGroupId: "collection",
+      indexId: "abc123",
+    });
+  });
+
+  it("should parse a field name correctly", () => {
+    const name =
+      "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123/";
+    expect(idx.parseFieldName(name)).to.eql({
+      projectId: "myproject",
+      collectionGroupId: "collection",
+      fieldPath: "abc123",
+    });
+  });
+});
+
+describe("IndexSpecMatching", () => {
+  it("should identify a positive index spec match", () => {
+    const apiIndex: API.Index = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123",
+      queryScope: API.QueryScope.COLLECTION,
+      fields: [
+        { fieldPath: "foo", order: API.Order.ASCENDING },
+        { fieldPath: "bar", arrayConfig: API.ArrayConfig.CONTAINS },
+      ],
+      state: API.State.READY,
+    };
+
+    const specIndex = {
+      collectionGroup: "collection",
+      queryScope: "COLLECTION",
+      fields: [
+        { fieldPath: "foo", order: "ASCENDING" },
+        { fieldPath: "bar", arrayConfig: "CONTAINS" },
+      ],
+    } as Spec.Index;
+
+    expect(idx.indexMatchesSpec(apiIndex, specIndex)).to.eql(true);
+  });
+
+  it("should identify a negative index spec match", () => {
+    const apiIndex = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123",
+      queryScope: "COLLECTION",
+      fields: [
+        { fieldPath: "foo", order: "DESCENDING" },
+        { fieldPath: "bar", arrayConfig: "CONTAINS" },
+      ],
+      state: API.State.READY,
+    } as API.Index;
+
+    const specIndex = {
+      collectionGroup: "collection",
+      queryScope: "COLLECTION",
+      fields: [
+        { fieldPath: "foo", order: "ASCENDING" },
+        { fieldPath: "bar", arrayConfig: "CONTAINS" },
+      ],
+    } as Spec.Index;
+
+    // The second spec contains ASCENDING where the former contains DESCENDING
+    expect(idx.indexMatchesSpec(apiIndex, specIndex)).to.eql(false);
+  });
+
+  it("should identify a positive field spec match", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123",
+      indexConfig: {
+        indexes: [
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", order: "ASCENDING" }],
+          },
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", arrayConfig: "CONTAINS" }],
+          },
+        ],
+      },
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "abc123",
+      indexes: [
+        { order: "ASCENDING", queryScope: "COLLECTION" },
+        { arrayConfig: "CONTAINS", queryScope: "COLLECTION" },
+      ],
+    } as Spec.FieldOverride;
+
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
+  });
+
+  it("should identify a negative field spec match", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123",
+      indexConfig: {
+        indexes: [
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", order: "ASCENDING" }],
+          },
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", arrayConfig: "CONTAINS" }],
+          },
+        ],
+      },
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "abc123",
+      indexes: [
+        { order: "DESCENDING", queryScope: "COLLECTION" },
+        { arrayConfig: "CONTAINS", queryScope: "COLLECTION" },
+      ],
+    } as Spec.FieldOverride;
+
+    // The second spec contains "DESCENDING" where the first contains "ASCENDING"
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(false);
+  });
+});

templates/init/firestore/firestore.indexes.json

@@ -3,12 +3,24 @@
   //
   // "indexes": [
   //   {
-  //     "collectionId": "widgets",
+  //     "collectionGroup": "widgets",
+  //     "queryScope": "COLLECTION",
   //     "fields": [
-  //       { "fieldPath": "foo", "mode": "ASCENDING" },
+  //       { "fieldPath": "foo", "arrayConfig": "CONTAINS" },
   //       { "fieldPath": "bar", "mode": "DESCENDING" }
   //     ]
-  //   }
+  //   },
+  //
+  //  "fieldOverrides": [
+  //    {
+  //      "collectionGroup": "widgets",
+  //      "fieldPath": "baz",
+  //      "indexes": [
+  //        { "order": "ASCENDING", "queryScope": "COLLECTION" }
+  //      ]
+  //    },
+  //   ]
   // ]
-  "indexes": []
+  "indexes": [],
+  "fieldOverrides": []
 }