diff --git a/src/commands/firestore-indexes-list.js b/src/commands/firestore-indexes-list.js deleted file mode 100644 index 798d46d4..00000000 --- a/src/commands/firestore-indexes-list.js +++ /dev/null @@ -1,43 +0,0 @@ -"use strict"; - -var Command = require("../command"); -var firestoreIndexes = require("../firestore/indexes.js"); -var requirePermissions = require("../requirePermissions"); -var logger = require("../logger"); -var _ = require("lodash"); - -var _prettyPrint = function(indexes) { - indexes.forEach(function(index) { - logger.info(firestoreIndexes.toPrettyString(index)); - }); -}; - -var _makeJsonSpec = function(indexes) { - return { - indexes: indexes.map(function(index) { - return _.pick(index, ["collectionId", "fields"]); - }), - }; -}; - -module.exports = new Command("firestore:indexes") - .description("List indexes in your project's Cloud Firestore database.") - .option( - "--pretty", - "Pretty print. When not specified the indexes are printed in the " + - "JSON specification format." - ) - .before(requirePermissions, ["datastore.indexes.list"]) - .action(function(options) { - return firestoreIndexes.list(options.project).then(function(indexes) { - var jsonSpec = _makeJsonSpec(indexes); - - if (options.pretty) { - _prettyPrint(indexes); - } else { - logger.info(JSON.stringify(jsonSpec, undefined, 2)); - } - - return jsonSpec; - }); - }); diff --git a/src/commands/firestore-indexes-list.ts b/src/commands/firestore-indexes-list.ts new file mode 100644 index 00000000..87778c25 --- /dev/null +++ b/src/commands/firestore-indexes-list.ts @@ -0,0 +1,37 @@ +import * as Command from "../command"; +import * as clc from "cli-color"; +import * as fsi from "../firestore/indexes"; +import * as logger from "../logger"; +import * as requirePermissions from "../requirePermissions"; + +module.exports = new Command("firestore:indexes") + .description("List indexes in your project's Cloud Firestore database.") + .option( + "--pretty", + "Pretty print. When not specified the indexes are printed in the " + + "JSON specification format." + ) + .before(requirePermissions, ["datastore.indexes.list"]) + .action(async (options: any) => { + const indexApi = new fsi.FirestoreIndexes(); + + const indexes = await indexApi.listIndexes(options.project); + const fieldOverrides = await indexApi.listFieldOverrides(options.project); + + const indexSpec = indexApi.makeIndexSpec(indexes, fieldOverrides); + + if (options.pretty) { + logger.info(clc.bold.white("Compound Indexes")); + indexApi.prettyPrintIndexes(indexes); + + if (fieldOverrides) { + logger.info(); + logger.info(clc.bold.white("Field Overrides")); + indexApi.printFieldOverrides(fieldOverrides); + } + } else { + logger.info(JSON.stringify(indexSpec, undefined, 2)); + } + + return indexSpec; + }); diff --git a/src/deploy/firestore/deploy.js b/src/deploy/firestore/deploy.js index b1c6d9a0..d0239361 100644 --- a/src/deploy/firestore/deploy.js +++ b/src/deploy/firestore/deploy.js @@ -2,85 +2,12 @@ var _ = require("lodash"); var clc = require("cli-color"); -var firestoreIndexes = require("../../firestore/indexes"); + +var fsi = require("../../firestore/indexes"); var logger = require("../../logger"); var utils = require("../../utils"); -/** - * Create an index if it does not already exist on the specified project. - * - * @param {string} projectId the Firestore project Id. - * @param {object} index a Firestore index speficiation. - * @param {object[]} existingIndexes array of existing indexes on the project. - * @return {Promise} a promise for index creation. - */ -var _createIfMissing = function(projectId, index, existingIndexes) { - var exists = existingIndexes.some(function(x) { - return firestoreIndexes.equal(x, index); - }); - - if (exists) { - logger.debug("Skipping existing index: " + JSON.stringify(index)); - return Promise.resolve(); - } - - logger.debug("Creating new index: " + JSON.stringify(index)); - return firestoreIndexes.create(projectId, index); -}; - -/** - * Create a number of indexes on a project, chaining the operations - * in sequence. - * - * @param {string} projectId the Firestore project Id. - * @param {object[]} indexes array of indexes to create. - * @param {object[]} existingIndexes array of existing indexes on the project. - * @return {Promise} a promise representing the entire creation chain. - */ -var _createAllChained = function(projectId, indexes, existingIndexes) { - if (indexes.length === 0) { - return Promise.resolve(); - } - - var index = indexes.shift(); - return _createIfMissing(projectId, index, existingIndexes).then(function() { - return _createAllChained(projectId, indexes, existingIndexes); - }); -}; - -/** - * Warn the user of any indexes that exist in the project but not - * in their index file. - * - * @param {object[]} indexes array of Firestore indexes to be created. - * @param {object[]} existingIndexes array of Firestore indexes that exit. - */ -var _logAllMissing = function(indexes, existingIndexes) { - var hashes = {}; - - indexes.forEach(function(index) { - hashes[firestoreIndexes.hash(index)] = true; - }); - - var missingIndexes = existingIndexes.filter(function(index) { - return !hashes[firestoreIndexes.hash(index)]; - }); - - if (missingIndexes.length > 0) { - logger.info(); - logger.info( - clc.bold("NOTE: ") + - "The following indexes are already deployed but not present in the specified indexes file:" - ); - - missingIndexes.forEach(function(index) { - logger.info(firestoreIndexes.toPrettyString(index)); - }); - logger.info(); - } -}; - function _deployRules(context) { var rulesDeploy = _.get(context, "firestore.rulesDeploy"); if (!context.firestoreRules || !rulesDeploy) { @@ -94,6 +21,7 @@ function _deployIndexes(context, options) { return Promise.resolve(); } + var indexesFileName = _.get(context, "firestore.indexes.name"); var indexesSrc = _.get(context, "firestore.indexes.content"); if (!indexesSrc) { logger.debug("No Firestore indexes present."); @@ -105,10 +33,18 @@ function _deployIndexes(context, options) { return utils.reject('Index file must contain an "indexes" property.'); } - return firestoreIndexes.list(options.project).then(function(existingIndexes) { - _logAllMissing(indexes, existingIndexes); - return _createAllChained(options.project, indexes, existingIndexes); - }); + var fieldOverrides = indexesSrc.fieldOverrides || []; + + return new fsi.FirestoreIndexes() + .deploy(options.project, indexes, fieldOverrides) + .then(function() { + utils.logSuccess( + clc.bold.green("firestore:") + + " deployed indexes in " + + clc.bold(indexesFileName) + + " successfully" + ); + }); } /** diff --git a/src/deploy/firestore/prepare.js b/src/deploy/firestore/prepare.js index 6202eb97..ab7a6944 100644 --- a/src/deploy/firestore/prepare.js +++ b/src/deploy/firestore/prepare.js @@ -1,9 +1,11 @@ "use strict"; var _ = require("lodash"); +var clc = require("cli-color"); -var firestoreIndexes = require("../../firestore/indexes"); +var loadCJSON = require("../../loadCJSON"); var RulesDeploy = require("../../RulesDeploy"); +var utils = require("../../utils"); function _prepareRules(context, options) { var prepare = Promise.resolve(); @@ -24,7 +26,19 @@ function _prepareIndexes(context, options) { return Promise.resolve(); } - return firestoreIndexes.prepare(context, options); + var indexesFileName = options.config.get("firestore.indexes"); + var indexesPath = options.config.path(indexesFileName); + var parsedSrc = loadCJSON(indexesPath); + + utils.logBullet( + clc.bold.cyan("firestore:") + " reading indexes from " + clc.bold(indexesFileName) + "..." + ); + + context.firestore = context.firestore || {}; + context.firestore.indexes = { + name: indexesFileName, + content: parsedSrc, + }; } module.exports = function(context, options) { diff --git a/src/firestore/indexes-api.ts b/src/firestore/indexes-api.ts new file mode 100644 index 00000000..1b5e8568 --- /dev/null +++ b/src/firestore/indexes-api.ts @@ -0,0 +1,69 @@ +/** + * The v1beta1 indexes API used a 'mode' field to represent the indexing mode. + * This information has now been split into the fields 'arrayConfig' and 'order'. + * We allow use of 'mode' (for now) so that the move to v1beta2/v1 is not + * breaking when we can understand the developer's intent. + */ +export enum Mode { + ASCENDING = "ASCENDING", + DESCENDING = "DESCENDING", + ARRAY_CONTAINS = "ARRAY_CONTAINS", +} + +export enum QueryScope { + COLLECTION = "COLLECTION", + COLLECTION_GROUP = "COLLECTION_GROUP", +} + +export enum Order { + ASCENDING = "ASCENDING", + DESCENDING = "DESCENDING", +} + +export enum ArrayConfig { + CONTAINS = "CONTAINS", +} + +export enum State { + CREATING = "CREATING", + READY = "READY", + NEEDS_REPAIR = "NEEDS_REPAIR", +} + +/** + * An Index as it is represented in the Firestore v1beta2 indexes API. + */ +export interface Index { + name?: string; + queryScope: QueryScope; + fields: IndexField[]; + state?: State; +} + +/** + * A field in an index. + */ +export interface IndexField { + fieldPath: string; + order?: Order; + arrayConfig?: ArrayConfig; +} + +/** + * Represents a single field in the database. + * + * If a field has an empty indexConfig, that means all + * default indexes are exempted. + */ +export interface Field { + name: string; + indexConfig: IndexConfig; +} + +/** + * Index configuration overrides for a field. + */ +export interface IndexConfig { + ancestorField?: string; + indexes: Index[]; +} diff --git a/src/firestore/indexes-spec.ts b/src/firestore/indexes-spec.ts new file mode 100644 index 00000000..eb714a8e --- /dev/null +++ b/src/firestore/indexes-spec.ts @@ -0,0 +1,36 @@ +import * as API from "./indexes-api"; + +/** + * An entry specifying a compound or other non-default index. + */ +export interface Index { + collectionGroup: string; + queryScope: API.QueryScope; + fields: API.IndexField[]; +} + +/** + * An entry specifying field index configuration override. + */ +export interface FieldOverride { + collectionGroup: string; + fieldPath: string; + indexes: FieldIndex[]; +} + +/** + * Entry specifying a single-field index. + */ +export interface FieldIndex { + queryScope: API.QueryScope; + order: API.Order | undefined; + arrayConfig: API.ArrayConfig | undefined; +} + +/** + * Specification for the JSON file that is used for index deployment, + */ +export interface IndexFile { + indexes: Index[]; + fieldOverrides: FieldOverride[] | undefined; +} diff --git a/src/firestore/indexes.js b/src/firestore/indexes.js deleted file mode 100644 index f47f1e9b..00000000 --- a/src/firestore/indexes.js +++ /dev/null @@ -1,243 +0,0 @@ -"use strict"; - -var api = require("../api"); -var clc = require("cli-color"); -var FirebaseError = require("../error"); -var loadCJSON = require("../loadCJSON"); - -var VALID_INDEX_MODES = ["ASCENDING", "DESCENDING", "ARRAY_CONTAINS"]; - -/** - * Validate an index is correctly formed, throws an exception for all - * fatal errors. - * - * See: - * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index - * - * @param {string} index.collectionId index collection Id. - * @param {object[]} index.fields array of index field specifications. - */ -var _validate = function(index) { - var indexString = clc.cyan(JSON.stringify(index)); - if (!index.collectionId) { - throw new FirebaseError('Index must contain "collectionId": ' + indexString); - } - - if (!index.fields) { - throw new FirebaseError('Index must contain "fields": ' + indexString); - } - - for (var i = 0; i < index.fields.length; i++) { - var field = index.fields[i]; - - if (!field.fieldPath) { - throw new FirebaseError('All index fields must contain "fieldPath": ' + indexString); - } - - if (!field.mode) { - throw new FirebaseError('All index fields must contain "mode": ' + indexString); - } - - if (VALID_INDEX_MODES.indexOf(field.mode) < 0) { - throw new FirebaseError( - "Index field mode must be one of " + VALID_INDEX_MODES.join(", ") + ": " + indexString - ); - } - } -}; - -/** - * Create an index in the specified project. - * - * See: - * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index - * - * @param {string} project the Firestore project Id. - * @param {string} index.collectionId index collection Id. - * @param {object[]} index.fields array of index field specifications. - * @return {Promise} a promise for index creation. - */ -var create = function(project, index) { - _validate(index); - - var url = "projects/" + project + "/databases/(default)/indexes"; - return api.request("POST", "/v1beta1/" + url, { - auth: true, - data: index, - origin: api.firestoreOrigin, - }); -}; - -/** - * List all indexes in the specified project. - * - * See: - * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.indexes#Index - * - * @param {string} project the Firestore project Id. - * @return {Promise} a promise for an array of indexes. - */ -var list = function(project) { - var url = "projects/" + project + "/databases/(default)/indexes"; - - return api - .request("GET", "/v1beta1/" + url, { - auth: true, - origin: api.firestoreOrigin, - }) - .then(function(res) { - var indexes = res.body.indexes || []; - var result = []; - - // Clean up the index metadata so that they appear in the same - // format as they would be specified in firestore.indexes.json - for (var i = 0; i < indexes.length; i++) { - var index = indexes[i]; - var sanitized = {}; - - sanitized.collectionId = index.collectionId; - sanitized.state = index.state; - - sanitized.fields = index.fields.filter(function(field) { - return field.fieldPath !== "__name__"; - }); - - result.push(sanitized); - } - - return result; - }); -}; - -/** - * Determines if two indexes are equal by comparing their collectionId - * and field specifications. All other properties are ignored. - * - * @param {object} a a Firestore index. - * @param {object} b a Firestore index. - * @return {boolean} true if the indexes are equal, false otherwise. - */ -var equal = function(a, b) { - if (a.collectionId !== b.collectionId) { - return false; - } - - if (a.fields.length !== b.fields.length) { - return false; - } - - for (var i = 0; i < a.fields.length; i++) { - var aField = a.fields[i]; - var bField = b.fields[i]; - - if (aField.fieldPath !== bField.fieldPath) { - return false; - } - - if (aField.mode !== bField.mode) { - return false; - } - } - - return true; -}; - -/** - * Create a unique hash for a Firestore index that can be used - * for deduplication. - * - * @param {object} index a Firestore index. - * @return {string} a unique hash. - */ -var hash = function(index) { - var result = ""; - result += index.collectionId; - result += "["; - - for (var i = 0; i < index.fields.length; i++) { - var field = index.fields[i]; - - // Skip __name__ fields - if (field.fieldPath === "__name__") { - continue; - } - - // Append the field description - result += "("; - result += field.fieldPath + "," + field.mode; - result += ")"; - } - - result += "]"; - - return result; -}; - -/** - * Get a colored, pretty printed representation of an index. - * - * @param {object} index a Firestore index. - * @return {string} string for logging. - */ -var toPrettyString = function(index) { - var result = ""; - - if (index.state) { - var stateMsg = "[" + index.state + "] "; - - if (index.state === "READY") { - result += clc.green(stateMsg); - } else if (index.state === "CREATING") { - result += clc.yellow(stateMsg); - } else { - result += clc.red(stateMsg); - } - } - - result += clc.cyan("(" + index.collectionId + ")"); - result += " -- "; - - index.fields.forEach(function(field) { - if (field.fieldPath === "__name__") { - return; - } - - result += "(" + field.fieldPath + "," + field.mode + ") "; - }); - - return result; -}; - -/** - * Prepare indexes for deployment. - */ -var prepare = function(context, options) { - var indexesFileName = options.config.get("firestore.indexes"); - var indexesPath = options.config.path(indexesFileName); - var parsedSrc = loadCJSON(indexesPath); - - if (!parsedSrc.indexes) { - throw new FirebaseError('Indexes file must contain "indexes" property: ' + indexesPath); - } - - parsedSrc.indexes.forEach(function(index) { - _validate(index); - }); - - context.firestore = context.firestore || {}; - context.firestore.indexes = { - name: indexesFileName, - content: parsedSrc, - }; - - return Promise.resolve(); -}; - -module.exports = { - create: create, - list: list, - equal: equal, - hash: hash, - toPrettyString: toPrettyString, - prepare: prepare, -}; diff --git a/src/firestore/indexes.ts b/src/firestore/indexes.ts new file mode 100644 index 00000000..b9714f9a --- /dev/null +++ b/src/firestore/indexes.ts @@ -0,0 +1,564 @@ +import * as clc from "cli-color"; + +import * as api from "../api"; +import * as FirebaseError from "../error"; +import * as logger from "../logger"; +import * as utils from "../utils"; +import * as validator from "./validator"; + +import * as API from "./indexes-api"; +import * as Spec from "./indexes-spec"; + +// projects/$PROJECT_ID/databases/(default)/collectionGroups/$COLLECTION_GROUP_ID/indexes/$INDEX_ID +const INDEX_NAME_REGEX = /projects\/([^\/]+?)\/databases\/\(default\)\/collectionGroups\/([^\/]+?)\/indexes\/([^\/]*)/; + +// projects/$PROJECT_ID/databases/(default)/collectionGroups/$COLLECTION_GROUP_ID/fields/$FIELD_ID +const FIELD_NAME_REGEX = /projects\/([^\/]+?)\/databases\/\(default\)\/collectionGroups\/([^\/]+?)\/fields\/([^\/]*)/; + +interface IndexName { + projectId: string; + collectionGroupId: string; + indexId: string; +} + +interface FieldName { + projectId: string; + collectionGroupId: string; + fieldPath: string; +} + +export class FirestoreIndexes { + /** + * Deploy an index specification to the specified project. + * @param project the Firebase project ID. + * @param indexes an array of objects, each will be validated and then converted + * to an {@link Spec.Index}. + * @param fieldOverrides an array of objects, each will be validated and then + * converted to an {@link Spec.FieldOverride}. + */ + async deploy(project: string, indexes: any[], fieldOverrides: any[]): Promise { + const spec = this.upgradeOldSpec({ + indexes, + fieldOverrides, + }); + + this.validateSpec(spec); + + // Now that the spec is validated we can safely assert these types. + const indexesToDeploy: Spec.Index[] = spec.indexes; + const fieldOverridesToDeploy: Spec.FieldOverride[] = spec.fieldOverrides; + + const existingIndexes = await this.listIndexes(project); + const existingFieldOverrides = await this.listFieldOverrides(project); + + if (existingIndexes.length > indexesToDeploy.length) { + utils.logBullet( + clc.bold.cyan("firestore:") + + " there are some indexes defined in your project that are not present in your " + + "firestore indexes file. Run firebase firestore:indexes and save the result to correct the discrepancy." + ); + } + + indexesToDeploy.forEach(async (index) => { + const exists = existingIndexes.some((x) => this.indexMatchesSpec(x, index)); + if (exists) { + logger.debug(`Skipping existing index: ${JSON.stringify(index)}`); + return; + } + + logger.debug(`Creating new index: ${JSON.stringify(index)}`); + await this.createIndex(project, index); + }); + + if (existingFieldOverrides.length > fieldOverridesToDeploy.length) { + utils.logBullet( + clc.bold.cyan("firestore:") + + " there are some field overrides defined in your project that are not present in your " + + "firestore indexes file. Run firebase firestore:indexes and save the result to correct the discrepancy." + ); + } + + fieldOverridesToDeploy.forEach(async (field) => { + const exists = existingFieldOverrides.some((x) => this.fieldMatchesSpec(x, field)); + if (exists) { + logger.debug(`Skipping existing field override: ${JSON.stringify(field)}`); + return; + } + + logger.debug(`Updating field override: ${JSON.stringify(field)}`); + await this.patchField(project, field); + }); + } + + /** + * List all indexes that exist on a given project. + * @param project the Firebase project id. + */ + async listIndexes(project: string): Promise { + const url = `projects/${project}/databases/(default)/collectionGroups/-/indexes`; + + const res = await api.request("GET", `/v1beta2/${url}`, { + auth: true, + origin: api.firestoreOrigin, + }); + + const indexes = res.body.indexes; + return indexes.map( + (index: any): API.Index => { + // Ignore any fields that point at the document ID, as those are implied + // in all indexes. + const fields = index.fields.filter((field: API.IndexField) => { + return field.fieldPath !== "__name__"; + }); + + return { + name: index.name, + state: index.state, + queryScope: index.queryScope, + fields, + }; + } + ); + } + + /** + * List all field configuration overrides defined on the given project. + * @param project the Firebase project. + */ + async listFieldOverrides(project: string): Promise { + const parent = `projects/${project}/databases/(default)/collectionGroups/-`; + const url = `${parent}/fields?filter=indexConfig.usesAncestorConfig=false`; + + const res = await api.request("GET", `/v1beta2/${url}`, { + auth: true, + origin: api.firestoreOrigin, + }); + + const fields = res.body.fields as API.Field[]; + + // Ignore the default config, only list other fields. + return fields.filter((field) => { + return field.name.indexOf("__default__") < 0; + }); + } + + /** + * Turn an array of indexes and field overrides into a {@link Spec.IndexFile} suitable for use + * in an indexes.json file. + */ + makeIndexSpec(indexes: API.Index[], fields?: API.Field[]): Spec.IndexFile { + const indexesJson = indexes.map((index) => { + return { + collectionGroup: this.parseIndexName(index.name).collectionGroupId, + queryScope: index.queryScope, + fields: index.fields, + }; + }); + + if (!fields) { + logger.debug("No field overrides specified, using []."); + fields = []; + } + + const fieldsJson = fields.map((field) => { + const parsedName = this.parseFieldName(field.name); + return { + collectionGroup: parsedName.collectionGroupId, + fieldPath: parsedName.fieldPath, + + indexes: field.indexConfig.indexes.map((index) => { + const firstField = index.fields[0]; + return { + order: firstField.order, + arrayConfig: firstField.arrayConfig, + queryScope: index.queryScope, + }; + }), + }; + }); + + return { + indexes: indexesJson, + fieldOverrides: fieldsJson, + }; + } + + /** + * Print an array of indexes to the console. + * @param indexes the array of indexes. + */ + prettyPrintIndexes(indexes: API.Index[]): void { + indexes.forEach((index) => { + logger.info(this.prettyIndexString(index)); + }); + } + + /** + * Print an array of field overrides to the console. + * @param fields the array of field overrides. + */ + printFieldOverrides(fields: API.Field[]): void { + fields.forEach((field) => { + logger.info(this.prettyFieldString(field)); + }); + } + + /** + * Validate that an object is a valid index specification. + * @param spec the object, normally parsed from JSON. + */ + validateSpec(spec: any): void { + validator.assertHas(spec, "indexes"); + + spec.indexes.forEach((index: any) => { + this.validateIndex(index); + }); + + if (spec.fieldOverrides) { + spec.fieldOverrides.forEach((field: any) => { + this.validateField(field); + }); + } + } + + /** + * Validate that an arbitrary object is safe to use as an {@link API.Field}. + */ + validateIndex(index: any): void { + validator.assertHas(index, "collectionGroup"); + validator.assertHas(index, "queryScope"); + validator.assertEnum(index, "queryScope", Object.keys(API.QueryScope)); + + validator.assertHas(index, "fields"); + + index.fields.forEach((field: any) => { + validator.assertHas(field, "fieldPath"); + validator.assertHasOneOf(field, ["order", "arrayConfig"]); + + if (field.order) { + validator.assertEnum(field, "order", Object.keys(API.Order)); + } + + if (field.arrayConfig) { + validator.assertEnum(field, "arrayConfig", Object.keys(API.ArrayConfig)); + } + }); + } + + /** + * Validate that an arbitrary object is safe to use as an {@link Spec.FieldOverride}. + * @param field + */ + validateField(field: any): void { + validator.assertHas(field, "collectionGroup"); + validator.assertHas(field, "fieldPath"); + validator.assertHas(field, "indexes"); + + field.indexes.forEach((index: any) => { + validator.assertHasOneOf(index, ["arrayConfig", "order"]); + + if (index.arrayConfig) { + validator.assertEnum(index, "arrayConfig", Object.keys(API.ArrayConfig)); + } + + if (index.order) { + validator.assertEnum(index, "order", Object.keys(API.Order)); + } + + if (index.queryScope) { + validator.assertEnum(index, "queryScope", Object.keys(API.QueryScope)); + } + }); + } + + /** + * Update the configuration of a field. Note that this kicks off a long-running + * operation for index creation/deletion so the update is complete when this + * method returns. + * @param project the Firebase project. + * @param spec the new field override specification. + */ + async patchField(project: string, spec: Spec.FieldOverride): Promise { + const url = `projects/${project}/databases/(default)/collectionGroups/${ + spec.collectionGroup + }/fields/${spec.fieldPath}`; + + const indexes = spec.indexes.map((index) => { + return { + queryScope: index.queryScope, + fields: [ + { + fieldPath: spec.fieldPath, + arrayConfig: index.arrayConfig, + order: index.order, + }, + ], + }; + }); + + const data = { + indexConfig: { + indexes, + }, + }; + + await api.request("PATCH", `/v1beta2/${url}`, { + auth: true, + origin: api.firestoreOrigin, + data, + }); + } + + /** + * Create a new index on the specified project. + */ + createIndex(project: string, index: Spec.Index): Promise { + const url = `projects/${project}/databases/(default)/collectionGroups/${ + index.collectionGroup + }/indexes`; + return api.request("POST", "/v1beta2/" + url, { + auth: true, + data: { + fields: index.fields, + queryScope: index.queryScope, + }, + origin: api.firestoreOrigin, + }); + } + + /** + * Determine if an API Index and a Spec Index are functionally equivalent. + */ + indexMatchesSpec(index: API.Index, spec: Spec.Index): boolean { + const collection = this.parseIndexName(index.name).collectionGroupId; + if (collection !== spec.collectionGroup) { + return false; + } + + if (index.queryScope !== spec.queryScope) { + return false; + } + + if (index.fields.length !== spec.fields.length) { + return false; + } + + let i = 0; + while (i < index.fields.length) { + const iField = index.fields[i]; + const sField = spec.fields[i]; + + if (iField.fieldPath !== sField.fieldPath) { + return false; + } + + if (iField.order !== sField.order) { + return false; + } + + if (iField.arrayConfig !== sField.arrayConfig) { + return false; + } + + i++; + } + + return true; + } + + /** + * Determine if an API Field and a Spec Field are functionally equivalent. + */ + fieldMatchesSpec(field: API.Field, spec: Spec.FieldOverride): boolean { + const parsedName = this.parseFieldName(field.name); + + if (parsedName.collectionGroupId !== spec.collectionGroup) { + return false; + } + + if (parsedName.fieldPath !== spec.fieldPath) { + return false; + } + + if (field.indexConfig.indexes.length !== spec.indexes.length) { + return false; + } + + const fieldModes = field.indexConfig.indexes.map((index) => { + const firstField = index.fields[0]; + return firstField.order || firstField.arrayConfig; + }); + + const specModes = spec.indexes.map((index) => { + return index.order || index.arrayConfig; + }); + + // Confirms that the two objects have the same set of enabled indexes without + // caring about specification order. + for (const mode of fieldModes) { + if (specModes.indexOf(mode) < 0) { + return false; + } + } + + return true; + } + + /** + * Parse an Index name into useful pieces. + */ + parseIndexName(name?: string): IndexName { + if (!name) { + throw new FirebaseError(`Cannot parse undefined index name.`); + } + + const m = name.match(INDEX_NAME_REGEX); + if (!m || m.length < 4) { + throw new FirebaseError(`Error parsing index name: ${name}`); + } + + return { + projectId: m[1], + collectionGroupId: m[2], + indexId: m[3], + }; + } + + /** + * Parse an Field name into useful pieces. + */ + parseFieldName(name: string): FieldName { + const m = name.match(FIELD_NAME_REGEX); + if (!m || m.length < 4) { + throw new FirebaseError(`Error parsing field name: ${name}`); + } + + return { + projectId: m[1], + collectionGroupId: m[2], + fieldPath: m[3], + }; + } + + /** + * Take a object that may represent an old v1beta1 indexes spec + * and convert it to the new v1beta2/v1 spec format. + * + * This function is meant to be run **before** validation and + * works on a purely best-effort basis. + */ + upgradeOldSpec(spec: any): any { + const result = { + indexes: [], + fieldOverrides: spec.fieldOverrides || [], + }; + + if (!spec.indexes) { + return; + } + + // Try to detect use of the old API, warn the users. + if (spec.indexes[0].collectionId) { + utils.logBullet( + clc.bold.cyan("firestore:") + + " your indexes indexes are specified in the v1beta1 API format. " + + "Please upgrade to the new index API format by running " + + clc.bold("firebase firestore:indexes") + + " again and saving the result." + ); + } + + result.indexes = spec.indexes.map((index: any) => { + const i = { + collectionGroup: index.collectionGroup || index.collectionId, + queryScope: index.queryScope || API.QueryScope.COLLECTION, + fields: [], + }; + + if (index.fields) { + i.fields = index.fields.map((field: any) => { + const f: any = { + fieldPath: field.fieldPath, + }; + + if (field.order) { + f.order = field.order; + } else if (field.arrayConfig) { + f.arrayConfig = field.arrayConfig; + } else if (field.mode === API.Mode.ARRAY_CONTAINS) { + f.arrayConfig = API.ArrayConfig.CONTAINS; + } else { + f.order = field.mode; + } + + return f; + }); + } + + return i; + }); + + return result; + } + + /** + * Get a colored, pretty-printed representation of an index. + */ + private prettyIndexString(index: API.Index): string { + let result = ""; + + if (index.state) { + const stateMsg = `[${index.state}] `; + + if (index.state === API.State.READY) { + result += clc.green(stateMsg); + } else if (index.state === API.State.CREATING) { + result += clc.yellow(stateMsg); + } else { + result += clc.red(stateMsg); + } + } + + const nameInfo = this.parseIndexName(index.name); + + result += clc.cyan(`(${nameInfo.collectionGroupId})`); + result += " -- "; + + index.fields.forEach((field) => { + if (field.fieldPath === "__name__") { + return; + } + + // Normal field indexes have an "order" while array indexes have an "arrayConfig", + // we want to display whichever one is present. + const orderOrArrayConfig = field.order ? field.order : field.arrayConfig; + result += `(${field.fieldPath},${orderOrArrayConfig}) `; + }); + + return result; + } + + /** + * Get a colored, pretty-printed representation of a field + */ + private prettyFieldString(field: API.Field): string { + let result = ""; + + const parsedName = this.parseFieldName(field.name); + + result += + "[" + + clc.cyan(parsedName.collectionGroupId) + + "." + + clc.yellow(parsedName.fieldPath) + + "] --"; + + field.indexConfig.indexes.forEach((index) => { + const firstField = index.fields[0]; + const mode = firstField.order || firstField.arrayConfig; + result += " (" + mode + ")"; + }); + + return result; + } +} diff --git a/src/firestore/validator.ts b/src/firestore/validator.ts new file mode 100644 index 00000000..054c4303 --- /dev/null +++ b/src/firestore/validator.ts @@ -0,0 +1,41 @@ +import * as clc from "cli-color"; +import * as FirebaseError from "../error"; + +/** + * Throw an error if 'obj' does not have a value for the property 'prop'. + */ +export function assertHas(obj: any, prop: string): void { + const objString = clc.cyan(JSON.stringify(obj)); + if (!obj[prop]) { + throw new FirebaseError(`Must contain "${prop}": ${objString}`); + } +} + +/** + * throw an error if 'obj' does not have a value for exactly one of the + * properties in 'props'. + */ +export function assertHasOneOf(obj: any, props: string[]): void { + const objString = clc.cyan(JSON.stringify(obj)); + let count = 0; + props.forEach((prop) => { + if (obj[prop]) { + count++; + } + }); + + if (count !== 1) { + throw new FirebaseError(`Must contain exactly one of "${props.join(",")}": ${objString}`); + } +} + +/** + * Throw an error if the value of the property 'prop' on 'obj' is not one of + * the values in the the array 'valid'. + */ +export function assertEnum(obj: any, prop: string, valid: any[]): void { + const objString = clc.cyan(JSON.stringify(obj)); + if (valid.indexOf(obj[prop]) < 0) { + throw new FirebaseError(`Field "${prop}" must be one of ${valid.join(", ")}: ${objString}`); + } +} diff --git a/src/init/features/firestore.js b/src/init/features/firestore.js index 10da0177..e3385fc2 100644 --- a/src/init/features/firestore.js +++ b/src/init/features/firestore.js @@ -5,7 +5,7 @@ var fs = require("fs"); var FirebaseError = require("../../error"); var gcp = require("../../gcp"); -var indexes = require("../../firestore/indexes"); +var iv2 = require("../../firestore/indexes"); var fsutils = require("../../fsutils"); var prompt = require("../../prompt"); var logger = require("../../logger"); @@ -13,6 +13,8 @@ var utils = require("../../utils"); var requireAccess = require("../../requireAccess"); var scopes = require("../../scopes"); +var indexes = new iv2.FirestoreIndexes(); + var RULES_TEMPLATE = fs.readFileSync( __dirname + "/../../../templates/init/firestore/firestore.rules", "utf8" @@ -141,10 +143,12 @@ var _initIndexes = function(setup, config) { }; var _getIndexesFromConsole = function(projectId) { - return indexes - .list(projectId) - .then(function(indexes) { - return JSON.stringify({ indexes: indexes }, null, 2); + var indexesPromise = indexes.listIndexes(projectId); + var fieldOverridesPromise = indexes.listFieldOverrides(projectId); + + return Promise.all([indexesPromise, fieldOverridesPromise]) + .then(function(res) { + return indexes.makeIndexSpec(res[0], res[1]); }) .catch(function(e) { if (e.message.indexOf("is not a Cloud Firestore enabled project") >= 0) { diff --git a/src/test/firestore/indexes.spec.ts b/src/test/firestore/indexes.spec.ts new file mode 100644 index 00000000..a9df9783 --- /dev/null +++ b/src/test/firestore/indexes.spec.ts @@ -0,0 +1,222 @@ +import { expect } from "chai"; +import { FirestoreIndexes } from "../../firestore/indexes"; +import * as FirebaseError from "../../error"; +import * as API from "../../firestore/indexes-api"; +import * as Spec from "../../firestore/indexes-spec"; + +const idx = new FirestoreIndexes(); + +const VALID_SPEC = { + indexes: [ + { + collectionGroup: "collection", + queryScope: "COLLECTION", + fields: [ + { fieldPath: "foo", order: "ASCENDING" }, + { fieldPath: "bar", order: "DESCENDING" }, + { fieldPath: "baz", arrayConfig: "CONTAINS" }, + ], + }, + ], + fieldOverrides: [ + { + collectionGroup: "collection", + fieldPath: "foo", + indexes: [ + { order: "ASCENDING", scope: "COLLECTION" }, + { arrayConfig: "CONTAINS", scope: "COLLECTION" }, + ], + }, + ], +}; + +describe("IndexValidation", () => { + it("should accept a valid v1beta2 index spec", () => { + idx.validateSpec(VALID_SPEC); + }); + + it("should not change a valid v1beta2 index spec after upgrade", () => { + const upgraded = idx.upgradeOldSpec(VALID_SPEC); + expect(upgraded).to.eql(VALID_SPEC); + }); + + it("should accept a valid v1beta1 index spec after upgrade", () => { + idx.validateSpec( + idx.upgradeOldSpec({ + indexes: [ + { + collectionId: "collection", + fields: [ + { fieldPath: "foo", mode: "ASCENDING" }, + { fieldPath: "bar", mode: "DESCENDING" }, + { fieldPath: "baz", mode: "ARRAY_CONTAINS" }, + ], + }, + ], + }) + ); + }); + + it("should reject an incomplete index spec", () => { + expect(() => { + idx.validateSpec({ + indexes: [ + { + collectionGroup: "collection", + fields: [ + { fieldPath: "foo", order: "ASCENDING" }, + { fieldPath: "bar", order: "DESCENDING" }, + ], + }, + ], + }); + }).to.throw(FirebaseError, /Must contain "queryScope"/); + }); + + it("should reject an overspecified index spec", () => { + expect(() => { + idx.validateSpec({ + indexes: [ + { + collectionGroup: "collection", + queryScope: "COLLECTION", + fields: [ + { fieldPath: "foo", order: "ASCENDING", arrayConfig: "CONTAINES" }, + { fieldPath: "bar", order: "DESCENDING" }, + ], + }, + ], + }); + }).to.throw(FirebaseError, /Must contain exactly one of "order,arrayConfig"/); + }); +}); + +describe("IndexNameParsing", () => { + it("should parse an index name correctly", () => { + const name = + "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123/"; + expect(idx.parseIndexName(name)).to.eql({ + projectId: "myproject", + collectionGroupId: "collection", + indexId: "abc123", + }); + }); + + it("should parse a field name correctly", () => { + const name = + "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123/"; + expect(idx.parseFieldName(name)).to.eql({ + projectId: "myproject", + collectionGroupId: "collection", + fieldPath: "abc123", + }); + }); +}); + +describe("IndexSpecMatching", () => { + it("should identify a positive index spec match", () => { + const apiIndex: API.Index = { + name: "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123", + queryScope: API.QueryScope.COLLECTION, + fields: [ + { fieldPath: "foo", order: API.Order.ASCENDING }, + { fieldPath: "bar", arrayConfig: API.ArrayConfig.CONTAINS }, + ], + state: API.State.READY, + }; + + const specIndex = { + collectionGroup: "collection", + queryScope: "COLLECTION", + fields: [ + { fieldPath: "foo", order: "ASCENDING" }, + { fieldPath: "bar", arrayConfig: "CONTAINS" }, + ], + } as Spec.Index; + + expect(idx.indexMatchesSpec(apiIndex, specIndex)).to.eql(true); + }); + + it("should identify a negative index spec match", () => { + const apiIndex = { + name: "/projects/myproject/databases/(default)/collectionGroups/collection/indexes/abc123", + queryScope: "COLLECTION", + fields: [ + { fieldPath: "foo", order: "DESCENDING" }, + { fieldPath: "bar", arrayConfig: "CONTAINS" }, + ], + state: API.State.READY, + } as API.Index; + + const specIndex = { + collectionGroup: "collection", + queryScope: "COLLECTION", + fields: [ + { fieldPath: "foo", order: "ASCENDING" }, + { fieldPath: "bar", arrayConfig: "CONTAINS" }, + ], + } as Spec.Index; + + // The second spec contains ASCENDING where the former contains DESCENDING + expect(idx.indexMatchesSpec(apiIndex, specIndex)).to.eql(false); + }); + + it("should identify a positive field spec match", () => { + const apiField = { + name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123", + indexConfig: { + indexes: [ + { + queryScope: "COLLECTION", + fields: [{ fieldPath: "abc123", order: "ASCENDING" }], + }, + { + queryScope: "COLLECTION", + fields: [{ fieldPath: "abc123", arrayConfig: "CONTAINS" }], + }, + ], + }, + } as API.Field; + + const specField = { + collectionGroup: "collection", + fieldPath: "abc123", + indexes: [ + { order: "ASCENDING", queryScope: "COLLECTION" }, + { arrayConfig: "CONTAINS", queryScope: "COLLECTION" }, + ], + } as Spec.FieldOverride; + + expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true); + }); + + it("should identify a negative field spec match", () => { + const apiField = { + name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123", + indexConfig: { + indexes: [ + { + queryScope: "COLLECTION", + fields: [{ fieldPath: "abc123", order: "ASCENDING" }], + }, + { + queryScope: "COLLECTION", + fields: [{ fieldPath: "abc123", arrayConfig: "CONTAINS" }], + }, + ], + }, + } as API.Field; + + const specField = { + collectionGroup: "collection", + fieldPath: "abc123", + indexes: [ + { order: "DESCENDING", queryScope: "COLLECTION" }, + { arrayConfig: "CONTAINS", queryScope: "COLLECTION" }, + ], + } as Spec.FieldOverride; + + // The second spec contains "DESCENDING" where the first contains "ASCENDING" + expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(false); + }); +}); diff --git a/templates/init/firestore/firestore.indexes.json b/templates/init/firestore/firestore.indexes.json index 2a6adac1..4ff4fab5 100644 --- a/templates/init/firestore/firestore.indexes.json +++ b/templates/init/firestore/firestore.indexes.json @@ -3,12 +3,24 @@ // // "indexes": [ // { - // "collectionId": "widgets", + // "collectionGroup": "widgets", + // "queryScope": "COLLECTION", // "fields": [ - // { "fieldPath": "foo", "mode": "ASCENDING" }, + // { "fieldPath": "foo", "arrayConfig": "CONTAINS" }, // { "fieldPath": "bar", "mode": "DESCENDING" } // ] - // } + // }, + // + // "fieldOverrides": [ + // { + // "collectionGroup": "widgets", + // "fieldPath": "baz", + // "indexes": [ + // { "order": "ASCENDING", "queryScope": "COLLECTION" } + // ] + // }, + // ] // ] - "indexes": [] + "indexes": [], + "fieldOverrides": [] } \ No newline at end of file