mirror of
https://github.com/zhigang1992/firebase-tools.git
synced 2026-04-28 20:05:23 +08:00
Add module to parse dotenv file (#3625)
Small regex-based implementation for parsing dotenv files, e.g. turns ``` FOO=bar MY_SERCVICE_URL=http://example.com ``` into a JS objects: ```js { FOO: "bar", MY_SERVICE_URL: "http://example.com" } ``` The parser is used to load environment variables for Cloud Functions for Firebase by searching for `.env` and `.env.<project or alias or local>` in the function source directory. We add additional requirements that the dotenv file be free of errors (i.e. don't have any lines with invalid contents) and not contain keys that are reserved for internal use. There is already popular NPM package for parsing dotenv files (https://www.npmjs.com/package/dotenv), but I decided to not use it because it lacks few capabilities like multiline string support (we want this! See https://github.com/firebase/firebase-tools/issues/3613) and inspecting invalid lines for clearer error messages.
This commit is contained in:
Daniel Lee
239
src/functions/env.ts
Normal file
239
src/functions/env.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import { FirebaseError } from "../error";
|
||||
import { logger } from "../logger";
|
||||
|
||||
const RESERVED_KEYS = [
|
||||
// Cloud Functions for Firebase
|
||||
"FIREBASE_CONFIG",
|
||||
"CLOUD_RUNTIME_CONFIG",
|
||||
// Cloud Functions - old runtimes:
|
||||
// https://cloud.google.com/functions/docs/env-var#nodejs_8_python_37_and_go_111
|
||||
"ENTRY_POINT",
|
||||
"GCP_PROJECT",
|
||||
"GCLOUD_PROJECT",
|
||||
"GOOGLE_CLOUD_PROJECT",
|
||||
"FUNCTION_TRIGGER_TYPE",
|
||||
"FUNCTION_NAME",
|
||||
"FUNCTION_MEMORY_MB",
|
||||
"FUNCTION_TIMEOUT_SEC",
|
||||
"FUNCTION_IDENTITY",
|
||||
"FUNCTION_REGION",
|
||||
// Cloud Functions - new runtimes:
|
||||
// https://cloud.google.com/functions/docs/env-var#newer_runtimes
|
||||
"FUNCTION_TARGET",
|
||||
"FUNCTION_SIGNATURE_TYPE",
|
||||
"K_SERVICE",
|
||||
"K_REVISION",
|
||||
"PORT",
|
||||
// Cloud Run:
|
||||
// https://cloud.google.com/run/docs/reference/container-contract#env-vars
|
||||
"K_CONFIGURATION",
|
||||
];
|
||||
|
||||
// Regex to capture key, value pair in a dotenv file.
|
||||
// Inspired by:
|
||||
// https://github.com/bkeepers/dotenv/blob/master/lib/dotenv/parser.rb
|
||||
// prettier-ignore
|
||||
const LINE_RE = new RegExp(
|
||||
"^" + // begin line
|
||||
"\\s*" + // leading whitespaces
|
||||
"(\\w+)" + // key
|
||||
"\\s*=\\s*" + // separator (=)
|
||||
"(" + // begin optional value
|
||||
"\\s*'(?:\\'|[^'])*'|" + // single quoted or
|
||||
'\\s*"(?:\\"|[^"])*"|' + // double quoted or
|
||||
"[^\\#\\r\\n]+" + // unquoted
|
||||
")?" + // end optional value
|
||||
"\\s*" + // trailing whitespaces
|
||||
"(?:#[^\\n]*)?" + // optional comment
|
||||
"$", // end line
|
||||
"gms" // flags: global, multiline, dotall
|
||||
);
|
||||
|
||||
interface ParseResult {
|
||||
envs: Record<string, string>;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse contents of a dotenv file.
|
||||
*
|
||||
* Each line should contain key, value pairs, e.g.:
|
||||
*
|
||||
* SERVICE_URL=https://example.com
|
||||
*
|
||||
* Values can be double quoted, e.g.:
|
||||
*
|
||||
* SERVICE_URL="https://example.com"
|
||||
*
|
||||
* Double quoted values can include newlines, e.g.:
|
||||
*
|
||||
* PUBLIC_KEY="-----BEGIN PUBLIC KEY-----\nABC\nEFG\n-----BEGIN PUBLIC KEY-----""
|
||||
*
|
||||
* or span multiple lines, e.g.:
|
||||
*
|
||||
* PUBLIC_KEY="-----BEGIN PUBLIC KEY-----
|
||||
* ABC
|
||||
* EFG
|
||||
* -----BEGIN PUBLIC KEY-----"
|
||||
*
|
||||
* See test for more examples.
|
||||
*
|
||||
* @return {ParseResult} Result containing parsed key, value pairs and errored lines.
|
||||
*/
|
||||
export function parse(data: string): ParseResult {
|
||||
const envs: Record<string, string> = {};
|
||||
const errors: string[] = [];
|
||||
|
||||
data = data.replace(/\r\n?/, "\n"); // For Windows support.
|
||||
let match;
|
||||
while ((match = LINE_RE.exec(data))) {
|
||||
let [, k, v] = match;
|
||||
v = (v || "").trim();
|
||||
|
||||
let quotesMatch;
|
||||
if ((quotesMatch = /^(["'])(.*)\1$/ms.exec(v)) != null) {
|
||||
// Remove surrounding single/double quotes.
|
||||
v = quotesMatch[2];
|
||||
if (quotesMatch[1] === '"') {
|
||||
// Unescape newlines and tabs.
|
||||
v = v.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\v", "\v");
|
||||
// Unescape other escapable characters.
|
||||
v = v.replace(/\\([\\'"])/g, "$1");
|
||||
}
|
||||
}
|
||||
envs[k] = v;
|
||||
}
|
||||
|
||||
const nonmatches = data.replace(LINE_RE, "");
|
||||
for (let line of nonmatches.split(/[\r\n]+/)) {
|
||||
line = line.trim();
|
||||
if (line.startsWith("#")) {
|
||||
// Ignore comments
|
||||
continue;
|
||||
}
|
||||
if (line.length) errors.push(line);
|
||||
}
|
||||
|
||||
return { envs, errors };
|
||||
}
|
||||
|
||||
class KeyValidationError extends Error {}
|
||||
|
||||
/**
|
||||
* Validates string for use as an env var key.
|
||||
*
|
||||
* We restrict key names to ones that conform to POSIX standards.
|
||||
* This is more restrictive than what is allowed in Cloud Functions or Cloud Run.
|
||||
*/
|
||||
export function validateKey(key: string): void {
|
||||
if (RESERVED_KEYS.includes(key)) {
|
||||
throw new KeyValidationError(`Key ${key} is reserved for internal use.`);
|
||||
}
|
||||
if (!/^[A-Z_][A-Z0-9_]*$/.test(key)) {
|
||||
throw new KeyValidationError(
|
||||
`Key ${key} must start with an uppercase ASCII letter or underscore` +
|
||||
", and then consist of uppercase ASCII letters, digits, and underscores."
|
||||
);
|
||||
}
|
||||
if (key.startsWith("X_GOOGLE_") || key.startsWith("FIREBASE_")) {
|
||||
throw new KeyValidationError(
|
||||
`Key ${key} starts with a reserved prefix (X_GOOGLE_ or FIREBASE_)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse dotenv file, but throw errors if:
|
||||
// 1. Input has any invalid lines.
|
||||
// 2. Any env key fails validation.
|
||||
function parseStrict(data: string): Record<string, string> {
|
||||
const { envs, errors } = parse(data);
|
||||
|
||||
if (errors.length) {
|
||||
throw new FirebaseError(`Invalid dotenv file, error on lines: ${errors.join(",")}`);
|
||||
}
|
||||
|
||||
const validationErrors: KeyValidationError[] = [];
|
||||
for (const key of Object.keys(envs)) {
|
||||
try {
|
||||
validateKey(key);
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to validate key ${key}: ${err}`);
|
||||
if (err instanceof KeyValidationError) {
|
||||
validationErrors.push(err);
|
||||
} else {
|
||||
// Unexpected error. Throw.
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (validationErrors.length > 0) {
|
||||
throw new FirebaseError("Validation failed", { children: validationErrors });
|
||||
}
|
||||
|
||||
return envs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads environment variables for a project.
|
||||
*
|
||||
* Load looks for .env files at the root of functions source directory
|
||||
* and loads the contents of the .env files.
|
||||
*
|
||||
* .env files are searched and merged in the following order:
|
||||
*
|
||||
* 1. .env
|
||||
* 2. .env.<project or alias>
|
||||
*
|
||||
* If both .env.<project> and .env.<alias> files are found, an error is thrown.
|
||||
*
|
||||
* @return {Record<string, string>} Environment variables for the project.
|
||||
*/
|
||||
export function load(options: {
|
||||
functionsSource: string;
|
||||
projectId: string;
|
||||
projectAlias?: string;
|
||||
}): Record<string, string> {
|
||||
const targetFiles = [".env"];
|
||||
targetFiles.push(`.env.${options.projectId}`);
|
||||
if (options.projectAlias && options.projectAlias.length) {
|
||||
targetFiles.push(`.env.${options.projectAlias}`);
|
||||
}
|
||||
|
||||
const targetPaths = targetFiles
|
||||
.map((f) => path.join(options.functionsSource, f))
|
||||
.filter(fs.existsSync);
|
||||
|
||||
// Check if both .env.<project> and .env.<alias> exists.
|
||||
if (targetPaths.some((p) => path.basename(p) === `.env.${options.projectId}`)) {
|
||||
if (options.projectAlias && options.projectAlias.length) {
|
||||
for (const p of targetPaths) {
|
||||
if (path.basename(p) === `.env.${options.projectAlias}`) {
|
||||
throw new FirebaseError(
|
||||
`Can't have both .env.${options.projectId} and .env.${options.projectAlias}> files.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let envs: Record<string, string> = {};
|
||||
for (const targetPath of targetPaths) {
|
||||
try {
|
||||
const data = fs.readFileSync(targetPath, "utf8");
|
||||
envs = { ...envs, ...parseStrict(data) };
|
||||
} catch (err) {
|
||||
throw new FirebaseError(`Failed to load environment variables from ${targetPath}.`, {
|
||||
exit: 2,
|
||||
children: err.children?.length > 0 ? err.children : [err],
|
||||
});
|
||||
}
|
||||
}
|
||||
logger.debug(
|
||||
`Loaded environment variables ${JSON.stringify(envs)} from ${targetPaths.join(",")}.`
|
||||
);
|
||||
|
||||
return envs;
|
||||
}
|
||||
339
src/test/functions/env.spec.ts
Normal file
339
src/test/functions/env.spec.ts
Normal file
@@ -0,0 +1,339 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as os from "os";
|
||||
import { sync as rimraf } from "rimraf";
|
||||
import { expect } from "chai";
|
||||
|
||||
import * as env from "../../functions/env";
|
||||
|
||||
describe("functions/env", () => {
|
||||
describe("parse", () => {
|
||||
const tests: { description: string; input: string; want: Record<string, string> }[] = [
|
||||
{
|
||||
description: "should parse values with trailing spaces",
|
||||
input: "FOO=foo ",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should parse values with trailing spaces (single quotes)",
|
||||
input: "FOO='foo' ",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should parse values with trailing spaces (double quotes)",
|
||||
input: 'FOO="foo" ',
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should parse double quoted, multi-line values",
|
||||
input: `
|
||||
FOO="foo1
|
||||
foo2"
|
||||
BAR=bar
|
||||
`,
|
||||
want: { FOO: "foo1\nfoo2", BAR: "bar" },
|
||||
},
|
||||
{
|
||||
description: "should parse double quoted with escaped newlines",
|
||||
input: 'FOO="foo1\\nfoo2"\nBAR=bar',
|
||||
want: { FOO: "foo1\nfoo2", BAR: "bar" },
|
||||
},
|
||||
{
|
||||
description: "should unescape escape characters for double quoted values",
|
||||
input: 'FOO="foo1\\"foo2"',
|
||||
want: { FOO: 'foo1"foo2' },
|
||||
},
|
||||
{
|
||||
description: "should leave escape characters intact for single quoted values",
|
||||
input: "FOO='foo1\\'foo2'",
|
||||
want: { FOO: "foo1\\'foo2" },
|
||||
},
|
||||
{
|
||||
description: "should leave escape characters intact for unquoted values",
|
||||
input: "FOO=foo1\\'foo2",
|
||||
want: { FOO: "foo1\\'foo2" },
|
||||
},
|
||||
{
|
||||
description: "should parse empty value",
|
||||
input: "FOO=",
|
||||
want: { FOO: "" },
|
||||
},
|
||||
{
|
||||
description: "should parse keys with leading spaces",
|
||||
input: " FOO=foo ",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should parse values with trailing spaces (unquoted)",
|
||||
input: "FOO=foo ",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should parse values with trailing spaces (single quoted)",
|
||||
input: "FOO='foo ' ",
|
||||
want: { FOO: "foo " },
|
||||
},
|
||||
{
|
||||
description: "should parse values with trailing spaces (double quoted)",
|
||||
input: 'FOO="foo " ',
|
||||
want: { FOO: "foo " },
|
||||
},
|
||||
{
|
||||
description: "should throw away unquoted values following #",
|
||||
input: "FOO=foo#bar",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should keep values following # in singqle quotes",
|
||||
input: "FOO='foo#bar'",
|
||||
want: { FOO: "foo#bar" },
|
||||
},
|
||||
{
|
||||
description: "should keep values following # in double quotes",
|
||||
input: 'FOO="foo#bar"',
|
||||
want: { FOO: "foo#bar" },
|
||||
},
|
||||
{
|
||||
description: "should ignore leading/trailing spaces before the separator (unquoted)",
|
||||
input: "FOO = foo",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should ignore leading/trailing spaces before the separator (single quotes)",
|
||||
input: "FOO = 'foo'",
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should ignore leading/trailing spaces before the separator (double quotes)",
|
||||
input: 'FOO = "foo"',
|
||||
want: { FOO: "foo" },
|
||||
},
|
||||
{
|
||||
description: "should ignore comments",
|
||||
input: `
|
||||
FOO=foo # comment
|
||||
# line comment 1
|
||||
# line comment 2
|
||||
BAR=bar # another comment
|
||||
`,
|
||||
want: { FOO: "foo", BAR: "bar" },
|
||||
},
|
||||
{
|
||||
description: "should ignore empty lines",
|
||||
input: `
|
||||
FOO=foo
|
||||
|
||||
|
||||
BAR=bar
|
||||
|
||||
`,
|
||||
want: { FOO: "foo", BAR: "bar" },
|
||||
},
|
||||
];
|
||||
|
||||
tests.forEach(({ description, input, want }) => {
|
||||
it(description, () => {
|
||||
const { envs, errors } = env.parse(input);
|
||||
expect(envs).to.deep.equal(want);
|
||||
expect(errors).to.be.empty;
|
||||
});
|
||||
});
|
||||
|
||||
it("should catch invalid lines", () => {
|
||||
expect(
|
||||
env.parse(`
|
||||
BAR###
|
||||
FOO=foo
|
||||
// not a comment
|
||||
=missing key
|
||||
`)
|
||||
).to.deep.equal({
|
||||
envs: { FOO: "foo" },
|
||||
errors: ["BAR###", "// not a comment", "=missing key"],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateKey", () => {
|
||||
it("accepts valid keys", () => {
|
||||
const keys = ["FOO", "ABC_EFG", "A1_B2"];
|
||||
keys.forEach((key) => {
|
||||
expect(() => {
|
||||
env.validateKey(key);
|
||||
}).not.to.throw();
|
||||
});
|
||||
});
|
||||
|
||||
it("throws error given invalid keys", () => {
|
||||
const keys = ["", "1F", "B=C"];
|
||||
keys.forEach((key) => {
|
||||
expect(() => {
|
||||
env.validateKey(key);
|
||||
}).to.throw("must start with");
|
||||
});
|
||||
});
|
||||
|
||||
it("throws error given reserved keys", () => {
|
||||
const keys = [
|
||||
"FIREBASE_CONFIG",
|
||||
"FUNCTION_TARGET",
|
||||
"FUNCTION_SIGNATURE_TYPE",
|
||||
"K_SERVICE",
|
||||
"K_REVISION",
|
||||
"PORT",
|
||||
"K_CONFIGURATION",
|
||||
];
|
||||
keys.forEach((key) => {
|
||||
expect(() => {
|
||||
env.validateKey(key);
|
||||
}).to.throw("reserved for internal use");
|
||||
});
|
||||
});
|
||||
|
||||
it("throws error given keys with a reserved prefix", () => {
|
||||
expect(() => {
|
||||
env.validateKey("X_GOOGLE_FOOBAR");
|
||||
}).to.throw("starts with a reserved prefix");
|
||||
|
||||
expect(() => {
|
||||
env.validateKey("FIREBASE_FOOBAR");
|
||||
}).to.throw("starts with a reserved prefix");
|
||||
});
|
||||
});
|
||||
|
||||
describe("load", () => {
|
||||
const createEnvFiles = (sourceDir: string, envs: Record<string, string>): void => {
|
||||
for (const [filename, data] of Object.entries(envs)) {
|
||||
fs.writeFileSync(path.join(sourceDir, filename), data);
|
||||
}
|
||||
};
|
||||
const projectInfo = { projectId: "my-project", projectAlias: "dev" };
|
||||
let tmpdir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), "test"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rimraf(tmpdir);
|
||||
expect(() => {
|
||||
fs.statSync(tmpdir);
|
||||
}).to.throw;
|
||||
});
|
||||
|
||||
it("loads nothing if .env files are missing", () => {
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({});
|
||||
});
|
||||
|
||||
it("loads envs from .env file", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=foo\nBAR=bar",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "foo",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads envs from .env file, ignoring comments", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "# THIS IS A COMMENT\nFOO=foo # inline comments\nBAR=bar",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "foo",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads envs from .env.<project> file", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
[`.env.${projectInfo.projectId}`]: "FOO=foo\nBAR=bar",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "foo",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads envs from .env.<alias> file", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
[`.env.${projectInfo.projectAlias}`]: "FOO=foo\nBAR=bar",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "foo",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads envs, preferring ones from .env.<project>", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=bad\nBAR=bar",
|
||||
[`.env.${projectInfo.projectId}`]: "FOO=good",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "good",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads envs, preferring ones from .env.<alias>", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=bad\nBAR=bar",
|
||||
[`.env.${projectInfo.projectAlias}`]: "FOO=good",
|
||||
});
|
||||
|
||||
expect(env.load({ ...projectInfo, functionsSource: tmpdir })).to.be.deep.equal({
|
||||
FOO: "good",
|
||||
BAR: "bar",
|
||||
});
|
||||
});
|
||||
|
||||
it("throws an error if both .env.<project> and .env.<alias> exists", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=foo\nBAR=bar",
|
||||
[`.env.${projectInfo.projectId}`]: "FOO=not-foo",
|
||||
[`.env.${projectInfo.projectAlias}`]: "FOO=not-foo",
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
env.load({ ...projectInfo, functionsSource: tmpdir });
|
||||
}).to.throw("Can't have both");
|
||||
});
|
||||
|
||||
it("throws an error .env file is invalid", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "BAH: foo",
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
env.load({ ...projectInfo, functionsSource: tmpdir });
|
||||
}).to.throw("Failed to load");
|
||||
});
|
||||
|
||||
it("throws an error .env file contains invalid keys", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=foo",
|
||||
[`.env.${projectInfo.projectId}`]: "Foo=bad-key",
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
env.load({ ...projectInfo, functionsSource: tmpdir });
|
||||
}).to.throw("Failed to load");
|
||||
});
|
||||
|
||||
it("throws an error .env file contains reserved keys", () => {
|
||||
createEnvFiles(tmpdir, {
|
||||
".env": "FOO=foo\nPORT=100",
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
env.load({ ...projectInfo, functionsSource: tmpdir });
|
||||
}).to.throw("Failed to load");
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user