/** * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * * @flow * @format */ 'use strict'; const crypto = require('crypto'); const debugRead = require('debug')('RNP:TransformCache:Read'); const fs = require('fs'); const mkdirp = require('mkdirp'); const path = require('path'); const rimraf = require('rimraf'); const terminal = require('../lib/terminal'); const writeFileAtomicSync = require('write-file-atomic').sync; import type {Options as WorkerOptions} from '../JSTransformer/worker/worker'; import type {MappingsMap} from './SourceMap'; import type {Reporter} from './reporting'; type CacheFilePaths = {transformedCode: string, metadata: string}; export type GetTransformCacheKey = (options: {}) => string; const CACHE_NAME = 'react-native-packager-cache'; const CACHE_SUB_DIR = 'cache'; export type CachedResult = { code: string, dependencies: Array, dependencyOffsets: Array, map?: ?MappingsMap, }; export type TransformCacheResult = {| +result: ?CachedResult, +outdatedDependencies: $ReadOnlyArray, |}; export type CacheOptions = { reporter: Reporter, resetCache?: boolean, }; export type ReadTransformProps = { filePath: string, sourceCode: string, transformOptions: WorkerOptions, transformOptionsKey: string, getTransformCacheKey: GetTransformCacheKey, cacheOptions: CacheOptions, }; const EMPTY_ARRAY = []; /* 1 day */ const GARBAGE_COLLECTION_PERIOD = 24 * 60 * 60 * 1000; /* 4 days */ const CACHE_FILE_MAX_LAST_ACCESS_TIME = GARBAGE_COLLECTION_PERIOD * 4; class TransformCache { _cacheWasReset: boolean; _dirPath: string; _lastCollected: ?number; constructor() { this._cacheWasReset = false; } /** * We store the transformed JS because it is likely to be much bigger than the * rest of the data JSON. Probably the map should be stored separately as * well. * * We make the write operation as much atomic as possible: indeed, if another * process is reading the cache at the same time, there would be a risk it * reads new transformed code, but old metadata. This is avoided by removing * the files first. * * There is still a risk of conflincting writes, that is mitigated by hashing * the result code, that is verified at the end. In case of writes happening * close to each others, one of the workers is going to loose its results no * matter what. */ writeSync(props: { filePath: string, sourceCode: string, getTransformCacheKey: GetTransformCacheKey, transformOptions: WorkerOptions, transformOptionsKey: string, result: CachedResult, }): void { const cacheFilePath = this._getCacheFilePaths(props); mkdirp.sync(path.dirname(cacheFilePath.transformedCode)); const {result} = props; unlinkIfExistsSync(cacheFilePath.transformedCode); unlinkIfExistsSync(cacheFilePath.metadata); writeFileAtomicSync(cacheFilePath.transformedCode, result.code); writeFileAtomicSync( cacheFilePath.metadata, JSON.stringify([ crypto.createHash('sha1').update(result.code).digest('hex'), hashSourceCode(props), result.dependencies, result.dependencyOffsets, result.map, ]), ); } readSync(props: ReadTransformProps): TransformCacheResult { const result = this._readSync(props); const msg = result ? 'Cache hit: ' : 'Cache miss: '; debugRead(msg + props.filePath); return result; } /** * We verify the source hash matches to ensure we always favor rebuilding when * source change (rather than just using fs.mtime(), a bit less robust). * * That means when the source changes, we override the old transformed code * with the new one. This is, I believe, preferable, so as to avoid bloating * the cache during development cycles, where people changes files all the * time. If we implement a global cache ability at some point, we'll be able * to store old artifacts as well. * * Meanwhile we store transforms with different options in different files so * that it is fast to switch between ex. minified, or not. */ _readSync(props: ReadTransformProps): TransformCacheResult { this._collectIfNecessarySync(props.cacheOptions); const cacheFilePaths = this._getCacheFilePaths(props); let metadata, transformedCode; try { metadata = readMetadataFileSync(cacheFilePaths.metadata); if (metadata == null) { return {result: null, outdatedDependencies: EMPTY_ARRAY}; } const sourceHash = hashSourceCode(props); if (sourceHash !== metadata.cachedSourceHash) { return {result: null, outdatedDependencies: metadata.dependencies}; } transformedCode = fs.readFileSync(cacheFilePaths.transformedCode, 'utf8'); const codeHash = crypto .createHash('sha1') .update(transformedCode) .digest('hex'); if (metadata.cachedResultHash !== codeHash) { return {result: null, outdatedDependencies: metadata.dependencies}; } } catch (error) { if (error.code === 'ENOENT') { return {result: null, outdatedDependencies: EMPTY_ARRAY}; } throw error; } return { result: { code: transformedCode, dependencies: metadata.dependencies, dependencyOffsets: metadata.dependencyOffsets, map: metadata.sourceMap, }, outdatedDependencies: EMPTY_ARRAY, }; } /** * Temporary folder is never cleaned up automatically, we need to clean up old * stuff ourselves. This code should be safe even if two different React * Native projects are running at the same time. */ _collectIfNecessarySync(options: CacheOptions) { if (options.resetCache && !this._cacheWasReset) { this._resetCache(options.reporter); return; } const lastCollected = this._lastCollected; if ( lastCollected == null || Date.now() - lastCollected > GARBAGE_COLLECTION_PERIOD ) { this._collectSyncNoThrow(); } } _resetCache(reporter: Reporter) { rimraf.sync(this._getCacheDirPath()); reporter.update({type: 'transform_cache_reset'}); this._cacheWasReset = true; this._lastCollected = Date.now(); } /** * We want to avoid preventing tool use if the cleanup fails for some reason, * but still provide some chance for people to report/fix things. */ _collectSyncNoThrow() { try { this._collectCacheIfOldSync(); } catch (error) { terminal.log(error.stack); terminal.log( 'Error: Cleaning up the cache folder failed. Continuing anyway.', ); terminal.log('The cache folder is: %s', this._getCacheDirPath()); } this._lastCollected = Date.now(); } /** * When restarting packager we want to avoid running the collection over * again, so we store the last collection time in a file and we check that * first. */ _collectCacheIfOldSync() { const cacheDirPath = this._getCacheDirPath(); mkdirp.sync(cacheDirPath); const cacheCollectionFilePath = path.join(cacheDirPath, 'last_collected'); const lastCollected = Number.parseInt( tryReadFileSync(cacheCollectionFilePath, 'utf8'), 10, ); if ( Number.isInteger(lastCollected) && Date.now() - lastCollected > GARBAGE_COLLECTION_PERIOD ) { return; } const effectiveCacheDirPath = path.join(cacheDirPath, CACHE_SUB_DIR); mkdirp.sync(effectiveCacheDirPath); collectCacheSync(effectiveCacheDirPath); fs.writeFileSync(cacheCollectionFilePath, Date.now().toString()); } /** * The path, built as a hash, does not take the source code itself into * account because it would generate lots of file during development. (The * source hash is stored in the metadata instead). */ _getCacheFilePaths(props: { filePath: string, transformOptionsKey: string, }): CacheFilePaths { const hasher = crypto .createHash('sha1') .update(props.filePath) .update(props.transformOptionsKey); const hash = hasher.digest('hex'); const prefix = hash.substr(0, 2); const fileName = `${hash.substr(2)}`; const base = path.join( this._getCacheDirPath(), CACHE_SUB_DIR, prefix, fileName, ); return {transformedCode: base, metadata: base + '.meta'}; } /** * If packager is running for two different directories, we don't want the * caches to conflict with each other. `__dirname` carries that because * packager will be, for example, installed in a different `node_modules/` * folder for different projects. */ _getCacheDirPath() { if (this._dirPath != null) { return this._dirPath; } const hash = crypto.createHash('sha1').update(__dirname); if (process.getuid != null) { hash.update(process.getuid().toString()); } this._dirPath = path.join( require('os').tmpdir(), CACHE_NAME + '-' + hash.digest('hex'), ); require('debug')('RNP:TransformCache:Dir')( `transform cache directory: ${this._dirPath}`, ); return this._dirPath; } } /** * Remove all the cache files from the specified folder that are older than a * certain duration. */ function collectCacheSync(dirPath: string) { const prefixDirs = fs.readdirSync(dirPath); for (let i = 0; i < prefixDirs.length; ++i) { const prefixDir = path.join(dirPath, prefixDirs[i]); const cacheFileNames = fs.readdirSync(prefixDir); for (let j = 0; j < cacheFileNames.length; ++j) { const cacheFilePath = path.join(prefixDir, cacheFileNames[j]); const stats = fs.lstatSync(cacheFilePath); const timeSinceLastAccess = Date.now() - stats.atime.getTime(); if ( stats.isFile() && timeSinceLastAccess > CACHE_FILE_MAX_LAST_ACCESS_TIME ) { fs.unlinkSync(cacheFilePath); } } } } function tryReadFileSync(filePath: string): string { try { return fs.readFileSync(filePath, 'utf8'); } catch (error) { if (error.code !== 'ENOENT') { throw error; } return ''; } } function readMetadataFileSync( metadataFilePath: string, ): ?{ cachedResultHash: string, cachedSourceHash: string, dependencies: Array, dependencyOffsets: Array, sourceMap: ?MappingsMap, } { const metadataStr = fs.readFileSync(metadataFilePath, 'utf8'); let metadata; try { metadata = JSON.parse(metadataStr); } catch (error) { if (error instanceof SyntaxError) { return null; } throw error; } if (!Array.isArray(metadata)) { return null; } const [ cachedResultHash, cachedSourceHash, dependencies, dependencyOffsets, sourceMap, ] = metadata; if ( typeof cachedResultHash !== 'string' || typeof cachedSourceHash !== 'string' || !(Array.isArray(dependencies) && dependencies.every(dep => typeof dep === 'string')) || !(Array.isArray(dependencyOffsets) && dependencyOffsets.every(offset => typeof offset === 'number')) || !(sourceMap == null || typeof sourceMap === 'object') ) { return null; } return { cachedResultHash, cachedSourceHash, dependencies, dependencyOffsets, sourceMap, }; } function hashSourceCode(props: { filePath: string, sourceCode: string, getTransformCacheKey: GetTransformCacheKey, transformOptions: WorkerOptions, transformOptionsKey: string, }): string { return crypto .createHash('sha1') .update(props.getTransformCacheKey(props.transformOptions)) .update(props.sourceCode) .digest('hex'); } /** * We want to unlink all cache files before writing, so that it is as much * atomic as possible. */ function unlinkIfExistsSync(filePath: string) { try { fs.unlinkSync(filePath); } catch (error) { if (error.code === 'ENOENT') { return; } throw error; } } module.exports = TransformCache;