Open Source symbolication code

Summary: This moves the internal symbolication code out to open source so that people can look up source locations for stack traces in RAM bundles. Wix needs this, and there is no reason to keep this code internal.

Reviewed By: bthood

Differential Revision: D14083307

fbshipit-source-id: afd2435c1b549b04ae7a04340ceeac1c5e4c99f0
This commit is contained in:
Christoph Nakazawa
2019-02-15 07:17:26 -08:00
committed by Facebook Github Bot
parent bd67254af1
commit 3d83540973
17 changed files with 641 additions and 0 deletions

View File

@@ -259,6 +259,14 @@
},
"overrides": [
{
"files": [
"packages/react-native-symbolicate/**/*.js",
],
"env": {
"node": true,
},
},
{
"files": [
"**/__fixtures__/**/*.js",

View File

@@ -0,0 +1,54 @@
load("@fbsource//tools/build_defs:fb_native_wrapper.bzl", "fb_native")
load("@fbsource//tools/build_defs/third_party:node_defs.bzl", "nodejs_binary")
load("@fbsource//tools/build_defs/third_party:yarn_defs.bzl", "yarn_install", "yarn_workspace")
yarn_workspace(
name = "yarn-workspace",
srcs = glob(
["**/*.js"],
exclude = [
"**/__fixtures__/**",
"**/__flowtests__/**",
"**/__mocks__/**",
"**/__tests__/**",
"**/node_modules/**",
"**/node_modules/.bin/**",
"**/.*",
"**/.*/**",
"**/.*/.*",
"**/*.xcodeproj/**",
"**/*.xcworkspace/**",
],
),
visibility = ["PUBLIC"],
)
yarn_install(
name = "node-modules",
extra_srcs = [
"symbolicate.js",
"Symbolication.js",
],
)
nodejs_binary(
name = "symbolicate",
main = [
":node-modules",
"symbolicate.js",
],
node_args = [
"--max-old-space-size=8192",
"--stack_size=10000",
],
visibility = ["PUBLIC"],
)
fb_native.sh_test(
name = "symbolicate-integration-test",
args = ["$(exe :symbolicate)"],
test = "tests/symbolicate_test.sh",
deps = [
":symbolicate",
],
)

View File

@@ -0,0 +1,263 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
'use strict';
/* eslint-disable no-console */
const fs = require('fs');
const UNKNOWN_MODULE_IDS = {
segmentId: 0,
localId: undefined,
};
/*
* If the file name of a stack frame is numeric (+ ".js"), we assume it's a
* lazily injected module coming from a "random access bundle". We are using
* special source maps for these bundles, so that we can symbolicate stack
* traces for multiple injected files with a single source map.
*
* There is also a convention for callsites that are in split segments of a
* bundle, named either `seg-3.js` for segment #3 for example, or `seg-3_5.js`
* for module #5 of segment #3 of a segmented RAM bundle.
*/
function parseFileName(str) {
const modMatch = str.match(/^(\d+).js$/);
if (modMatch != null) {
return {segmentId: 0, localId: Number(modMatch[1])};
}
const segMatch = str.match(/^seg-(\d+)(?:_(\d+))?.js$/);
if (segMatch != null) {
return {
segmentId: Number(segMatch[1]),
localId: segMatch[2] && Number(segMatch[2]),
};
}
return UNKNOWN_MODULE_IDS;
}
/*
* A helper function to return a mapping {line, column} object for a given input
* line and column, and optionally a module ID.
*/
function getOriginalPositionFor(lineNumber, columnNumber, moduleIds, context) {
var moduleLineOffset = 0;
var metadata = context.segments[moduleIds.segmentId];
const {localId} = moduleIds;
if (localId != null) {
const {moduleOffsets} = metadata;
if (!moduleOffsets) {
throw new Error(
'Module ID given for a source map that does not have ' +
'an x_facebook_offsets field',
);
}
if (moduleOffsets[localId] == null) {
throw new Error('Unknown module ID: ' + localId);
}
moduleLineOffset = moduleOffsets[localId];
}
return metadata.consumer.originalPositionFor({
line: Number(lineNumber) + moduleLineOffset,
column: Number(columnNumber),
});
}
function createContext(SourceMapConsumer, sourceMapContent) {
var sourceMapJson = JSON.parse(sourceMapContent.replace(/^\)\]\}'/, ''));
return {
segments: Object.entries(sourceMapJson.x_facebook_segments || {}).reduce(
(acc, seg) => {
acc[seg[0]] = {
consumer: new SourceMapConsumer(seg[1]),
moduleOffsets: seg[1].x_facebook_offsets || {},
};
return acc;
},
{
'0': {
consumer: new SourceMapConsumer(sourceMapJson),
moduleOffsets: sourceMapJson.x_facebook_offsets || {},
},
},
),
};
}
// parse stack trace with String.replace
// replace the matched part of stack trace to symbolicated result
// sample stack trace:
// IOS: foo@4:18131, Android: bar:4:18063
// sample stack trace with module id:
// IOS: foo@123.js:4:18131, Android: bar:123.js:4:18063
// sample result:
// IOS: foo.js:57:foo, Android: bar.js:75:bar
function symbolicate(stackTrace, context) {
return stackTrace.replace(
/(?:([^@: \n]+)(@|:))?(?:(?:([^@: \n]+):)?(\d+):(\d+)|\[native code\])/g,
function(match, func, delimiter, fileName, line, column) {
var original = getOriginalPositionFor(
line,
column,
parseFileName(fileName || ''),
context,
);
return original.source + ':' + original.line + ':' + original.name;
},
);
}
// Taking in a map like
// trampoline offset (optional js function name)
// JS_0158_xxxxxxxxxxxxxxxxxxxxxx fe 91081
// JS_0159_xxxxxxxxxxxxxxxxxxxxxx Ft 68651
// JS_0160_xxxxxxxxxxxxxxxxxxxxxx value 50700
// JS_0161_xxxxxxxxxxxxxxxxxxxxxx setGapAtCursor 0
// JS_0162_xxxxxxxxxxxxxxxxxxxxxx (unknown) 50818
// JS_0163_xxxxxxxxxxxxxxxxxxxxxx value 108267
function symbolicateProfilerMap(mapFile, context) {
return fs
.readFileSync(mapFile, 'utf8')
.split('\n')
.slice(0, -1)
.map(function(line) {
const line_list = line.split(' ');
const trampoline = line_list[0];
const js_name = line_list[1];
const offset = parseInt(line_list[2], 10);
if (!offset) {
return trampoline + ' ' + trampoline;
}
var original = getOriginalPositionFor(
1,
offset,
UNKNOWN_MODULE_IDS,
context,
);
return (
trampoline +
' ' +
(original.name || js_name) +
'::' +
[original.source, original.line, original.column].join(':')
);
})
.join('\n');
}
function symbolicateAttribution(obj, context) {
var loc = obj.location;
var line = loc.line || 1;
var column = loc.column || loc.virtualOffset;
var file = loc.filename ? parseFileName(loc.filename) : UNKNOWN_MODULE_IDS;
var original = getOriginalPositionFor(line, column, file, context);
obj.location = {
file: original.source,
line: original.line,
column: original.column,
};
}
// Symbolicate chrome trace "stackFrames" section.
// Each frame in it has three fields: name, funcVirtAddr(optional), offset(optional).
// funcVirtAddr and offset are only available if trace is generated from
// hbc bundle without debug info.
function symbolicateChromeTrace(traceFile, context) {
const contentJson = JSON.parse(fs.readFileSync(traceFile, 'utf8'));
if (contentJson.stackFrames == null) {
console.error('Unable to locate `stackFrames` section in trace.');
process.exit(1);
}
console.log(
'Processing ' + Object.keys(contentJson.stackFrames).length + ' frames',
);
Object.values(contentJson.stackFrames).forEach(function(entry) {
let line;
let column;
// Function entrypoint line/column; used for symbolicating function name.
let funcLine;
let funcColumn;
if (entry.funcVirtAddr != null && entry.offset != null) {
// Without debug information.
const funcVirtAddr = parseInt(entry.funcVirtAddr, 10);
const offsetInFunction = parseInt(entry.offset, 10);
// Main bundle always use hard-coded line value 1.
// TODO: support multiple bundle/module.
line = 1;
column = funcVirtAddr + offsetInFunction;
funcLine = 1;
funcColumn = funcVirtAddr;
} else if (entry.line != null && entry.column != null) {
// For hbc bundle with debug info, name field may already have source
// information for the bundle; we still can use babel/metro/prepack
// source map to symbolicate the bundle frame addresses further to its
// original source code.
line = entry.line;
column = entry.column;
funcLine = entry.funcLine;
funcColumn = entry.funcColumn;
} else {
// Native frames.
return;
}
// Symbolicate original file/line/column.
const addressOriginal = getOriginalPositionFor(
line,
column,
UNKNOWN_MODULE_IDS,
context,
);
let frameName = entry.name;
// Symbolicate function name.
if (funcLine != null && funcColumn != null) {
const funcOriginal = getOriginalPositionFor(
funcLine,
funcColumn,
UNKNOWN_MODULE_IDS,
context,
);
if (funcOriginal.name != null) {
frameName = funcOriginal.name;
}
} else {
// No function line/column info.
console.warn(
'Warning: no function prolog line/column info; name may be wrong',
);
}
// Output format is: funcName(file:line:column)
const sourceLocation = `(${addressOriginal.source}:${
addressOriginal.line
}:${addressOriginal.column})`;
entry.name = frameName + sourceLocation;
});
console.log('Writing to ' + traceFile);
fs.writeFileSync(traceFile, JSON.stringify(contentJson));
}
module.exports = {
createContext,
getOriginalPositionFor,
parseFileName,
symbolicate,
symbolicateProfilerMap,
symbolicateAttribution,
symbolicateChromeTrace,
};

View File

@@ -0,0 +1,8 @@
{
"name": "react-native-symbolicate",
"version": "0.0.1",
"dependencies": {
"source-map": "^0.5.6",
"through2": "^2.0.1"
}
}

View File

@@ -0,0 +1,103 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* Symbolicates a JavaScript stack trace using a source map.
* In our first form, we read a stack trace from stdin and symbolicate it via
* the provided source map.
* In our second form, we symbolicate using an explicit line number, and
* optionally a column.
* In our third form, we symbolicate using a module ID, a line number, and
* optionally a column.
*
* See https://our.intern.facebook.com/intern/dex/symbolicating-javascript-stack-traces-for-react-native/
*
* @flow
* @format
*/
'use strict';
var SourceMapConsumer = require('source-map').SourceMapConsumer;
var Symbolication = require('./Symbolication.js');
var fs = require('fs');
var through2 = require('through2');
var argv = process.argv.slice(2);
if (argv.length < 1 || argv.length > 3) {
/* eslint no-path-concat: "off" */
var usages = [
'Usage: ' + __filename + ' <source-map-file>',
' ' + __filename + ' <source-map-file> <line> [column]',
' ' + __filename + ' <source-map-file> <moduleId>.js <line> [column]',
' ' + __filename + ' <source-map-file> <mapfile>.profmap',
' ' +
__filename +
' <source-map-file> --attribution < attribution.jsonl > symbolicated.jsonl',
' ' + __filename + ' <source-map-file> <tracefile>.cpuprofile',
];
console.error(usages.join('\n'));
process.exit(1);
}
// Read the source map.
var sourceMapFileName = argv.shift();
var content = fs.readFileSync(sourceMapFileName, 'utf8');
var context = Symbolication.createContext(SourceMapConsumer, content);
if (argv.length === 0) {
// read-from-stdin form.
var stackTrace = fs.readFileSync('/dev/stdin', 'utf8');
var result = Symbolication.symbolicate(stackTrace, context);
process.stdout.write(result);
} else if (argv[0].endsWith('.profmap')) {
process.stdout.write(Symbolication.symbolicateProfilerMap(argv[0], context));
} else if (argv[0] === '--attribution') {
var buffer = '';
process.stdin
.pipe(
through2(function(data, enc, callback) {
// Take arbitrary strings, output single lines
buffer += data;
var lines = buffer.split('\n');
for (var i = 0, e = lines.length - 1; i < e; i++) {
this.push(lines[i]);
}
buffer = lines[lines.length - 1];
callback();
}),
)
.pipe(
through2.obj(function(data, enc, callback) {
// This is JSONL, so each line is a separate JSON object
var obj = JSON.parse(data);
Symbolication.symbolicateAttribution(obj, context);
this.push(JSON.stringify(obj) + '\n');
callback();
}),
)
.pipe(process.stdout);
} else if (argv[0].endsWith('.cpuprofile')) {
Symbolication.symbolicateChromeTrace(argv[0], context);
} else {
// read-from-argv form.
var moduleIds, lineNumber, columnNumber;
if (argv[0].endsWith('.js')) {
moduleIds = Symbolication.parseFileName(argv[1]);
argv.shift();
} else {
moduleIds = {segmentId: 0, localId: undefined};
}
lineNumber = argv.shift();
columnNumber = argv.shift() || 0;
var original = Symbolication.getOriginalPositionFor(
lineNumber,
columnNumber,
moduleIds,
context,
);
console.log(original.source + ':' + original.line + ':' + original.name);
}

View File

@@ -0,0 +1,44 @@
#!/bin/bash
SYMBOLICATE_CMD="$1"
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
exitcode=0
trap 'exit $exitcode' EXIT
can() {
/usr/bin/diff >&2 -U5 "${@:2}" || {
exitcode=$?
printf >&2 "FAIL: can't %s\\n" "$1"
}
}
symbolicate() {
$SYMBOLICATE_CMD "$@"
}
can "symbolicate a stack trace" \
<(symbolicate testfile.js.map < testfile.stack) \
testfile.symbolicated.stack
can "symbolicate a single entry" \
<(symbolicate testfile.js.map 1 161) \
<(echo thrower.js:18:null)
can "symbolicate a profiler map" \
<(symbolicate testfile.js.map testfile.profmap) \
testfile.symbolicated.profmap
can "symbolicate an attribution file" \
<(symbolicate testfile.js.map --attribution < testfile.attribution.input) \
testfile.attribution.output
cp testfile.cpuprofile testfile.temp.cpuprofile
($SYMBOLICATE_CMD testfile.cpuprofile.map testfile.temp.cpuprofile)
if ! /usr/bin/diff testfile.temp.cpuprofile -U5 \
testfile.symbolicated.cpuprofile; then
rm testfile.temp.cpuprofile
exit 1
fi
rm testfile.temp.cpuprofile

View File

@@ -0,0 +1,2 @@
{"functionId":1,"location":{"virtualOffset":22},"usage":[]}
{"functionId":2,"location":{"virtualOffset":99},"usage":[]}

View File

@@ -0,0 +1,2 @@
{"functionId":1,"location":{"file":"thrower.js","line":4,"column":11},"usage":[]}
{"functionId":2,"location":{"file":"thrower.js","line":14,"column":14},"usage":[]}

View File

@@ -0,0 +1,42 @@
{
"stackFrames": {
"1": {
"funcVirtAddr": "0",
"offset": "0",
"name": "global+0",
"category": "JavaScript"
},
"2": {
"funcVirtAddr": "0",
"offset": "55",
"name": "global+55",
"category": "JavaScript"
},
"3": {
"funcVirtAddr": "67",
"offset": "16",
"name": "entryPoint+16",
"category": "JavaScript",
"parent": 2
},
"4": {
"funcVirtAddr": "89",
"offset": "0",
"name": "helper+0",
"category": "JavaScript",
"parent": 3
},
"5": {
"funcVirtAddr": "89",
"offset": "146",
"name": "helper+146",
"category": "JavaScript",
"parent": 3
},
"6": {
"name": "[Native]4367295792",
"category": "Native",
"parent": 5
}
}
}

View File

@@ -0,0 +1,7 @@
{
"version": 3,
"sources": [
"temp/bench.js"
],
"mappings": "AACC,uDAmBU,YAnBY,gBACd,MAGU,wFAKA,WACT,iBACC,IAAD,YACU,cAAA,YAHY,eAAb;"
}

View File

@@ -0,0 +1,2 @@
{"version":3,"sources":["thrower.js"],"names":["notCalled","alsoNotCalled","throws6a","Error","throws6","throws6b","throws4","obj","throws5","apply","this","throws2","throws3","throws1","throws0","t","eval","o","forEach","call","arguments","arg","err","print","stack"],"mappings":"CAAA,WAGE,SAASA,YACP,SAASC,IACPA,IAEFA,IACAD,YAIF,SAASE,WACP,MAAM,IAAIC,MAAM,wBAGlB,SAASC,UACP,MAAM,IAAID,MAAM,wBAGlB,SAASE,WACP,MAAM,IAAIF,MAAM,wBAYlB,SAASG,UACPC,IAAIC,EAAQC,MAAMC,MAAON,QAASA,QAASA,UAG7C,SAASO,UACPJ,IAAIK,IAGN,SAASC,UAELF,UAIJ,SAASG,UACPD,UAxBF,IAAIN,KACFQ,EAAS,WACPC,KAAK,eAEPC,EAAS,cACJC,QAAQC,KAAKC,UAAW,SAASC,GAAOA,QAsB/C,IACEP,UACA,MAAOQ,GACPC,MAAMD,EAAIE,QAtDd"}

View File

@@ -0,0 +1,4 @@
JS_0000_xxxxxxxxxxxxxxxxxxxxxx (unknown) 373
JS_0001_xxxxxxxxxxxxxxxxxxxxxx garbage 296
JS_0002_xxxxxxxxxxxxxxxxxxxxxx name 1234
JS_0003_xxxxxxxxxxxxxxxxxxxxxx (unknown) 5678

View File

@@ -0,0 +1,13 @@
throws6@thrower.min.js:1:161
thrower.min.js:1:488
forEach@[native code]
o@thrower.min.js:1:464
throws4@thrower.min.js:1:276
eval code
eval@[native code]
t@thrower.min.js:1:420
throws2@thrower.min.js:1:333
throws1@thrower.min.js:1:362
throws0@thrower.min.js:1:391
thrower.min.js:1:506
global code@thrower.min.js:1:534

View File

@@ -0,0 +1 @@
{"stackFrames":{"1":{"funcVirtAddr":"0","offset":"0","name":"global+0(temp/bench.js:2:1)","category":"JavaScript"},"2":{"funcVirtAddr":"0","offset":"55","name":"global+55(temp/bench.js:21:11)","category":"JavaScript"},"3":{"funcVirtAddr":"67","offset":"16","name":"entryPoint+16(temp/bench.js:3:9)","category":"JavaScript","parent":2},"4":{"funcVirtAddr":"89","offset":"0","name":"helper+0(temp/bench.js:6:19)","category":"JavaScript","parent":3},"5":{"funcVirtAddr":"89","offset":"146","name":"helper+146(temp/bench.js:14:20)","category":"JavaScript","parent":3},"6":{"name":"[Native]4367295792","category":"Native","parent":5}}}

View File

@@ -0,0 +1,4 @@
JS_0000_xxxxxxxxxxxxxxxxxxxxxx throws0::thrower.js:48:11
JS_0001_xxxxxxxxxxxxxxxxxxxxxx throws6::thrower.js:35:38
JS_0002_xxxxxxxxxxxxxxxxxxxxxx name::thrower.js:1:0
JS_0003_xxxxxxxxxxxxxxxxxxxxxx (unknown)::thrower.js:1:0

View File

@@ -0,0 +1,13 @@
thrower.js:18:null
thrower.js:30:arg
null:null:null
thrower.js:30:arguments
thrower.js:35:this
eval code
null:null:null
thrower.js:27:null
thrower.js:39:throws3
thrower.js:44:throws2
thrower.js:49:throws1
thrower.js:53:throws0
global thrower.js:1:null

View File

@@ -0,0 +1,71 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
core-util-is@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
inherits@~2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
process-nextick-args@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa"
integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==
readable-stream@~2.3.6:
version "2.3.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.3"
isarray "~1.0.0"
process-nextick-args "~2.0.0"
safe-buffer "~5.1.1"
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.2"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
source-map@^0.5.6:
version "0.5.7"
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
dependencies:
safe-buffer "~5.1.0"
through2@^2.0.1:
version "2.0.5"
resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
dependencies:
readable-stream "~2.3.6"
xtend "~4.0.1"
util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
xtend@~4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68=