From 3ac3d9c5694b9d1b8272e309a4298c85108e0ce6 Mon Sep 17 00:00:00 2001 From: Khang Hoang Date: Sat, 13 Jul 2019 12:49:50 -0700 Subject: [PATCH] Add network tab --- front_end/ndb.json | 28 +++-- front_end/ndb/NdbMain.js | 96 +++++++++++++--- front_end/ndb/module.json | 9 +- lib/preload/ndb/httpMonkeyPatching.js | 153 ++++++++++++++++++++++++++ services/ndd_service.js | 22 +++- 5 files changed, 277 insertions(+), 31 deletions(-) create mode 100644 lib/preload/ndb/httpMonkeyPatching.js diff --git a/front_end/ndb.json b/front_end/ndb.json index bccce14b..79b509ab 100644 --- a/front_end/ndb.json +++ b/front_end/ndb.json @@ -1,15 +1,21 @@ { - "modules" : [ - { "name": "ndb_sdk", "type": "autostart" }, - { "name": "ndb", "type": "autostart" }, - { "name": "layer_viewer" }, - { "name": "timeline_model" }, - { "name": "timeline" }, - { "name": "product_registry" }, - { "name": "mobile_throttling" }, - { "name": "ndb_ui" }, - { "name": "xterm" } - ], + "modules": [ + { "name": "ndb_sdk", "type": "autostart" }, + { "name": "ndb", "type": "autostart" }, + { "name": "layer_viewer" }, + { "name": "timeline_model" }, + { "name": "timeline" }, + { "name": "product_registry" }, + { "name": "mobile_throttling" }, + { "name": "ndb_ui" }, + { "name": "xterm" }, + { "name": "emulation", "type": "autostart" }, + { "name": "inspector_main", "type": "autostart" }, + { "name": "mobile_throttling", "type": "autostart" }, + { "name": "cookie_table" }, + { "name": "har_importer" }, + { "name": "network" } + ], "extends": "shell", "has_html": true } diff --git a/front_end/ndb/NdbMain.js b/front_end/ndb/NdbMain.js index f8204af0..1d96b3b8 100644 --- a/front_end/ndb/NdbMain.js +++ b/front_end/ndb/NdbMain.js @@ -5,14 +5,20 @@ */ Ndb.nodeExecPath = function() { - if (!Ndb._nodeExecPathPromise) - Ndb._nodeExecPathPromise = Ndb.backend.which('node').then(result => result.resolvedPath); + if (!Ndb._nodeExecPathPromise) { + Ndb._nodeExecPathPromise = Ndb.backend + .which('node') + .then(result => result.resolvedPath); + } return Ndb._nodeExecPathPromise; }; Ndb.npmExecPath = function() { - if (!Ndb._npmExecPathPromise) - Ndb._npmExecPathPromise = Ndb.backend.which('npm').then(result => result.resolvedPath); + if (!Ndb._npmExecPathPromise) { + Ndb._npmExecPathPromise = Ndb.backend + .which('npm') + .then(result => result.resolvedPath); + } return Ndb._npmExecPathPromise; }; @@ -27,39 +33,60 @@ Ndb.processInfo = function() { */ Ndb.NdbMain = class extends Common.Object { /** - * @override - */ + * @override + */ async run() { InspectorFrontendAPI.setUseSoftMenu(true); document.title = 'ndb'; - Common.moduleSetting('blackboxInternalScripts').addChangeListener(Ndb.NdbMain._calculateBlackboxState); + Common.moduleSetting('blackboxInternalScripts').addChangeListener( + Ndb.NdbMain._calculateBlackboxState + ); Ndb.NdbMain._calculateBlackboxState(); const setting = Persistence.isolatedFileSystemManager.workspaceFolderExcludePatternSetting(); setting.set(Ndb.NdbMain._defaultExcludePattern().join('|')); - Ndb.nodeProcessManager = await Ndb.NodeProcessManager.create(SDK.targetManager); + Ndb.nodeProcessManager = await Ndb.NodeProcessManager.create( + SDK.targetManager + ); - const {cwd} = await Ndb.processInfo(); + const { cwd } = await Ndb.processInfo(); await Ndb.nodeProcessManager.addFileSystem(cwd); // TODO(ak239): we do not want to create this model for workers, so we need a way to add custom capabilities. - SDK.SDKModel.register(NdbSdk.NodeWorkerModel, SDK.Target.Capability.JS, true); - SDK.SDKModel.register(NdbSdk.NodeRuntimeModel, SDK.Target.Capability.JS, true); + SDK.SDKModel.register( + NdbSdk.NodeWorkerModel, + SDK.Target.Capability.JS, + true + ); + SDK.SDKModel.register( + NdbSdk.NodeRuntimeModel, + SDK.Target.Capability.JS, + true + ); await new Promise(resolve => SDK.initMainConnection(resolve)); - SDK.targetManager.createTarget('', ls`Root`, SDK.Target.Type.Browser, null); + SDK.targetManager.createTarget( + '', + ls`Root`, + SDK.Target.Type.Browser, + null + ); + if (Common.moduleSetting('autoStartMain').get()) { const main = await Ndb.mainConfiguration(); if (main) { - if (main.prof) - await Ndb.nodeProcessManager.profile(main.execPath, main.args); - else - Ndb.nodeProcessManager.debug(main.execPath, main.args); + if (main.prof) { + await Ndb.nodeProcessManager.profile( + main.execPath, + main.args + ); + } else {Ndb.nodeProcessManager.debug(main.execPath, main.args);} } } Ndb.nodeProcessManager.startRepl(); } + static _defaultExcludePattern() { const defaultCommonExcludedFolders = [ '/bower_components/', '/\\.devtools', '/\\.git/', '/\\.sass-cache/', '/\\.hg/', '/\\.idea/', @@ -125,7 +152,6 @@ Ndb.mainConfiguration = async() => { prof }; }; - /** * @implements {UI.ContextMenu.Provider} * @unrestricted @@ -172,9 +198,21 @@ Ndb.NodeProcessManager = class extends Common.Object { static async create(targetManager) { const manager = new Ndb.NodeProcessManager(targetManager); manager._service = await Ndb.backend.createService('ndd_service.js', rpc.handle(manager)); + InspectorFrontendHost.sendMessageToBackend = manager.sendMessageToBackend.bind(manager); + return manager; } + /** + * @param {object} message + * + * @return {Promise} void + */ + async sendMessageToBackend(message) { + if (this._service && this._service.sendMessage) + return this._service.sendMessage(message); + } + env() { return this._service.env(); } @@ -246,6 +284,30 @@ Ndb.NodeProcessManager = class extends Common.Object { } } + sendLoadingFinished({ type, payload }) { + SDK._mainConnection._onMessage(JSON.stringify({ + method: 'Network.loadingFinished', + params: payload + })); + } + + responseToFrontEnd(id, result) { + InspectorFrontendHost.events.dispatchEventToListeners( + InspectorFrontendHostAPI.Events.DispatchMessage, + { + id, + result + } + ); + } + + sendNetworkData({ type, payload }) { + SDK._mainConnection._onMessage(JSON.stringify({ + method: type, + params: payload + })); + } + async terminalData(stream, data) { const content = await(await fetch(`data:application/octet-stream;base64,${data}`)).text(); if (content.startsWith('Debugger listening on') || content.startsWith('Debugger attached.') || content.startsWith('Waiting for the debugger to disconnect...')) diff --git a/front_end/ndb/module.json b/front_end/ndb/module.json index 4113c13c..baff7549 100644 --- a/front_end/ndb/module.json +++ b/front_end/ndb/module.json @@ -52,7 +52,14 @@ "className": "Ndb.ContextMenuProvider" } ], - "dependencies": ["common", "sdk", "ndb_sdk", "bindings", "persistence", "components"], + "dependencies": [ + "common", + "sdk", + "ndb_sdk", + "bindings", + "persistence", + "components" + ], "scripts": [ "InspectorFrontendHostOverrides.js", "Connection.js", diff --git a/lib/preload/ndb/httpMonkeyPatching.js b/lib/preload/ndb/httpMonkeyPatching.js new file mode 100644 index 00000000..930bca1f --- /dev/null +++ b/lib/preload/ndb/httpMonkeyPatching.js @@ -0,0 +1,153 @@ +const zlib = require('zlib'); +const http = require('http'); +const https = require('https'); + +const initTime = process.hrtime(); + +// DT requires us to use relative time in a strange format (xxx.xxx) +const getTime = () => { + const diff = process.hrtime(initTime); + + return diff[0] + diff[1] / 1e9; +}; + +const formatRequestHeaders = req => { + if (!req.headers) return {}; + return Object.keys(req.headers).reduce((acc, k) => { + if (typeof req.headers[k] === 'string') acc[k] = req.headers[k]; + return acc; + }, {}); +}; + +const formatResponseHeaders = res => { + if (!res.headers) return {}; + return Object.keys(res.headers).reduce((acc, k) => { + if (typeof res.headers[k] === 'string') acc[k] = res.headers[k]; + return acc; + }, {}); +}; + +const getMineType = mimeType => { + // nasty hack for ASF + if (mimeType === 'OPENJSON') + return 'application/json;charset=UTF-8'; + + + return mimeType; +}; + +const cacheRequests = {}; +let id = 1; +const getId = () => id++; + +const callbackWrapper = (callback, req) => res => { + const requestId = getId(); + res.req.__requestId = requestId; + + process.send({ + payload: { + requestId: requestId, + loaderId: requestId, + documentURL: req.href, + request: { + url: req.href, + method: req.method, + headers: formatRequestHeaders(req), + mixedContentType: 'none', + initialPriority: 'VeryHigh', + referrerPolicy: 'no-referrer-when-downgrade', + postData: req.body + }, + timestamp: getTime(), + wallTime: Date.now(), + initiator: { + type: 'other' + }, + type: 'Document' + }, + type: 'Network.requestWillBeSent' + }); + + const encoding = res.headers['content-encoding']; + let rawData = []; + + const onEnd = function() { + rawData = Buffer.concat(rawData); + rawData = rawData.toString('base64'); + + cacheRequests[res.req.__requestId] = { + ...res, + __rawData: rawData, + base64Encoded: true + }; + const payload = { + id: res.req.__requestId, + requestId: res.req.__requestId, + loaderId: res.req.__requestId, + base64Encoded: true, + data: cacheRequests[res.req.__requestId].__rawData, + timestamp: getTime(), + type: 'XHR', + encodedDataLength: 100, + response: { + url: req.href, + status: res.statusCode, + statusText: res.statusText, + // set-cookie prop in the header has value as an array + // for example: ["__cfduid=dbfe006ef71658bf4dba321343c227f9a15449556…20:29 GMT; path=/; domain=.typicode.com; HttpOnly"] + headers: formatResponseHeaders(res), + mimeType: getMineType( + res.headers['content-encoding'] || + res.headers['content-type'] + ), + requestHeaders: formatRequestHeaders(req) + } + }; + + // Send the response back. + process.send({ payload: payload, type: 'Network.responseReceived' }); + process.send({ payload: payload, type: 'Network.loadingFinished' }); + }; + + if (encoding === 'gzip' || encoding === 'x-gzip') { + const gunzip = zlib.createGunzip(); + res.pipe(gunzip); + + gunzip.on('data', function(data) { + rawData.push(data); + }); + gunzip.on('end', onEnd); + } else { + res.on('data', chunk => { + rawData.push(chunk); + }); + res.on('end', onEnd); + } + + callback && callback(res); +}; + +const originHTTPRequest = http.request; +http.request = function wrapMethodRequest(req, callback) { + const request = originHTTPRequest.call( + this, + req, + callbackWrapper(callback, req) + ); + return request; +}; + +const originHTTPSRequest = https.request; +https.request = function wrapMethodRequest(req, callback) { + const request = originHTTPSRequest.call( + this, + req, + callbackWrapper(callback, req) + ); + const originWrite = request.write.bind(request); + request.write = data => { + req.body = data.toString(); + originWrite(data); + }; + return request; +}; diff --git a/services/ndd_service.js b/services/ndd_service.js index 41beb06f..bfdb5f30 100644 --- a/services/ndd_service.js +++ b/services/ndd_service.js @@ -23,6 +23,8 @@ function silentRpcErrors(error) { process.on('uncaughtException', silentRpcErrors); process.on('unhandledRejection', silentRpcErrors); +const catchedRequests = {}; + const DebugState = { WS_OPEN: 1, WS_ERROR: 2, @@ -134,24 +136,40 @@ class NddService { } env() { + const pathToHttpPatch = path.resolve(__dirname, '..', './lib/preload/ndb/httpMonkeyPatching.js'); + return { - NODE_OPTIONS: `--require ndb/preload.js`, + NODE_OPTIONS: `--require ndb/preload.js --require ${pathToHttpPatch}`, NODE_PATH: `${process.env.NODE_PATH || ''}${path.delimiter}${path.join(__dirname, '..', 'lib', 'preload')}`, NDD_IPC: this._pipe }; } + sendMessage(rawMessage) { + const message = JSON.parse(rawMessage); + // send message to frontend directly + // (eg: getResponseBody) + const { base64Encoded, data } = catchedRequests[message.params.requestId]; + this._frontend.responseToFrontEnd(message.id, { base64Encoded, body: data }); + } + async debug(execPath, args, options) { const env = this.env(); if (options.data) env.NDD_DATA = options.data; + const p = spawn(execPath, args, { cwd: options.cwd, env: { ...process.env, ...env }, - stdio: options.ignoreOutput ? 'ignore' : ['inherit', 'pipe', 'pipe'], + stdio: options.ignoreOutput ? ['ignore', 'ignore', 'ignore', 'ipc'] : ['pipe', 'pipe', 'pipe', 'ipc'], windowsHide: true }); if (!options.ignoreOutput) { + p.on('message', ({ type, payload }) => { + if (!(type && payload)) return; + catchedRequests[payload.id] = payload; + this._frontend.sendNetworkData({ type, payload }); + }); p.stderr.on('data', data => { if (process.connected) this._frontend.terminalData('stderr', data.toString('base64'));