diff --git a/lib/index.js b/lib/index.js index ae744d3..83dd7d2 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,4 +1,4 @@ -const Chrome = require('chrome-remote-interface'); +const CDP = require('chrome-remote-interface'); const ChromePage = require('./page'); const fs = require('fs'); const _ = require('underscore') @@ -78,7 +78,7 @@ module.exports = exports = function(options) { ChromeClient.flush = function(fn) { // connect to client - var scopedClient = Chrome({ + CDP({ host: options.host, port: options.port @@ -98,12 +98,22 @@ module.exports = exports = function(options) { // the entries var entries = (result || {}).targetInfos || []; + // how many specials ? + var specialCount = 0; + // close targets for(var i = 0; i < entries.length; i++) { if(entries[i].url.indexOf('chrome://') === 0 || - entries[i].url.indexOf('about://') === 0) - continue; + entries[i].url.indexOf('about://') === 0) { + + // check if open + specialCount++; + + // if special count is 1 or less + if(specialCount == 1) continue; + + } promises.push(Target.closeTarget({ @@ -157,7 +167,7 @@ module.exports = exports = function(options) { var callback = params.callback; // connect to client - var scopedClient = Chrome({ + CDP({ host: options.host, port: options.port, @@ -212,35 +222,50 @@ module.exports = exports = function(options) { })); page.exit = function(cb) { - // close the target - Chrome.Close({ + try { - host: options.host, - port: options.port, - id: targetid + // close the target + client.close(); + fn(null); + cb(null); + return + Chrome.Close({ - }) - .then(function() { + host: options.host, + port: options.port, + id: targetid - fn(null); - if(cb) cb(null); + }) + .then(function() { - }) - .catch(function() { + fn(null); + cb(null); + + }) + .catch(function() { + + fn(null); + cb(null); + + }) + + } catch(err){ fn(null); - if(cb) cb(null); + cb(null); - }) + } }; page.close = function(cb) { // close the page - page.disable(); + page.disable(function() { - // totally close the page and client - page.exit(cb); + // close it + page.exit(cb || function(){}); + + }); }; @@ -253,16 +278,6 @@ module.exports = exports = function(options) { }); - scopedClient.on('error', function(err) { - - // output the error - callback(err); - - // totally close the page and client - page.exit(); - - }); - }; /** @@ -280,78 +295,12 @@ module.exports = exports = function(options) { }; - /** - - // connect to client - var scopedClient = Chrome({ - - host: options.host, - port: options.port - - }, (client) => { - - // remove the notification - process.removeListener('unhandledRejection', rejectionHandler) - - // check if we got a client ... ? - if(!client) { - - // return error - fn(new Error('Problem connecting to ' + options.host + ':' + options.port)) - - // nope.. - return - - } - - // extract what we want - const {Target} = client; - - // create a target - Target.createTarget({ - - url: 'about:blank' - - }) - .then((result)=>{ - - // then close it - client.close() - - // great pass this along - ChromeClient.handlePageRequest(result.targetId, params, fn); - - }) - .catch((err)=>{ - - // then close it - client.close() - - // return it - callback(err); - - // finish up - return fn(); - - }) - - }); - - // handle any errors - scopedClient.on('error', function (err){ - - // return the error - fn(err); - - }); - **/ - // there seems to be a issue in the chrome-remote-interface library // where a rejection error is not caught process.once('unhandledRejection', rejectionHandler); // create a new target - Chrome.New({ + CDP.New({ host: options.host, port: options.port, diff --git a/lib/page.js b/lib/page.js index f042787..f63ab0b 100644 --- a/lib/page.js +++ b/lib/page.js @@ -1,6 +1,7 @@ const _ = require('underscore') const async = require('async'); const hexRgb = require('hex-rgb') +const uuid = require('node-uuid'); const EventEmitter = require('events'); module.exports = exports = function(options) { @@ -25,8 +26,9 @@ module.exports = exports = function(options) { frames: [], size: 0, transferSize: 0, - duration: null, + duration: 0, started: null, + documents: [], document: null, timing: null, @@ -37,6 +39,8 @@ module.exports = exports = function(options) { firstRequest: null, lastRequest: null, + loadtime: 0, + result: null, timingKeys: [ @@ -59,7 +63,7 @@ module.exports = exports = function(options) { /** * All done **/ - ChromePage.disable = function() { + ChromePage.disable = function(fn) { // stop all timers if(VARS.timeoutTimer) clearTimeout(VARS.timeoutTimer); @@ -80,7 +84,13 @@ module.exports = exports = function(options) { ]) .then(() => { + + fn(null); + }).catch((err) => { + + fn(null); + }); }; @@ -144,20 +154,85 @@ module.exports = exports = function(options) { Network.requestWillBeSent((params) => { VARS.requests[params.requestId] = {} + VARS.requests[params.requestId].type = params.type; VARS.requests[params.requestId].id = params.requestId; VARS.requests[params.requestId].request = params.request; VARS.requests[params.requestId].size = 0; + VARS.requests[params.requestId].responseLength = 0; VARS.requests[params.requestId].transferSize = 0; VARS.requests[params.requestId].initiator = params.initiator - VARS.requests[params.requestId].created = params.wallTime; + VARS.requests[params.requestId].created = new Date().getTime(); VARS.requestCount += 1 + // check if we did not have the doc + if(!VARS.document) { + + if(params.type == 'Document' && + (params.initiator || {}).type == 'script') { + + // add it to the list + VARS.documents.push({ + + type: 'client', + url: params.request.url + + }) + + } else if(params.type == 'Document' && + (params.initiator || {}).type == 'other') { + + // add it to the list + VARS.documents.push({ + + type: 'server', + url: params.request.url + + }) + + } + + } + }); // Disabled as we don't need it for now - // Network.dataReceived((params) => { }); + Network.dataReceived(({requestId, dataLength}) => { + const entry = VARS.requests[requestId]; + if (!entry) { + return; + } + entry.responseLength += dataLength; + }); + + Network.loadingFinished(async ({requestId, timestamp, encodedDataLength}) => { + const entry = VARS.requests[requestId]; + if (!entry) { + return; + } + + try { + + const params = await Network.getResponseBody({requestId}); + const {body, base64Encoded} = params; + entry.responseBody = body; + entry.responseBodyIsBase64 = base64Encoded; + + } catch (err) { + + // reject(err); + + } - Network.responseReceived((params) => { + VARS.size += encodedDataLength || 0; + entry.size += encodedDataLength || 0; + + entry.resolved = new Date().getTime(); + entry.duration = entry.resolved - entry.created; + VARS.duration = (VARS.duration || 0) + entry.duration; + + }); + + Network.responseReceived(function(params) { // the resource var resource = VARS.requests[params.requestId]; @@ -170,29 +245,23 @@ module.exports = exports = function(options) { size = parseFloat(params.response.headers['content-length'] || 0) - } catch(err) { - - // use the encoded value + } catch(err) {} - } - - VARS.size += size || 0; - resource.size += size || 0; + VARS.size += resource.responseLength || size; + resource.size += resource.responseLength || size; VARS.transferSize += params.response.encodedDataLength || 0; resource.transferSize += params.response.encodedDataLength || 0; if(params.type == 'Document' && (resource.initiator || {}).type == 'other' && - (params.response || {}).status == 200) { + (params.response || {}).status == 200 && + !VARS.document) { // set the document VARS.document = resource; } - - resource.resolved = params.timestamp; - resource.duration = resource.resolved - resource.created; // check if this response has timing information if((params.response || {}).timing) { @@ -221,6 +290,7 @@ module.exports = exports = function(options) { } }); + Page.loadEventFired(() => { // stop capturing steps @@ -394,6 +464,7 @@ module.exports = exports = function(options) { promises.push(Network.enable()) promises.push(Page.enable()) promises.push(Console.enable()) + promises.push(Emulation.clearDeviceMetricsOverride()); if(options.disableCache === true) { promises.push(Network.setCacheDisabled({ @@ -409,51 +480,32 @@ module.exports = exports = function(options) { })) } - if(options.bypassServiceWorkers === true) { - promises.push(Network.setBypassServiceWorker({ + // should we override the user agent ? + if(options.userAgent) { + promises.push(Network.setUserAgentOverride({ - bypass: true + userAgent: options.userAgent })) } - promises.push(ChromePage.emulate(options)) - - /*if(options.width || - options.height || - options.scale || - options.mobile != false) { - - var metricConfig = { - - width: options.width || 0, - height: options.height || 0, - mobile: options.mobile === true, - fitWindow: options.fitWindow === true - - }; - - // set the scale - metricConfig.deviceScaleFactor = 1.0 * options.scale; - if(metricConfig.deviceScaleFactor === NaN || - metricConfig.deviceScaleFactor === null || - metricConfig.deviceScaleFactor === undefined) { - - metricConfig.deviceScaleFactor = 1; - - } - metricConfig.deviceScaleFactor = parseFloat(metricConfig.deviceScaleFactor) + // set the headers + if(options.headers) { + promises.push(Network.setExtraHTTPHeaders({ - if(options.orientation) { + headers: options.headers || {} - // set it - metricConfig.screenOrientation = options.orientation; + })) + } - } + // should we allow service workers ... ? + if(options.bypassServiceWorkers === true) { + promises.push(Network.setBypassServiceWorker({ - promises.push(Emulation.setDeviceMetricsOverride(metricConfig)) + bypass: true - }*/ + })) + } if(options.background) { @@ -474,16 +526,6 @@ module.exports = exports = function(options) { } - /* if(options.media) { - - promises.push(Emulation.setEmulatedMedia({ - - media: options.media - - })) - - } */ - if(options.downloadRate || options.uploadRate || options.latency || @@ -526,16 +568,6 @@ module.exports = exports = function(options) { })) - if(options.headers) { - - promises.push(Network.setExtraHTTPHeaders({ - - headers: options.headers - - })) - - } - if(options.userAgent) { promises.push(Network.setUserAgentOverride({ @@ -596,7 +628,11 @@ module.exports = exports = function(options) { return Page.navigate(navigateOptions); }).catch((err) => { - + + // output the error + console.dir(err) + + // close it all ChromePage.close(); }); @@ -629,7 +665,7 @@ module.exports = exports = function(options) { }) .then((params)=>{ - fn(null, params.exceptionDetails || null, (params.result || {}).value); + fn(params.exceptionDetails || null, (params.result || {}).value); }) .catch((err)=>{ @@ -650,7 +686,45 @@ module.exports = exports = function(options) { /** * Returns the content of the page **/ - ChromePage.getContent = function(fn) {}; + ChromePage.getContent = function(rendered, fn) { + + // should we return the server side content ? + if(rendered === false) { + + // get the document HAR item + return ChromePage.getDocument(function(err, document) { + + // did we get the document ? + if(err) { + + // stop here + return fn(new Error('Something went wrong while trying to get the document content for #getContent')) + + } + + // did we get the document ? + if(!document) { + + // stop here + return fn(new Error('Problem getting the document, no document was returned ...')) + + } + + // done + return fn(null, document); + + }); + + } + + // just return the messages we have been tracking + return ChromePage.exec(function() { + + return ((document || {}).documentElement || {}).outerHTML || ''; + + }, {}, fn); + + }; /** * Returns the memory usage from the page @@ -660,7 +734,21 @@ module.exports = exports = function(options) { // just return the messages we have been tracking return ChromePage.exec(function() { - return ((window.performance || {}).memory || {}).usedJSHeapSize; + return ((window.performance || {}).memory || {}).usedJSHeapSize || 0; + + }, {}, fn); + + }; + + /** + * Returns the title of the page + **/ + ChromePage.getTitle = function(fn) { + + // just return the messages we have been tracking + return ChromePage.exec(function() { + + return document.title; }, {}, fn); @@ -674,7 +762,7 @@ module.exports = exports = function(options) { // just return the messages we have been tracking return ChromePage.exec(function() { - return document.location; + return document.location.toString(); }, {}, fn); @@ -744,13 +832,23 @@ module.exports = exports = function(options) { }; + /** + * Returns a screenshot + **/ + ChromePage.getDocuments = function(fn) { + + // send it back + fn(null, VARS.documents || []); + + }; + /** * Returns a screenshot **/ ChromePage.getDocument = function(fn) { - // return the document - fn(null, VARS.document || null); + // send it back + ChromePage.getHarItem((VARS.document || {}).id || null, fn) }; @@ -856,85 +954,81 @@ module.exports = exports = function(options) { **/ ChromePage.emulate = function(params, fn) { - // check if the any emulation information was given - if(params.width || - params.height || - params.mobile || - params.orientation || - params.fitWindow) { + // add to clear any other emulation + return Emulation.clearDeviceMetricsOverride() + .then(()=>{ - // add to clear any other emulation - return Emulation.clearDeviceMetricsOverride() - .then(()=>{ + var promises = []; - var promises = []; + // set the metrics + var metricConfig = { - // set the metrics - var metricConfig = { + width: params.width || options.width || 1280, + height: params.height || options.height || 800, + mobile: params.mobile === true || options.mobile === true, + fitWindow: true - width: options.width || 0, - height: options.height || 0, - mobile: options.mobile === true, - fitWindow: options.fitWindow === true + }; - }; + // set the scale + metricConfig.deviceScaleFactor = 1.0 * (params.scale || options.scale); + if(metricConfig.deviceScaleFactor === NaN || + metricConfig.deviceScaleFactor === null || + metricConfig.deviceScaleFactor === undefined) { - // set the scale - metricConfig.deviceScaleFactor = 1.0 * options.scale; - if(metricConfig.deviceScaleFactor === NaN || - metricConfig.deviceScaleFactor === null || - metricConfig.deviceScaleFactor === undefined) { + metricConfig.deviceScaleFactor = 1; - metricConfig.deviceScaleFactor = 1; + } + metricConfig.deviceScaleFactor = parseFloat(metricConfig.deviceScaleFactor) - } - metricConfig.deviceScaleFactor = parseFloat(metricConfig.deviceScaleFactor) + if(options.orientation) { - if(options.orientation) { + // set it + metricConfig.screenOrientation = options.orientation; - // set it - metricConfig.screenOrientation = options.orientation; + } - } + promises.push(Emulation.setVisibleSize({ - promises.push(Emulation.setDeviceMetricsOverride(metricConfig)) + width: params.width || options.width || 1280, + height: params.height || options.height || 800 - if(promises.media) { + })); - promises.push(Emulation.setEmulatedMedia({ + promises.push(Emulation.setDeviceMetricsOverride(metricConfig)) - media: promises.media + if(params.media) { - })) + promises.push(Emulation.setEmulatedMedia({ - } + media: params.media - return Promises.all(promises) - .catch((err)=>{ + })) - // done - if(fn) fn(err); + } - }) + if(promises.length == 0) return fn(null); - }) + return Promises.all(promises) .then(()=>{ - // done - if(fn) fn(null); + fn(null); }) .catch((err)=>{ // done - if(fn) fn(err); + fn(err); }) - } + }) + .catch((err)=>{ - // done - if(fn) fn(null); + // done + fn(err); + + }) }; @@ -943,10 +1037,11 @@ module.exports = exports = function(options) { **/ ChromePage.end = function(params) { - if(VARS.result) return; + // skip first end + if(!params.result) return; // set the result - VARS.result = params.result; + VARS.result = params.result || 'error'; // finish up ChromePage.loaded() @@ -983,12 +1078,24 @@ module.exports = exports = function(options) { var resource = VARS.requests[requestId] || null; // check the resource - if(!resource) return fn(null, null); + if(!resource) return fn(null); // set local variables var response = resource.response; var request = resource.request; + // check the resource + if(!request) { + + return fn(null); + + } + if(!response) { + + return fn(null); + + } + // the entry to return var entry = { @@ -1018,10 +1125,20 @@ module.exports = exports = function(options) { "content": {}, "redirectURL": "", "headersSize" : 160, - "bodySize" : 850, - "comment" : "" + "bodySize" : resource.size, + "comment" : "", + "content": { + + "size": resource.responseLength, + "mimeType": '', + "compression": response.compression, + "text": '', + "encoding": '' + + } }, + "timing": response.timing, "cache": {}, "timings": response.timing, "serverIPAddress": response.remoteIPAddress, @@ -1030,26 +1147,8 @@ module.exports = exports = function(options) { }; - // get the resources - Network.getResponseBody({ - - requestId: requestId - - }) - .then((result) => { - - entry.response.content.text = result.body; - if(result.base64Encoded === true) - entry.response.content.encoding = 'base64'; - entry.response.content.mimeType = response.mimeType; - fn(null, entry); - - }) - .catch((err)=>{ - - fn(err); - - }) + // done + fn(null, entry) }; @@ -1113,7 +1212,10 @@ module.exports = exports = function(options) { ChromePage.render = function(params, fn) { // first try to emulate if any - ChromePage.emulate(params, ()=>{ + ChromePage.emulate(params, function(){ + + // create the callback + var callback = _.once(fn); // righto check format if(params.format === 'pdf') { @@ -1122,12 +1224,12 @@ module.exports = exports = function(options) { Page.printToPDF() .then((result)=>{ - fn(null, new Buffer(result.data, 'base64')) + callback(null, new Buffer(result.data, 'base64')) }) .catch((err)=>{ - fn(err) + callback(err) }) @@ -1137,12 +1239,12 @@ module.exports = exports = function(options) { Page.captureScreenshot('png') .then((result)=>{ - fn(null, new Buffer(result.data, 'base64')) + callback(null, new Buffer(result.data, 'base64')) }) .catch((err)=>{ - fn(err) + callback(err) }) @@ -1152,12 +1254,14 @@ module.exports = exports = function(options) { Page.captureScreenshot('jpeg', params.quality || 30) .then((result)=>{ - fn(null, new Buffer(result.data, 'base64')) + callback(null, new Buffer(result.data, 'base64')) }) .catch((err)=>{ - fn(err) + console.dir(err) + + callback(err) }) @@ -1182,7 +1286,8 @@ module.exports = exports = function(options) { var step = { delay: new Date().getTime() - VARS.started.getTime(), - index: VARS.currentStep + index: VARS.currentStep, + uid: uuid.v1() }; diff --git a/package.json b/package.json index d4c252d..579bc54 100644 --- a/package.json +++ b/package.json @@ -32,12 +32,13 @@ "author": "Passmarked ", "license": "Apache-2.0", "dependencies": { - "async": "^2.4.0", - "chrome-remote-interface": "^0.22.0", + "async": "^2.5.0", + "chrome-remote-interface": "^0.24.3", "hex-rgb": "^1.0.0", + "node-uuid": "^1.4.8", "underscore": "^1.8.3" }, "devDependencies": { - "mocha": "^3.3.0" + "mocha": "^3.5.0" } }