axemclion/perfjankie

Checking browser rendering performance regression

perfjankie's Issues

Getting error "The environment you requested was unavailable"

I am trying to run perfJankie and getting this error every time.

Code:

var login = function(){
return browser.get('http://localhost:8081').then(function(){
return browser.setLocalStorageKey('jarvisMock','true')
});
}

perfjankie({
suite: 'Performance analysis',
name: "Test",
run: "1.0",
preScript: function(browser) {
return login();
},
time: new Date().getTime(),
callback: function (err, res) {
if (err) {
throw err;
}
runQueue(i + 1);
},
repeat: 1,
selenium: 'http://localhost:4444/wd/hub',
couch: COUCH,
browsers: [{
browserName: 'chrome',
chromeOptions: {
args: ['--start-maximized',"--enable-gpu-benchmarking"],
}
}],
metrics : ['TimelineMetrics','NetworkResources', 'NetworkTimings', 'RafRenderingStats','ChromeTracingMetrics'],
actions: function (browser) {
if (job.component === 'HomePageRefesh') {
return browser.execute('chrome.gpuBenchmarking.smoothScrollBy(2000, function(){})').then(function(){
return browser.sleep(5000).then(function(){
return browser.refresh();
});
});
}
if (job.component === 'UpdatePageRefresh') {
return browser.get(job.url + '/index.html#/auth/updates/checkForUpdate').then(function(){
return browser.sleep(3000).then(function(){
return browser.execute('chrome.gpuBenchmarking.smoothScrollBy(2000, function(){})').then(function(){
return browser.sleep(5000).then(function(){
return browser.refresh();
});
});
});
});
}
}
});

Error logs:

Running "shell:perfjankie" (shell) task
selenium standalone is up to date.
chromedriver is up to date.
C:\XXX\node_modules\webdriver-manager\selenium\chromedriver_2.16.zip C:\XXX\node_modules\webdriver-manager\selenium
The following files were installed: C:\XXX\node_modules\webdriver-manager\selenium\selenium-server-standalone-2.46.0.jar C:\XXX\node_modules\webdriver-manager\selenium\chromedriver
selenium.pid: 832
Running [1/2] HomePageRefesh@923328
Wed, 20 Jan 2016 14:04:54 GMT bp:index Selenium is on localhost
Wed, 20 Jan 2016 14:04:54 GMT bp:metrics Initializing Metrics
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe PerfLogProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes:AndroidTracingProbe Initialize
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe AndroidTracingProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe NetworkResourcesProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe NavTimingProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe RafBenchmarkingProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe PerfLogProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:probes Registering probe AndroidTracingProbe
Wed, 20 Jan 2016 14:04:54 GMT bp:metrics TimelineMetrics setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:metrics NetworkResources setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:metrics ChromeTracingMetrics setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:probes setup
Wed, 20 Jan 2016 14:04:54 GMT bp:probes PerfLogProbe setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:probes AndroidTracingProbe setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:probes NetworkResourcesProbe setup called
Wed, 20 Jan 2016 14:04:54 GMT bp:index Stating browser with {"browserName":"chrome","chromeOptions":{"args":["--start-maximized","--enable-gpu-benchmarking"],"perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame,benchmark"}},"loggingPrefs":{"performance":"ALL"}}
Wed, 20 Jan 2016 14:04:54 GMT bp:selenium CALL init({"browserName":"chrome","chromeOptions":{"args":["--start-maximiz
Wed, 20 Jan 2016 14:04:56 GMT bp:selenium CALL quit()

C:\XXX\node_modules\perfjankie\node_modules\q\q.js:155
throw e;
^
Error: [init({"browserName":"chrome","chromeOptions":{"args":["--start-maximized","--enable-gpu-benchmarking"],"perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame,benchmark"}},"loggingPrefs":{"performance":"ALL"}})] The environment you requested was unavailable.

Have anyone faced this issue ?

Left menu always redirect to the homepage

Hello!
I can access locally dashboard and see statistics pushed there by perfjankie but links in left menu always redirect to the home/intro page.

I have exactly the same issue even in official demo:
http://nparashuram.com/perfslides/perfjankie/#/page-select

Only one link: All metrics works good.

I attached simple gif which shows this bug

Chrome 39.0.2171.71 (64-bit)
OS X 10.10.1

Project looks promising. It'd be great to get beauty stats from browser-perf with perfjankie help.

browsers firefox isnt working

When im using this local.config.json:
{
"browsers": [{
"browserName": "firefox"
}],
"selenium": {
"hostname": "localhost",
"port": 4444
},
"couch": {
"server": "http://localhost:5984",
"username": "name",
"pwd": "password",
"database": "autotest"
}
}
And run first
perfjankie -c local.config.json --only-update-site
then
perfjankie --config-file=local.config.json --name nametest --suite namesuite
Then check:
http://localhost:5984/autotest/_design/site/index.html#/page-select
and click on nametest it says:
Error loading page

An error occured when trying to load this page.
Please refresh this page, or go back to the home page .

Until i remove the last session out of my database i cant see my other results from chrome by example, i havent checked internet explorer.

Comparison

Currently i'm using this to research different existing approaches to the markup and css of different components as preparation to creating a new css framework. It would be really nice if there was a comparison view where I could compare the results of different tests. perhaps a table + graph with all of the selected pages plotted?

preScript example problem

In the follow url:
https://github.com/axemclion/browser-perf/wiki/FAQ#how-can-i-test-a-page-that-requires-login
prescript should be preScript,
Is there any way to warn user because it is difficult to discover those issues.

URLs with parameters not working

If I use an url with parameters the report site is not working, e.g https://www.google.de/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=angular

Jenkins integration

Hi there,

First, thanks for this - it's a really useful tool! Second, I was wondering if there's any work being done with CI integration. For instance, can we pull results into a JUnit style XML result file for Jenkins jobs that use perfjankie?

perfjankie could't keep browser alive

in the perfTest.js :
debug: config.debug, ==>debugBrowser: config.debug

Query: Basic setup

Could you pls provide the basic setup details. I have installed perfkankie as node module and couchdb. With just url and couchdb configuration, its showing following error. Can you pls provide details how to setup it with minimal confirguration on localhost to check howit works.

ERROR:
[{
[Error: [quit()] connect ECONNREFUSED]
code: 'ECONNREFUSED',
errno: 'ECONNREFUSED',
syscall: 'connect'
}]

RESPONSE: null

Frame rate graph not working

Have been working with this for some jQuery projects and the frame rate graph does not seem to be working. http://104.236.81.132:5984/css-perf/_design/site/index.html#/page-select

Layout and Recalculate styles not working

Graphs do not work on some pages / runs but do on others no differences in the runs or output not errors.

No data displayed while trying to work a sample perfjankie code

I am trying to run the following code :

var perfjankie = require('../AppData/Roaming/npm/node_modules/perfjankie');
perfjankie({
"url": "http://localhost:3033/ui", // URL of the page that you would like to test.

/* The next set of values identify the test */
name: "Component or Webpage Name", // A friendly name for the URL. This is shown as component name in the dashboard
suite: "optional suite name", // Displayed as the title in the dashboard. Only 1 suite name for all components
time: new Date().getTime(), // Used to sort the data when displaying graph. Can be the time when a commit was made
run: "commit#Hash", // A hash for the commit, displayed in the x-axis in the dashboard
repeat: 3, // Run the tests 3 times. Default is 1 time

/* Identifies where the data and the dashboard are saved */
couch: {
server: 'http://localhost:5984',
database: 'performance',
updateSite: !process.env.CI, // If true, updates the couchApp that shows the dashboard. Set to false in when running Continuous integration, run this the first time using command line.
onlyUpdateSite: true // No data to upload, just update the site. Recommended to do from dev box as couchDB instance may require special access to create views.
},

callback: function(err, res) {
console.log(err);
console.log(res);
// The callback function, err is falsy if all of the following happen
// 1. Browsers perf tests ran
// 2. Data has been saved in couchDB
// err is not falsy even if update site fails.
},

/* OPTIONS PASSED TO BROWSER-PERF */
// Properties identifying the test environment */
browsers: ["chrome", "firefox" ], // This can also be a ["chrome", "firefox"] or "chrome,firefox"// See browser perf browser configuration for all options.

selenium: {
hostname: 'http://localhost:4444/wd/hub', // or localhost or hub.browserstack.com
//port: 4444,
},
});

---

When I log the result on the console in the above code I am getting something like

[ { state: 'fulfilled', value: undefined },
{ state: 'fulfilled', value: undefined } ]

---

I am unable to see any graphical interface (Metric data) on the following URL upon running the above code:

http://localhost:5984/performance/_design/site/index.html#/page-select

---

Please let me know if I am missing something or may be doing things in a wrong way that is not allowing perfjankie to display the data on the console and at the same time on the GUI.

Make setup easier

All of the dependencies of this ( except couchdb ) are available as NPM packages ( and or grunt modules ). It would be nice to make these dependencies to ease setup to just an NPM install plus downloading the right couchdb binary and clicking it, for basic local setup. You can see an example here https://github.com/jquery/css-chassis/pull/34/files. Would be happy to do a PR if your interested.

Logging in a user before the tests are run

How can i pass the actions.login object to perfjankie from browser-perf in the grunt task? I want to login a user before the tests are run.

Multiple data points for same run name is plotted as the same point

If the same run label is used at different times, they are plotted along the same x axis.

Using perfjankie with browser-perf runner

Hello,

is it possible to use perfjankie with browser-perf runner? I have Electron application and Spectron E2E tests for that. I was able to use browser-perf runner for connecting to the Chromedriver instance created by Spectron and gather some performance data. Is it possible to use perfjankie in a same way or is the only possibility to let it create a new browser-perf instance along the initialization of perfjankie?

The issue seems to be that it's not possible to pass session ID to the browser-perf instance created by perfjankie. Also it would be great if there was a possibility to pass an existing browser-perf runner instance to perfjankie.

Y Axis label does not show up

Missing data points from browser-perf should still show up on the charts

Make change / select dropdowns rather then buttons

When using this for comparison it would be nice if you could easily switch test pages when your already within a test if the change and or select buttons were made to be drop downs with a list of the available pages this would be a lot nicer.

running perfjankie for first time doesn't get passed "Starting PerfJankie" message.

set up perfjankie for first time and running the sample script provided in readme, I am not able to get passed "Starting PerfJankie" log message written to console which is present in line 33 of ./lib/index.js.

Its as if it completed executing the script and nothing happens after the message is printed.

Error starting perfjankie

I'm trying to use perfjankie on OS X 10.11 and the task Browser Perf is not working, it's just starting and never finishes:

Running tasks: perfjankie:local

Running "perfjankie:local" (perfjankie) task
Verifying property perfjankie.local exists in config...OK
File: [no files]
Options: log={}, time=1463135076042, selenium="http://localhost:5555/wd/hub", browsers=["firefox","chrome"], couch={"server":"https://.../couchdb/","requestOptions":{"auth":{"user":"...","pass":".."}},"username":"..","pwd":"...","database":"perfjankie","updateSite":true}, suite="Project A", urls=["http://localhost:41180/"], preScript=undefined
Testing File  http://localhost:41180/
>> Starting PerfJankie
Trying to see if the database exists
Starting Browser Perf

Consider making title configurable

It seems like right now the test page title is just the file name from the url. This causes several potential issues.

1.) A page which contains a query string breaks results. I tried to use a page like component.html?component=button&framework=bootstrap&count=1000 to have a pages which generates the pages to be tested. however because the results are displayed as the filename nad used in a query string to show results you end up with something like

http://104.236.81.132:5984/css-perf/_design/site/index.html#/summary?pagename=component.html?component=button&framework=bootstrap&count=1000&browser=chrome which is not valid and just displays an error

2.) it can make for ugly names and organization when you have a lot of tests your running for comparison of individual components.

For now to work around this im using urls like http://localhost/component/button/framework/bootstrap/count/1000/bootstrap:button
to work around this but it requires all the urls to be re-written. The easy solution seems like it would be to just use the actual page title instead of the file from the url?

Add a summary page for the results

There are too many graphs displayed, but it would be good to have a summary page

Running Perf Jankie launches Chrome.exe

When i try to run Perf jankie, it launches chrome browser along with chrome.exe.
And the page which i am trying to test is not loaded.

Reg: Perfjankie usage

Hi @axemclion could you please help me in setting up perfjankie .. I am able to run browser-perf and get the metrics but I am not able to understand perfjankie usage and how to start with? I have angular appplication running in my local and I am interested to see the performance metrics in graphical format.

Add description/explanation to each metric

• Add descriptions to each metric
• Group metrics by more meaningful categories
• Make NavMetrics into Area charts
• Make paint events into 100% area charts

Parse HTML data missing

Getting "Could not plot graph due to error in data" no errors or issues while running

Cannot run a tool to profiling on Android / Nexus 7 - results are not saved in couchdb

Hey,
I was able to run browser-perf and connect to android device and get results. Awesome.

When I use perfjankie, results are not saved in couchdb. While page is scrolled, Chrome shows a message on android device which can be translated as:

Cannot run a tool to profiling

When scrolling is finished, nothing happens.
I'm not sure but from my perspective perfjankie waits for a response or an action.
After tens seconds chrome closes on android device and logs are returned to terminal, the bottom of logs:

[27.973][INFO]: RESPONSE GetSessionCapabilities {
   "acceptSslCerts": true,
   "applicationCacheEnabled": false,
   "browserConnectionEnabled": false,
   "browserName": "chrome",
   "chrome": {

   },
   "cssSelectorsEnabled": true,
   "databaseEnabled": false,
   "handlesAlerts": true,
   "javascriptEnabled": true,
   "locationContextEnabled": true,
   "mobileEmulationEnabled": false,
   "nativeEvents": true,
   "platform": "ANDROID",
   "rotatable": false,
   "takesHeapSnapshot": true,
   "takesScreenshot": true,
   "version": "40.0.2214.89",
   "webStorageEnabled": true
}
[148.913][INFO]: COMMAND Quit {

}
[148.913][DEBUG]: Sending adb command: host:transport:015d24a42d442004|shell:am force-stop com.android.chrome
[150.003][DEBUG]: Received adb response: 
[150.003][INFO]: RESPONSE Quit
[150.003][DEBUG]: Log type 'driver' lost 0 entries on destruction
[150.003][DEBUG]: Log type 'performance' lost 0 entries on destruction
[150.003][DEBUG]: Log type 'browser' lost 2 entries on destruction

Full logs: https://www.dropbox.com/s/9sec5zwxn5agkdo/perfjankie-chromedriver-logs-dump.txt?dl=0 (Attention, 2.6mb)

• chromeDriver 2.13.307650
• perfjankie 1.1.2
• Nexus 7, Android 5.0.2 + Chrome 40.0.2214.89

Thanks.

Error running

When I run from the command line browser-perf --verbose --config-file=local.config.json I get this error:

bp:index Selenium is on localhost +0ms
  bp:metrics Initializing Metrics +2ms
  bp:probes Registering probe +66ms PerfLogProbe
  bp:probes:AndroidTracingProbe Initialize +1ms
  bp:probes Registering probe +0ms AndroidTracingProbe
  bp:probes Registering probe +2ms PerfLogProbe
  bp:probes Registering probe +0ms AndroidTracingProbe
  bp:probes Registering probe +1ms RafBenchmarkingProbe
  bp:probes Registering probe +2ms NavTimingProbe
  bp:probes Registering probe +1ms NetworkResourcesProbe
  bp:metrics TimelineMetrics +2ms setup called
  bp:metrics ChromeTracingMetrics +0ms setup called
  bp:metrics NetworkResources +0ms setup called
  bp:probes setup +1ms
  bp:probes PerfLogProbe +0ms setup called
  bp:probes AndroidTracingProbe +0ms setup called
  bp:probes NetworkResourcesProbe +0ms setup called
  bp:index Stating browser with +1ms {"browserName":"android","platform":"OS X","chromeOptions":{"androidActivity":"com.ionicframework.clientapp176148.MainActivity","androidPackage":"com.ionicframework.clientapp176148","perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame"},"args":["--enable-gpu-benchmarking","--enable-thread-composting"]},"loggingPrefs":{"performance":"ALL"}}
  bp:selenium CALL +0ms init({"browserName":"android","platform":"OS X","chromeOptions":{"andr
  bp:selenium CALL +19ms quit()
[ { [Error: [init({"browserName":"android","platform":"OS X","chromeOptions":{"androidActivity":"com.ionicframework.clientapp176148.MainActivity","androidPackage":"com.ionicframework.clientapp176148","perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame"},"args":["--enable-gpu-benchmarking","--enable-thread-composting"]},"loggingPrefs":{"performance":"ALL"}})] The environment you requested was unavailable.] data: 'unknown command: wd/hub/session' } ]
Error: [init({"browserName":"android","platform":"OS X","chromeOptions":{"androidActivity":"com.ionicframework.clientapp176148.MainActivity","androidPackage":"com.ionicframework.clientapp176148","perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame"},"args":["--enable-gpu-benchmarking","--enable-thread-composting"]},"loggingPrefs":{"performance":"ALL"}})] The environment you requested was unavailable.
    at /usr/local/lib/node_modules/browser-perf/node_modules/wd/lib/webdriver.js:129:15
    at Request._callback (/usr/local/lib/node_modules/browser-perf/node_modules/wd/lib/http-utils.js:87:7)
    at Request.self.callback (/usr/local/lib/node_modules/browser-perf/node_modules/wd/node_modules/request/request.js:368:22)
    at emitTwo (events.js:100:13)
    at Request.emit (events.js:185:7)
    at Request.<anonymous> (/usr/local/lib/node_modules/browser-perf/node_modules/wd/node_modules/request/request.js:1219:14)
    at emitOne (events.js:95:20)
    at Request.emit (events.js:182:7)
    at IncomingMessage.<anonymous> (/usr/local/lib/node_modules/browser-perf/node_modules/wd/node_modules/request/request.js:1167:12)
    at emitNone (events.js:85:20)

And when I run perfjankie --config-file=local.config.json I get this error:

Starting PerfJankie
Trying to see if the database exists
Starting Browser Perf
[ { [Error: [init({"browserName":"android","platform":"OS X","chromeOptions":{"androidActivity":"com.ionicframework.clientapp176148.MainActivity","androidPackage":"com.ionicframework.clientapp176148","perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame"},"args":["--enable-gpu-benchmarking","--enable-thread-composting"]},"loggingPrefs":{"performance":"ALL"}})] The environment you requested was unavailable.] data: 'unknown command: wd/hub/session' } ]
[ { [Error: [init({"browserName":"android","platform":"OS X","chromeOptions":{"androidActivity":"com.ionicframework.clientapp176148.MainActivity","androidPackage":"com.ionicframework.clientapp176148","perfLoggingPrefs":{"traceCategories":",blink.console,devtools.timeline,disabled-by-default-devtools.timeline,toplevel,disabled-by-default-devtools.timeline.frame"},"args":["--enable-gpu-benchmarking","--enable-thread-composting"]},"loggingPrefs":{"performance":"ALL"}})] The environment you requested was unavailable.] data: 'unknown command: wd/hub/session' } ] null

Here is my local.config.json file contents:

{
    "url": "http://localhost:8100/",
    "name": "Performance Testing",
    "repeat": 3,
    "browsers": [{
        "browserName": "android",
        "platform": "OS X",
        "chromeOptions": {
            "androidActivity": "com.ionicframework.XYZ.MainActivity",
            "androidPackage": "com.ionicframework.XYZ"
        }
    }],
    "selenium": {
        "hostname": "localhost",
        "port": 9515
    },
    "couch": {
        "server": "http://localhost:5984",
        "username": "XXXXX",
        "pwd": "YYYYY",
        "database": "ZZZZZ"
    }
}

Is anyone else getting issues like this? Or does someone possibly know what I have set up wrong?

Thanks!
Alex.

Actions option

Hi, I'am trying to mesure the performance not only of the scrolling of the page. I know that browser-perf has the actions option where you can create custom actions. Is this option is available in perfjankie? If yes, how can I configure this?
Thanks,

Frame Rate data not found on page refresh

Hi,
When i am trying to perform page refresh under actions using "browser.refresh()", the perf graph for frame rate trend is not plotted.
Regards,
Atul