protractor-jasmine2-html-reporter doesn't consolidate results for all test when tests are shared using 'shardTestFiles': true in conf file

Question

Recently we have configured our e2e-tests to be on Jenkins & soon we realized that we have to use shared test files: true options as complete suite run is taking very long time for us peeking 9-10hrs on daily basis. but when we configured below two options in conf file. tests are running fine but final report displays only the last specs run results in save path. consolidate all options is not giving the full reports.

please find our conf file details. any help will be appreciated.

Edit the conf file as per solution provide by Aditya. please help

   var Jasmine2HtmlReporter = require('protractor-jasmine2-html-reporter');
var log4js = require('log4js');
var params = process.argv;
var args = process.argv.slice(3);

exports.config = {
  //seleniumServerJar: './node_modules/gulp-protractor/node_modules/protractor/selenium/selenium-server-standalone-2.48.2.jar',
  seleniumAddress: 'http://localhost:4444/wd/hub',
  allScriptsTimeout: 100000,
  framework: 'jasmine2',

  onPrepare: function () {

    return new Promise(function(fulfill, reject) {
      browser.getCapabilities().then(function(value) {
        reportName = value.get(Math.random(8,2)) + '_' + value.get('browserName') + '_' + Math.floor(Math.random() * 1E16);
        jasmine.getEnv().addReporter(
          new Jasmine2HtmlReporter({
            //cleanDestination: false,
            savePath: __dirname+'/target',
            //docTitle: 'Web UI Test Report',
            screenshotsFolder: 'image',
            //takeScreenshots: true,
            takeScreenshotsOnlyOnFailures: true,
            consolidate: true,
            consolidateAll: true,
             preserveDirectory: true,
            //fixedScreenshotName: true,
            filePrefix: reportName + ".html"
          })
        );
        fulfill();
      });
    });

    // browser.manage().timeouts().implicitlyWait(11000);
    var width = 768;
    var height = 1366;
    browser.driver.manage().window().setSize(768, 1366);
    browser.ignoreSynchronization = false; 
  },

  afterLaunch: function afterLaunch() {
    var fs = require('fs');
    var output = '';
    fs.readdirSync('target/').forEach(function (file) {
      if (!(fs.lstatSync('target/' + file).isDirectory()))
        output = output + fs.readFileSync('target/' + file);
    });
    fs.writeFileSync('target/ConsolidatedReport.html', output, 'utf8');

  },

  suites:{

    example:['./test/e2e/specs/**/*Spec.js',]
  },


  /*  capabilities: {
      'browserName': 'chrome'
    },*/

  multiCapabilities: [
    {
      'browserName': 'chrome'
    },
    {
      'browserName': 'firefox'
    }
  ],


  resultJsonOutputFile:'./results.json',

  // Options to be passed to Jasmine-node.
  jasmineNodeOpts: {
    showColors: true,
    defaultTimeoutInterval: 100000
  }
};

Answer 1

Here is another solution, building on top of this answer, using protractor-html-reporter-2 which works on the xml file produced by jasmine-reporters. But jasmine-reporters does not have any option to work with reports generated by multiple instances of browsers. After failing to find an ideal solution, I ended up doing following in the protractor config js file:

// add relevant packages in package.json
'use strict';
const HTMLReport = require('protractor-html-reporter-2');
const jasmineReporters = require('jasmine-reporters');
const moment = require('moment');
const os = require('os');
const xmldoc = require('xmldoc');
...
const DATE_FORMAT = 'YYYYMMDD-HHmmss-SSS'; // use any other format that gives unique timestamp
const reportDir = path.join(__dirname, '../report');
...
exports.config = {
    ...
    framework: 'jasmine',
    capabilities: {
        browserName: 'chrome',
        maxInstances: 2,
        shardTestFiles: true,
    },
    beforeLaunch: async function () {
        // clean up report directory
        fs.emptyDirSync(reportDir);
    }, 
    onPrepare: async function () {
         const NOW = moment().format(DATE_FORMAT);
         const reportName = 'index-' + NOW;
         jasmine.getEnv().addReporter(new jasmineReporters.JUnitXmlReporter({
             consolidateAll: true,
             savePath: reportDir,
             filePrefix: reportName,
         }));

        // for screenshots
        jasmine.getEnv().addReporter({
    specDone: function (result) {
        if (result.status === 'failed') {
            browser.getCapabilities().then(function (caps) {
                const browserName = caps.get('browserName');
                browser.takeScreenshot().then(function (png) {
                    const stream = fs.createWriteStream(reportDir + '/screenshots/' + browserName + '-' + result.fullName.replace(/[.:]/g, ' ') + '.png');
                    stream.write(Buffer.from(png, 'base64'));
                    stream.end();
                });
            });
        }
    }
});
    },
    onComplete: async function () {
        // do something after each instance of browser is closed
    },
    afterLaunch: async function (exitCode) {
        // do something after ALL instances of browser are closed
        await consolidateJasmineXmlReports();
    },
    ...     
},
...
async function consolidateJasmineXmlReports() {
    // there may be better ways to write xml out but this works for me
    const files = fs.readdirSync(reportDir).filter(fn => fn.endsWith('.xml'));
    let disabledSum = 0;
    let errorsSum = 0;
    let failuresSum = 0;
    let testsSum = 0;
    let timeSum = 0;
    const allTestSuiteNodes = [];
    for (const file of files) {
        const pathToXml = reportDir + path.sep + file;
        console.log('Reading xml report file: ' + pathToXml);
        const xml = fs.readFileSync(pathToXml);
        const xmlDoc = new xmldoc.XmlDocument(xml);
        const disabled = parseInt(xmlDoc.attr.disabled);
        const errors = parseInt(xmlDoc.attr.errors);
        const failures = parseInt(xmlDoc.attr.failures);
        const tests = parseInt(xmlDoc.attr.tests);
        const time = parseFloat(xmlDoc.attr.time);
        disabledSum += disabled;
        errorsSum += errors;
        failuresSum += failures;
        testsSum += tests;
        timeSum += time;

        const testSuiteNodes = xmlDoc.childrenNamed('testsuite');
        allTestSuiteNodes.push(testSuiteNodes);
    }

    let startXml = `<?xml version="1.0" encoding="UTF-8" ?>`;
    startXml += `<testsuites disabled="` + disabledSum + `" errors="` + errorsSum + `" failures="` + failuresSum + `" tests="` + testsSum + `" time="` + timeSum + `">`;
    const endXml = '</testsuites>';
    allTestSuiteNodes.push(endXml);
    const finalXml = startXml + allTestSuiteNodes.join('\n');
    fs.writeFileSync(reportDir + path.sep + 'consolidated.xml', finalXml, 'utf8');
        
    const testConfig = {            
        outputPath: reportDir,
        outputFilename: 'consolidated',
        screenshotPath: './screenshots',
        screenshotsOnlyOnFailure: true,
        ...
    };

    new HTMLReport().from(reportDir + path.sep + 'consolidated.xml', testConfig);
}

The logic is

1. Ensure unique names for all xml files.
2. Combine all the xml files to one valid xml in the afterLaunch.
3. Use any package that uses that xml file to produce html in the afterLaunch.

We use Jenkins to run the tests and the report created by above displays well in Jenkins and also shows up accurately on the report displayed by Open Blue Ocean Jenkins plugin.

Note: I have tested with shardTestFiles and not with multiCapabilities but I think it should work with that as well.

See Also:

• Protractor config file
• Setting up jasmine reporter

Answer 2

The limitation is with the 'Jasmine2HtmlReporter' as it overwrites the html report file when tests run in parallel. But avoiding this is definitely possible and there are couple of ways of doing it. Pick up the right way based on your convenience

1) Tinker with 'index.js' of Jasmine2HtmlReporter to append file instead of PhantomJs overwrite its using

2) Generate unique HTML reports by configuring Jasmine2HTML reporter from onPrepare() function and consolidate all the reports later

SOLUTION 1: The current code base of Jasmine2HtmlReporter - index.js uses two functions - phantomWrite() & nodeWrite() to write data. Refer here

I have created a new function - appendwrite() to append instead of overwriting and have modified code to pickup this function Check out my github code forked out of protractor-jasmine2-html-reporter

        function appendwrite(path, filename, text){
            var fs = require("fs");
            var nodejs_path = require("path");
            require("mkdirp").sync(path); // make sure the path exists
            var filepath = nodejs_path.join(path, filename);
            fs.appendFileSync(filepath,text)
            return;
        }

And modify the self.writeFile function in 'node_modules/protractor-jasmine2-html-reporter/index.js' to pickup the new function

        try {
            appendwrite(path, filename, text);
            //phantomWrite(path, filename, text);
            return;
        } catch (e) { errors.push('  PhantomJs attempt: ' + e.message); }
        try {
            nodeWrite(path, filename, text);
            return;
        } catch (f) { errors.push('  NodeJS attempt: ' + f.message); }

And Comment the below code which cleans reports on new run so that you dont see any error cleanup error - CleanUpCode

    rmdir(self.savePath);

SOLUTION 2: Generate separate reports based on sessionID for parallel instances by configuring the Jasmine reporter in OnPrepare function

onPrepare: function() {
        return new Promise(function (fulfill, reject) {
            browser.getCapabilities().then(function (value) {
                reportName = value.get('webdriver.remote.sessionid') + '_' + value.get('browserName') + '_' + Math.floor(Math.random()*1E16);
                jasmine.getEnv().addReporter(
                    new Jasmine2HtmlReporter({
                        savePath: 'target/',
                        screenshotsFolder: 'images',
                        consolidate: true,
                        consolidateAll: true,
                        filePrefix: reportName + ".html"
                    })
                );
                fulfill();
            })
        });
    },

Step 2: Consolidate the reports generated across parallel instances in afterLaunch() method after complete tests are done and all webdriver sessions are closed

afterLaunch: function afterLaunch() {
        var fs = require('fs');
        var output = '';
       fs.readdirSync('target/').forEach(function(file){
           if(!(fs.lstatSync('target/' + file).isDirectory()))
            output = output + fs.readFileSync('target/' + file);
       });
        fs.writeFileSync('target/ConsolidatedReport.html', output, 'utf8');
    },

You will see reports generated something like below with one ConsolidatedReport also PS: Please ignore any typo and syntax errors. this is just to serve as an example and can be customized

EDIT1: The 'sessionID' we are using to name the HTML report is the webdriver remote sessionID and if you have doubt that it may not remain unique through multiple sessions, Just generate a random number for the individual HTML reports and consolidate later

I have modified the code above

Answer 3

I used below solution to create unique folder with unique timestamp. This will save HTML report in date-timestamp folder instead Jasmin-2-html-reporter deleting it.

var today = new Date();
   var timeStamp = today.getMonth() + 1 + '-' + today.getDate() + '-' + today.getFullYear() + '-' + 

today.getHours() + 'h-' + today.getMinutes() + 'm-' +today.getSeconds()+'s';

 jasmine.getEnv().addReporter(
            new Jasmine2HtmlReporter({
                savePath: './Reports/testResultsReport '+timeStamp,
                screenshotsFolder: 'screenPrints',
                takeScreenshots: true,
                takeScreenshotsOnlyOnFailures: true,
            })
        ); 

`

Answer 4

Solution from Aditya works fine for me. My example config file: var Jasmine2HtmlReporter = require('protractor-jasmine2-html-reporter');

exports.config = { framework: 'jasmine2', seleniumAddress: 'http://localhost:4444/wd/hub', /multiCapabilities: [ { 'browserName': 'chrome', 'shardTestFiles': true, 'maxInstances': 2, chromeOptions: { args: ['chrome.switches', '--disable-extensions'] } }, { 'browserName': 'firefox' } ],/ capabilities: { 'browserName': 'chrome', 'shardTestFiles': true, 'maxInstances': 2, chromeOptions: { args: ['chrome.switches', '--disable-extensions'] } }, suites: { loginpage: 'login.js', addproduct: 'addproduct.js' }, //specs: ['addproduct.js'], jasmineNodeOpts: { onComplete: null, isVerbose: false, includeStackTrace: true, showColors: true, defaultTimeoutInterval: 30000 },

onPrepare: function() {
    return new Promise(function(fulfill, reject) {
        browser.getCapabilities().then(function(value) {
            reportName = value.get(Math.random(8,2)) + '_' + value.get('browserName') + '_' + Math.floor(Math.random() * 1E16);
            jasmine.getEnv().addReporter(
                new Jasmine2HtmlReporter({
                    //cleanDestination: false,
                    savePath: 'target/',
                    //docTitle: 'Web UI Test Report',
                    screenshotsFolder: 'image',
                    //takeScreenshots: true,
                    //takeScreenshotsOnlyOnFailures: true,
                    consolidate: true,
                    consolidateAll: true,
                   // preserveDirectory: true,
                    //fixedScreenshotName: true,
                    filePrefix: reportName + ".html"
                })
            );
            fulfill();
        });
    });
},
afterLaunch: function afterLaunch() {
    var fs = require('fs');
    var output = '';
    fs.readdirSync('target/').forEach(function(file) {
        if (!(fs.lstatSync('target/' + file).isDirectory()))
            output = output + fs.readFileSync('target/' + file);
    });
    fs.writeFileSync('target/ConsolidatedReport.html', output, 'utf8');
}

}

Answer 5

I am currently struggling with the same problem, however in my prototype setup it IS working, and I am using the BASE config, and nothing more.

var Jasmine2HtmlReporter = require('protractor-jasmine2-html-reporter');


exports.config = {
framework: 'jasmine',
seleniumAddress: 'http://localhost:4444/wd/hub',
specs: ['**-spec**.js'],

capabilities: {
    browserName: 'chrome',                                                      
    shardTestFiles: true,                                                        
    maxInstances: 2                                                             
},

onPrepare: function() {
    jasmine.getEnv().addReporter(
      new Jasmine2HtmlReporter({
        savePath: 'target/',
        screenshotsFolder: 'images',
        consolidate: true,
        consolidateAll: false  // false in my saved config - true in tut.
    })
  );
}
}

Can you see if you can get shard reporting working with just the bare minimum ?

Edit : Also keep a eye on what is happening in the target folder, Jasmine might be overwriting/cleaning when you dont want it.

Edit 2: if you go for the solution below me, make sure you are launching enough browsers as you have specs - the solution below me makes reports based on browser ID, if you split your specs :

capabilities: {
    browserName: 'chrome',                                                      
    shardTestFiles: true,                                                       
    maxInstances: **2**                                                             
},

it will still overwrite your reporter HTML if you want more than 2. It creates a HTML file based on the browser/session ID used to test it - If you use 2 browsers instances you will get 2 HTML files, no more no less.

Edit 3 : A quick fix would be to not let jasmine clean up...

cleanDestination: false,

But this does not seem to do anything, so after searching and searching - I do not think the jasmine html reporter is going to let us consolidate more specs than we have shards. Issue tracker on github is not showing any progress.

So the only solution I can think of, is to use the solution below me, with enough shards to support your ammount of specs, and then parse it back to one file when your done.

Edit 4: You can always abuse Jenkins to concate the HTML files between actual test runs.