TestNG failing most tests after update

Question

I've looked at other channels but it I'm not having much luck getting an answer to my question.

I'm working on a testing code base that was originally written using TestNG 5.14.10 and have updated it in my build.gradle file to 6.8.7 (I'm using Gradle + Maven for the dependancies).

My build file is as follows (includes the new version # for TestNG):

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'eclipse'

repositories {
    mavenCentral()
}

dependencies {
    compile "org.seleniumhq.selenium:selenium-java:2.38.0"
    compile "org.testng:testng:6.8.7"
    testCompile('org.uncommons:reportng:1.1.2') {
        exclude group: 'org.testng'
    }
    testCompile "junit:junit:4.8.2"
    compile "com.jayway.restassured:rest-assured:1.8.1"
}

//initialize thread count variable for parallel testing and default to 1
def threadCount = System.getProperty("MAXTHREADS", "1")

tasks.withType(Test) {
    maxParallelForks = 1
    forkEvery = 1000
    ignoreFailures = false

    // Pass all system properties to the tests
    systemProperties = System.getProperties()

    // Makes the standard streams (err and out) visible at console when running tests
    testLogging.showStandardStreams = true

    exclude '**/tasks/'

    classpath += configurations.testCompile
}

task firefox(type: Test) {
    maxParallelForks = Integer.valueOf(threadCount) //default is 1 if not specified
    testLogging.events "started"
    testLogging {
        events "started", "passed", "skipped", "failed", "standardOut", "standardError"
        exceptionFormat "full" // default is "short"
    }
    useTestNG() {
        excludeGroups 'chrome', 'broken'
        useDefaultListeners = false
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/firefox")
    testReportDir = file("${reporting.baseDir}/firefox")

    systemProperties.BROWSER = System.getProperty('BROWSER', 'firefox')

    exclude '**/selenium/'
    exclude '**/setupscripts/'
}

task chrome(type: Test) {
    maxParallelForks = Integer.valueOf(threadCount) //default is 1 if not specified
    testLogging {
        events "started", "passed", "skipped", "failed", "standardOut", "standardError"
        exceptionFormat "full"
    }
    useTestNG() {
        excludeGroups 'broken'
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/chrome")
    testReportDir = file("${reporting.baseDir}/chrome")

    systemProperties.BROWSER = System.getProperty('BROWSER', 'chrome')

    exclude '**/selenium/'
    exclude '**/setupscripts/'
}

task internetExplorer(type: Test) {
    useTestNG() {
        excludeGroups 'chrome', 'broken'
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }
    testReportDir = file("${reporting.baseDir}/internetExplorer")
    testResultsDir = file("${buildDir}/test-results/internetExplorer")

    systemProperties.BROWSER = System.getProperty('BROWSER', 'internetExplorer')

    exclude '**/selenium/'
    exclude '**/setupscripts/'
}

task applylicense(type: Test) {
    useTestNG() {
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/applyLicense")
    testReportDir = file("${reporting.baseDir}/applyLicense")

    scanForTestClasses = false
    include '**/setupscripts/ApplyLicense.class'
}

task uatsetup(type: Test) {
    useTestNG() {
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/uatSetup")
    testReportDir = file("${reporting.baseDir}/uatSetup")

    scanForTestClasses = false
    include '**/setupscripts/UatSetup.class'
}

task restsetup(type: Test) {
    useTestNG() {
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/restSetup")
    testReportDir = file("${reporting.baseDir}/restSetup")

    scanForTestClasses = false
    include '**/setupscripts/RestSetup.class'
}

task soapsetup(type: Test) {
    useTestNG() {
        useDefaultListeners = false;
        listeners << 'org.uncommons.reportng.HTMLReporter'
        listeners << 'org.uncommons.reportng.JUnitXMLReporter'
        listeners << 'com.xmatters.testng.Listener'
    }

    testResultsDir = file("${buildDir}/test-results/soapSetup")
    testReportDir = file("${reporting.baseDir}/soapSetup")

    scanForTestClasses = false
    include '**/setupscripts/SoapSetup.class'
}

task selenium(type: Test) {
    testResultsDir = file("${buildDir}/test-results/selenium")
    testReportDir = file("${reporting.baseDir}/selenium")

    include '**/selenium/'
    exclude '**/webdriver/'
    exclude '**/*$*', '**/Abstract*'
    exclude '**/messagepanel/'
    exclude '**/propertylibrary/'
}

javadoc() {
    title = "xMod Page Objects API"
}

task wrapper(type: Wrapper) {
    gradleVersion = '1.7'
}

I ran a build using the --debug and --stacktrace arguments for one task (contains two setup scripts that are annotated and treated like a standard test). It appears that the test is being set to a pass and then a fail. Very bizarre behavior. It appears this is some sort of backwards compatibility issue. I generally run this against a suite of ~500 tests without issue, the only change has been my TestNG bersion. Here is my shell command and resulting output (I trimmed it down to the lines that I believe are relevant since debug mode is VERY verbose):

./gradlew uatsetup -DREMOTE_DRIVER=false -DWEB_SERVER=10.3.1.226:8888 --stacktrace --debug

Had to use PasteBin for my debug output due to size: http://pastebin.com/wsczwgT9

Any help would be great.

Cheers,

Darwin

Darwin Allen · Accepted Answer

Well, after some digging, I've found the answer to this one. The newer versions of TestNG appear to require the Google guice library. I added this to my dependencies and it appears to have resolved the issue. What was happening is that the tests were passing but the reporting was broken due to the missing dependency and marked the entire build (and tests) as a failure.

I added the following line to my dependencies declarations:

compile "com.google.inject:guice:3.0"

No idea why it's not a little more clear that this library is needed but I suppose that's just how it is. Anyways, problem solved.

TestNG failing most tests after update

Answers (2)

Related Questions