Some of QUnit tests should be replaced with functional tests

[AlexanderMoskovkin]

No description provided.

[georgiy-abbasov]

After the research we have an interface that we are going to implement for functional tests running.

Implement functional harness

/cc @inikulin Is it all ok?

[inikulin]

@georgiy-abbasov Yay, looks like a long reading. I'll review on Monday if you wouldn't mind? Need a clear head for this.

[inikulin]

@georgiy-abbasov Can you ping me then, please?

[georgiy-abbasov]

@inikulin, ok, we will discuss it together on Monday.

[AlexanderMoskovkin]

@georgiy-abbasov Please write here the comments where we describe the interface you've written about. Don't force guys to look at our working long issue. Put the main things to this issue

[georgiy-abbasov]

testcafe-functional-harness

Allows to run functional tests in local browsers or on SauceLabs.

What it's doing

1. Start server for test pages.
2. Open the specified browsers (locally or on SauceLabs).
3. Run mocha tests.
4. Close the browsers and stop the server after the tests.

Install:

$ npm install testcafe-functional-harness

Usage:

functionalHarness
    .src(...sources)
    .sitePorts(port1, port2)
    .browsers(...{alias, url, name, platform})    //name and platform - for saucelabs
    //optional
    .saucelabs(username, accessKey, jobName)
    .timeout(sec) //by default 180s
    //run tests
    .run();

[georgiy-abbasov]

TestCafe

Gulpfile.js

var functionalHarness = require('test/functional/harness.js');

var FUNCTIONAL_TESTS_SETTINGS = {
    testCafePort1: 2000,
    testCafePort2: 2001,
    sitePort1: 3000,
    sitePort2: 3001
};

var FUNCTIONAL_TESTS_BROWSERS = {
    platform:    'Windows 10',
    browserName: 'microsoftedge',
    alias:       'edge'
};

var SAUCELABS_FUNCTIONAL_SETTINGS = {
    username:  process.env.SAUCE_USERNAME,
    accessKey: process.env.SAUCE_ACCESS_KEY,
    build:     process.env.TRAVIS_JOB_ID || '',
    tags:      [process.env.TRAVIS_BRANCH || 'master']
    name:      'testcafe functional tests'
};

gulp.task('test-functional', function () {
    var testsTimeout = 180;

    return gulp
        .src('test/functional/fixtures/**/*-test.js')
        .pipe(functionalHarness.run(FUNCTIONAL_TESTS_SETTINGS, FUNCTIONAL_TESTS_BROWSERS,
                                testsTimeout, SAUCELABS_FUNCTIONAL_SETTINGS));
});

test/functional/harness.js

What it's doing:

1. Create testcafe instance from the sources.
2. Exports testcafe and browsersInfo for test-runner.
3. Run tests via testcafe-functional-harness.

test/functional/test-runner.js

What it's doing:

1. Provides the runTests function that runs testCafe tests via the testCafe instance from harness.js. It is called from mocha tests.
2. Returns the object with failed tests in the following format:

{
    testName1: 'test1Error',
    testName2: 'test2Error'...
}

How it works:

var harness  = require(test/functional/harness.js);

var testCafe    = harness.testCafe;
var browserInfo = harness.browserInfo;

export runTests(fixture, testName) {
    var tc     = harness.testCafe;
    var runner =  tc.createRunner();

    return runner
    .filter(function(){
    ...
    })
    .src(fixture)
    .browsers(browserInfo)
    .run()
    .then(function()
    {
        // modify testcafe report to the following way:
        //<testName>: [testError]
        // throwing report if there was errors
    });
}

test/function/fixtures/

--- index.html    // tested page
--- test1.test.js // testcafe test
--- test1-test.js // mocha test

mocha test example

var runTests = require('../../../../test-runner.js').runTests;
var expect   = require('chai').expect;

describe('api click test', function () {
    it('Should fail when the first argument is invisible', function () {
        return runTests('./click.test.js', 'Should fail when the first argument is invisible')
            .then(function () {
                throw new Error('Test should fail but was succeed');
            })
            .catch(function (err) {
                var expectedError = [
                    'Error at step "1.Click on invisible element":',
                    '',
                    'act.click($input);',
                    '',
                    'A target element \<input id="input"\> of the click action is not visible.',
                    'If this element should appear when you are hovering over another',
                    'element, make sure that you properly recorded the hover action.'
                ].join(' ');

                expect(err['Should fail when the first argument is invisible']).eql(expectedError);
            });
    });

    it('Pointer events test (T191183) [ONLY:ie]', function () {
        return runTests('./click.test.js', 'Pointer events test (T191183) [ONLY:ie]')
            .catch(function (err) {
                expect(err['Pointer events test (T191183) [ONLY:ie]']).eql('');
            });
    });
});

[inikulin]

it('Should fail when the first argument is invisible', function () {
        return runTests('./click.test.js', 'Should fail when the first argument is invisible')
            .then(function () {
                throw new Error('Test should fail but was succeed');
            })
            .catch(function (err) {
                var expectedError = [
                    'Error at step "1.Click on invisible element":',
                    '',
                    'act.click($input);',
                    '',
                    'A target element \<input id="input"\> of the click action is not visible.',
                    'If this element should appear when you are hovering over another',
                    'element, make sure that you properly recorded the hover action.'
                ].join(' ');

                expect(err['Should fail when the first argument is invisible']).eql(expectedError);
            });
    });

BTW, maybe we can get rid of this part:

 .then(function () {
                throw new Error('Test should fail but was succeed');
 })

By encapsulating it in runTests, e.g.:

it('Should fail when the first argument is invisible', function () {
        return runTests('./click.test.js', 'Should fail when the first argument is invisible', { shouldFail: true })
            .catch(function (err) {
                var expectedError = [
                    'Error at step "1.Click on invisible element":',
                    '',
                    'act.click($input);',
                    '',
                    'A target element \<input id="input"\> of the click action is not visible.',
                    'If this element should appear when you are hovering over another',
                    'element, make sure that you properly recorded the hover action.'
                ].join(' ');

                expect(err['Should fail when the first argument is invisible']).eql(expectedError);
            });
    });

Also, maybe if test name is explicitly specified we could return just this test's error, so it will be:

expect(err).eql(expectedError)

What do you guys think?

[AlexanderMoskovkin]

I suppose, it'll be useful. @georgiy-abbasov let's implement this. If we specify the test name, err - it is a string with an error. It we specify just fixture - it's a dictionary like now.
The shouldFail flag I like too

[inikulin]

I guess we can close it now, since we use functional testing heavily to cover functionality.

[lock]

This thread has been automatically locked since it is closed and there has not been any recent activity. Please open a new issue for related bugs or feature requests. We recommend you ask TestCafe API, usage and configuration inquiries on StackOverflow.