palemoon-dev/platform/testing/xpcshell/selftest.py

#!/usr/bin/env python
#
# Any copyright is dedicated to the Public Domain.
# http://creativecommons.org/publicdomain/zero/1.0/
#

import mozinfo
import mozunit
import os
import pprint
import re
import shutil
import sys
import tempfile
import unittest

from buildconfig import substs
from StringIO import StringIO
from mozlog import structured
from mozbuild.base import MozbuildObject
os.environ.pop('MOZ_OBJDIR', None)
build_obj = MozbuildObject.from_environment()

from runxpcshelltests import XPCShellTests

mozinfo.find_and_update_from_json()

objdir = build_obj.topobjdir.encode("utf-8")

if mozinfo.isMac:
  xpcshellBin = os.path.join(objdir, "dist", substs['MOZ_MACBUNDLE_NAME'], "Contents", "MacOS", "xpcshell")
else:
  xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
  if sys.platform == "win32":
    xpcshellBin += ".exe"

TEST_PASS_STRING = "TEST-PASS"
TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"

SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }"
SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }"

SIMPLE_UNCAUGHT_REJECTION_TEST = '''
function run_test() {
  Promise.reject(new Error("Test rejection."));
  do_check_true(true);
}
'''

SIMPLE_UNCAUGHT_REJECTION_JSM_TEST = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

Promise.reject(new Error("Test rejection."));

function run_test() {
  do_check_true(true);
}
'''

ADD_TEST_SIMPLE = '''
function run_test() { run_next_test(); }

add_test(function test_simple() {
  do_check_true(true);
  run_next_test();
});
'''

ADD_TEST_FAILING = '''
function run_test() { run_next_test(); }

add_test(function test_failing() {
  do_check_true(false);
  run_next_test();
});
'''

ADD_TEST_UNCAUGHT_REJECTION = '''
function run_test() { run_next_test(); }

add_test(function test_uncaught_rejection() {
  Promise.reject(new Error("Test rejection."));
  run_next_test();
});
'''

ADD_TEST_UNCAUGHT_REJECTION_JSM = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

function run_test() { run_next_test(); }

add_test(function test_uncaught_rejection() {
  Promise.reject(new Error("Test rejection."));
  run_next_test();
});
'''

CHILD_TEST_PASSING = '''
function run_test () { run_next_test(); }

add_test(function test_child_simple () {
  run_test_in_child("test_pass.js");
  run_next_test();
});
'''

CHILD_TEST_FAILING = '''
function run_test () { run_next_test(); }

add_test(function test_child_simple () {
  run_test_in_child("test_fail.js");
  run_next_test();
});
'''

CHILD_HARNESS_SIMPLE = '''
function run_test () { run_next_test(); }

add_test(function test_child_assert () {
  do_load_child_test_harness();
  do_test_pending("test child assertion");
  sendCommand("Assert.ok(true);", do_test_finished);
  run_next_test();
});
'''

CHILD_TEST_HANG = '''
function run_test () { run_next_test(); }

add_test(function test_child_simple () {
  do_test_pending("hang test");
  do_load_child_test_harness();
  sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
              + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
              do_test_finished);
  run_next_test();
});
'''

SIMPLE_LOOPING_TEST = '''
function run_test () { run_next_test(); }

add_test(function test_loop () {
  do_test_pending()
});
'''

PASSING_TEST_UNICODE = '''
function run_test () { run_next_test(); }

add_test(function test_unicode_print () {
  do_check_eq("\u201c\u201d", "\u201c\u201d");
  run_next_test();
});
'''

ADD_TASK_SINGLE = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

function run_test() { run_next_test(); }

add_task(function test_task() {
  yield Promise.resolve(true);
  yield Promise.resolve(false);
});
'''

ADD_TASK_MULTIPLE = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

function run_test() { run_next_test(); }

add_task(function test_task() {
  yield Promise.resolve(true);
});

add_task(function test_2() {
  yield Promise.resolve(true);
});
'''

ADD_TASK_REJECTED = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

function run_test() { run_next_test(); }

add_task(function test_failing() {
  yield Promise.reject(new Error("I fail."));
});
'''

ADD_TASK_FAILURE_INSIDE = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

function run_test() { run_next_test(); }

add_task(function test() {
  let result = yield Promise.resolve(false);

  do_check_true(result);
});
'''

ADD_TASK_RUN_NEXT_TEST = '''
function run_test() { run_next_test(); }

add_task(function () {
  Assert.ok(true);

  run_next_test();
});
'''

ADD_TASK_STACK_TRACE = '''
Components.utils.import("resource://gre/modules/Promise.jsm", this);

function run_test() { run_next_test(); }

add_task(function* this_test_will_fail() {
  for (let i = 0; i < 10; ++i) {
    yield Promise.resolve();
  }
  Assert.ok(false);
});
'''

ADD_TASK_STACK_TRACE_WITHOUT_STAR = '''
Components.utils.import("resource://gre/modules/Promise.jsm", this);

function run_test() { run_next_test(); }

add_task(function this_test_will_fail() {
  for (let i = 0; i < 10; ++i) {
    yield Promise.resolve();
  }
  Assert.ok(false);
});
'''

ADD_TEST_THROW_STRING = '''
function run_test() {do_throw("Passing a string to do_throw")};
'''

ADD_TEST_THROW_OBJECT = '''
let error = {
  message: "Error object",
  fileName: "failure.js",
  stack: "ERROR STACK",
  toString: function() {return this.message;}
};
function run_test() {do_throw(error)};
'''

ADD_TEST_REPORT_OBJECT = '''
let error = {
  message: "Error object",
  fileName: "failure.js",
  stack: "ERROR STACK",
  toString: function() {return this.message;}
};
function run_test() {do_report_unexpected_exception(error)};
'''

ADD_TEST_VERBOSE = '''
function run_test() {do_print("a message from do_print")};
'''

# A test for genuine JS-generated Error objects
ADD_TEST_REPORT_REF_ERROR = '''
function run_test() {
  let obj = {blah: 0};
  try {
    obj.noSuchFunction();
  }
  catch (error) {
    do_report_unexpected_exception(error);
  }
};
'''

# A test for failure to load a test due to a syntax error
LOAD_ERROR_SYNTAX_ERROR = '''
function run_test(
'''

# A test for failure to load a test due to an error other than a syntax error
LOAD_ERROR_OTHER_ERROR = '''
function run_test() {
    yield "foo";
    return "foo"; // can't use return in a generator!
};
'''

# A test for asynchronous cleanup functions
ASYNC_CLEANUP = '''
function run_test() {
  Components.utils.import("resource://gre/modules/Promise.jsm", this);

  // The list of checkpoints in the order we encounter them.
  let checkpoints = [];

  // Cleanup tasks, in reverse order
  do_register_cleanup(function cleanup_checkout() {
    do_check_eq(checkpoints.join(""), "1234");
    do_print("At this stage, the test has succeeded");
    do_throw("Throwing an error to force displaying the log");
  });

  do_register_cleanup(function sync_cleanup_2() {
    checkpoints.push(4);
  });

  do_register_cleanup(function async_cleanup_2() {
    let deferred = Promise.defer();
    do_execute_soon(deferred.resolve);
    return deferred.promise.then(function() {
      checkpoints.push(3);
    });
  });

  do_register_cleanup(function sync_cleanup() {
    checkpoints.push(2);
  });

  do_register_cleanup(function async_cleanup() {
    let deferred = Promise.defer();
    do_execute_soon(deferred.resolve);
    return deferred.promise.then(function() {
      checkpoints.push(1);
    });
  });

}
'''

# A test to check that add_test() tests run without run_test()
NO_RUN_TEST_ADD_TEST = '''
add_test(function no_run_test_add_test() {
  do_check_true(true);
  run_next_test();
});
'''

# A test to check that add_task() tests run without run_test()
NO_RUN_TEST_ADD_TASK = '''
add_task(function no_run_test_add_task() {
  do_check_true(true);
});
'''

# A test to check that both add_task() and add_test() work without run_test()
NO_RUN_TEST_ADD_TEST_ADD_TASK = '''
add_test(function no_run_test_add_test() {
  do_check_true(true);
  run_next_test();
});

add_task(function no_run_test_add_task() {
  do_check_true(true);
});
'''

# A test to check that an empty test file without run_test(),
# add_test() or add_task() works.
NO_RUN_TEST_EMPTY_TEST = '''
// This is an empty test file.
'''

NO_RUN_TEST_ADD_TEST_FAIL = '''
add_test(function no_run_test_add_test_fail() {
  do_check_true(false);
  run_next_test();
});
'''

NO_RUN_TEST_ADD_TASK_FAIL = '''
add_task(function no_run_test_add_task_fail() {
  do_check_true(false);
});
'''

NO_RUN_TEST_ADD_TASK_MULTIPLE = '''
Components.utils.import("resource://gre/modules/Promise.jsm");

add_task(function test_task() {
  yield Promise.resolve(true);
});

add_task(function test_2() {
  yield Promise.resolve(true);
});
'''

LOAD_MOZINFO = '''
function run_test() {
  do_check_neq(typeof mozinfo, undefined);
  do_check_neq(typeof mozinfo.os, undefined);
}
'''

CHILD_MOZINFO = '''
function run_test () { run_next_test(); }

add_test(function test_child_mozinfo () {
  run_test_in_child("test_mozinfo.js");
  run_next_test();
});
'''
class XPCShellTestsTests(unittest.TestCase):
    """
    Yes, these are unit tests for a unit test harness.
    """
    def setUp(self):
        self.log = StringIO()
        self.tempdir = tempfile.mkdtemp()
        self.utility_path = os.path.join(objdir, 'dist', 'bin')
        logger = structured.commandline.setup_logging("selftest%s" % id(self),
                                                      {},
                                                      {"tbpl": self.log})
        self.x = XPCShellTests(logger)
        self.x.harness_timeout = 15
        self.symbols_path = None
        candidate_path = os.path.join(build_obj.distdir, 'crashreporter-symbols')
        if (os.path.isdir(candidate_path)):
          self.symbols_path = candidate_path

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def writeFile(self, name, contents):
        """
        Write |contents| to a file named |name| in the temp directory,
        and return the full path to the file.
        """
        fullpath = os.path.join(self.tempdir, name)
        with open(fullpath, "w") as f:
            f.write(contents)
        return fullpath

    def writeManifest(self, tests):
        """
        Write an xpcshell.ini in the temp directory and set
        self.manifest to its pathname. |tests| is a list containing
        either strings (for test names), or tuples with a test name
        as the first element and manifest conditions as the following
        elements.
        """
        testlines = []
        for t in tests:
            testlines.append("[%s]" % (t if isinstance(t, basestring)
                                       else t[0]))
            if isinstance(t, tuple):
                testlines.extend(t[1:])
        self.manifest = self.writeFile("xpcshell.ini", """
[DEFAULT]
head =
tail =

""" + "\n".join(testlines))

    def assertTestResult(self, expected, shuffle=False, verbose=False):
        """
        Assert that self.x.runTests with manifest=self.manifest
        returns |expected|.
        """
        self.assertEquals(expected,
                          self.x.runTests(xpcshellBin,
                                          symbolsPath=self.symbols_path,
                                          manifest=self.manifest,
                                          mozInfo=mozinfo.info,
                                          shuffle=shuffle,
                                          verbose=verbose,
                                          sequential=True,
                                          testingModulesDir=os.path.join(objdir, '_tests', 'modules'),
                                          utility_path=self.utility_path),
                          msg="""Tests should have %s, log:
========
%s
========
""" % ("passed" if expected else "failed", self.log.getvalue()))

    def _assertLog(self, s, expected):
        l = self.log.getvalue()
        self.assertEqual(expected, s in l,
                         msg="""Value %s %s in log:
========
%s
========""" % (s, "expected" if expected else "not expected", l))

    def assertInLog(self, s):
        """
        Assert that the string |s| is contained in self.log.
        """
        self._assertLog(s, True)

    def assertNotInLog(self, s):
        """
        Assert that the string |s| is not contained in self.log.
        """
        self._assertLog(s, False)

    def testPass(self):
        """
        Check that a simple test without any manifest conditions passes.
        """
        self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
        self.writeManifest(["test_basic.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testFail(self):
        """
        Check that a simple failing test without any manifest conditions fails.
        """
        self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
        self.writeManifest(["test_basic.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    @unittest.skipIf(mozinfo.isWin or not mozinfo.info.get('debug'),
                     'We don\'t have a stack fixer on hand for windows.')
    def testAssertStack(self):
        """
        When an assertion is hit, we should produce a useful stack.
        """
        self.writeFile("test_assert.js", '''
          add_test(function test_asserts_immediately() {
            Components.classes["@mozilla.org/xpcom/debug;1"]
                      .getService(Components.interfaces.nsIDebug2)
                      .assertion("foo", "assertion failed", "test.js", 1)
            run_next_test();
          });
        ''')

        self.writeManifest(["test_assert.js"])
        self.assertTestResult(False)

        self.assertInLog("###!!! ASSERTION")
        log_lines = self.log.getvalue().splitlines()
        line_pat = "#\d\d:"
        unknown_pat = "#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
        self.assertFalse(any(re.search(unknown_pat, line) for line in log_lines),
                         "An stack frame without symbols was found in\n%s" % pprint.pformat(log_lines))
        self.assertTrue(any(re.search(line_pat, line) for line in log_lines),
                        "No line resembling a stack frame was found in\n%s" % pprint.pformat(log_lines))

    def testChildPass(self):
        """
        Check that a simple test running in a child process passes.
        """
        self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
        self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
        self.writeManifest(["test_child_pass.js"])

        self.assertTestResult(True, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertInLog("CHILD-TEST-STARTED")
        self.assertInLog("CHILD-TEST-COMPLETED")
        self.assertNotInLog(TEST_FAIL_STRING)


    def testChildFail(self):
        """
        Check that a simple failing test running in a child process fails.
        """
        self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
        self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
        self.writeManifest(["test_child_fail.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("CHILD-TEST-STARTED")
        self.assertInLog("CHILD-TEST-COMPLETED")
        self.assertNotInLog(TEST_PASS_STRING)

    def testChildHang(self):
        """
        Check that incomplete output from a child process results in a
        test failure.
        """
        self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
        self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
        self.writeManifest(["test_child_hang.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("CHILD-TEST-STARTED")
        self.assertNotInLog("CHILD-TEST-COMPLETED")
        self.assertNotInLog(TEST_PASS_STRING)

    def testChild(self):
        """
        Checks that calling do_load_child_test_harness without run_test_in_child
        results in a usable test state. This test has a spurious failure when
        run using |mach python-test|. See bug 1103226.
        """
        self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
        self.writeManifest(["test_child_assertions.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testSkipForAddTest(self):
        """
        Check that add_test is skipped if |skip_if| condition is true
        """
        self.writeFile("test_skip.js", """
add_test({
  skip_if: () => true,
}, function test_should_be_skipped() {
  do_check_true(false);
  run_next_test();
});
""")
        self.writeManifest(["test_skip.js"])
        self.assertTestResult(True, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertInLog("TEST-SKIP")
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNotSkipForAddTask(self):
        """
        Check that add_task is not skipped if |skip_if| condition is false
        """
        self.writeFile("test_not_skip.js", """
add_task({
  skip_if: () => false,
}, function test_should_not_be_skipped() {
  do_check_true(true);
});
""")
        self.writeManifest(["test_not_skip.js"])
        self.assertTestResult(True, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog("TEST-SKIP")
        self.assertNotInLog(TEST_FAIL_STRING)

    def testSkipForAddTask(self):
        """
        Check that add_task is skipped if |skip_if| condition is true
        """
        self.writeFile("test_skip.js", """
add_task({
  skip_if: () => true,
}, function test_should_be_skipped() {
  do_check_true(false);
});
""")
        self.writeManifest(["test_skip.js"])
        self.assertTestResult(True, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertInLog("TEST-SKIP")
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNotSkipForAddTest(self):
        """
        Check that add_test is not skipped if |skip_if| condition is false
        """
        self.writeFile("test_not_skip.js", """
add_test({
  skip_if: () => false,
}, function test_should_not_be_skipped() {
  do_check_true(true);
  run_next_test();
});
""")
        self.writeManifest(["test_not_skip.js"])
        self.assertTestResult(True, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog("TEST-SKIP")
        self.assertNotInLog(TEST_FAIL_STRING)

    def testSyntaxError(self):
        """
        Check that running a test file containing a syntax error produces
        a test failure and expected output.
        """
        self.writeFile("test_syntax_error.js", '"')
        self.writeManifest(["test_syntax_error.js"])

        self.assertTestResult(False, verbose=True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testUnicodeInAssertMethods(self):
        """
        Check that passing unicode characters through an assertion method works.
        """
        self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE)
        self.writeManifest(["test_unicode_assert.js"])

        self.assertTestResult(True, verbose=True)

    @unittest.skipIf('MOZ_AUTOMATION' in os.environ,
                     'Timeout code path occasionally times out (bug 1098121)')
    def testHangingTimeout(self):
        """
        Check that a test that never finishes results in the correct error log.
        """
        self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
        self.writeManifest(["test_loop.js"])

        old_timeout = self.x.harness_timeout
        self.x.harness_timeout = 1

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog("TEST-UNEXPECTED-TIMEOUT")

        self.x.harness_timeout = old_timeout

    def testPassFail(self):
        """
        Check that running more than one test works.
        """
        self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
        self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
        self.writeManifest(["test_pass.js", "test_fail.js"])

        self.assertTestResult(False)
        self.assertEquals(2, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertInLog(TEST_FAIL_STRING)

    def testSkip(self):
        """
        Check that a simple failing test skipped in the manifest does
        not cause failure.
        """
        self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
        self.writeManifest([("test_basic.js", "skip-if = true")])
        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertNotInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testKnownFail(self):
        """
        Check that a simple failing test marked as known-fail in the manifest
        does not cause failure.
        """
        self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
        self.writeManifest([("test_basic.js", "fail-if = true")])
        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(1, self.x.todoCount)
        self.assertInLog("TEST-FAIL")
        # This should be suppressed because the harness doesn't include
        # the full log from the xpcshell run when things pass.
        self.assertNotInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testUnexpectedPass(self):
        """
        Check that a simple failing test marked as known-fail in the manifest
        that passes causes an unexpected pass.
        """
        self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
        self.writeManifest([("test_basic.js", "fail-if = true")])
        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        # From the outer (Python) harness
        self.assertInLog("TEST-UNEXPECTED-PASS")
        self.assertNotInLog("TEST-KNOWN-FAIL")

    def testReturnNonzero(self):
        """
        Check that a test where xpcshell returns nonzero fails.
        """
        self.writeFile("test_error.js", "throw 'foo'")
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testUncaughtRejection(self):
        """
        Ensure a simple test with an uncaught rejection is reported.
        """
        self.writeFile("test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST)
        self.writeManifest(["test_simple_uncaught_rejection.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        if not substs.get('RELEASE_OR_BETA'):
          # async stacks are currently not enabled in release builds.
          self.assertInLog("test_simple_uncaught_rejection.js:3:3")
        self.assertInLog("Test rejection.")
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testUncaughtRejectionJSM(self):
        """
        Ensure a simple test with an uncaught rejection from Promise.jsm is reported.
        """
        self.writeFile("test_simple_uncaught_rejection_jsm.js", SIMPLE_UNCAUGHT_REJECTION_JSM_TEST)
        self.writeManifest(["test_simple_uncaught_rejection_jsm.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("test_simple_uncaught_rejection_jsm.js:4:16")
        self.assertInLog("Test rejection.")
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTestSimple(self):
        """
        Ensure simple add_test() works.
        """
        self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
        self.writeManifest(["test_add_test_simple.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)

    def testLogCorrectFileName(self):
        """
        Make sure a meaningful filename and line number is logged
        by a passing test.
        """
        self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
        self.writeManifest(["test_add_test_simple.js"])

        self.assertTestResult(True, verbose=True)
        self.assertInLog("true == true")
        self.assertNotInLog("[do_check_true :")
        self.assertInLog("[test_simple : 5]")

    def testAddTestFailing(self):
        """
        Ensure add_test() with a failing test is reported.
        """
        self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
        self.writeManifest(["test_add_test_failing.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTestUncaughtRejection(self):
        """
        Ensure add_test() with an uncaught rejection is reported.
        """
        self.writeFile("test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION)
        self.writeManifest(["test_add_test_uncaught_rejection.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTestUncaughtRejectionJSM(self):
        """
        Ensure add_test() with an uncaught rejection from Promise.jsm is reported.
        """
        self.writeFile("test_add_test_uncaught_rejection_jsm.js", ADD_TEST_UNCAUGHT_REJECTION_JSM)
        self.writeManifest(["test_add_test_uncaught_rejection_jsm.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTaskTestSingle(self):
        """
        Ensure add_test_task() with a single passing test works.
        """
        self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
        self.writeManifest(["test_add_task_simple.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)

    def testAddTaskTestMultiple(self):
        """
        Ensure multiple calls to add_test_task() work as expected.
        """
        self.writeFile("test_add_task_multiple.js",
            ADD_TASK_MULTIPLE)
        self.writeManifest(["test_add_task_multiple.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)

    def testAddTaskTestRejected(self):
        """
        Ensure rejected task reports as failure.
        """
        self.writeFile("test_add_task_rejected.js",
            ADD_TASK_REJECTED)
        self.writeManifest(["test_add_task_rejected.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTaskTestFailureInside(self):
        """
        Ensure tests inside task are reported as failures.
        """
        self.writeFile("test_add_task_failure_inside.js",
            ADD_TASK_FAILURE_INSIDE)
        self.writeManifest(["test_add_task_failure_inside.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTaskRunNextTest(self):
        """
        Calling run_next_test() from inside add_task() results in failure.
        """
        self.writeFile("test_add_task_run_next_test.js",
            ADD_TASK_RUN_NEXT_TEST)
        self.writeManifest(["test_add_task_run_next_test.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)

    def testAddTaskStackTrace(self):
        """
        Ensuring that calling Assert.ok(false) from inside add_task()
        results in a human-readable stack trace.
        """
        self.writeFile("test_add_task_stack_trace.js",
            ADD_TASK_STACK_TRACE)
        self.writeManifest(["test_add_task_stack_trace.js"])

        self.assertTestResult(False)
        self.assertInLog("this_test_will_fail")
        self.assertInLog("run_next_test")
        self.assertInLog("run_test")
        self.assertNotInLog("Task.jsm")

    def testAddTaskStackTraceWithoutStar(self):
        """
        Ensuring that calling Assert.ok(false) from inside add_task()
        results in a human-readable stack trace. This variant uses deprecated
        `function()` syntax instead of now standard `function*()`.
        """
        self.writeFile("test_add_task_stack_trace_without_star.js",
            ADD_TASK_STACK_TRACE)
        self.writeManifest(["test_add_task_stack_trace_without_star.js"])

        self.assertTestResult(False)
        self.assertInLog("this_test_will_fail")
        self.assertInLog("run_next_test")
        self.assertInLog("run_test")
        self.assertNotInLog("Task.jsm")

    def testMissingHeadFile(self):
        """
        Ensure that missing head file results in fatal error.
        """
        self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
        self.writeManifest([("test_basic.js", "head = missing.js")])

        raised = False

        try:
            # The actual return value is never checked because we raise.
            self.assertTestResult(True)
        except Exception, ex:
            raised = True
            self.assertEquals(ex.message[0:9], "head file")

        self.assertTrue(raised)

    def testMissingTailFile(self):
        """
        Ensure that missing tail file results in fatal error.
        """
        self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
        self.writeManifest([("test_basic.js", "tail = missing.js")])

        raised = False

        try:
            self.assertTestResult(True)
        except Exception, ex:
            raised = True
            self.assertEquals(ex.message[0:9], "tail file")

        self.assertTrue(raised)

    def testRandomExecution(self):
        """
        Check that random execution doesn't break.
        """
        manifest = []
        for i in range(0, 10):
            filename = "test_pass_%d.js" % i
            self.writeFile(filename, SIMPLE_PASSING_TEST)
            manifest.append(filename)

        self.writeManifest(manifest)
        self.assertTestResult(True, shuffle=True)
        self.assertEquals(10, self.x.testCount)
        self.assertEquals(10, self.x.passCount)

    def testDoThrowString(self):
        """
        Check that do_throw produces reasonable messages when the
        input is a string instead of an object
        """
        self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("Passing a string to do_throw")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoThrowForeignObject(self):
        """
        Check that do_throw produces reasonable messages when the
        input is a generic object with 'filename', 'message' and 'stack' attributes
        but 'object instanceof Error' returns false
        """
        self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("failure.js")
        self.assertInLog("Error object")
        self.assertInLog("ERROR STACK")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoReportForeignObject(self):
        """
        Check that do_report_unexpected_exception produces reasonable messages when the
        input is a generic object with 'filename', 'message' and 'stack' attributes
        but 'object instanceof Error' returns false
        """
        self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("failure.js")
        self.assertInLog("Error object")
        self.assertInLog("ERROR STACK")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoReportRefError(self):
        """
        Check that do_report_unexpected_exception produces reasonable messages when the
        input is a JS-generated Error
        """
        self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("test_error.js")
        self.assertInLog("obj.noSuchFunction is not a function")
        self.assertInLog("run_test@")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoReportSyntaxError(self):
        """
        Check that attempting to load a test file containing a syntax error
        generates details of the error in the log
        """
        self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("test_error.js:3")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoReportNonSyntaxError(self):
        """
        Check that attempting to load a test file containing an error other
        than a syntax error generates details of the error in the log
        """
        self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
        self.writeManifest(["test_error.js"])

        self.assertTestResult(False)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertInLog("TypeError: generator function can't return a value at")
        self.assertInLog("test_error.js:4")
        self.assertNotInLog(TEST_PASS_STRING)

    def testDoPrintWhenVerboseNotExplicit(self):
        """
        Check that do_print() and similar calls that generate output do
        not have the output when not run verbosely.
        """
        self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
        self.writeManifest(["test_verbose.js"])

        self.assertTestResult(True)
        self.assertNotInLog("a message from do_print")

    def testDoPrintWhenVerboseExplicit(self):
        """
        Check that do_print() and similar calls that generate output have the
        output shown when run verbosely.
        """
        self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
        self.writeManifest(["test_verbose.js"])
        self.assertTestResult(True, verbose=True)
        self.assertInLog("a message from do_print")

    def testDoPrintWhenVerboseInManifest(self):
        """
        Check that do_print() and similar calls that generate output have the
        output shown when 'verbose = true' is in the manifest, even when
        not run verbosely.
        """
        self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
        self.writeManifest([("test_verbose.js", "verbose = true")])

        self.assertTestResult(True)
        self.assertInLog("a message from do_print")

    def testAsyncCleanup(self):
        """
        Check that do_register_cleanup handles nicely cleanup tasks that
        return a promise
        """
        self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
        self.writeManifest(["test_asyncCleanup.js"])
        self.assertTestResult(False)
        self.assertInLog("\"1234\" == \"1234\"")
        self.assertInLog("At this stage, the test has succeeded")
        self.assertInLog("Throwing an error to force displaying the log")

    def testNoRunTestAddTest(self):
        """
        Check that add_test() works fine without run_test() in the test file.
        """
        self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
        self.writeManifest(["test_noRunTestAddTest.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNoRunTestAddTask(self):
        """
        Check that add_task() works fine without run_test() in the test file.
        """
        self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
        self.writeManifest(["test_noRunTestAddTask.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNoRunTestAddTestAddTask(self):
        """
        Check that both add_test() and add_task() work without run_test()
        in the test file.
        """
        self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
        self.writeManifest(["test_noRunTestAddTestAddTask.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNoRunTestEmptyTest(self):
        """
        Check that the test passes on an empty file that contains neither
        run_test() nor add_test(), add_task().
        """
        self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
        self.writeManifest(["test_noRunTestEmptyTest.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testNoRunTestAddTestFail(self):
        """
        Check that test fails on using add_test() without run_test().
        """
        self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
        self.writeManifest(["test_noRunTestAddTestFail.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testNoRunTestAddTaskFail(self):
        """
        Check that test fails on using add_task() without run_test().
        """
        self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
        self.writeManifest(["test_noRunTestAddTaskFail.js"])

        self.assertTestResult(False)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(0, self.x.passCount)
        self.assertEquals(1, self.x.failCount)
        self.assertInLog(TEST_FAIL_STRING)
        self.assertNotInLog(TEST_PASS_STRING)

    def testNoRunTestAddTaskMultiple(self):
        """
        Check that multple add_task() tests work without run_test().
        """
        self.writeFile("test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE)
        self.writeManifest(["test_noRunTestAddTaskMultiple.js"])

        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testMozinfo(self):
        """
        Check that mozinfo.json is loaded
        """
        self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
        self.writeManifest(["test_mozinfo.js"])
        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

    def testChildMozinfo(self):
        """
        Check that mozinfo.json is loaded in child process
        """
        self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
        self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
        self.writeManifest(["test_child_mozinfo.js"])
        self.assertTestResult(True)
        self.assertEquals(1, self.x.testCount)
        self.assertEquals(1, self.x.passCount)
        self.assertEquals(0, self.x.failCount)
        self.assertEquals(0, self.x.todoCount)
        self.assertInLog(TEST_PASS_STRING)
        self.assertNotInLog(TEST_FAIL_STRING)

if __name__ == "__main__":
    mozunit.main()