1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345 |
- #!/usr/bin/env python
- #
- # Any copyright is dedicated to the Public Domain.
- # http://creativecommons.org/publicdomain/zero/1.0/
- #
- import mozinfo
- import mozunit
- import os
- import pprint
- import re
- import shutil
- import sys
- import tempfile
- import unittest
- from buildconfig import substs
- from StringIO import StringIO
- from mozlog import structured
- from mozbuild.base import MozbuildObject
- os.environ.pop('MOZ_OBJDIR', None)
- build_obj = MozbuildObject.from_environment()
- from runxpcshelltests import XPCShellTests
- mozinfo.find_and_update_from_json()
- objdir = build_obj.topobjdir.encode("utf-8")
- if mozinfo.isMac:
- xpcshellBin = os.path.join(objdir, "dist", substs['MOZ_MACBUNDLE_NAME'], "Contents", "MacOS", "xpcshell")
- else:
- xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
- if sys.platform == "win32":
- xpcshellBin += ".exe"
- TEST_PASS_STRING = "TEST-PASS"
- TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
- SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }"
- SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }"
- SIMPLE_UNCAUGHT_REJECTION_TEST = '''
- function run_test() {
- Promise.reject(new Error("Test rejection."));
- do_check_true(true);
- }
- '''
- SIMPLE_UNCAUGHT_REJECTION_JSM_TEST = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- Promise.reject(new Error("Test rejection."));
- function run_test() {
- do_check_true(true);
- }
- '''
- ADD_TEST_SIMPLE = '''
- function run_test() { run_next_test(); }
- add_test(function test_simple() {
- do_check_true(true);
- run_next_test();
- });
- '''
- ADD_TEST_FAILING = '''
- function run_test() { run_next_test(); }
- add_test(function test_failing() {
- do_check_true(false);
- run_next_test();
- });
- '''
- ADD_TEST_UNCAUGHT_REJECTION = '''
- function run_test() { run_next_test(); }
- add_test(function test_uncaught_rejection() {
- Promise.reject(new Error("Test rejection."));
- run_next_test();
- });
- '''
- ADD_TEST_UNCAUGHT_REJECTION_JSM = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- function run_test() { run_next_test(); }
- add_test(function test_uncaught_rejection() {
- Promise.reject(new Error("Test rejection."));
- run_next_test();
- });
- '''
- CHILD_TEST_PASSING = '''
- function run_test () { run_next_test(); }
- add_test(function test_child_simple () {
- run_test_in_child("test_pass.js");
- run_next_test();
- });
- '''
- CHILD_TEST_FAILING = '''
- function run_test () { run_next_test(); }
- add_test(function test_child_simple () {
- run_test_in_child("test_fail.js");
- run_next_test();
- });
- '''
- CHILD_HARNESS_SIMPLE = '''
- function run_test () { run_next_test(); }
- add_test(function test_child_assert () {
- do_load_child_test_harness();
- do_test_pending("test child assertion");
- sendCommand("Assert.ok(true);", do_test_finished);
- run_next_test();
- });
- '''
- CHILD_TEST_HANG = '''
- function run_test () { run_next_test(); }
- add_test(function test_child_simple () {
- do_test_pending("hang test");
- do_load_child_test_harness();
- sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
- + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
- do_test_finished);
- run_next_test();
- });
- '''
- SIMPLE_LOOPING_TEST = '''
- function run_test () { run_next_test(); }
- add_test(function test_loop () {
- do_test_pending()
- });
- '''
- PASSING_TEST_UNICODE = '''
- function run_test () { run_next_test(); }
- add_test(function test_unicode_print () {
- do_check_eq("\u201c\u201d", "\u201c\u201d");
- run_next_test();
- });
- '''
- ADD_TASK_SINGLE = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- function run_test() { run_next_test(); }
- add_task(function test_task() {
- yield Promise.resolve(true);
- yield Promise.resolve(false);
- });
- '''
- ADD_TASK_MULTIPLE = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- function run_test() { run_next_test(); }
- add_task(function test_task() {
- yield Promise.resolve(true);
- });
- add_task(function test_2() {
- yield Promise.resolve(true);
- });
- '''
- ADD_TASK_REJECTED = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- function run_test() { run_next_test(); }
- add_task(function test_failing() {
- yield Promise.reject(new Error("I fail."));
- });
- '''
- ADD_TASK_FAILURE_INSIDE = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- function run_test() { run_next_test(); }
- add_task(function test() {
- let result = yield Promise.resolve(false);
- do_check_true(result);
- });
- '''
- ADD_TASK_RUN_NEXT_TEST = '''
- function run_test() { run_next_test(); }
- add_task(function () {
- Assert.ok(true);
- run_next_test();
- });
- '''
- ADD_TASK_STACK_TRACE = '''
- Components.utils.import("resource://gre/modules/Promise.jsm", this);
- function run_test() { run_next_test(); }
- add_task(function* this_test_will_fail() {
- for (let i = 0; i < 10; ++i) {
- yield Promise.resolve();
- }
- Assert.ok(false);
- });
- '''
- ADD_TASK_STACK_TRACE_WITHOUT_STAR = '''
- Components.utils.import("resource://gre/modules/Promise.jsm", this);
- function run_test() { run_next_test(); }
- add_task(function this_test_will_fail() {
- for (let i = 0; i < 10; ++i) {
- yield Promise.resolve();
- }
- Assert.ok(false);
- });
- '''
- ADD_TEST_THROW_STRING = '''
- function run_test() {do_throw("Passing a string to do_throw")};
- '''
- ADD_TEST_THROW_OBJECT = '''
- let error = {
- message: "Error object",
- fileName: "failure.js",
- stack: "ERROR STACK",
- toString: function() {return this.message;}
- };
- function run_test() {do_throw(error)};
- '''
- ADD_TEST_REPORT_OBJECT = '''
- let error = {
- message: "Error object",
- fileName: "failure.js",
- stack: "ERROR STACK",
- toString: function() {return this.message;}
- };
- function run_test() {do_report_unexpected_exception(error)};
- '''
- ADD_TEST_VERBOSE = '''
- function run_test() {do_print("a message from do_print")};
- '''
- # A test for genuine JS-generated Error objects
- ADD_TEST_REPORT_REF_ERROR = '''
- function run_test() {
- let obj = {blah: 0};
- try {
- obj.noSuchFunction();
- }
- catch (error) {
- do_report_unexpected_exception(error);
- }
- };
- '''
- # A test for failure to load a test due to a syntax error
- LOAD_ERROR_SYNTAX_ERROR = '''
- function run_test(
- '''
- # A test for failure to load a test due to an error other than a syntax error
- LOAD_ERROR_OTHER_ERROR = '''
- function run_test() {
- yield "foo";
- return "foo"; // can't use return in a generator!
- };
- '''
- # A test for asynchronous cleanup functions
- ASYNC_CLEANUP = '''
- function run_test() {
- Components.utils.import("resource://gre/modules/Promise.jsm", this);
- // The list of checkpoints in the order we encounter them.
- let checkpoints = [];
- // Cleanup tasks, in reverse order
- do_register_cleanup(function cleanup_checkout() {
- do_check_eq(checkpoints.join(""), "1234");
- do_print("At this stage, the test has succeeded");
- do_throw("Throwing an error to force displaying the log");
- });
- do_register_cleanup(function sync_cleanup_2() {
- checkpoints.push(4);
- });
- do_register_cleanup(function async_cleanup_2() {
- let deferred = Promise.defer();
- do_execute_soon(deferred.resolve);
- return deferred.promise.then(function() {
- checkpoints.push(3);
- });
- });
- do_register_cleanup(function sync_cleanup() {
- checkpoints.push(2);
- });
- do_register_cleanup(function async_cleanup() {
- let deferred = Promise.defer();
- do_execute_soon(deferred.resolve);
- return deferred.promise.then(function() {
- checkpoints.push(1);
- });
- });
- }
- '''
- # A test to check that add_test() tests run without run_test()
- NO_RUN_TEST_ADD_TEST = '''
- add_test(function no_run_test_add_test() {
- do_check_true(true);
- run_next_test();
- });
- '''
- # A test to check that add_task() tests run without run_test()
- NO_RUN_TEST_ADD_TASK = '''
- add_task(function no_run_test_add_task() {
- do_check_true(true);
- });
- '''
- # A test to check that both add_task() and add_test() work without run_test()
- NO_RUN_TEST_ADD_TEST_ADD_TASK = '''
- add_test(function no_run_test_add_test() {
- do_check_true(true);
- run_next_test();
- });
- add_task(function no_run_test_add_task() {
- do_check_true(true);
- });
- '''
- # A test to check that an empty test file without run_test(),
- # add_test() or add_task() works.
- NO_RUN_TEST_EMPTY_TEST = '''
- // This is an empty test file.
- '''
- NO_RUN_TEST_ADD_TEST_FAIL = '''
- add_test(function no_run_test_add_test_fail() {
- do_check_true(false);
- run_next_test();
- });
- '''
- NO_RUN_TEST_ADD_TASK_FAIL = '''
- add_task(function no_run_test_add_task_fail() {
- do_check_true(false);
- });
- '''
- NO_RUN_TEST_ADD_TASK_MULTIPLE = '''
- Components.utils.import("resource://gre/modules/Promise.jsm");
- add_task(function test_task() {
- yield Promise.resolve(true);
- });
- add_task(function test_2() {
- yield Promise.resolve(true);
- });
- '''
- LOAD_MOZINFO = '''
- function run_test() {
- do_check_neq(typeof mozinfo, undefined);
- do_check_neq(typeof mozinfo.os, undefined);
- }
- '''
- CHILD_MOZINFO = '''
- function run_test () { run_next_test(); }
- add_test(function test_child_mozinfo () {
- run_test_in_child("test_mozinfo.js");
- run_next_test();
- });
- '''
- class XPCShellTestsTests(unittest.TestCase):
- """
- Yes, these are unit tests for a unit test harness.
- """
- def setUp(self):
- self.log = StringIO()
- self.tempdir = tempfile.mkdtemp()
- self.utility_path = os.path.join(objdir, 'dist', 'bin')
- logger = structured.commandline.setup_logging("selftest%s" % id(self),
- {},
- {"tbpl": self.log})
- self.x = XPCShellTests(logger)
- self.x.harness_timeout = 15
- self.symbols_path = None
- candidate_path = os.path.join(build_obj.distdir, 'crashreporter-symbols')
- if (os.path.isdir(candidate_path)):
- self.symbols_path = candidate_path
- def tearDown(self):
- shutil.rmtree(self.tempdir)
- def writeFile(self, name, contents):
- """
- Write |contents| to a file named |name| in the temp directory,
- and return the full path to the file.
- """
- fullpath = os.path.join(self.tempdir, name)
- with open(fullpath, "w") as f:
- f.write(contents)
- return fullpath
- def writeManifest(self, tests):
- """
- Write an xpcshell.ini in the temp directory and set
- self.manifest to its pathname. |tests| is a list containing
- either strings (for test names), or tuples with a test name
- as the first element and manifest conditions as the following
- elements.
- """
- testlines = []
- for t in tests:
- testlines.append("[%s]" % (t if isinstance(t, basestring)
- else t[0]))
- if isinstance(t, tuple):
- testlines.extend(t[1:])
- self.manifest = self.writeFile("xpcshell.ini", """
- [DEFAULT]
- head =
- tail =
- """ + "\n".join(testlines))
- def assertTestResult(self, expected, shuffle=False, verbose=False):
- """
- Assert that self.x.runTests with manifest=self.manifest
- returns |expected|.
- """
- self.assertEquals(expected,
- self.x.runTests(xpcshellBin,
- symbolsPath=self.symbols_path,
- manifest=self.manifest,
- mozInfo=mozinfo.info,
- shuffle=shuffle,
- verbose=verbose,
- sequential=True,
- testingModulesDir=os.path.join(objdir, '_tests', 'modules'),
- utility_path=self.utility_path),
- msg="""Tests should have %s, log:
- ========
- %s
- ========
- """ % ("passed" if expected else "failed", self.log.getvalue()))
- def _assertLog(self, s, expected):
- l = self.log.getvalue()
- self.assertEqual(expected, s in l,
- msg="""Value %s %s in log:
- ========
- %s
- ========""" % (s, "expected" if expected else "not expected", l))
- def assertInLog(self, s):
- """
- Assert that the string |s| is contained in self.log.
- """
- self._assertLog(s, True)
- def assertNotInLog(self, s):
- """
- Assert that the string |s| is not contained in self.log.
- """
- self._assertLog(s, False)
- def testPass(self):
- """
- Check that a simple test without any manifest conditions passes.
- """
- self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
- self.writeManifest(["test_basic.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testFail(self):
- """
- Check that a simple failing test without any manifest conditions fails.
- """
- self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
- self.writeManifest(["test_basic.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- @unittest.skipIf(mozinfo.isWin or not mozinfo.info.get('debug'),
- 'We don\'t have a stack fixer on hand for windows.')
- def testAssertStack(self):
- """
- When an assertion is hit, we should produce a useful stack.
- """
- self.writeFile("test_assert.js", '''
- add_test(function test_asserts_immediately() {
- Components.classes["@mozilla.org/xpcom/debug;1"]
- .getService(Components.interfaces.nsIDebug2)
- .assertion("foo", "assertion failed", "test.js", 1)
- run_next_test();
- });
- ''')
- self.writeManifest(["test_assert.js"])
- self.assertTestResult(False)
- self.assertInLog("###!!! ASSERTION")
- log_lines = self.log.getvalue().splitlines()
- line_pat = "#\d\d:"
- unknown_pat = "#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
- self.assertFalse(any(re.search(unknown_pat, line) for line in log_lines),
- "An stack frame without symbols was found in\n%s" % pprint.pformat(log_lines))
- self.assertTrue(any(re.search(line_pat, line) for line in log_lines),
- "No line resembling a stack frame was found in\n%s" % pprint.pformat(log_lines))
- def testChildPass(self):
- """
- Check that a simple test running in a child process passes.
- """
- self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
- self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
- self.writeManifest(["test_child_pass.js"])
- self.assertTestResult(True, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertInLog("CHILD-TEST-STARTED")
- self.assertInLog("CHILD-TEST-COMPLETED")
- self.assertNotInLog(TEST_FAIL_STRING)
- def testChildFail(self):
- """
- Check that a simple failing test running in a child process fails.
- """
- self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
- self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
- self.writeManifest(["test_child_fail.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("CHILD-TEST-STARTED")
- self.assertInLog("CHILD-TEST-COMPLETED")
- self.assertNotInLog(TEST_PASS_STRING)
- def testChildHang(self):
- """
- Check that incomplete output from a child process results in a
- test failure.
- """
- self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
- self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
- self.writeManifest(["test_child_hang.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("CHILD-TEST-STARTED")
- self.assertNotInLog("CHILD-TEST-COMPLETED")
- self.assertNotInLog(TEST_PASS_STRING)
- def testChild(self):
- """
- Checks that calling do_load_child_test_harness without run_test_in_child
- results in a usable test state. This test has a spurious failure when
- run using |mach python-test|. See bug 1103226.
- """
- self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
- self.writeManifest(["test_child_assertions.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testSkipForAddTest(self):
- """
- Check that add_test is skipped if |skip_if| condition is true
- """
- self.writeFile("test_skip.js", """
- add_test({
- skip_if: () => true,
- }, function test_should_be_skipped() {
- do_check_true(false);
- run_next_test();
- });
- """)
- self.writeManifest(["test_skip.js"])
- self.assertTestResult(True, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertInLog("TEST-SKIP")
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNotSkipForAddTask(self):
- """
- Check that add_task is not skipped if |skip_if| condition is false
- """
- self.writeFile("test_not_skip.js", """
- add_task({
- skip_if: () => false,
- }, function test_should_not_be_skipped() {
- do_check_true(true);
- });
- """)
- self.writeManifest(["test_not_skip.js"])
- self.assertTestResult(True, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog("TEST-SKIP")
- self.assertNotInLog(TEST_FAIL_STRING)
- def testSkipForAddTask(self):
- """
- Check that add_task is skipped if |skip_if| condition is true
- """
- self.writeFile("test_skip.js", """
- add_task({
- skip_if: () => true,
- }, function test_should_be_skipped() {
- do_check_true(false);
- });
- """)
- self.writeManifest(["test_skip.js"])
- self.assertTestResult(True, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertInLog("TEST-SKIP")
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNotSkipForAddTest(self):
- """
- Check that add_test is not skipped if |skip_if| condition is false
- """
- self.writeFile("test_not_skip.js", """
- add_test({
- skip_if: () => false,
- }, function test_should_not_be_skipped() {
- do_check_true(true);
- run_next_test();
- });
- """)
- self.writeManifest(["test_not_skip.js"])
- self.assertTestResult(True, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog("TEST-SKIP")
- self.assertNotInLog(TEST_FAIL_STRING)
- def testSyntaxError(self):
- """
- Check that running a test file containing a syntax error produces
- a test failure and expected output.
- """
- self.writeFile("test_syntax_error.js", '"')
- self.writeManifest(["test_syntax_error.js"])
- self.assertTestResult(False, verbose=True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testUnicodeInAssertMethods(self):
- """
- Check that passing unicode characters through an assertion method works.
- """
- self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE)
- self.writeManifest(["test_unicode_assert.js"])
- self.assertTestResult(True, verbose=True)
- @unittest.skipIf('MOZ_AUTOMATION' in os.environ,
- 'Timeout code path occasionally times out (bug 1098121)')
- def testHangingTimeout(self):
- """
- Check that a test that never finishes results in the correct error log.
- """
- self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
- self.writeManifest(["test_loop.js"])
- old_timeout = self.x.harness_timeout
- self.x.harness_timeout = 1
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog("TEST-UNEXPECTED-TIMEOUT")
- self.x.harness_timeout = old_timeout
- def testPassFail(self):
- """
- Check that running more than one test works.
- """
- self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
- self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
- self.writeManifest(["test_pass.js", "test_fail.js"])
- self.assertTestResult(False)
- self.assertEquals(2, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertInLog(TEST_FAIL_STRING)
- def testSkip(self):
- """
- Check that a simple failing test skipped in the manifest does
- not cause failure.
- """
- self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
- self.writeManifest([("test_basic.js", "skip-if = true")])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertNotInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testKnownFail(self):
- """
- Check that a simple failing test marked as known-fail in the manifest
- does not cause failure.
- """
- self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
- self.writeManifest([("test_basic.js", "fail-if = true")])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(1, self.x.todoCount)
- self.assertInLog("TEST-FAIL")
- # This should be suppressed because the harness doesn't include
- # the full log from the xpcshell run when things pass.
- self.assertNotInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testUnexpectedPass(self):
- """
- Check that a simple failing test marked as known-fail in the manifest
- that passes causes an unexpected pass.
- """
- self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
- self.writeManifest([("test_basic.js", "fail-if = true")])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- # From the outer (Python) harness
- self.assertInLog("TEST-UNEXPECTED-PASS")
- self.assertNotInLog("TEST-KNOWN-FAIL")
- def testReturnNonzero(self):
- """
- Check that a test where xpcshell returns nonzero fails.
- """
- self.writeFile("test_error.js", "throw 'foo'")
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testUncaughtRejection(self):
- """
- Ensure a simple test with an uncaught rejection is reported.
- """
- self.writeFile("test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST)
- self.writeManifest(["test_simple_uncaught_rejection.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- if not substs.get('RELEASE_OR_BETA'):
- # async stacks are currently not enabled in release builds.
- self.assertInLog("test_simple_uncaught_rejection.js:3:3")
- self.assertInLog("Test rejection.")
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testUncaughtRejectionJSM(self):
- """
- Ensure a simple test with an uncaught rejection from Promise.jsm is reported.
- """
- self.writeFile("test_simple_uncaught_rejection_jsm.js", SIMPLE_UNCAUGHT_REJECTION_JSM_TEST)
- self.writeManifest(["test_simple_uncaught_rejection_jsm.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("test_simple_uncaught_rejection_jsm.js:4:16")
- self.assertInLog("Test rejection.")
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTestSimple(self):
- """
- Ensure simple add_test() works.
- """
- self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
- self.writeManifest(["test_add_test_simple.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- def testLogCorrectFileName(self):
- """
- Make sure a meaningful filename and line number is logged
- by a passing test.
- """
- self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
- self.writeManifest(["test_add_test_simple.js"])
- self.assertTestResult(True, verbose=True)
- self.assertInLog("true == true")
- self.assertNotInLog("[do_check_true :")
- self.assertInLog("[test_simple : 5]")
- def testAddTestFailing(self):
- """
- Ensure add_test() with a failing test is reported.
- """
- self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
- self.writeManifest(["test_add_test_failing.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTestUncaughtRejection(self):
- """
- Ensure add_test() with an uncaught rejection is reported.
- """
- self.writeFile("test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION)
- self.writeManifest(["test_add_test_uncaught_rejection.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTestUncaughtRejectionJSM(self):
- """
- Ensure add_test() with an uncaught rejection from Promise.jsm is reported.
- """
- self.writeFile("test_add_test_uncaught_rejection_jsm.js", ADD_TEST_UNCAUGHT_REJECTION_JSM)
- self.writeManifest(["test_add_test_uncaught_rejection_jsm.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTaskTestSingle(self):
- """
- Ensure add_test_task() with a single passing test works.
- """
- self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
- self.writeManifest(["test_add_task_simple.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- def testAddTaskTestMultiple(self):
- """
- Ensure multiple calls to add_test_task() work as expected.
- """
- self.writeFile("test_add_task_multiple.js",
- ADD_TASK_MULTIPLE)
- self.writeManifest(["test_add_task_multiple.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- def testAddTaskTestRejected(self):
- """
- Ensure rejected task reports as failure.
- """
- self.writeFile("test_add_task_rejected.js",
- ADD_TASK_REJECTED)
- self.writeManifest(["test_add_task_rejected.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTaskTestFailureInside(self):
- """
- Ensure tests inside task are reported as failures.
- """
- self.writeFile("test_add_task_failure_inside.js",
- ADD_TASK_FAILURE_INSIDE)
- self.writeManifest(["test_add_task_failure_inside.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTaskRunNextTest(self):
- """
- Calling run_next_test() from inside add_task() results in failure.
- """
- self.writeFile("test_add_task_run_next_test.js",
- ADD_TASK_RUN_NEXT_TEST)
- self.writeManifest(["test_add_task_run_next_test.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- def testAddTaskStackTrace(self):
- """
- Ensuring that calling Assert.ok(false) from inside add_task()
- results in a human-readable stack trace.
- """
- self.writeFile("test_add_task_stack_trace.js",
- ADD_TASK_STACK_TRACE)
- self.writeManifest(["test_add_task_stack_trace.js"])
- self.assertTestResult(False)
- self.assertInLog("this_test_will_fail")
- self.assertInLog("run_next_test")
- self.assertInLog("run_test")
- self.assertNotInLog("Task.jsm")
- def testAddTaskStackTraceWithoutStar(self):
- """
- Ensuring that calling Assert.ok(false) from inside add_task()
- results in a human-readable stack trace. This variant uses deprecated
- `function()` syntax instead of now standard `function*()`.
- """
- self.writeFile("test_add_task_stack_trace_without_star.js",
- ADD_TASK_STACK_TRACE)
- self.writeManifest(["test_add_task_stack_trace_without_star.js"])
- self.assertTestResult(False)
- self.assertInLog("this_test_will_fail")
- self.assertInLog("run_next_test")
- self.assertInLog("run_test")
- self.assertNotInLog("Task.jsm")
- def testMissingHeadFile(self):
- """
- Ensure that missing head file results in fatal error.
- """
- self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
- self.writeManifest([("test_basic.js", "head = missing.js")])
- raised = False
- try:
- # The actual return value is never checked because we raise.
- self.assertTestResult(True)
- except Exception, ex:
- raised = True
- self.assertEquals(ex.message[0:9], "head file")
- self.assertTrue(raised)
- def testMissingTailFile(self):
- """
- Ensure that missing tail file results in fatal error.
- """
- self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
- self.writeManifest([("test_basic.js", "tail = missing.js")])
- raised = False
- try:
- self.assertTestResult(True)
- except Exception, ex:
- raised = True
- self.assertEquals(ex.message[0:9], "tail file")
- self.assertTrue(raised)
- def testRandomExecution(self):
- """
- Check that random execution doesn't break.
- """
- manifest = []
- for i in range(0, 10):
- filename = "test_pass_%d.js" % i
- self.writeFile(filename, SIMPLE_PASSING_TEST)
- manifest.append(filename)
- self.writeManifest(manifest)
- self.assertTestResult(True, shuffle=True)
- self.assertEquals(10, self.x.testCount)
- self.assertEquals(10, self.x.passCount)
- def testDoThrowString(self):
- """
- Check that do_throw produces reasonable messages when the
- input is a string instead of an object
- """
- self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("Passing a string to do_throw")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoThrowForeignObject(self):
- """
- Check that do_throw produces reasonable messages when the
- input is a generic object with 'filename', 'message' and 'stack' attributes
- but 'object instanceof Error' returns false
- """
- self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("failure.js")
- self.assertInLog("Error object")
- self.assertInLog("ERROR STACK")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoReportForeignObject(self):
- """
- Check that do_report_unexpected_exception produces reasonable messages when the
- input is a generic object with 'filename', 'message' and 'stack' attributes
- but 'object instanceof Error' returns false
- """
- self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("failure.js")
- self.assertInLog("Error object")
- self.assertInLog("ERROR STACK")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoReportRefError(self):
- """
- Check that do_report_unexpected_exception produces reasonable messages when the
- input is a JS-generated Error
- """
- self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("test_error.js")
- self.assertInLog("obj.noSuchFunction is not a function")
- self.assertInLog("run_test@")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoReportSyntaxError(self):
- """
- Check that attempting to load a test file containing a syntax error
- generates details of the error in the log
- """
- self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("test_error.js:3")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoReportNonSyntaxError(self):
- """
- Check that attempting to load a test file containing an error other
- than a syntax error generates details of the error in the log
- """
- self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
- self.writeManifest(["test_error.js"])
- self.assertTestResult(False)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertInLog("TypeError: generator function can't return a value at")
- self.assertInLog("test_error.js:4")
- self.assertNotInLog(TEST_PASS_STRING)
- def testDoPrintWhenVerboseNotExplicit(self):
- """
- Check that do_print() and similar calls that generate output do
- not have the output when not run verbosely.
- """
- self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
- self.writeManifest(["test_verbose.js"])
- self.assertTestResult(True)
- self.assertNotInLog("a message from do_print")
- def testDoPrintWhenVerboseExplicit(self):
- """
- Check that do_print() and similar calls that generate output have the
- output shown when run verbosely.
- """
- self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
- self.writeManifest(["test_verbose.js"])
- self.assertTestResult(True, verbose=True)
- self.assertInLog("a message from do_print")
- def testDoPrintWhenVerboseInManifest(self):
- """
- Check that do_print() and similar calls that generate output have the
- output shown when 'verbose = true' is in the manifest, even when
- not run verbosely.
- """
- self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
- self.writeManifest([("test_verbose.js", "verbose = true")])
- self.assertTestResult(True)
- self.assertInLog("a message from do_print")
- def testAsyncCleanup(self):
- """
- Check that do_register_cleanup handles nicely cleanup tasks that
- return a promise
- """
- self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
- self.writeManifest(["test_asyncCleanup.js"])
- self.assertTestResult(False)
- self.assertInLog("\"1234\" == \"1234\"")
- self.assertInLog("At this stage, the test has succeeded")
- self.assertInLog("Throwing an error to force displaying the log")
- def testNoRunTestAddTest(self):
- """
- Check that add_test() works fine without run_test() in the test file.
- """
- self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
- self.writeManifest(["test_noRunTestAddTest.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNoRunTestAddTask(self):
- """
- Check that add_task() works fine without run_test() in the test file.
- """
- self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
- self.writeManifest(["test_noRunTestAddTask.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNoRunTestAddTestAddTask(self):
- """
- Check that both add_test() and add_task() work without run_test()
- in the test file.
- """
- self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
- self.writeManifest(["test_noRunTestAddTestAddTask.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNoRunTestEmptyTest(self):
- """
- Check that the test passes on an empty file that contains neither
- run_test() nor add_test(), add_task().
- """
- self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
- self.writeManifest(["test_noRunTestEmptyTest.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testNoRunTestAddTestFail(self):
- """
- Check that test fails on using add_test() without run_test().
- """
- self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
- self.writeManifest(["test_noRunTestAddTestFail.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testNoRunTestAddTaskFail(self):
- """
- Check that test fails on using add_task() without run_test().
- """
- self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
- self.writeManifest(["test_noRunTestAddTaskFail.js"])
- self.assertTestResult(False)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(0, self.x.passCount)
- self.assertEquals(1, self.x.failCount)
- self.assertInLog(TEST_FAIL_STRING)
- self.assertNotInLog(TEST_PASS_STRING)
- def testNoRunTestAddTaskMultiple(self):
- """
- Check that multple add_task() tests work without run_test().
- """
- self.writeFile("test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE)
- self.writeManifest(["test_noRunTestAddTaskMultiple.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testMozinfo(self):
- """
- Check that mozinfo.json is loaded
- """
- self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
- self.writeManifest(["test_mozinfo.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- def testChildMozinfo(self):
- """
- Check that mozinfo.json is loaded in child process
- """
- self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
- self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
- self.writeManifest(["test_child_mozinfo.js"])
- self.assertTestResult(True)
- self.assertEquals(1, self.x.testCount)
- self.assertEquals(1, self.x.passCount)
- self.assertEquals(0, self.x.failCount)
- self.assertEquals(0, self.x.todoCount)
- self.assertInLog(TEST_PASS_STRING)
- self.assertNotInLog(TEST_FAIL_STRING)
- if __name__ == "__main__":
- mozunit.main()
|