test_api.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582
  1. # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
  2. # For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt
  3. """Tests for coverage.py's API."""
  4. import fnmatch
  5. import os
  6. import sys
  7. import textwrap
  8. import warnings
  9. import coverage
  10. from coverage import env
  11. from coverage.backward import StringIO
  12. from coverage.misc import CoverageException
  13. from coverage.report import Reporter
  14. from tests.coveragetest import CoverageTest
  15. class ApiTest(CoverageTest):
  16. """Api-oriented tests for coverage.py."""
  17. def clean_files(self, files, pats):
  18. """Remove names matching `pats` from `files`, a list of file names."""
  19. good = []
  20. for f in files:
  21. for pat in pats:
  22. if fnmatch.fnmatch(f, pat):
  23. break
  24. else:
  25. good.append(f)
  26. return good
  27. def assertFiles(self, files):
  28. """Assert that the files here are `files`, ignoring the usual junk."""
  29. here = os.listdir(".")
  30. here = self.clean_files(here, ["*.pyc", "__pycache__"])
  31. self.assertCountEqual(here, files)
  32. def test_unexecuted_file(self):
  33. cov = coverage.Coverage()
  34. self.make_file("mycode.py", """\
  35. a = 1
  36. b = 2
  37. if b == 3:
  38. c = 4
  39. d = 5
  40. """)
  41. self.make_file("not_run.py", """\
  42. fooey = 17
  43. """)
  44. # Import the Python file, executing it.
  45. self.start_import_stop(cov, "mycode")
  46. _, statements, missing, _ = cov.analysis("not_run.py")
  47. self.assertEqual(statements, [1])
  48. self.assertEqual(missing, [1])
  49. def test_filenames(self):
  50. self.make_file("mymain.py", """\
  51. import mymod
  52. a = 1
  53. """)
  54. self.make_file("mymod.py", """\
  55. fooey = 17
  56. """)
  57. # Import the Python file, executing it.
  58. cov = coverage.Coverage()
  59. self.start_import_stop(cov, "mymain")
  60. filename, _, _, _ = cov.analysis("mymain.py")
  61. self.assertEqual(os.path.basename(filename), "mymain.py")
  62. filename, _, _, _ = cov.analysis("mymod.py")
  63. self.assertEqual(os.path.basename(filename), "mymod.py")
  64. filename, _, _, _ = cov.analysis(sys.modules["mymain"])
  65. self.assertEqual(os.path.basename(filename), "mymain.py")
  66. filename, _, _, _ = cov.analysis(sys.modules["mymod"])
  67. self.assertEqual(os.path.basename(filename), "mymod.py")
  68. # Import the Python file, executing it again, once it's been compiled
  69. # already.
  70. cov = coverage.Coverage()
  71. self.start_import_stop(cov, "mymain")
  72. filename, _, _, _ = cov.analysis("mymain.py")
  73. self.assertEqual(os.path.basename(filename), "mymain.py")
  74. filename, _, _, _ = cov.analysis("mymod.py")
  75. self.assertEqual(os.path.basename(filename), "mymod.py")
  76. filename, _, _, _ = cov.analysis(sys.modules["mymain"])
  77. self.assertEqual(os.path.basename(filename), "mymain.py")
  78. filename, _, _, _ = cov.analysis(sys.modules["mymod"])
  79. self.assertEqual(os.path.basename(filename), "mymod.py")
  80. def test_ignore_stdlib(self):
  81. self.make_file("mymain.py", """\
  82. import colorsys
  83. a = 1
  84. hls = colorsys.rgb_to_hls(1.0, 0.5, 0.0)
  85. """)
  86. # Measure without the stdlib.
  87. cov1 = coverage.Coverage()
  88. self.assertEqual(cov1.config.cover_pylib, False)
  89. self.start_import_stop(cov1, "mymain")
  90. # some statements were marked executed in mymain.py
  91. _, statements, missing, _ = cov1.analysis("mymain.py")
  92. self.assertNotEqual(statements, missing)
  93. # but none were in colorsys.py
  94. _, statements, missing, _ = cov1.analysis("colorsys.py")
  95. self.assertEqual(statements, missing)
  96. # Measure with the stdlib.
  97. cov2 = coverage.Coverage(cover_pylib=True)
  98. self.start_import_stop(cov2, "mymain")
  99. # some statements were marked executed in mymain.py
  100. _, statements, missing, _ = cov2.analysis("mymain.py")
  101. self.assertNotEqual(statements, missing)
  102. # and some were marked executed in colorsys.py
  103. _, statements, missing, _ = cov2.analysis("colorsys.py")
  104. self.assertNotEqual(statements, missing)
  105. def test_include_can_measure_stdlib(self):
  106. self.make_file("mymain.py", """\
  107. import colorsys, random
  108. a = 1
  109. r, g, b = [random.random() for _ in range(3)]
  110. hls = colorsys.rgb_to_hls(r, g, b)
  111. """)
  112. # Measure without the stdlib, but include colorsys.
  113. cov1 = coverage.Coverage(cover_pylib=False, include=["*/colorsys.py"])
  114. self.start_import_stop(cov1, "mymain")
  115. # some statements were marked executed in colorsys.py
  116. _, statements, missing, _ = cov1.analysis("colorsys.py")
  117. self.assertNotEqual(statements, missing)
  118. # but none were in random.py
  119. _, statements, missing, _ = cov1.analysis("random.py")
  120. self.assertEqual(statements, missing)
  121. def test_exclude_list(self):
  122. cov = coverage.Coverage()
  123. cov.clear_exclude()
  124. self.assertEqual(cov.get_exclude_list(), [])
  125. cov.exclude("foo")
  126. self.assertEqual(cov.get_exclude_list(), ["foo"])
  127. cov.exclude("bar")
  128. self.assertEqual(cov.get_exclude_list(), ["foo", "bar"])
  129. self.assertEqual(cov._exclude_regex('exclude'), "(?:foo)|(?:bar)")
  130. cov.clear_exclude()
  131. self.assertEqual(cov.get_exclude_list(), [])
  132. def test_exclude_partial_list(self):
  133. cov = coverage.Coverage()
  134. cov.clear_exclude(which='partial')
  135. self.assertEqual(cov.get_exclude_list(which='partial'), [])
  136. cov.exclude("foo", which='partial')
  137. self.assertEqual(cov.get_exclude_list(which='partial'), ["foo"])
  138. cov.exclude("bar", which='partial')
  139. self.assertEqual(cov.get_exclude_list(which='partial'), ["foo", "bar"])
  140. self.assertEqual(
  141. cov._exclude_regex(which='partial'), "(?:foo)|(?:bar)"
  142. )
  143. cov.clear_exclude(which='partial')
  144. self.assertEqual(cov.get_exclude_list(which='partial'), [])
  145. def test_exclude_and_partial_are_separate_lists(self):
  146. cov = coverage.Coverage()
  147. cov.clear_exclude(which='partial')
  148. cov.clear_exclude(which='exclude')
  149. cov.exclude("foo", which='partial')
  150. self.assertEqual(cov.get_exclude_list(which='partial'), ['foo'])
  151. self.assertEqual(cov.get_exclude_list(which='exclude'), [])
  152. cov.exclude("bar", which='exclude')
  153. self.assertEqual(cov.get_exclude_list(which='partial'), ['foo'])
  154. self.assertEqual(cov.get_exclude_list(which='exclude'), ['bar'])
  155. cov.exclude("p2", which='partial')
  156. cov.exclude("e2", which='exclude')
  157. self.assertEqual(cov.get_exclude_list(which='partial'), ['foo', 'p2'])
  158. self.assertEqual(cov.get_exclude_list(which='exclude'), ['bar', 'e2'])
  159. cov.clear_exclude(which='partial')
  160. self.assertEqual(cov.get_exclude_list(which='partial'), [])
  161. self.assertEqual(cov.get_exclude_list(which='exclude'), ['bar', 'e2'])
  162. cov.clear_exclude(which='exclude')
  163. self.assertEqual(cov.get_exclude_list(which='partial'), [])
  164. self.assertEqual(cov.get_exclude_list(which='exclude'), [])
  165. def test_datafile_default(self):
  166. # Default data file behavior: it's .coverage
  167. self.make_file("datatest1.py", """\
  168. fooey = 17
  169. """)
  170. self.assertFiles(["datatest1.py"])
  171. cov = coverage.Coverage()
  172. self.start_import_stop(cov, "datatest1")
  173. cov.save()
  174. self.assertFiles(["datatest1.py", ".coverage"])
  175. def test_datafile_specified(self):
  176. # You can specify the data file name.
  177. self.make_file("datatest2.py", """\
  178. fooey = 17
  179. """)
  180. self.assertFiles(["datatest2.py"])
  181. cov = coverage.Coverage(data_file="cov.data")
  182. self.start_import_stop(cov, "datatest2")
  183. cov.save()
  184. self.assertFiles(["datatest2.py", "cov.data"])
  185. def test_datafile_and_suffix_specified(self):
  186. # You can specify the data file name and suffix.
  187. self.make_file("datatest3.py", """\
  188. fooey = 17
  189. """)
  190. self.assertFiles(["datatest3.py"])
  191. cov = coverage.Coverage(data_file="cov.data", data_suffix="14")
  192. self.start_import_stop(cov, "datatest3")
  193. cov.save()
  194. self.assertFiles(["datatest3.py", "cov.data.14"])
  195. def test_datafile_from_rcfile(self):
  196. # You can specify the data file name in the .coveragerc file
  197. self.make_file("datatest4.py", """\
  198. fooey = 17
  199. """)
  200. self.make_file(".coveragerc", """\
  201. [run]
  202. data_file = mydata.dat
  203. """)
  204. self.assertFiles(["datatest4.py", ".coveragerc"])
  205. cov = coverage.Coverage()
  206. self.start_import_stop(cov, "datatest4")
  207. cov.save()
  208. self.assertFiles(["datatest4.py", ".coveragerc", "mydata.dat"])
  209. def test_empty_reporting(self):
  210. # empty summary reports raise exception, just like the xml report
  211. cov = coverage.Coverage()
  212. cov.erase()
  213. self.assertRaises(CoverageException, cov.report)
  214. def make_code1_code2(self):
  215. """Create the code1.py and code2.py files."""
  216. self.make_file("code1.py", """\
  217. code1 = 1
  218. """)
  219. self.make_file("code2.py", """\
  220. code2 = 1
  221. code2 = 2
  222. """)
  223. def check_code1_code2(self, cov):
  224. """Check the analysis is correct for code1.py and code2.py."""
  225. _, statements, missing, _ = cov.analysis("code1.py")
  226. self.assertEqual(statements, [1])
  227. self.assertEqual(missing, [])
  228. _, statements, missing, _ = cov.analysis("code2.py")
  229. self.assertEqual(statements, [1, 2])
  230. self.assertEqual(missing, [])
  231. def test_start_stop_start_stop(self):
  232. self.make_code1_code2()
  233. cov = coverage.Coverage()
  234. self.start_import_stop(cov, "code1")
  235. cov.save()
  236. self.start_import_stop(cov, "code2")
  237. self.check_code1_code2(cov)
  238. def test_start_save_stop(self):
  239. self.skipTest("Expected failure: https://bitbucket.org/ned/coveragepy/issue/79")
  240. self.make_code1_code2()
  241. cov = coverage.Coverage()
  242. cov.start()
  243. self.import_local_file("code1")
  244. cov.save()
  245. self.import_local_file("code2")
  246. cov.stop()
  247. self.check_code1_code2(cov)
  248. def make_corrupt_data_files(self):
  249. """Make some good and some bad data files."""
  250. self.make_code1_code2()
  251. cov = coverage.Coverage(data_suffix=True)
  252. self.start_import_stop(cov, "code1")
  253. cov.save()
  254. cov = coverage.Coverage(data_suffix=True)
  255. self.start_import_stop(cov, "code2")
  256. cov.save()
  257. self.make_file(".coverage.foo", """La la la, this isn't coverage data!""")
  258. def test_combining_corrupt_data(self):
  259. # If you combine a corrupt data file, then you will get a warning,
  260. # and the file will remain.
  261. self.make_corrupt_data_files()
  262. cov = coverage.Coverage()
  263. warning_regex = (
  264. r"Couldn't read data from '.*\.coverage\.foo': "
  265. r"CoverageException: Doesn't seem to be a coverage\.py data file"
  266. )
  267. with self.assert_warnings(cov, [warning_regex]):
  268. cov.combine()
  269. # We got the results from code1 and code2 properly.
  270. self.check_code1_code2(cov)
  271. # The bad file still exists.
  272. self.assert_exists(".coverage.foo")
  273. class NamespaceModuleTest(CoverageTest):
  274. """Test PEP-420 namespace modules."""
  275. def setUp(self):
  276. super(NamespaceModuleTest, self).setUp()
  277. if env.PYVERSION < (3, 3):
  278. self.skipTest("Python before 3.3 doesn't have namespace packages")
  279. def test_explicit_namespace_module(self):
  280. self.make_file("namespace/package/module.py", "VAR = 1\n")
  281. self.make_file("main.py", "import namespace\n")
  282. cov = coverage.Coverage()
  283. self.start_import_stop(cov, "main")
  284. with self.assertRaisesRegex(CoverageException, r"Module .* has no file"):
  285. cov.analysis(sys.modules['namespace'])
  286. class UsingModulesMixin(object):
  287. """A mixin for importing modules from test/modules and test/moremodules."""
  288. run_in_temp_dir = False
  289. def setUp(self):
  290. super(UsingModulesMixin, self).setUp()
  291. old_dir = os.getcwd()
  292. os.chdir(self.nice_file(os.path.dirname(__file__), 'modules'))
  293. self.addCleanup(os.chdir, old_dir)
  294. # Parent class saves and restores sys.path, we can just modify it.
  295. sys.path.append(".")
  296. sys.path.append("../moremodules")
  297. class OmitIncludeTestsMixin(UsingModulesMixin):
  298. """Test methods for coverage methods taking include and omit."""
  299. def filenames_in(self, summary, filenames):
  300. """Assert the `filenames` are in the keys of `summary`."""
  301. for filename in filenames.split():
  302. self.assertIn(filename, summary)
  303. def filenames_not_in(self, summary, filenames):
  304. """Assert the `filenames` are not in the keys of `summary`."""
  305. for filename in filenames.split():
  306. self.assertNotIn(filename, summary)
  307. def test_nothing_specified(self):
  308. result = self.coverage_usepkgs()
  309. self.filenames_in(result, "p1a p1b p2a p2b othera otherb osa osb")
  310. self.filenames_not_in(result, "p1c")
  311. # Because there was no source= specified, we don't search for
  312. # unexecuted files.
  313. def test_include(self):
  314. result = self.coverage_usepkgs(include=["*/p1a.py"])
  315. self.filenames_in(result, "p1a")
  316. self.filenames_not_in(result, "p1b p1c p2a p2b othera otherb osa osb")
  317. def test_include_2(self):
  318. result = self.coverage_usepkgs(include=["*a.py"])
  319. self.filenames_in(result, "p1a p2a othera osa")
  320. self.filenames_not_in(result, "p1b p1c p2b otherb osb")
  321. def test_include_as_string(self):
  322. result = self.coverage_usepkgs(include="*a.py")
  323. self.filenames_in(result, "p1a p2a othera osa")
  324. self.filenames_not_in(result, "p1b p1c p2b otherb osb")
  325. def test_omit(self):
  326. result = self.coverage_usepkgs(omit=["*/p1a.py"])
  327. self.filenames_in(result, "p1b p2a p2b")
  328. self.filenames_not_in(result, "p1a p1c")
  329. def test_omit_2(self):
  330. result = self.coverage_usepkgs(omit=["*a.py"])
  331. self.filenames_in(result, "p1b p2b otherb osb")
  332. self.filenames_not_in(result, "p1a p1c p2a othera osa")
  333. def test_omit_as_string(self):
  334. result = self.coverage_usepkgs(omit="*a.py")
  335. self.filenames_in(result, "p1b p2b otherb osb")
  336. self.filenames_not_in(result, "p1a p1c p2a othera osa")
  337. def test_omit_and_include(self):
  338. result = self.coverage_usepkgs(include=["*/p1*"], omit=["*/p1a.py"])
  339. self.filenames_in(result, "p1b")
  340. self.filenames_not_in(result, "p1a p1c p2a p2b")
  341. class SourceOmitIncludeTest(OmitIncludeTestsMixin, CoverageTest):
  342. """Test using `source`, `omit` and `include` when measuring code."""
  343. def coverage_usepkgs(self, **kwargs):
  344. """Run coverage on usepkgs and return the line summary.
  345. Arguments are passed to the `coverage.Coverage` constructor.
  346. """
  347. cov = coverage.Coverage(**kwargs)
  348. cov.start()
  349. import usepkgs # pragma: nested # pylint: disable=import-error
  350. cov.stop() # pragma: nested
  351. data = cov.get_data()
  352. summary = data.line_counts()
  353. for k, v in list(summary.items()):
  354. assert k.endswith(".py")
  355. summary[k[:-3]] = v
  356. return summary
  357. def test_source_package(self):
  358. lines = self.coverage_usepkgs(source=["pkg1"])
  359. self.filenames_in(lines, "p1a p1b")
  360. self.filenames_not_in(lines, "p2a p2b othera otherb osa osb")
  361. # Because source= was specified, we do search for unexecuted files.
  362. self.assertEqual(lines['p1c'], 0)
  363. def test_source_package_dotted(self):
  364. lines = self.coverage_usepkgs(source=["pkg1.p1b"])
  365. self.filenames_in(lines, "p1b")
  366. self.filenames_not_in(lines, "p1a p1c p2a p2b othera otherb osa osb")
  367. def test_source_package_part_omitted(self):
  368. # https://bitbucket.org/ned/coveragepy/issue/218
  369. # Used to be if you omitted something executed and inside the source,
  370. # then after it was executed but not recorded, it would be found in
  371. # the search for unexecuted files, and given a score of 0%.
  372. lines = self.coverage_usepkgs(source=["pkg1"], omit=["pkg1/p1b.py"])
  373. self.filenames_in(lines, "p1a")
  374. self.filenames_not_in(lines, "p1b")
  375. self.assertEqual(lines['p1c'], 0)
  376. class ReportIncludeOmitTest(OmitIncludeTestsMixin, CoverageTest):
  377. """Tests of the report include/omit functionality."""
  378. def coverage_usepkgs(self, **kwargs):
  379. """Try coverage.report()."""
  380. cov = coverage.Coverage()
  381. cov.start()
  382. import usepkgs # pragma: nested # pylint: disable=import-error
  383. cov.stop() # pragma: nested
  384. report = StringIO()
  385. cov.report(file=report, **kwargs)
  386. return report.getvalue()
  387. class XmlIncludeOmitTest(OmitIncludeTestsMixin, CoverageTest):
  388. """Tests of the XML include/omit functionality.
  389. This also takes care of the HTML and annotate include/omit, by virtue
  390. of the structure of the code.
  391. """
  392. def coverage_usepkgs(self, **kwargs):
  393. """Try coverage.xml_report()."""
  394. cov = coverage.Coverage()
  395. cov.start()
  396. import usepkgs # pragma: nested # pylint: disable=import-error
  397. cov.stop() # pragma: nested
  398. cov.xml_report(outfile="-", **kwargs)
  399. return self.stdout()
  400. class AnalysisTest(CoverageTest):
  401. """Test the numerical analysis of results."""
  402. def test_many_missing_branches(self):
  403. cov = coverage.Coverage(branch=True)
  404. self.make_file("missing.py", """\
  405. def fun1(x):
  406. if x == 1:
  407. print("one")
  408. else:
  409. print("not one")
  410. print("done") # pragma: nocover
  411. def fun2(x):
  412. print("x")
  413. fun2(3)
  414. """)
  415. # Import the Python file, executing it.
  416. self.start_import_stop(cov, "missing")
  417. nums = cov._analyze("missing.py").numbers
  418. self.assertEqual(nums.n_files, 1)
  419. self.assertEqual(nums.n_statements, 7)
  420. self.assertEqual(nums.n_excluded, 1)
  421. self.assertEqual(nums.n_missing, 3)
  422. self.assertEqual(nums.n_branches, 2)
  423. self.assertEqual(nums.n_partial_branches, 0)
  424. self.assertEqual(nums.n_missing_branches, 2)
  425. class TestRunnerPluginTest(CoverageTest):
  426. """Test that the API works properly the way various third-party plugins call it.
  427. We don't actually use the plugins, but these tests call the API the same
  428. way they do.
  429. """
  430. def pretend_to_be_nose_with_cover(self, erase):
  431. """This is what the nose --with-cover plugin does."""
  432. cov = coverage.Coverage()
  433. self.make_file("no_biggie.py", """\
  434. a = 1
  435. b = 2
  436. if b == 1:
  437. c = 4
  438. """)
  439. if erase:
  440. cov.combine()
  441. cov.erase()
  442. cov.load()
  443. self.start_import_stop(cov, "no_biggie")
  444. cov.combine()
  445. cov.save()
  446. cov.report(["no_biggie.py"], show_missing=True)
  447. self.assertEqual(self.stdout(), textwrap.dedent("""\
  448. Name Stmts Miss Cover Missing
  449. --------------------------------------------
  450. no_biggie.py 4 1 75% 4
  451. """))
  452. def test_nose_plugin(self):
  453. self.pretend_to_be_nose_with_cover(erase=False)
  454. def test_nose_plugin_with_erase(self):
  455. self.pretend_to_be_nose_with_cover(erase=True)
  456. class ReporterDeprecatedAttributeTest(CoverageTest):
  457. """Test that Reporter.file_reporters has been deprecated."""
  458. run_in_temp_dir = False
  459. def test_reporter_file_reporters(self):
  460. rep = Reporter(None, None)
  461. with warnings.catch_warnings(record=True) as warns:
  462. warnings.simplefilter("always")
  463. # Accessing this attribute will raise a DeprecationWarning.
  464. rep.file_reporters # pylint: disable=pointless-statement
  465. self.assertEqual(len(warns), 1)
  466. self.assertTrue(issubclass(warns[0].category, DeprecationWarning))