test_memory_leaks.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. #!/usr/bin/env python
  2. # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """
  6. A test script which attempts to detect memory leaks by calling C
  7. functions many times and compare process memory usage before and
  8. after the calls. It might produce false positives.
  9. """
  10. import functools
  11. import gc
  12. import os
  13. import socket
  14. import sys
  15. import threading
  16. import time
  17. import psutil
  18. import psutil._common
  19. from psutil._compat import xrange, callable
  20. from test_psutil import (WINDOWS, POSIX, OSX, LINUX, SUNOS, BSD, TESTFN,
  21. RLIMIT_SUPPORT, TRAVIS)
  22. from test_psutil import (reap_children, supports_ipv6, safe_remove,
  23. get_test_subprocess)
  24. if sys.version_info < (2, 7):
  25. import unittest2 as unittest # https://pypi.python.org/pypi/unittest2
  26. else:
  27. import unittest
  28. LOOPS = 1000
  29. TOLERANCE = 4096
  30. SKIP_PYTHON_IMPL = True
  31. def skip_if_linux():
  32. return unittest.skipIf(LINUX and SKIP_PYTHON_IMPL,
  33. "not worth being tested on LINUX (pure python)")
  34. class Base(unittest.TestCase):
  35. proc = psutil.Process()
  36. def execute(self, function, *args, **kwargs):
  37. def call_many_times():
  38. for x in xrange(LOOPS - 1):
  39. self.call(function, *args, **kwargs)
  40. del x
  41. gc.collect()
  42. return self.get_mem()
  43. self.call(function, *args, **kwargs)
  44. self.assertEqual(gc.garbage, [])
  45. self.assertEqual(threading.active_count(), 1)
  46. # RSS comparison
  47. # step 1
  48. rss1 = call_many_times()
  49. # step 2
  50. rss2 = call_many_times()
  51. difference = rss2 - rss1
  52. if difference > TOLERANCE:
  53. # This doesn't necessarily mean we have a leak yet.
  54. # At this point we assume that after having called the
  55. # function so many times the memory usage is stabilized
  56. # and if there are no leaks it should not increase any
  57. # more.
  58. # Let's keep calling fun for 3 more seconds and fail if
  59. # we notice any difference.
  60. stop_at = time.time() + 3
  61. while True:
  62. self.call(function, *args, **kwargs)
  63. if time.time() >= stop_at:
  64. break
  65. del stop_at
  66. gc.collect()
  67. rss3 = self.get_mem()
  68. difference = rss3 - rss2
  69. if rss3 > rss2:
  70. self.fail("rss2=%s, rss3=%s, difference=%s"
  71. % (rss2, rss3, difference))
  72. def execute_w_exc(self, exc, function, *args, **kwargs):
  73. kwargs['_exc'] = exc
  74. self.execute(function, *args, **kwargs)
  75. def get_mem(self):
  76. return psutil.Process().memory_info()[0]
  77. def call(self, function, *args, **kwargs):
  78. raise NotImplementedError("must be implemented in subclass")
  79. class TestProcessObjectLeaks(Base):
  80. """Test leaks of Process class methods and properties"""
  81. def setUp(self):
  82. gc.collect()
  83. def tearDown(self):
  84. reap_children()
  85. def call(self, function, *args, **kwargs):
  86. if callable(function):
  87. if '_exc' in kwargs:
  88. exc = kwargs.pop('_exc')
  89. self.assertRaises(exc, function, *args, **kwargs)
  90. else:
  91. try:
  92. function(*args, **kwargs)
  93. except psutil.Error:
  94. pass
  95. else:
  96. meth = getattr(self.proc, function)
  97. if '_exc' in kwargs:
  98. exc = kwargs.pop('_exc')
  99. self.assertRaises(exc, meth, *args, **kwargs)
  100. else:
  101. try:
  102. meth(*args, **kwargs)
  103. except psutil.Error:
  104. pass
  105. @skip_if_linux()
  106. def test_name(self):
  107. self.execute('name')
  108. @skip_if_linux()
  109. def test_cmdline(self):
  110. self.execute('cmdline')
  111. @skip_if_linux()
  112. def test_exe(self):
  113. self.execute('exe')
  114. @skip_if_linux()
  115. def test_ppid(self):
  116. self.execute('ppid')
  117. @unittest.skipUnless(POSIX, "POSIX only")
  118. @skip_if_linux()
  119. def test_uids(self):
  120. self.execute('uids')
  121. @unittest.skipUnless(POSIX, "POSIX only")
  122. @skip_if_linux()
  123. def test_gids(self):
  124. self.execute('gids')
  125. @skip_if_linux()
  126. def test_status(self):
  127. self.execute('status')
  128. def test_nice_get(self):
  129. self.execute('nice')
  130. def test_nice_set(self):
  131. niceness = psutil.Process().nice()
  132. self.execute('nice', niceness)
  133. @unittest.skipUnless(hasattr(psutil.Process, 'ionice'),
  134. "Linux and Windows Vista only")
  135. def test_ionice_get(self):
  136. self.execute('ionice')
  137. @unittest.skipUnless(hasattr(psutil.Process, 'ionice'),
  138. "Linux and Windows Vista only")
  139. def test_ionice_set(self):
  140. if WINDOWS:
  141. value = psutil.Process().ionice()
  142. self.execute('ionice', value)
  143. else:
  144. from psutil._pslinux import cext
  145. self.execute('ionice', psutil.IOPRIO_CLASS_NONE)
  146. fun = functools.partial(cext.proc_ioprio_set, os.getpid(), -1, 0)
  147. self.execute_w_exc(OSError, fun)
  148. @unittest.skipIf(OSX or SUNOS, "feature not supported on this platform")
  149. @skip_if_linux()
  150. def test_io_counters(self):
  151. self.execute('io_counters')
  152. @unittest.skipUnless(WINDOWS, "not worth being tested on posix")
  153. def test_username(self):
  154. self.execute('username')
  155. @skip_if_linux()
  156. def test_create_time(self):
  157. self.execute('create_time')
  158. @skip_if_linux()
  159. def test_num_threads(self):
  160. self.execute('num_threads')
  161. @unittest.skipUnless(WINDOWS, "Windows only")
  162. def test_num_handles(self):
  163. self.execute('num_handles')
  164. @unittest.skipUnless(POSIX, "POSIX only")
  165. @skip_if_linux()
  166. def test_num_fds(self):
  167. self.execute('num_fds')
  168. @skip_if_linux()
  169. def test_threads(self):
  170. self.execute('threads')
  171. @skip_if_linux()
  172. def test_cpu_times(self):
  173. self.execute('cpu_times')
  174. @skip_if_linux()
  175. def test_memory_info(self):
  176. self.execute('memory_info')
  177. @skip_if_linux()
  178. def test_memory_info_ex(self):
  179. self.execute('memory_info_ex')
  180. @unittest.skipUnless(POSIX, "POSIX only")
  181. @skip_if_linux()
  182. def test_terminal(self):
  183. self.execute('terminal')
  184. @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL,
  185. "not worth being tested on POSIX (pure python)")
  186. def test_resume(self):
  187. self.execute('resume')
  188. @skip_if_linux()
  189. def test_cwd(self):
  190. self.execute('cwd')
  191. @unittest.skipUnless(WINDOWS or LINUX or BSD,
  192. "Windows or Linux or BSD only")
  193. def test_cpu_affinity_get(self):
  194. self.execute('cpu_affinity')
  195. @unittest.skipUnless(WINDOWS or LINUX or BSD,
  196. "Windows or Linux or BSD only")
  197. def test_cpu_affinity_set(self):
  198. affinity = psutil.Process().cpu_affinity()
  199. self.execute('cpu_affinity', affinity)
  200. if not TRAVIS:
  201. self.execute_w_exc(ValueError, 'cpu_affinity', [-1])
  202. @skip_if_linux()
  203. def test_open_files(self):
  204. safe_remove(TESTFN) # needed after UNIX socket test has run
  205. with open(TESTFN, 'w'):
  206. self.execute('open_files')
  207. # OSX implementation is unbelievably slow
  208. @unittest.skipIf(OSX, "OSX implementation is too slow")
  209. @skip_if_linux()
  210. def test_memory_maps(self):
  211. self.execute('memory_maps')
  212. @unittest.skipUnless(LINUX, "Linux only")
  213. @unittest.skipUnless(LINUX and RLIMIT_SUPPORT,
  214. "only available on Linux >= 2.6.36")
  215. def test_rlimit_get(self):
  216. self.execute('rlimit', psutil.RLIMIT_NOFILE)
  217. @unittest.skipUnless(LINUX, "Linux only")
  218. @unittest.skipUnless(LINUX and RLIMIT_SUPPORT,
  219. "only available on Linux >= 2.6.36")
  220. def test_rlimit_set(self):
  221. limit = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)
  222. self.execute('rlimit', psutil.RLIMIT_NOFILE, limit)
  223. self.execute_w_exc(OSError, 'rlimit', -1)
  224. @skip_if_linux()
  225. # Windows implementation is based on a single system-wide function
  226. @unittest.skipIf(WINDOWS, "tested later")
  227. def test_connections(self):
  228. def create_socket(family, type):
  229. sock = socket.socket(family, type)
  230. sock.bind(('', 0))
  231. if type == socket.SOCK_STREAM:
  232. sock.listen(1)
  233. return sock
  234. socks = []
  235. socks.append(create_socket(socket.AF_INET, socket.SOCK_STREAM))
  236. socks.append(create_socket(socket.AF_INET, socket.SOCK_DGRAM))
  237. if supports_ipv6():
  238. socks.append(create_socket(socket.AF_INET6, socket.SOCK_STREAM))
  239. socks.append(create_socket(socket.AF_INET6, socket.SOCK_DGRAM))
  240. if hasattr(socket, 'AF_UNIX'):
  241. safe_remove(TESTFN)
  242. s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  243. s.bind(TESTFN)
  244. s.listen(1)
  245. socks.append(s)
  246. kind = 'all'
  247. # TODO: UNIX sockets are temporarily implemented by parsing
  248. # 'pfiles' cmd output; we don't want that part of the code to
  249. # be executed.
  250. if SUNOS:
  251. kind = 'inet'
  252. try:
  253. self.execute('connections', kind=kind)
  254. finally:
  255. for s in socks:
  256. s.close()
  257. p = get_test_subprocess()
  258. DEAD_PROC = psutil.Process(p.pid)
  259. DEAD_PROC.kill()
  260. DEAD_PROC.wait()
  261. del p
  262. class TestProcessObjectLeaksZombie(TestProcessObjectLeaks):
  263. """Same as above but looks for leaks occurring when dealing with
  264. zombie processes raising NoSuchProcess exception.
  265. """
  266. proc = DEAD_PROC
  267. def call(self, *args, **kwargs):
  268. try:
  269. TestProcessObjectLeaks.call(self, *args, **kwargs)
  270. except psutil.NoSuchProcess:
  271. pass
  272. if not POSIX:
  273. def test_kill(self):
  274. self.execute('kill')
  275. def test_terminate(self):
  276. self.execute('terminate')
  277. def test_suspend(self):
  278. self.execute('suspend')
  279. def test_resume(self):
  280. self.execute('resume')
  281. def test_wait(self):
  282. self.execute('wait')
  283. class TestModuleFunctionsLeaks(Base):
  284. """Test leaks of psutil module functions."""
  285. def setUp(self):
  286. gc.collect()
  287. def call(self, function, *args, **kwargs):
  288. fun = getattr(psutil, function)
  289. fun(*args, **kwargs)
  290. @skip_if_linux()
  291. def test_cpu_count_logical(self):
  292. psutil.cpu_count = psutil._psplatform.cpu_count_logical
  293. self.execute('cpu_count')
  294. @skip_if_linux()
  295. def test_cpu_count_physical(self):
  296. psutil.cpu_count = psutil._psplatform.cpu_count_physical
  297. self.execute('cpu_count')
  298. @skip_if_linux()
  299. def test_boot_time(self):
  300. self.execute('boot_time')
  301. @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL,
  302. "not worth being tested on POSIX (pure python)")
  303. def test_pid_exists(self):
  304. self.execute('pid_exists', os.getpid())
  305. def test_virtual_memory(self):
  306. self.execute('virtual_memory')
  307. # TODO: remove this skip when this gets fixed
  308. @unittest.skipIf(SUNOS,
  309. "not worth being tested on SUNOS (uses a subprocess)")
  310. def test_swap_memory(self):
  311. self.execute('swap_memory')
  312. @skip_if_linux()
  313. def test_cpu_times(self):
  314. self.execute('cpu_times')
  315. @skip_if_linux()
  316. def test_per_cpu_times(self):
  317. self.execute('cpu_times', percpu=True)
  318. @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL,
  319. "not worth being tested on POSIX (pure python)")
  320. def test_disk_usage(self):
  321. self.execute('disk_usage', '.')
  322. def test_disk_partitions(self):
  323. self.execute('disk_partitions')
  324. @skip_if_linux()
  325. def test_net_io_counters(self):
  326. self.execute('net_io_counters')
  327. @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'),
  328. '/proc/diskstats not available on this Linux version')
  329. @skip_if_linux()
  330. def test_disk_io_counters(self):
  331. self.execute('disk_io_counters')
  332. # XXX - on Windows this produces a false positive
  333. @unittest.skipIf(WINDOWS, "XXX produces a false positive on Windows")
  334. def test_users(self):
  335. self.execute('users')
  336. @unittest.skipIf(LINUX,
  337. "not worth being tested on Linux (pure python)")
  338. def test_net_connections(self):
  339. self.execute('net_connections')
  340. def test_net_if_addrs(self):
  341. self.execute('net_if_addrs')
  342. @unittest.skipIf(TRAVIS, "EPERM on travis")
  343. def test_net_if_stats(self):
  344. self.execute('net_if_stats')
  345. def main():
  346. test_suite = unittest.TestSuite()
  347. tests = [TestProcessObjectLeaksZombie,
  348. TestProcessObjectLeaks,
  349. TestModuleFunctionsLeaks]
  350. for test in tests:
  351. test_suite.addTest(unittest.makeSuite(test))
  352. result = unittest.TextTestRunner(verbosity=2).run(test_suite)
  353. return result.wasSuccessful()
  354. if __name__ == '__main__':
  355. if not main():
  356. sys.exit(1)