From 494d8b84f601a19f4f298ffbb3b47a647384d521 Mon Sep 17 00:00:00 2001 From: Ryan Carsten Schmidt Date: Fri, 26 Jan 2024 02:42:14 -0600 Subject: [PATCH 01/46] Include net/if.h before net/if_dl.h (#2361) In old versions of macOS, net/if_dl.h neglects to include sys/types.h, which results in build failure: error: unknown type name 'u_char'; did you mean 'char'? Including net/if.h first works around the problem because net/if.h includes sys/types.h. Fixes #2360 --- psutil/arch/osx/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/osx/net.c b/psutil/arch/osx/net.c index e9cc61e9b..24ce1b834 100644 --- a/psutil/arch/osx/net.c +++ b/psutil/arch/osx/net.c @@ -9,11 +9,11 @@ // https://github.com/giampaolo/psutil/blame/efd7ed3/psutil/_psutil_osx.c #include +#include #include #include #include #include -#include #include "../../_psutil_common.h" From 2f1cd0cafa895786183cf43bb5f02fceeb68de55 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 26 Jan 2024 09:46:03 +0100 Subject: [PATCH 02/46] update CREDITS + mention @c0m4r for sponsorship (thanks!) --- CREDITS | 4 ++++ HISTORY.rst | 7 +++++++ README.rst | 1 + 3 files changed, 12 insertions(+) diff --git a/CREDITS b/CREDITS index 59715ac76..ca5847131 100644 --- a/CREDITS +++ b/CREDITS @@ -817,3 +817,7 @@ I: 2010 N: Oliver Tomé W: https://github.com/snom3ad I: 2222 + +N: Ryan Carsten Schmidt +W: https://github.com/ryandesign +I: 2361 diff --git a/HISTORY.rst b/HISTORY.rst index fae3b43b2..3877d437a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,12 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.9.9 (IN DEVELOPMENT) +====================== + +**Bug fixes** + +- 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) + 5.9.8 ===== diff --git a/README.rst b/README.rst index 12bf64254..0d14b99cd 100644 --- a/README.rst +++ b/README.rst @@ -150,6 +150,7 @@ Supporters + add your avatar From 5f0a409607d8105e380c05e4d94d52474fbb73d8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Feb 2024 00:28:18 +0100 Subject: [PATCH 03/46] adapt to new ruff config directives --- pyproject.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 677722dbe..59c2b6a22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,15 @@ [tool.black] +target-version = ["py37"] line-length = 79 skip-string-normalization = true preview = true -target-version = ["py37"] [tool.ruff] # https://beta.ruff.rs/docs/settings/ target-version = "py37" line-length = 79 + +[tool.ruff.lint] select = [ # To get a list of all values: `python3 -m ruff linter`. "ALL", @@ -29,6 +31,7 @@ ignore = [ "ARG002", # unused-method-argument "B007", # Loop control variable `x` not used within loop body "B904", # Within an `except` clause, raise exceptions with `raise ... from err` (PYTHON2.7 COMPAT) + "B904", # Use `raise from` to specify exception cause (PYTHON2.7 COMPAT) "C4", # flake8-comprehensions (PYTHON2.7 COMPAT) "C90", # mccabe (function `X` is too complex) "COM812", # Trailing comma missing @@ -66,7 +69,6 @@ ignore = [ "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements "SLF", # flake8-self "TD", # all TODOs, XXXs, etc. - "TRY200", # Use `raise from` to specify exception cause (PYTHON2.7 COMPAT) "TRY300", # Consider moving this statement to an `else` block "TRY301", # Abstract `raise` to an inner function "UP009", # [*] UTF-8 encoding declaration is unnecessary (PYTHON2.7 COMPAT) @@ -77,7 +79,7 @@ ignore = [ "UP032", # [*] Use f-string instead of `format` call (PYTHON2.7 COMPAT) ] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] # T201 == print(), T203 == pprint() # EM101 == raw-string-in-exception # TRY003 == raise-vanilla-args @@ -89,7 +91,7 @@ ignore = [ "scripts/internal/*" = ["T201", "T203"] "setup.py" = ["T201", "T203"] -[tool.ruff.isort] +[tool.ruff.lint.isort] # https://beta.ruff.rs/docs/settings/#isort force-single-line = true # one import per line lines-after-imports = 2 From 61dfd67808efc8ee99a7d6ea385abfb0bb97558b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Feb 2024 10:20:40 +0100 Subject: [PATCH 04/46] more ruff rules --- docs/conf.py | 6 +++++- psutil/_pslinux.py | 3 +-- psutil/_pswindows.py | 3 +-- psutil/tests/__init__.py | 6 +++--- psutil/tests/test_linux.py | 3 --- psutil/tests/test_process.py | 4 ++-- psutil/tests/test_system.py | 2 +- pyproject.toml | 21 +++++++++++++++++++- scripts/disk_usage.py | 3 ++- scripts/ifconfig.py | 2 +- scripts/internal/download_wheels_appveyor.py | 2 +- scripts/internal/print_announce.py | 1 + scripts/internal/print_api_speed.py | 2 +- scripts/internal/print_downloads.py | 2 +- scripts/internal/winmake.py | 4 ++-- scripts/procsmem.py | 5 +++-- scripts/winservices.py | 2 +- 17 files changed, 46 insertions(+), 25 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f2b05483c..3ebc64178 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- -# + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + # psutil documentation build configuration file, created by # sphinx-quickstart on Wed Oct 19 21:54:30 2016. # diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 798dd3651..5d0deb403 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -60,7 +60,6 @@ # fmt: off __extra__all__ = [ - # 'PROCFS_PATH', # io prio constants "IOPRIO_CLASS_NONE", "IOPRIO_CLASS_RT", "IOPRIO_CLASS_BE", @@ -1355,7 +1354,7 @@ def disk_partitions(all=False): if device in ("/dev/root", "rootfs"): device = RootFsDeviceFinder().find() or device if not all: - if device == '' or fstype not in fstypes: + if not device or fstype not in fstypes: continue maxfile = maxpath = None # set later ntuple = _common.sdiskpart( diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 2d3a0c9fd..6199e5748 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -239,7 +239,6 @@ def virtual_memory(): """System virtual memory as a namedtuple.""" mem = cext.virtual_mem() totphys, availphys, totsys, availsys = mem - # total = totphys avail = availphys free = availphys @@ -515,7 +514,7 @@ def win_service_get(name): return service -class WindowsService: +class WindowsService: # noqa: PLW1641 """Represents an installed Windows service.""" def __init__(self, name, display_name): diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 83bcfa5a9..5d74c185e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -737,9 +737,9 @@ def wrapper(*args, **kwargs): self.sleep() continue if PY3: - raise exc + raise exc # noqa: PLE0704 else: - raise + raise # noqa: PLE0704 # This way the user of the decorated function can change config # parameters. @@ -1944,7 +1944,7 @@ def is_namedtuple(x): """Check if object is an instance of namedtuple.""" t = type(x) b = t.__bases__ - if len(b) != 1 or b[0] != tuple: + if len(b) != 1 or b[0] is not tuple: return False f = getattr(t, '_fields', None) if not isinstance(f, tuple): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 0aa04f121..9fe399d52 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1342,7 +1342,6 @@ def test_emulate_exclude_partitions(self): ret = psutil.disk_io_counters(perdisk=False, nowrap=False) self.assertIsNone(ret) - # def is_storage_device(name): return name == 'nvme0n1' @@ -1936,7 +1935,6 @@ def get_test_file(fname): break raise RuntimeError("timeout looking for test file") - # testfn = self.get_testfn() with open(testfn, "w"): self.assertEqual(get_test_file(testfn).mode, "w") @@ -1944,7 +1942,6 @@ def get_test_file(fname): self.assertEqual(get_test_file(testfn).mode, "r") with open(testfn, "a"): self.assertEqual(get_test_file(testfn).mode, "a") - # with open(testfn, "r+"): self.assertEqual(get_test_file(testfn).mode, "r+") with open(testfn, "w+"): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 451c49200..35beb41e0 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -965,7 +965,7 @@ def test_cpu_affinity(self): self.assertEqual( p.cpu_affinity(), list(os.sched_getaffinity(p.pid)) ) - # + self.assertRaises(TypeError, p.cpu_affinity, 1) p.cpu_affinity(initial) # it should work with all iterables, not only lists @@ -1344,7 +1344,7 @@ def assert_raises_nsp(fun, fun_name): @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process(self): - parent, zombie = self.spawn_zombie() + _parent, zombie = self.spawn_zombie() self.assertProcessZombie(zombie) @unittest.skipIf(not POSIX, 'POSIX only') diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 152e378fd..6656c19ba 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -344,7 +344,7 @@ def test_cpu_count_logical(self): self.assertIsNotNone(logical) self.assertEqual(logical, len(psutil.cpu_times(percpu=True))) self.assertGreaterEqual(logical, 1) - # + if os.path.exists("/proc/cpuinfo"): with open("/proc/cpuinfo") as fd: cpuinfo_data = fd.read() diff --git a/pyproject.toml b/pyproject.toml index 59c2b6a22..dd7e22603 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,10 +2,13 @@ target-version = ["py37"] line-length = 79 skip-string-normalization = true +# https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html preview = true +enable-unstable-feature = ["multiline_string_handling", "string_processing", "wrap_long_dict_values_in_parens"] [tool.ruff] # https://beta.ruff.rs/docs/settings/ +preview = true target-version = "py37" line-length = 79 @@ -38,9 +41,15 @@ ignore = [ "D", # pydocstyle "DTZ", # flake8-datetimez "ERA001", # Found commented-out code + "F841", # Local variable `parent` is assigned to but never used "FBT", # flake8-boolean-trap (makes zero sense) "FIX", # Line contains TODO / XXX / ..., consider resolving the issue "FLY", # flynt (PYTHON2.7 COMPAT) + "FURB101", # `open` and `read` should be replaced by `Path(src).read_text()` + "FURB113", # Use `x.extend(('a', 'b', 'c'))` instead of repeatedly calling `x.append()` + "FURB118", # [*] Use `operator.add` instead of defining a lambda + "FURB140", # [*] Use `itertools.starmap` instead of the generator + "FURB145", # [*] Prefer `copy` method over slicing (PYTHON2.7 COMPAT) "INP", # flake8-no-pep420 "N801", # Class name `async_chat` should use CapWords convention (ASYNCORE COMPAT) "N802", # Function name X should be lowercase. @@ -48,13 +57,23 @@ ignore = [ "N818", # Exception name `FooBar` should be named with an Error suffix "PERF", # Perflint "PGH004", # Use specific rule codes when using `noqa` + "PLC0415", # `import` should be at the top-level of a file + "PLC2701", # Private name import `x` from external module `y` + "PLR0904", # Too many public methods (x > y) "PLR0911", # Too many return statements (8 > 6) "PLR0912", # Too many branches (x > y) "PLR0913", # Too many arguments in function definition (x > y) - "PLR0915", # Too many statements (92 > 50) + "PLR0914", # Too many local variables (x/y) + "PLR0915", # Too many statements (x > y) + "PLR0917", # Too many positional arguments (x/y) + "PLR1702", # Too many nested blocks (x > y) + "PLR1704", # Redefining argument with the local name `type_` "PLR2004", # Magic value used in comparison, consider replacing X with a constant variable "PLR5501", # Use `elif` instead of `else` then `if`, to reduce indentation + "PLR6201", # Use a `set` literal when testing for membership + "PLR6301", # Method `x` could be a function, class method, or static method "PLW0603", # Using the global statement to update `lineno` is discouraged + "PLW1514", # `open` in text mode without explicit `encoding` argument "PLW2901", # `for` loop variable `x` overwritten by assignment target "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib diff --git a/scripts/disk_usage.py b/scripts/disk_usage.py index 934b24a1c..c75c4f61d 100755 --- a/scripts/disk_usage.py +++ b/scripts/disk_usage.py @@ -12,6 +12,7 @@ /dev/sda6 345.9G 83.8G 244.5G 24% ext4 /home /dev/sda1 296.0M 43.1M 252.9M 14% vfat /boot/efi /dev/sda2 600.0M 312.4M 287.6M 52% fuseblk /media/Recovery + """ import os @@ -26,7 +27,7 @@ def main(): print(templ % ("Device", "Total", "Used", "Free", "Use ", "Type", "Mount")) for part in psutil.disk_partitions(all=False): if os.name == 'nt': - if 'cdrom' in part.opts or part.fstype == '': + if 'cdrom' in part.opts or not part.fstype: # skip cd-rom drives with no disk in it; they may raise # ENOENT, pop-up a Windows GUI error for a non-ready # partition or just hang. diff --git a/scripts/ifconfig.py b/scripts/ifconfig.py index 4b4246aaa..e23472ba9 100755 --- a/scripts/ifconfig.py +++ b/scripts/ifconfig.py @@ -111,7 +111,7 @@ def main(): print(" netmask : %s" % addr.netmask) if addr.ptp: print(" p2p : %s" % addr.ptp) - print("") + print() if __name__ == '__main__': diff --git a/scripts/internal/download_wheels_appveyor.py b/scripts/internal/download_wheels_appveyor.py index e8b0c54e0..0e6490b39 100755 --- a/scripts/internal/download_wheels_appveyor.py +++ b/scripts/internal/download_wheels_appveyor.py @@ -65,7 +65,7 @@ def get_file_urls(): print_color("no artifacts found", 'red') sys.exit(1) else: - for url in sorted(urls, key=lambda x: os.path.basename(x)): + for url in sorted(urls, key=os.path.basename): yield url diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 9f89f635a..68201d7fc 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -6,6 +6,7 @@ """Prints release announce based on HISTORY.rst file content. See: https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode. + """ import os diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py index adbaa89a0..786644b5a 100755 --- a/scripts/internal/print_api_speed.py +++ b/scripts/internal/print_api_speed.py @@ -181,7 +181,7 @@ def main(): print_timings() # --- process - print("") + print() print_header("PROCESS APIS") ignore = [ 'send_signal', diff --git a/scripts/internal/print_downloads.py b/scripts/internal/print_downloads.py index 09169984d..b453f15b9 100755 --- a/scripts/internal/print_downloads.py +++ b/scripts/internal/print_downloads.py @@ -141,7 +141,7 @@ def main(): downs = downloads() print("# Download stats") - print("") + print() s = "psutil download statistics of the last %s days (last update " % DAYS s += "*%s*).\n" % LAST_UPDATE s += "Generated via [pypistats.py](%s) script.\n" % GITHUB_SCRIPT_URL diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 43db68fd9..b789aa80d 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -138,7 +138,7 @@ def safe_rmtree(path): def onerror(fun, path, excinfo): exc = excinfo[1] if exc.errno != errno.ENOENT: - raise + raise # noqa: PLE0704 existed = os.path.isdir(path) shutil.rmtree(path, onerror=onerror) @@ -181,7 +181,7 @@ def safe_rmtree(path): def onerror(fun, path, excinfo): exc = excinfo[1] if exc.errno != errno.ENOENT: - raise + raise # noqa: PLE0704 existed = os.path.isdir(path) shutil.rmtree(path, onerror=onerror) diff --git a/scripts/procsmem.py b/scripts/procsmem.py index 54e89b05e..eec5cd51a 100755 --- a/scripts/procsmem.py +++ b/scripts/procsmem.py @@ -32,6 +32,7 @@ 20513 giampao /opt/sublime_text/sublime_text 65.8M 73.0M 0B 87.9M 3976 giampao compiz 115.0M 117.0M 0B 130.9M 32486 giampao skype 145.1M 147.5M 0B 149.6M + """ from __future__ import print_function @@ -89,8 +90,8 @@ def main(): p.pid, p._info["username"][:7] if p._info["username"] else "", convert_bytes(p._uss), - convert_bytes(p._pss) if p._pss != "" else "", - convert_bytes(p._swap) if p._swap != "" else "", + convert_bytes(p._pss) if p._pss else "", + convert_bytes(p._swap) if p._swap else "", convert_bytes(p._rss), cmd, ) diff --git a/scripts/winservices.py b/scripts/winservices.py index 8d941d532..216d0a652 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -52,7 +52,7 @@ def main(): ) print(s) print("binpath: %s" % info['binpath']) - print("") + print() if __name__ == '__main__': From 87e08c8ef681c5c21d1cf06fd6cd6d44f4ac58e8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Feb 2024 12:28:22 +0100 Subject: [PATCH 05/46] make install-pip: fix installation on python 2 --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 34279c90a..1ab660119 100644 --- a/Makefile +++ b/Makefile @@ -103,11 +103,13 @@ install-pip: ## Install pip (no-op if already installed). @$(PYTHON) -c \ "import sys, ssl, os, pkgutil, tempfile, atexit; \ sys.exit(0) if pkgutil.find_loader('pip') else None; \ - pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \ + PY3 = sys.version_info[0] == 3; \ + pyexc = 'from urllib.request import urlopen' if PY3 else 'from urllib2 import urlopen'; \ exec(pyexc); \ ctx = ssl._create_unverified_context() if hasattr(ssl, '_create_unverified_context') else None; \ + url = 'https://bootstrap.pypa.io/pip/2.7/get-pip.py' if not PY3 else 'https://bootstrap.pypa.io/get-pip.py'; \ kw = dict(context=ctx) if ctx else {}; \ - req = urlopen('https://bootstrap.pypa.io/get-pip.py', **kw); \ + req = urlopen(url, **kw); \ data = req.read(); \ f = tempfile.NamedTemporaryFile(suffix='.py'); \ atexit.register(f.close); \ From db60971e76042cc1eb115be9d9cadb58b76a047f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Feb 2024 12:56:07 +0100 Subject: [PATCH 06/46] use unicode literals u"" instead of u("") --- psutil/_compat.py | 8 +------- psutil/tests/__init__.py | 3 +-- psutil/tests/test_linux.py | 37 ++++++++++++++++++------------------ psutil/tests/test_unicode.py | 3 +-- pyproject.toml | 1 + 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/psutil/_compat.py b/psutil/_compat.py index 3db56c601..6070c2a04 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -23,7 +23,7 @@ # builtins "long", "range", "super", "unicode", "basestring", # literals - "u", "b", + "b", # collections module "lru_cache", # shutil module @@ -47,9 +47,6 @@ basestring = str range = range - def u(s): - return s - def b(s): return s.encode("latin-1") @@ -59,9 +56,6 @@ def b(s): unicode = unicode basestring = basestring - def u(s): - return unicode(s, "unicode_escape") - def b(s): return s diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5d74c185e..dd5a6c856 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -56,7 +56,6 @@ from psutil._compat import FileNotFoundError from psutil._compat import range from psutil._compat import super -from psutil._compat import u from psutil._compat import unicode from psutil._compat import which @@ -186,7 +185,7 @@ def macos_version(): TESTFN_PREFIX = '$psutil-%s-' % os.getpid() else: TESTFN_PREFIX = '@psutil-%s-' % os.getpid() -UNICODE_SUFFIX = u("-ƒőő") +UNICODE_SUFFIX = u"-ƒőő" # An invalid unicode string. if PY3: INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 9fe399d52..0583e619d 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -28,7 +28,6 @@ from psutil._compat import PY3 from psutil._compat import FileNotFoundError from psutil._compat import basestring -from psutil._compat import u from psutil.tests import GITHUB_ACTIONS from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_BATTERY @@ -1218,7 +1217,7 @@ def test_zfs_fs(self): raise self.fail("couldn't find any ZFS partition") else: # No ZFS partitions on this system. Let's fake one. - fake_file = io.StringIO(u("nodev\tzfs\n")) + fake_file = io.StringIO(u"nodev\tzfs\n") with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m1: @@ -1654,7 +1653,7 @@ def open_mock(name, *args, **kwargs): if name.endswith(('AC0/online', 'AC/online')): raise IOError(errno.ENOENT, "") elif name.endswith("/status"): - return io.StringIO(u("charging")) + return io.StringIO(u"charging") else: return orig_open(name, *args, **kwargs) @@ -1685,7 +1684,7 @@ def open_mock(name, *args, **kwargs): if name.endswith(('AC0/online', 'AC/online')): raise IOError(errno.ENOENT, "") elif name.endswith("/status"): - return io.StringIO(u("discharging")) + return io.StringIO(u"discharging") else: return orig_open(name, *args, **kwargs) @@ -1759,11 +1758,11 @@ class TestSensorsBatteryEmulated(PsutilTestCase): def test_it(self): def open_mock(name, *args, **kwargs): if name.endswith("/energy_now"): - return io.StringIO(u("60000000")) + return io.StringIO(u"60000000") elif name.endswith("/power_now"): - return io.StringIO(u("0")) + return io.StringIO(u"0") elif name.endswith("/energy_full"): - return io.StringIO(u("60000001")) + return io.StringIO(u"60000001") else: return orig_open(name, *args, **kwargs) @@ -1781,9 +1780,9 @@ class TestSensorsTemperatures(PsutilTestCase): def test_emulate_class_hwmon(self): def open_mock(name, *args, **kwargs): if name.endswith('/name'): - return io.StringIO(u("name")) + return io.StringIO(u"name") elif name.endswith('/temp1_label'): - return io.StringIO(u("label")) + return io.StringIO(u"label") elif name.endswith('/temp1_input'): return io.BytesIO(b"30000") elif name.endswith('/temp1_max'): @@ -1813,9 +1812,9 @@ def open_mock(name, *args, **kwargs): elif name.endswith('temp'): return io.BytesIO(b"30000") elif name.endswith('0_type'): - return io.StringIO(u("critical")) + return io.StringIO(u"critical") elif name.endswith('type'): - return io.StringIO(u("name")) + return io.StringIO(u"name") else: return orig_open(name, *args, **kwargs) @@ -1849,11 +1848,11 @@ class TestSensorsFans(PsutilTestCase): def test_emulate_data(self): def open_mock(name, *args, **kwargs): if name.endswith('/name'): - return io.StringIO(u("name")) + return io.StringIO(u"name") elif name.endswith('/fan1_label'): - return io.StringIO(u("label")) + return io.StringIO(u"label") elif name.endswith('/fan1_input'): - return io.StringIO(u("2000")) + return io.StringIO(u"2000") else: return orig_open(name, *args, **kwargs) @@ -2033,13 +2032,13 @@ def test_terminal_mocked(self): def test_cmdline_mocked(self): # see: https://github.com/giampaolo/psutil/issues/639 p = psutil.Process() - fake_file = io.StringIO(u('foo\x00bar\x00')) + fake_file = io.StringIO(u'foo\x00bar\x00') with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m: self.assertEqual(p.cmdline(), ['foo', 'bar']) assert m.called - fake_file = io.StringIO(u('foo\x00bar\x00\x00')) + fake_file = io.StringIO(u'foo\x00bar\x00\x00') with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m: @@ -2049,13 +2048,13 @@ def test_cmdline_mocked(self): def test_cmdline_spaces_mocked(self): # see: https://github.com/giampaolo/psutil/issues/1179 p = psutil.Process() - fake_file = io.StringIO(u('foo bar ')) + fake_file = io.StringIO(u'foo bar ') with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m: self.assertEqual(p.cmdline(), ['foo', 'bar']) assert m.called - fake_file = io.StringIO(u('foo bar ')) + fake_file = io.StringIO(u'foo bar ') with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m: @@ -2066,7 +2065,7 @@ def test_cmdline_mixed_separators(self): # https://github.com/giampaolo/psutil/issues/ # 1179#issuecomment-552984549 p = psutil.Process() - fake_file = io.StringIO(u('foo\x20bar\x00')) + fake_file = io.StringIO(u'foo\x20bar\x00') with mock.patch( 'psutil._common.open', return_value=fake_file, create=True ) as m: diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index f4883bf9b..d09003d27 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -85,7 +85,6 @@ from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import super -from psutil._compat import u from psutil.tests import APPVEYOR from psutil.tests import ASCII_FS from psutil.tests import CI_TESTING @@ -190,7 +189,7 @@ class TestFSAPIs(BaseUnicodeTest): def expect_exact_path_match(self): # Do not expect psutil to correctly handle unicode paths on # Python 2 if os.listdir() is not able either. - here = '.' if isinstance(self.funky_name, str) else u('.') + here = '.' if isinstance(self.funky_name, str) else u'.' with warnings.catch_warnings(): warnings.simplefilter("ignore") return self.funky_name in os.listdir(here) diff --git a/pyproject.toml b/pyproject.toml index dd7e22603..5821c1a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ ignore = [ "UP009", # [*] UTF-8 encoding declaration is unnecessary (PYTHON2.7 COMPAT) "UP010", # [*] Unnecessary `__future__` import `print_function` for target Python version (PYTHON2.7 COMPAT) "UP024", # [*] Replace aliased errors with `OSError` (PYTHON2.7 COMPAT) + "UP025", # [*] Remove unicode literals from strings (PYTHON2.7 COMPAT) "UP028", # [*] Replace `yield` over `for` loop with `yield from` (PYTHON2.7 COMPAT) "UP031", # [*] Use format specifiers instead of percent format "UP032", # [*] Use f-string instead of `format` call (PYTHON2.7 COMPAT) From bb3d59a371b3e142d7bf2e98f20340ef603d70ce Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 9 Feb 2024 15:23:11 +0100 Subject: [PATCH 07/46] chore: update cibuildwheel on windows (#2370) close #2369 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38c164e7d..8a2644981 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ jobs: python-version: 3.11 - name: Create wheels + run tests - uses: pypa/cibuildwheel@v2.16.2 + uses: pypa/cibuildwheel@v2.16.5 env: CIBW_ARCHS: "${{ matrix.archs }}" CIBW_PRERELEASE_PYTHONS: True From 45c7fa4d74b624f5ff630684bfc3bab48bfcefe4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 18 Feb 2024 20:45:51 +0100 Subject: [PATCH 08/46] update style to latest black ver --- docs/conf.py | 20 ++++++++++-------- psutil/_common.py | 10 +++++---- psutil/_pslinux.py | 15 ++++++++------ psutil/_pssunos.py | 10 +++++---- psutil/tests/__init__.py | 12 ++++++----- psutil/tests/test_linux.py | 12 ++++++----- psutil/tests/test_process.py | 16 ++++++++------- psutil/tests/test_windows.py | 40 ++++++++++++++++++++---------------- scripts/top.py | 22 +++++++++++--------- setup.py | 20 ++++++++++-------- 10 files changed, 100 insertions(+), 77 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3ebc64178..d94b93c65 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -344,15 +344,17 @@ def get_version(): # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [( - master_doc, - 'psutil', - 'psutil Documentation', - author, - 'psutil', - 'One line description of project.', - 'Miscellaneous', -)] +texinfo_documents = [ + ( + master_doc, + 'psutil', + 'psutil Documentation', + author, + 'psutil', + 'One line description of project.', + 'Miscellaneous', + ) +] # Documents to append as an appendix to all manuals. # diff --git a/psutil/_common.py b/psutil/_common.py index 6989feafd..7a53cc13b 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -269,10 +269,12 @@ class BatteryTime(enum.IntEnum): } if AF_INET6 is not None: - conn_tmap.update({ - "tcp6": ([AF_INET6], [SOCK_STREAM]), - "udp6": ([AF_INET6], [SOCK_DGRAM]), - }) + conn_tmap.update( + { + "tcp6": ([AF_INET6], [SOCK_STREAM]), + "udp6": ([AF_INET6], [SOCK_DGRAM]), + } + ) if AF_UNIX is not None: conn_tmap.update({"unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM])}) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 5d0deb403..41ecf0c90 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1454,10 +1454,12 @@ def sensors_temperatures(): continue trip_paths = glob.glob(base + '/trip_point*') - trip_points = set([ - '_'.join(os.path.basename(p).split('_')[0:3]) - for p in trip_paths - ]) + trip_points = set( + [ + '_'.join(os.path.basename(p).split('_')[0:3]) + for p in trip_paths + ] + ) critical = None high = None for trip_point in trip_points: @@ -2100,7 +2102,7 @@ def get_blocks(lines, current_block): path ): path = path[:-10] - ls.append(( + item = ( decode(addr), decode(perms), path, @@ -2114,7 +2116,8 @@ def get_blocks(lines, current_block): data.get(b'Referenced:', 0), data.get(b'Anonymous:', 0), data.get(b'Swap:', 0), - )) + ) + ls.append(item) return ls @wrap_exceptions diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index dddbece1f..3e72967a3 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -682,10 +682,12 @@ def connections(self, kind='inet'): # UNIX sockets if kind in ('all', 'unix'): - ret.extend([ - _common.pconn(*conn) - for conn in self._get_unix_sockets(self.pid) - ]) + ret.extend( + [ + _common.pconn(*conn) + for conn in self._get_unix_sockets(self.pid) + ] + ) return ret nt_mmap_grouped = namedtuple('mmap', 'path rss anon locked') diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index dd5a6c856..10f71cd39 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -1286,11 +1286,13 @@ def print_sysinfo(): info['kernel'] = platform.uname()[2] # python - info['python'] = ', '.join([ - platform.python_implementation(), - platform.python_version(), - platform.python_compiler(), - ]) + info['python'] = ', '.join( + [ + platform.python_implementation(), + platform.python_version(), + platform.python_compiler(), + ] + ) info['pip'] = getattr(pip, '__version__', 'not installed') if wheel is not None: info['pip'] += " (wheel=%s)" % wheel.__version__ diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 0583e619d..d133cd06f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -120,7 +120,7 @@ def get_ipv4_broadcast(ifname): def get_ipv6_addresses(ifname): with open("/proc/net/if_inet6") as f: all_fields = [] - for line in f.readlines(): + for line in f: fields = line.split() if fields[-1] == ifname: all_fields.append(fields) @@ -1698,10 +1698,12 @@ def test_emulate_power_undetermined(self): # Pretend we can't know whether the AC power cable not # connected (assert fallback to False). def open_mock(name, *args, **kwargs): - if name.startswith(( - '/sys/class/power_supply/AC0/online', - '/sys/class/power_supply/AC/online', - )): + if name.startswith( + ( + '/sys/class/power_supply/AC0/online', + '/sys/class/power_supply/AC/online', + ) + ): raise IOError(errno.ENOENT, "") elif name.startswith("/sys/class/power_supply/BAT0/status"): return io.BytesIO(b"???") diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 35beb41e0..4e20f0e6b 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1436,13 +1436,15 @@ def clean_dict(d): d.pop("__CF_USER_TEXT_ENCODING", None) d.pop("VERSIONER_PYTHON_PREFER_32_BIT", None) d.pop("VERSIONER_PYTHON_VERSION", None) - return dict([ - ( - k.replace("\r", "").replace("\n", ""), - v.replace("\r", "").replace("\n", ""), - ) - for k, v in d.items() - ]) + return dict( + [ + ( + k.replace("\r", "").replace("\n", ""), + v.replace("\r", "").replace("\n", ""), + ) + for k, v in d.items() + ] + ) self.maxDiff = None p = psutil.Process() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 5983af70a..502d92638 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -869,25 +869,29 @@ def test_environ_64(self): @unittest.skipIf(not WINDOWS, "WINDOWS only") class TestServices(PsutilTestCase): def test_win_service_iter(self): - valid_statuses = set([ - "running", - "paused", - "start", - "pause", - "continue", - "stop", - "stopped", - ]) + valid_statuses = set( + [ + "running", + "paused", + "start", + "pause", + "continue", + "stop", + "stopped", + ] + ) valid_start_types = set(["automatic", "manual", "disabled"]) - valid_statuses = set([ - "running", - "paused", - "start_pending", - "pause_pending", - "continue_pending", - "stop_pending", - "stopped", - ]) + valid_statuses = set( + [ + "running", + "paused", + "start_pending", + "pause_pending", + "continue_pending", + "stop_pending", + "stopped", + ] + ) for serv in psutil.win_service_iter(): data = serv.as_dict() self.assertIsInstance(data['name'], str) diff --git a/scripts/top.py b/scripts/top.py index c0687ae1f..1caf9804d 100755 --- a/scripts/top.py +++ b/scripts/top.py @@ -82,16 +82,18 @@ def poll(interval): procs_status = {} for p in psutil.process_iter(): try: - p.dict = p.as_dict([ - 'username', - 'nice', - 'memory_info', - 'memory_percent', - 'cpu_percent', - 'cpu_times', - 'name', - 'status', - ]) + p.dict = p.as_dict( + [ + 'username', + 'nice', + 'memory_info', + 'memory_percent', + 'cpu_percent', + 'cpu_times', + 'name', + 'status', + ] + ) try: procs_status[p.dict['status']] += 1 except KeyError: diff --git a/setup.py b/setup.py index 7c59f5645..dac0a973a 100755 --- a/setup.py +++ b/setup.py @@ -213,15 +213,17 @@ def get_winver(): raise RuntimeError(msg) macros.append(("PSUTIL_WINDOWS", 1)) - macros.extend([ - # be nice to mingw, see: - # http://www.mingw.org/wiki/Use_more_recent_defined_functions - ('_WIN32_WINNT', get_winver()), - ('_AVAIL_WINVER_', get_winver()), - ('_CRT_SECURE_NO_WARNINGS', None), - # see: https://github.com/giampaolo/psutil/issues/348 - ('PSAPI_VERSION', 1), - ]) + macros.extend( + [ + # be nice to mingw, see: + # http://www.mingw.org/wiki/Use_more_recent_defined_functions + ('_WIN32_WINNT', get_winver()), + ('_AVAIL_WINVER_', get_winver()), + ('_CRT_SECURE_NO_WARNINGS', None), + # see: https://github.com/giampaolo/psutil/issues/348 + ('PSAPI_VERSION', 1), + ] + ) ext = Extension( 'psutil._psutil_windows', From da0b7c6880165fe45de68ee819c68e304958498a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Feb 2024 18:38:19 +0100 Subject: [PATCH 09/46] fix py2 failure --- psutil/tests/test_process_all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index ee1a9b58d..e59de3fb2 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -28,6 +28,7 @@ from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 +from psutil._compat import FileNotFoundError from psutil._compat import long from psutil._compat import unicode from psutil.tests import CI_TESTING From 4cf56e08c1bc883ec89758834b50954380759858 Mon Sep 17 00:00:00 2001 From: shadeyg56 <31134255+shadeyg56@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:55:35 -0600 Subject: [PATCH 10/46] Linux: skip offline cpu cores in cpu_freq (#2376) Signed-off-by: shadeyg56 --- CREDITS | 4 ++++ HISTORY.rst | 1 + psutil/_pslinux.py | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/CREDITS b/CREDITS index ca5847131..a3206bb9a 100644 --- a/CREDITS +++ b/CREDITS @@ -821,3 +821,7 @@ I: 2222 N: Ryan Carsten Schmidt W: https://github.com/ryandesign I: 2361 + +N: Shade Gladden +W: https://github.com/shadeyg56 +I: 2376 \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index 3877d437a..5ffa6151b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ **Bug fixes** - 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) +- 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) 5.9.8 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 41ecf0c90..cae515be8 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -779,6 +779,13 @@ def cpu_freq(): # https://github.com/giampaolo/psutil/issues/1071 curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None) if curr is None: + online_path = ( + "/sys/devices/system/cpu/cpu{}/online".format(i) + ) + # if cpu core is offline, set to all zeroes + if cat(online_path, fallback=None) == "0\n": + ret.append(_common.scpufreq(0.0, 0.0, 0.0)) + continue msg = "can't find current frequency file" raise NotImplementedError(msg) curr = int(curr) / 1000 From 3dc2fbd82d2f4c4b4f33e475007309243e4376c5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 26 Feb 2024 19:00:49 +0100 Subject: [PATCH 11/46] #2366 [Windows]: log debug message when using slower process APIs --- HISTORY.rst | 4 ++++ psutil/_pswindows.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5ffa6151b..ed4223374 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,10 @@ 5.9.9 (IN DEVELOPMENT) ====================== +**Enhancements** + +- 2366_, [Windows]: log debug message when using slower process APIs. + **Bug fixes** - 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 6199e5748..8db66f0a0 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -864,6 +864,7 @@ def _get_raw_meminfo(self): if is_permission_err(err): # TODO: the C ext can probably be refactored in order # to get this from cext.proc_info() + debug("attempting memory_info() fallback (slower)") info = self._proc_info() return ( info[pinfo_map['num_page_faults']], @@ -991,6 +992,7 @@ def create_time(self): return created except OSError as err: if is_permission_err(err): + debug("attempting create_time() fallback (slower)") return self._proc_info()[pinfo_map['create_time']] raise @@ -1014,6 +1016,7 @@ def cpu_times(self): except OSError as err: if not is_permission_err(err): raise + debug("attempting cpu_times() fallback (slower)") info = self._proc_info() user = info[pinfo_map['user_time']] system = info[pinfo_map['kernel_time']] @@ -1100,6 +1103,7 @@ def io_counters(self): except OSError as err: if not is_permission_err(err): raise + debug("attempting io_counters() fallback (slower)") info = self._proc_info() ret = ( info[pinfo_map['io_rcount']], @@ -1159,6 +1163,7 @@ def num_handles(self): return cext.proc_num_handles(self.pid) except OSError as err: if is_permission_err(err): + debug("attempting num_handles() fallback (slower)") return self._proc_info()[pinfo_map['num_handles']] raise From 004146a3a45f1fb26df73e99207c2fa40b185a46 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 27 Feb 2024 00:40:40 +0100 Subject: [PATCH 12/46] add black opt to make lines more compact Signed-off-by: Giampaolo Rodola --- docs/conf.py | 20 ++++++++---------- psutil/_common.py | 10 ++++----- psutil/_pslinux.py | 10 ++++----- psutil/_pssunos.py | 10 ++++----- psutil/tests/__init__.py | 12 +++++------ psutil/tests/test_linux.py | 10 ++++----- psutil/tests/test_process.py | 16 +++++++-------- psutil/tests/test_windows.py | 40 ++++++++++++++++-------------------- pyproject.toml | 2 +- scripts/top.py | 22 +++++++++----------- setup.py | 20 ++++++++---------- 11 files changed, 75 insertions(+), 97 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d94b93c65..3ebc64178 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -344,17 +344,15 @@ def get_version(): # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - 'psutil', - 'psutil Documentation', - author, - 'psutil', - 'One line description of project.', - 'Miscellaneous', - ) -] +texinfo_documents = [( + master_doc, + 'psutil', + 'psutil Documentation', + author, + 'psutil', + 'One line description of project.', + 'Miscellaneous', +)] # Documents to append as an appendix to all manuals. # diff --git a/psutil/_common.py b/psutil/_common.py index 7a53cc13b..6989feafd 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -269,12 +269,10 @@ class BatteryTime(enum.IntEnum): } if AF_INET6 is not None: - conn_tmap.update( - { - "tcp6": ([AF_INET6], [SOCK_STREAM]), - "udp6": ([AF_INET6], [SOCK_DGRAM]), - } - ) + conn_tmap.update({ + "tcp6": ([AF_INET6], [SOCK_STREAM]), + "udp6": ([AF_INET6], [SOCK_DGRAM]), + }) if AF_UNIX is not None: conn_tmap.update({"unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM])}) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index cae515be8..2a59bfe13 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1461,12 +1461,10 @@ def sensors_temperatures(): continue trip_paths = glob.glob(base + '/trip_point*') - trip_points = set( - [ - '_'.join(os.path.basename(p).split('_')[0:3]) - for p in trip_paths - ] - ) + trip_points = set([ + '_'.join(os.path.basename(p).split('_')[0:3]) + for p in trip_paths + ]) critical = None high = None for trip_point in trip_points: diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 3e72967a3..dddbece1f 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -682,12 +682,10 @@ def connections(self, kind='inet'): # UNIX sockets if kind in ('all', 'unix'): - ret.extend( - [ - _common.pconn(*conn) - for conn in self._get_unix_sockets(self.pid) - ] - ) + ret.extend([ + _common.pconn(*conn) + for conn in self._get_unix_sockets(self.pid) + ]) return ret nt_mmap_grouped = namedtuple('mmap', 'path rss anon locked') diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 10f71cd39..dd5a6c856 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -1286,13 +1286,11 @@ def print_sysinfo(): info['kernel'] = platform.uname()[2] # python - info['python'] = ', '.join( - [ - platform.python_implementation(), - platform.python_version(), - platform.python_compiler(), - ] - ) + info['python'] = ', '.join([ + platform.python_implementation(), + platform.python_version(), + platform.python_compiler(), + ]) info['pip'] = getattr(pip, '__version__', 'not installed') if wheel is not None: info['pip'] += " (wheel=%s)" % wheel.__version__ diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d133cd06f..af27095a3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1698,12 +1698,10 @@ def test_emulate_power_undetermined(self): # Pretend we can't know whether the AC power cable not # connected (assert fallback to False). def open_mock(name, *args, **kwargs): - if name.startswith( - ( - '/sys/class/power_supply/AC0/online', - '/sys/class/power_supply/AC/online', - ) - ): + if name.startswith(( + '/sys/class/power_supply/AC0/online', + '/sys/class/power_supply/AC/online', + )): raise IOError(errno.ENOENT, "") elif name.startswith("/sys/class/power_supply/BAT0/status"): return io.BytesIO(b"???") diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 4e20f0e6b..35beb41e0 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1436,15 +1436,13 @@ def clean_dict(d): d.pop("__CF_USER_TEXT_ENCODING", None) d.pop("VERSIONER_PYTHON_PREFER_32_BIT", None) d.pop("VERSIONER_PYTHON_VERSION", None) - return dict( - [ - ( - k.replace("\r", "").replace("\n", ""), - v.replace("\r", "").replace("\n", ""), - ) - for k, v in d.items() - ] - ) + return dict([ + ( + k.replace("\r", "").replace("\n", ""), + v.replace("\r", "").replace("\n", ""), + ) + for k, v in d.items() + ]) self.maxDiff = None p = psutil.Process() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 502d92638..5983af70a 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -869,29 +869,25 @@ def test_environ_64(self): @unittest.skipIf(not WINDOWS, "WINDOWS only") class TestServices(PsutilTestCase): def test_win_service_iter(self): - valid_statuses = set( - [ - "running", - "paused", - "start", - "pause", - "continue", - "stop", - "stopped", - ] - ) + valid_statuses = set([ + "running", + "paused", + "start", + "pause", + "continue", + "stop", + "stopped", + ]) valid_start_types = set(["automatic", "manual", "disabled"]) - valid_statuses = set( - [ - "running", - "paused", - "start_pending", - "pause_pending", - "continue_pending", - "stop_pending", - "stopped", - ] - ) + valid_statuses = set([ + "running", + "paused", + "start_pending", + "pause_pending", + "continue_pending", + "stop_pending", + "stopped", + ]) for serv in psutil.win_service_iter(): data = serv.as_dict() self.assertIsInstance(data['name'], str) diff --git a/pyproject.toml b/pyproject.toml index 5821c1a03..0625af1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ line-length = 79 skip-string-normalization = true # https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html preview = true -enable-unstable-feature = ["multiline_string_handling", "string_processing", "wrap_long_dict_values_in_parens"] +enable-unstable-feature = ["hug_parens_with_braces_and_square_brackets", "multiline_string_handling", "string_processing", "wrap_long_dict_values_in_parens"] [tool.ruff] # https://beta.ruff.rs/docs/settings/ diff --git a/scripts/top.py b/scripts/top.py index 1caf9804d..c0687ae1f 100755 --- a/scripts/top.py +++ b/scripts/top.py @@ -82,18 +82,16 @@ def poll(interval): procs_status = {} for p in psutil.process_iter(): try: - p.dict = p.as_dict( - [ - 'username', - 'nice', - 'memory_info', - 'memory_percent', - 'cpu_percent', - 'cpu_times', - 'name', - 'status', - ] - ) + p.dict = p.as_dict([ + 'username', + 'nice', + 'memory_info', + 'memory_percent', + 'cpu_percent', + 'cpu_times', + 'name', + 'status', + ]) try: procs_status[p.dict['status']] += 1 except KeyError: diff --git a/setup.py b/setup.py index dac0a973a..7c59f5645 100755 --- a/setup.py +++ b/setup.py @@ -213,17 +213,15 @@ def get_winver(): raise RuntimeError(msg) macros.append(("PSUTIL_WINDOWS", 1)) - macros.extend( - [ - # be nice to mingw, see: - # http://www.mingw.org/wiki/Use_more_recent_defined_functions - ('_WIN32_WINNT', get_winver()), - ('_AVAIL_WINVER_', get_winver()), - ('_CRT_SECURE_NO_WARNINGS', None), - # see: https://github.com/giampaolo/psutil/issues/348 - ('PSAPI_VERSION', 1), - ] - ) + macros.extend([ + # be nice to mingw, see: + # http://www.mingw.org/wiki/Use_more_recent_defined_functions + ('_WIN32_WINNT', get_winver()), + ('_AVAIL_WINVER_', get_winver()), + ('_CRT_SECURE_NO_WARNINGS', None), + # see: https://github.com/giampaolo/psutil/issues/348 + ('PSAPI_VERSION', 1), + ]) ext = Extension( 'psutil._psutil_windows', From 79cd04c83bd5094eb9b9912dff5b6c69ebcdd486 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 27 Feb 2024 21:14:05 +0100 Subject: [PATCH 13/46] enable ruff preview mode Signed-off-by: Giampaolo Rodola --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0625af1fd..94266a7bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ enable-unstable-feature = ["hug_parens_with_braces_and_square_brackets", "multil [tool.ruff] # https://beta.ruff.rs/docs/settings/ -preview = true target-version = "py37" line-length = 79 [tool.ruff.lint] +preview = true select = [ # To get a list of all values: `python3 -m ruff linter`. "ALL", From bd033622d5645233f8623ec7991666f3d9ed8199 Mon Sep 17 00:00:00 2001 From: Anthony Ryan Date: Fri, 15 Mar 2024 13:45:14 -0400 Subject: [PATCH 14/46] Add pickle support to psutil Exceptions (#2380) Add __reduce__ method to exceptions commonly thrown, and start testing that exceptions can be pickled in test_serialization. Fixes #2272 Signed-off-by: Anthony Ryan --- CREDITS | 6 +++++- HISTORY.rst | 1 + psutil/_common.py | 12 ++++++++++++ psutil/tests/test_misc.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index a3206bb9a..3520723d4 100644 --- a/CREDITS +++ b/CREDITS @@ -824,4 +824,8 @@ I: 2361 N: Shade Gladden W: https://github.com/shadeyg56 -I: 2376 \ No newline at end of file +I: 2376 + +N: Anthony Ryan +W: https://github.com/anthonyryan1 +I: 2272 diff --git a/HISTORY.rst b/HISTORY.rst index ed4223374..1d82aefe7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ - 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) - 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) +- 2272_, Add pickle support to psutil Exceptions 5.9.8 ===== diff --git a/psutil/_common.py b/psutil/_common.py index 6989feafd..f33f3e035 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -331,6 +331,9 @@ def __init__(self, pid, name=None, msg=None): self.name = name self.msg = msg or "process no longer exists" + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.msg)) + class ZombieProcess(NoSuchProcess): """Exception raised when querying a zombie process. This is @@ -347,6 +350,9 @@ def __init__(self, pid, name=None, ppid=None, msg=None): self.ppid = ppid self.msg = msg or "PID still exists but it's a zombie" + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.ppid, self.msg)) + class AccessDenied(Error): """Exception raised when permission to perform an action is denied.""" @@ -359,6 +365,9 @@ def __init__(self, pid=None, name=None, msg=None): self.name = name self.msg = msg or "" + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.msg)) + class TimeoutExpired(Error): """Raised on Process.wait(timeout) if timeout expires and process @@ -374,6 +383,9 @@ def __init__(self, seconds, pid=None, name=None): self.name = name self.msg = "timeout after %s seconds" % seconds + def __reduce__(self): + return (self.__class__, (self.seconds, self.pid, self.name)) + # =================================================================== # --- utils diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 700c054f8..bfd1f3535 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -280,6 +280,45 @@ def check(ret): check(psutil.disk_usage(os.getcwd())) check(psutil.users()) + b = pickle.loads( + pickle.dumps( + psutil.NoSuchProcess( + pid=4567, name='name test', msg='msg test' + ) + ) + ) + self.assertEqual(b.pid, 4567) + self.assertEqual(b.name, 'name test') + self.assertEqual(b.msg, 'msg test') + + b = pickle.loads( + pickle.dumps( + psutil.ZombieProcess( + pid=4567, name='name test', ppid=42, msg='msg test' + ) + ) + ) + self.assertEqual(b.pid, 4567) + self.assertEqual(b.ppid, 42) + self.assertEqual(b.name, 'name test') + self.assertEqual(b.msg, 'msg test') + + b = pickle.loads( + pickle.dumps(psutil.AccessDenied(pid=123, name='name', msg='msg')) + ) + self.assertEqual(b.pid, 123) + self.assertEqual(b.name, 'name') + self.assertEqual(b.msg, 'msg') + + b = pickle.loads( + pickle.dumps( + psutil.TimeoutExpired(seconds=33, pid=4567, name='name') + ) + ) + self.assertEqual(b.seconds, 33) + self.assertEqual(b.pid, 4567) + self.assertEqual(b.name, 'name') + # # XXX: https://github.com/pypa/setuptools/pull/2896 # @unittest.skipIf(APPVEYOR, "temporarily disabled due to setuptools bug") # def test_setup_script(self): From bf9e1c7c76b1601db147a375714980cc02cfe15e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 18:46:12 +0100 Subject: [PATCH 15/46] always use unittest.SkipTest where needed --- psutil/tests/test_bsd.py | 4 ++-- psutil/tests/test_contracts.py | 2 +- psutil/tests/test_linux.py | 4 ++-- psutil/tests/test_misc.py | 10 +++++----- psutil/tests/test_posix.py | 6 +++--- psutil/tests/test_process.py | 4 ++-- psutil/tests/test_system.py | 4 ++-- psutil/tests/test_unicode.py | 4 ++-- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 7b502bcb8..a714632dc 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -267,7 +267,7 @@ def test_cpu_frequency_against_sysctl(self): try: sysctl_result = int(sysctl(sensor)) except RuntimeError: - self.skipTest("frequencies not supported by kernel") + raise unittest.SkipTest("frequencies not supported by kernel") self.assertEqual(psutil.cpu_freq().current, sysctl_result) sensor = "dev.cpu.0.freq_levels" @@ -500,7 +500,7 @@ def test_sensors_temperatures_against_sysctl(self): try: sysctl_result = int(float(sysctl(sensor)[:-1])) except RuntimeError: - self.skipTest("temperatures not supported by kernel") + raise unittest.SkipTest("temperatures not supported by kernel") self.assertAlmostEqual( psutil.sensors_temperatures()["coretemp"][cpu].current, sysctl_result, diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 4736f5f1a..b2bfe29d6 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -239,7 +239,7 @@ def test_cpu_count(self): @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): if psutil.cpu_freq() is None: - raise self.skipTest("cpu_freq() returns None") + raise unittest.SkipTest("cpu_freq() returns None") self.assert_ntuple_of_nums(psutil.cpu_freq(), type_=(float, int, long)) def test_disk_io_counters(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index af27095a3..818a6be2f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -278,7 +278,7 @@ def test_used(self): # https://gitlab.com/procps-ng/procps/commit/ # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e if get_free_version_info() < (3, 3, 12): - raise self.skipTest("old free version") + raise unittest.SkipTest("old free version") cli_value = free_physmem().used psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( @@ -342,7 +342,7 @@ def test_used(self): # https://gitlab.com/procps-ng/procps/commit/ # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e if get_free_version_info() < (3, 3, 12): - raise self.skipTest("old free version") + raise unittest.SkipTest("old free version") vmstat_value = vmstat('used memory') * 1024 psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 700c054f8..69bce8aaf 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -285,7 +285,7 @@ def check(ret): # def test_setup_script(self): # setup_py = os.path.join(ROOT_DIR, 'setup.py') # if CI_TESTING and not os.path.exists(setup_py): - # return self.skipTest("can't find setup.py") + # raise unittest.SkipTest("can't find setup.py") # module = import_module_by_path(setup_py) # self.assertRaises(SystemExit, module.setup) # self.assertEqual(module.get_version(), psutil.__version__) @@ -850,7 +850,7 @@ def test_cache_clear(self): @unittest.skipIf(not HAS_NET_IO_COUNTERS, 'not supported') def test_cache_clear_public_apis(self): if not psutil.disk_io_counters() or not psutil.net_io_counters(): - return self.skipTest("no disks or NICs available") + raise unittest.SkipTest("no disks or NICs available") psutil.disk_io_counters() psutil.net_io_counters() caches = wrap_numbers.cache_info() @@ -959,7 +959,7 @@ def test_pmap(self): def test_procsmem(self): if 'uss' not in psutil.Process().memory_full_info()._fields: - raise self.skipTest("not supported") + raise unittest.SkipTest("not supported") self.assert_stdout('procsmem.py') def test_killall(self): @@ -988,13 +988,13 @@ def test_cpu_distribution(self): @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_temperatures(self): if not psutil.sensors_temperatures(): - self.skipTest("no temperatures") + raise unittest.SkipTest("no temperatures") self.assert_stdout('temperatures.py') @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") def test_fans(self): if not psutil.sensors_fans(): - self.skipTest("no fans") + raise unittest.SkipTest("no fans") self.assert_stdout('fans.py') @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 53852cef8..5203c2707 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -342,7 +342,7 @@ def test_nic_names(self): def test_users(self): out = sh("who -u") if not out.strip(): - raise self.skipTest("no users on this system") + raise unittest.SkipTest("no users on this system") lines = out.split('\n') users = [x.split()[0] for x in lines] terminals = [x.split()[1] for x in lines] @@ -358,7 +358,7 @@ def test_users(self): def test_users_started(self): out = sh("who -u") if not out.strip(): - raise self.skipTest("no users on this system") + raise unittest.SkipTest("no users on this system") tstamp = None # '2023-04-11 09:31' (Linux) started = re.findall(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d", out) @@ -444,7 +444,7 @@ def df(device): out = sh("df -k %s" % device).strip() except RuntimeError as err: if "device busy" in str(err).lower(): - raise self.skipTest("df returned EBUSY") + raise unittest.SkipTest("df returned EBUSY") raise line = out.split('\n')[1] fields = line.split() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 35beb41e0..bfba9f2d0 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -757,7 +757,7 @@ def test_long_cmdline(self): try: self.assertEqual(p.cmdline(), cmdline) except psutil.ZombieProcess: - raise self.skipTest("OPENBSD: process turned into zombie") + raise unittest.SkipTest("OPENBSD: process turned into zombie") else: self.assertEqual(p.cmdline(), cmdline) @@ -1165,7 +1165,7 @@ def test_children_duplicates(self): # this is the one, now let's make sure there are no duplicates pid = sorted(table.items(), key=lambda x: x[1])[-1][0] if LINUX and pid == 0: - raise self.skipTest("PID 0") + raise unittest.SkipTest("PID 0") p = psutil.Process(pid) try: c = p.children(recursive=True) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 6656c19ba..ce6786a43 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -355,7 +355,7 @@ def test_cpu_count_cores(self): logical = psutil.cpu_count() cores = psutil.cpu_count(logical=False) if cores is None: - raise self.skipTest("cpu_count_cores() is None") + raise unittest.SkipTest("cpu_count_cores() is None") if WINDOWS and sys.getwindowsversion()[:2] <= (6, 1): # <= Vista self.assertIsNone(cores) else: @@ -579,7 +579,7 @@ def check_ls(ls): ls = psutil.cpu_freq(percpu=True) if FREEBSD and not ls: - raise self.skipTest("returns empty list on FreeBSD") + raise unittest.SkipTest("returns empty list on FreeBSD") assert ls, ls check_ls([psutil.cpu_freq(percpu=False)]) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index d09003d27..cb4bccf7f 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -175,7 +175,7 @@ def setUpClass(cls): def setUp(self): super().setUp() if self.skip_tests: - raise self.skipTest("can't handle unicode str") + raise unittest.SkipTest("can't handle unicode str") @serialrun @@ -246,7 +246,7 @@ def test_proc_open_files(self): self.assertIsInstance(path, str) if BSD and not path: # XXX - see https://github.com/giampaolo/psutil/issues/595 - return self.skipTest("open_files on BSD is broken") + raise unittest.SkipTest("open_files on BSD is broken") if self.expect_exact_path_match(): self.assertEqual( os.path.normcase(path), os.path.normcase(self.funky_name) From bc18030af16c3d51b977b803fdd2ae5df31da321 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 19:19:20 +0100 Subject: [PATCH 16/46] refact serialization tests --- psutil/tests/test_misc.py | 68 +++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index d7ee038dc..983543640 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -19,7 +19,6 @@ import psutil import psutil.tests -from psutil import LINUX from psutil import POSIX from psutil import WINDOWS from psutil._common import bcat @@ -34,7 +33,6 @@ from psutil._compat import PY3 from psutil._compat import FileNotFoundError from psutil._compat import redirect_stderr -from psutil.tests import APPVEYOR from psutil.tests import CI_TESTING from psutil.tests import HAS_BATTERY from psutil.tests import HAS_MEMORY_MAPS @@ -47,8 +45,10 @@ from psutil.tests import SCRIPTS_DIR from psutil.tests import PsutilTestCase from psutil.tests import mock +from psutil.tests import process_namespace from psutil.tests import reload_module from psutil.tests import sh +from psutil.tests import system_namespace # =================================================================== @@ -259,53 +259,66 @@ def test_process_as_dict_no_new_names(self): def test_serialization(self): def check(ret): - if json is not None: - json.loads(json.dumps(ret)) + json.loads(json.dumps(ret)) + a = pickle.dumps(ret) b = pickle.loads(a) self.assertEqual(ret, b) + # --- process APIs + + proc = psutil.Process() check(psutil.Process().as_dict()) - check(psutil.virtual_memory()) - check(psutil.swap_memory()) - check(psutil.cpu_times()) - check(psutil.cpu_times_percent(interval=0)) - check(psutil.net_io_counters()) - if LINUX and not os.path.exists('/proc/diskstats'): - pass - else: - if not APPVEYOR: - check(psutil.disk_io_counters()) - check(psutil.disk_partitions()) - check(psutil.disk_usage(os.getcwd())) - check(psutil.users()) + + ns = process_namespace(proc) + for fun, name in ns.iter(ns.getters, clear_cache=True): + with self.subTest(proc=proc, name=name): + try: + ret = fun() + except psutil.Error: + pass + else: + check(ret) + + # --- system APIs + + ns = system_namespace() + for fun, name in ns.iter(ns.getters): + with self.subTest(name=name): + try: + ret = fun() + except psutil.AccessDenied: + pass + else: + check(ret) + + # --- exception classes b = pickle.loads( pickle.dumps( - psutil.NoSuchProcess( - pid=4567, name='name test', msg='msg test' - ) + psutil.NoSuchProcess(pid=4567, name='name', msg='msg') ) ) + self.assertIsInstance(b, psutil.NoSuchProcess) self.assertEqual(b.pid, 4567) - self.assertEqual(b.name, 'name test') - self.assertEqual(b.msg, 'msg test') + self.assertEqual(b.name, 'name') + self.assertEqual(b.msg, 'msg') b = pickle.loads( pickle.dumps( - psutil.ZombieProcess( - pid=4567, name='name test', ppid=42, msg='msg test' - ) + psutil.ZombieProcess(pid=4567, name='name', ppid=42, msg='msg') ) ) + self.assertIsInstance(b, psutil.ZombieProcess) self.assertEqual(b.pid, 4567) self.assertEqual(b.ppid, 42) - self.assertEqual(b.name, 'name test') - self.assertEqual(b.msg, 'msg test') + self.assertEqual(b.name, 'name') + self.assertEqual(b.msg, 'msg') b = pickle.loads( pickle.dumps(psutil.AccessDenied(pid=123, name='name', msg='msg')) ) + self.assertIsInstance(b, psutil.AccessDenied) self.assertEqual(b.pid, 123) self.assertEqual(b.name, 'name') self.assertEqual(b.msg, 'msg') @@ -315,6 +328,7 @@ def check(ret): psutil.TimeoutExpired(seconds=33, pid=4567, name='name') ) ) + self.assertIsInstance(b, psutil.TimeoutExpired) self.assertEqual(b.seconds, 33) self.assertEqual(b.pid, 4567) self.assertEqual(b.name, 'name') From 058b4ccc82dbcae965fb5f1931051b710e813f96 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 19:32:09 +0100 Subject: [PATCH 17/46] fix win tests --- psutil/tests/test_misc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 983543640..64fce7996 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -284,6 +284,8 @@ def check(ret): ns = system_namespace() for fun, name in ns.iter(ns.getters): + if name in {"win_service_iter", "win_service_get"}: + continue with self.subTest(name=name): try: ret = fun() From aab8d09d029420756f85df839b993e3f139dd2ac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 19:55:55 +0100 Subject: [PATCH 18/46] Makefile: define a PYTHON_ENV_VARS var to use with the $PYTHON var. Also try to address #2374. --- Makefile | 70 +++++++++++++++++++-------------------- psutil/tests/test_misc.py | 2 ++ 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 1ab660119..a1fdf6fd3 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ # Configurable. PYTHON = python3 +PYTHON_ENV_VARS = PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 ARGS = TSCRIPT = psutil/tests/runner.py @@ -47,7 +48,6 @@ BUILD_OPTS = `$(PYTHON) -c \ # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c \ "import sys; print('' if hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix else '--user')"` -TEST_PREFIX = PSUTIL_SCRIPTS_DIR=`pwd`/scripts PYTHONWARNINGS=always PSUTIL_DEBUG=1 # if make is invoked with no arg, default to `make help` .DEFAULT_GOAL := help @@ -87,17 +87,17 @@ build: ## Compile (in parallel) without installing. @# "build_ext -i" copies compiled *.so files in ./psutil directory in order @# to allow "import psutil" when using the interactive interpreter from @# within this directory. - PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i $(BUILD_OPTS) - $(PYTHON) -c "import psutil" # make sure it actually worked + $(PYTHON_ENV_VARS) $(PYTHON) setup.py build_ext -i $(BUILD_OPTS) + $(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil" # make sure it actually worked install: ## Install this package as current user in "edit" mode. ${MAKE} build - PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) - $(PYTHON) -c "import psutil" # make sure it actually worked + $(PYTHON_ENV_VARS) $(PYTHON) setup.py develop $(INSTALL_OPTS) + $(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil" # make sure it actually worked uninstall: ## Uninstall this package via pip. - cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true - $(PYTHON) scripts/internal/purge_installation.py + cd ..; $(PYTHON_ENV_VARS) $(PYTHON) -m pip uninstall -y -v psutil || true + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/purge_installation.py install-pip: ## Install pip (no-op if already installed). @$(PYTHON) -c \ @@ -123,8 +123,8 @@ install-pip: ## Install pip (no-op if already installed). setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). ${MAKE} install-git-hooks ${MAKE} install-pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade $(PY_DEPS) + $(PYTHON_ENV_VARS) $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip + $(PYTHON_ENV_VARS) $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade $(PY_DEPS) # =================================================================== # Tests @@ -132,65 +132,65 @@ setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). test: ## Run all tests. To run a specific test do "make test ARGS=psutil.tests.test_system.TestDiskAPIs" ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) test-parallel: ## Run all tests in parallel. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) --parallel + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) --parallel test-process: ## Run process-related API tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_process.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_process.py test-process-all: ## Run tests which iterate over all process PIDs. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_process_all.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_process_all.py test-system: ## Run system-related API tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_system.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_system.py test-misc: ## Run miscellaneous tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_misc.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_misc.py test-testutils: ## Run test utils tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_testutils.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_testutils.py test-unicode: ## Test APIs dealing with strings. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_unicode.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_unicode.py test-contracts: ## APIs sanity tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_contracts.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_contracts.py test-connections: ## Test net_connections() and Process.connections(). ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_connections.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_connections.py test-posix: ## POSIX specific tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_posix.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_posix.py test-platform: ## Run specific platform tests only. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_memleaks.py + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_memleaks.py test-last-failed: ## Re-run tests which failed on last run ${MAKE} build - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) --last-failed + $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) --last-failed test-coverage: ## Run test coverage. ${MAKE} build # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov - $(TEST_PREFIX) $(PYTHON) -m coverage run -m unittest -v + $(PYTHON_ENV_VARS) $(PYTHON) -m coverage run -m unittest -v $(PYTHON) -m coverage report @echo "writing results to htmlcov/index.html" $(PYTHON) -m coverage html @@ -210,7 +210,7 @@ _pylint: ## Python pylint (not mandatory, just run it from time to time) @git ls-files '*.py' | xargs $(PYTHON) -m pylint --rcfile=pyproject.toml --jobs=${NUM_WORKERS} lint-c: ## Run C linter. - @git ls-files '*.c' '*.h' | xargs $(PYTHON) scripts/internal/clinter.py + @git ls-files '*.c' '*.h' | xargs $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/clinter.py lint-rst: ## Run C linter. @git ls-files '*.rst' | xargs rstcheck --config=pyproject.toml @@ -261,20 +261,20 @@ install-git-hooks: ## Install GIT pre-commit hook. sdist: ## Create tar.gz source distribution. ${MAKE} generate-manifest - PYTHONWARNINGS=all $(PYTHON) setup.py sdist + $(PYTHON_ENV_VARS) $(PYTHON) setup.py sdist download-wheels-github: ## Download latest wheels hosted on github. - $(PYTHON) scripts/internal/download_wheels_github.py --tokenfile=~/.github.token + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/download_wheels_github.py --tokenfile=~/.github.token ${MAKE} print-dist download-wheels-appveyor: ## Download latest wheels hosted on appveyor. - $(PYTHON) scripts/internal/download_wheels_appveyor.py + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/download_wheels_appveyor.py ${MAKE} print-dist check-sdist: ## Check sanity of source distribution. - $(PYTHON) -m virtualenv --clear --no-wheel --quiet build/venv - build/venv/bin/python -m pip install -v --isolated --quiet dist/*.tar.gz - build/venv/bin/python -c "import os; os.chdir('build/venv'); import psutil" + $(PYTHON_ENV_VARS) $(PYTHON) -m virtualenv --clear --no-wheel --quiet build/venv + $(PYTHON_ENV_VARS) build/venv/bin/python -m pip install -v --isolated --quiet dist/*.tar.gz + $(PYTHON_ENV_VARS) build/venv/bin/python -c "import os; os.chdir('build/venv'); import psutil" $(PYTHON) -m twine check --strict dist/*.tar.gz check-wheels: ## Check sanity of wheels. @@ -335,11 +335,11 @@ print-timeline: ## Print releases' timeline. print-access-denied: ## Print AD exceptions ${MAKE} build - @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_access_denied.py + @$(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/print_access_denied.py print-api-speed: ## Benchmark all API calls ${MAKE} build - @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_api_speed.py $(ARGS) + @$(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/print_api_speed.py $(ARGS) print-downloads: ## Print PYPI download statistics $(PYTHON) scripts/internal/print_downloads.py @@ -356,11 +356,11 @@ grep-todos: ## Look for TODOs in the source files. bench-oneshot: ## Benchmarks for oneshot() ctx manager (see #799). ${MAKE} build - $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot.py + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/bench_oneshot.py bench-oneshot-2: ## Same as above but using perf module (supposed to be more precise) ${MAKE} build - $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot_2.py + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/bench_oneshot_2.py check-broken-links: ## Look for broken links in source files. git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 64fce7996..93204fa06 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -15,6 +15,7 @@ import pickle import socket import stat +import sys import unittest import psutil @@ -632,6 +633,7 @@ def test_debug(self): with redirect_stderr(StringIO()) as f: debug("hello") + sys.stderr.flush() msg = f.getvalue() assert msg.startswith("psutil-debug"), msg self.assertIn("hello", msg) From 03a0ecc04be165745cfd69ad20a9ebb99db261b5 Mon Sep 17 00:00:00 2001 From: Mayank Jha <44309707+maynk27@users.noreply.github.com> Date: Sat, 16 Mar 2024 00:47:30 +0530 Subject: [PATCH 19/46] Update to fix OSX older version build failure (#2379) * Update to fix OSX older version build failure Signed-off-by: Mayank Jha --- psutil/arch/osx/net.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/arch/osx/net.c b/psutil/arch/osx/net.c index 24ce1b834..d365676ce 100644 --- a/psutil/arch/osx/net.c +++ b/psutil/arch/osx/net.c @@ -9,11 +9,11 @@ // https://github.com/giampaolo/psutil/blame/efd7ed3/psutil/_psutil_osx.c #include +#include +#include #include #include #include -#include -#include #include "../../_psutil_common.h" From b6d8c20b2db38c63587c17cfbe520061bdb01132 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 15 Mar 2024 20:21:32 +0100 Subject: [PATCH 20/46] chore: build macOS arm64 wheels on macos-14 (#2375) macos-14 runners are running on arm64 processors. This allows to test macOS arm64 wheels. Signed-off-by: mayeut --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a2644981..2c2ac100d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ concurrency: jobs: # Linux + macOS + Windows Python 3 py3: - name: py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'windows') && matrix.archs || 'all' }} + name: py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'ubuntu') && 'all' || matrix.archs }} runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -28,7 +28,9 @@ jobs: - os: ubuntu-latest archs: "x86_64 i686" - os: macos-12 - archs: "x86_64 arm64" + archs: "x86_64" + - os: macos-14 + archs: "arm64" - os: windows-2019 archs: "AMD64" - os: windows-2019 @@ -40,7 +42,7 @@ jobs: python-version: 3.11 - name: Create wheels + run tests - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: CIBW_ARCHS: "${{ matrix.archs }}" CIBW_PRERELEASE_PYTHONS: True From 1f14eeda74ebfd2abd267f85c61d7dafaad2ff5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 15 Mar 2024 20:24:55 +0100 Subject: [PATCH 21/46] Tests: Compare floats less strictly (#2372) We see: ====================================================================== FAIL: psutil.tests.test_system.TestCpuAPIs.test_cpu_times ---------------------------------------------------------------------- Traceback (most recent call last): File "/builddir/build/BUILD/psutil-release-5.9.5/psutil/tests/test_system.py", line 351, in test_cpu_times self.assertAlmostEqual(total, sum(times)) AssertionError: 885725913.3 != 885725913.3000001 within 7 places (1.1920928955078125e-07 difference) ---------------------------------------------------------------------- Or: ====================================================================== FAIL: psutil.tests.test_system.TestCpuAPIs.test_cpu_times ---------------------------------------------------------------------- Traceback (most recent call last): File "/builddir/build/BUILD/psutil-release-5.9.5/psutil/tests/test_system.py", line 351, in test_cpu_times self.assertAlmostEqual(total, sum(times)) AssertionError: 324284741.90999997 != 324284741.91 within 7 places (5.960464477539063e-08 difference) ---------------------------------------------------------------------- In CentOS Stream 10 builds on i686 and x86_64. --- psutil/tests/test_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index ce6786a43..d517b7405 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -385,7 +385,7 @@ def test_cpu_times(self): self.assertIsInstance(cp_time, float) self.assertGreaterEqual(cp_time, 0.0) total += cp_time - self.assertAlmostEqual(total, sum(times)) + self.assertAlmostEqual(total, sum(times), places=6) str(times) # CPU times are always supposed to increase over time # or at least remain the same and that's because time @@ -424,7 +424,7 @@ def test_per_cpu_times(self): self.assertIsInstance(cp_time, float) self.assertGreaterEqual(cp_time, 0.0) total += cp_time - self.assertAlmostEqual(total, sum(times)) + self.assertAlmostEqual(total, sum(times), places=6) str(times) self.assertEqual( len(psutil.cpu_times(percpu=True)[0]), From 4fda16d63befb7432b56ad096731ed62d3a6c2b4 Mon Sep 17 00:00:00 2001 From: Ryan Carsten Schmidt Date: Fri, 15 Mar 2024 14:29:17 -0500 Subject: [PATCH 22/46] Include CoreFoundation/CoreFoundation.h (#2364) * Include CoreFoundation/CoreFoundation.h This is correct, since this file does use CoreFoundation types, and it is necessary because on old macOS versions IOKit/ps/IOPowerSources.h, which is already included and which also uses CoreFoundation types, forgot to include CoreFoundation/CoreFoundation.h. Fixes #2362 --- psutil/arch/osx/sensors.c | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/arch/osx/sensors.c b/psutil/arch/osx/sensors.c index a2faa157c..28ee5f0b2 100644 --- a/psutil/arch/osx/sensors.c +++ b/psutil/arch/osx/sensors.c @@ -12,6 +12,7 @@ #include +#include #include #include From 8e40fe64897f7e10224c6da20ad5b9f8e00a2f08 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 20:35:53 +0100 Subject: [PATCH 23/46] update HISTORY / CREDITS --- CREDITS | 4 +++- HISTORY.rst | 7 +++++-- Makefile | 2 +- docs/index.rst | 1 + psutil/__init__.py | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CREDITS b/CREDITS index 3520723d4..baff0c089 100644 --- a/CREDITS +++ b/CREDITS @@ -44,6 +44,8 @@ Github usernames of people to CC on github when in need of help. - alxchk, Oleksii Shevchuk - AIX: - wiggin15, Arnon Yaari (maintainer) +- wheels / packaging / CI matrix: + - mayeut, Matthieu Darbois Top contributors ------------------------------------------------------------------------------- @@ -820,7 +822,7 @@ I: 2222 N: Ryan Carsten Schmidt W: https://github.com/ryandesign -I: 2361 +I: 2361, 2365 N: Shade Gladden W: https://github.com/shadeyg56 diff --git a/HISTORY.rst b/HISTORY.rst index 1d82aefe7..af4d56253 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,12 +6,15 @@ **Enhancements** - 2366_, [Windows]: log debug message when using slower process APIs. +- 2375_, [macOS]: provide arm64 wheels. (patch by Matthieu Darbois) **Bug fixes** -- 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) - 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) -- 2272_, Add pickle support to psutil Exceptions +- 2272_: Add pickle support to psutil Exceptions. +- 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) +- 2362_, [macOS]: can't compile on macOS 10.11. (patch by Ryan Schmidt) +- 2365_, [macOS]: can't compile on macOS < 10.9. (patch by Ryan Schmidt) 5.9.8 ===== diff --git a/Makefile b/Makefile index a1fdf6fd3..8b81764cf 100644 --- a/Makefile +++ b/Makefile @@ -210,7 +210,7 @@ _pylint: ## Python pylint (not mandatory, just run it from time to time) @git ls-files '*.py' | xargs $(PYTHON) -m pylint --rcfile=pyproject.toml --jobs=${NUM_WORKERS} lint-c: ## Run C linter. - @git ls-files '*.c' '*.h' | xargs $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/clinter.py + @git ls-files '*.c' '*.h' | xargs $(PYTHON) scripts/internal/clinter.py lint-rst: ## Run C linter. @git ls-files '*.rst' | xargs rstcheck --config=pyproject.toml diff --git a/docs/index.rst b/docs/index.rst index fc4cddc24..c5ebf59e1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3048,6 +3048,7 @@ Timeline .. _`nettop.py`: https://github.com/giampaolo/psutil/blob/master/scripts/nettop.py .. _`open`: https://docs.python.org/3/library/functions.html#open .. _`os.cpu_count`: https://docs.python.org/3/library/os.html#os.cpu_count +.. _`os.getloadavg`: https://docs.python.org//library/os.html#os.getloadavg .. _`os.getpid`: https://docs.python.org/3/library/os.html#os.getpid .. _`os.getpriority`: https://docs.python.org/3/library/os.html#os.getpriority .. _`os.getresgid`: https://docs.python.org//library/os.html#os.getresgid diff --git a/psutil/__init__.py b/psutil/__init__.py index 8138db41e..91f3dd65c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -213,7 +213,7 @@ AF_LINK = _psplatform.AF_LINK __author__ = "Giampaolo Rodola'" -__version__ = "5.9.8" +__version__ = "5.9.9" version_info = tuple([int(num) for num in __version__.split('.')]) _timer = getattr(time, 'monotonic', time.time) From f51f62beef5e2ce0d64c9cb76fb7b4e9094b1864 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Mar 2024 21:17:06 +0100 Subject: [PATCH 24/46] fix doc style Signed-off-by: Giampaolo Rodola --- docs/_static/css/custom.css | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index e88f53077..49da69dfb 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -23,6 +23,11 @@ margin-bottom: 0px !important; } +.rst-content li { + list-style: outside; + margin-left: 15px; +} + .document td { padding-bottom: 0px !important; } @@ -536,3 +541,10 @@ div.body div.admonition, div.body div.impl-detail { .highlight .il { color: #208050 } + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + border: 1px solid #e1e4e5; + padding: 0px; + overflow-x: auto; + margin: 1px 0 0px 0; +} From cfe897e2e7f2e15be97c338a9237a300ffc6e88e Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Fri, 29 Mar 2024 19:15:00 +0800 Subject: [PATCH 25/46] Fix typos again (#2388) Found via `codespell -L bacic,nd,hda,numer,fo,ser,pytho,syste,ue` and `typos --hidden --format brief` Signed-off-by: Kian-Meng Ang --- docs/DEVGUIDE.rst | 2 +- docs/index.rst | 2 +- psutil/__init__.py | 2 +- psutil/_pslinux.py | 2 +- psutil/_pssunos.py | 2 +- psutil/arch/osx/sensors.c | 2 +- psutil/arch/solaris/environ.c | 2 +- psutil/tests/test_system.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/DEVGUIDE.rst b/docs/DEVGUIDE.rst index a53235dab..744ddadf4 100644 --- a/docs/DEVGUIDE.rst +++ b/docs/DEVGUIDE.rst @@ -28,7 +28,7 @@ Once you have a compiler installed run: make test-memleaks make test-coverage make lint-all # Run Python and C linter - make fix-all # Fix linting erors + make fix-all # Fix linting errors make uninstall make help diff --git a/docs/index.rst b/docs/index.rst index c5ebf59e1..4959801f6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1294,7 +1294,7 @@ Process class .. method:: cwd() The process current working directory as an absolute path. If cwd cannot be - determined for some internal reason (e.g. system process or directiory no + determined for some internal reason (e.g. system process or directory no longer exists) it may return an empty string. .. versionchanged:: 5.6.4 added support for NetBSD diff --git a/psutil/__init__.py b/psutil/__init__.py index 91f3dd65c..71ae4533a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2037,7 +2037,7 @@ def swap_memory(): # ===================================================================== -# --- disks/paritions related functions +# --- disks/partitions related functions # ===================================================================== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2a59bfe13..7c1a4f972 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -417,7 +417,7 @@ def calculate_avail_vmem(mems): def virtual_memory(): """Report virtual memory stats. - This implementation mimicks procps-ng-3.3.12, aka "free" CLI tool: + This implementation mimics procps-ng-3.3.12, aka "free" CLI tool: https://gitlab.com/procps-ng/procps/blob/ 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L778-791 The returned values are supposed to match both "free" and "vmstat -s" diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index dddbece1f..20987ecc8 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -730,7 +730,7 @@ def toaddr(start, end): # readlink() even if it exists (ls shows it). # If that's the case we just return the # unresolved link path. - # This seems an incosistency with /proc similar + # This seems an inconsistency with /proc similar # to: http://goo.gl/55XgO name = '%s/%s/path/%s' % (procfs_path, self.pid, name) hit_enoent = True diff --git a/psutil/arch/osx/sensors.c b/psutil/arch/osx/sensors.c index 28ee5f0b2..53626c2dc 100644 --- a/psutil/arch/osx/sensors.c +++ b/psutil/arch/osx/sensors.c @@ -60,7 +60,7 @@ psutil_sensors_battery(PyObject *self, PyObject *args) { power_sources_information, CFSTR(kIOPSCurrentCapacityKey)); if (!CFNumberGetValue(capacity_ref, kCFNumberSInt32Type, &capacity)) { PyErr_SetString(PyExc_RuntimeError, - "No battery capacity infomration in power sources info"); + "No battery capacity information in power sources info"); goto error; } diff --git a/psutil/arch/solaris/environ.c b/psutil/arch/solaris/environ.c index 4b4e041a4..a2c793855 100644 --- a/psutil/arch/solaris/environ.c +++ b/psutil/arch/solaris/environ.c @@ -240,7 +240,7 @@ ptr_size_by_psinfo(psinfo_t info) { /* * Count amount of pointers in a block which ends with NULL. - * @param fd a discriptor of /proc/PID/as special file. + * @param fd a descriptor of /proc/PID/as special file. * @param offt an offset of block of pointers at the file. * @param ptr_size a pointer size (allowed values: {4, 8}). * @return amount of non-NULL pointers or -1 in case of error. diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index d517b7405..c1b88f595 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -88,7 +88,7 @@ def test_process_iter(self): with self.assertRaises(psutil.AccessDenied): list(psutil.process_iter()) - def test_prcess_iter_w_attrs(self): + def test_process_iter_w_attrs(self): for p in psutil.process_iter(attrs=['pid']): self.assertEqual(list(p.info.keys()), ['pid']) with self.assertRaises(ValueError): From 034a1a6996d4ff5116fc45a9c5ed8477d0d73d37 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 Apr 2024 21:37:25 +0200 Subject: [PATCH 26/46] fix ruff errors --- Makefile | 4 ++-- psutil/_pswindows.py | 6 ++---- psutil/tests/test_linux.py | 4 +--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 8b81764cf..17d4c7570 100644 --- a/Makefile +++ b/Makefile @@ -230,10 +230,10 @@ lint-all: ## Run all linters # =================================================================== fix-black: - git ls-files '*.py' | xargs $(PYTHON) -m black + @git ls-files '*.py' | xargs $(PYTHON) -m black fix-ruff: - @git ls-files '*.py' | xargs $(PYTHON) -m ruff --no-cache --fix + @git ls-files '*.py' | xargs $(PYTHON) -m ruff check --no-cache --fix fix-unittests: ## Fix unittest idioms. @git ls-files '*test_*.py' | xargs $(PYTHON) -m teyit --show-stats diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 8db66f0a0..3c60a949a 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -702,12 +702,10 @@ def is_permission_err(exc): # On Python 2 OSError doesn't always have 'winerror'. Sometimes # it does, in which case the original exception was WindowsError # (which is a subclass of OSError). - if getattr(exc, "winerror", -1) in ( + return getattr(exc, "winerror", -1) in ( cext.ERROR_ACCESS_DENIED, cext.ERROR_PRIVILEGE_NOT_HELD, - ): - return True - return False + ) def convert_oserror(exc, pid=None, name=None): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 818a6be2f..203b5d487 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1360,9 +1360,7 @@ def is_storage_device(name): def test_emulate_use_sysfs(self): def exists(path): - if path == '/proc/diskstats': - return False - return True + return path == '/proc/diskstats' wprocfs = psutil.disk_io_counters(perdisk=True) with mock.patch( From 5bac1427631ee7c3a6ffb3c8071f3c11fef06524 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 6 Apr 2024 17:29:04 +0200 Subject: [PATCH 27/46] pid_exists() and Process() disagree on whether a pid exists when ERROR_ACCESS_DENIED (#2394) ## Summary * OS: Windows * Bug fix: yes * Type: core * Fixes: 2359 ## Description On Windows, `pid_exists()` may return True but `psutil.Process()` raises `NoSuchProcess`. Internally, this happens because of: https://github.com/giampaolo/psutil/blob/034a1a6996d4ff5116fc45a9c5ed8477d0d73d37/psutil/arch/windows/proc_utils.c#L176-L178. Differently from UNIX, the assumption in the code that ERROR_ACCESS_DENIED means there's a process to deny access to (hence it exists) is wrong. We therefore remove this assumption and also write a test case which ensures that `pid_exists()`, `Process()` and `pids()` APIs are all consistent with each other. As a bonus, I also discovered there are "hidden" PIDs on Windows (oh well!). --- .github/workflows/build.yml | 2 +- HISTORY.rst | 2 + Makefile | 2 +- psutil/arch/windows/proc_utils.c | 12 +++-- psutil/tests/__init__.py | 5 ++- psutil/tests/test_osx.py | 4 +- psutil/tests/test_process_all.py | 75 ++++++++++++++++++++++++++++++++ 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c2ac100d..5bd907fc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,7 +108,7 @@ jobs: python-version: 3.x - name: 'Run linters' run: | - python3 -m pip install ruff black rstcheck toml-sort sphinx + python3 -m pip install ruff==0.3.4 black rstcheck toml-sort sphinx make lint-all # Check sanity of .tar.gz + wheel files diff --git a/HISTORY.rst b/HISTORY.rst index af4d56253..0ae9a74e0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,8 @@ - 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) - 2272_: Add pickle support to psutil Exceptions. +- 2359_, [Windows], [CRITICAL]: `pid_exists()`_ disagrees with `Process`_ on + whether a pid exists when ERROR_ACCESS_DENIED. - 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) - 2362_, [macOS]: can't compile on macOS 10.11. (patch by Ryan Schmidt) - 2365_, [macOS]: can't compile on macOS < 10.9. (patch by Ryan Schmidt) diff --git a/Makefile b/Makefile index 17d4c7570..f09d83ec5 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ PY3_DEPS = \ pypinfo \ requests \ rstcheck \ - ruff \ + ruff==0.3.4 \ setuptools \ sphinx_rtd_theme \ teyit \ diff --git a/psutil/arch/windows/proc_utils.c b/psutil/arch/windows/proc_utils.c index dac1129c0..77b6dbf1e 100644 --- a/psutil/arch/windows/proc_utils.c +++ b/psutil/arch/windows/proc_utils.c @@ -173,17 +173,15 @@ psutil_pid_is_running(DWORD pid) { hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); - // Access denied means there's a process to deny access to. - if ((hProcess == NULL) && (GetLastError() == ERROR_ACCESS_DENIED)) - return 1; - - hProcess = psutil_check_phandle(hProcess, pid, 1); if (hProcess != NULL) { + hProcess = psutil_check_phandle(hProcess, pid, 1); + if (hProcess != NULL) { + CloseHandle(hProcess); + return 1; + } CloseHandle(hProcess); - return 1; } - CloseHandle(hProcess); PyErr_Clear(); return psutil_pid_in_pids(pid); } diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index dd5a6c856..5e50e1787 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -1551,7 +1551,10 @@ class system_namespace: ('virtual_memory', (), {}), ] if HAS_CPU_FREQ: - getters += [('cpu_freq', (), {'percpu': True})] + if MACOS and platform.machine() == 'arm64': # skipped due to #1892 + pass + else: + getters += [('cpu_freq', (), {'percpu': True})] if HAS_GETLOADAVG: getters += [('getloadavg', (), {})] if HAS_SENSORS_TEMPERATURES: diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 8fce8ae08..1fe02d3e4 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -120,7 +120,9 @@ def test_cpu_count_cores(self): self.assertEqual(num, psutil.cpu_count(logical=False)) # TODO: remove this once 1892 is fixed - @unittest.skipIf(platform.machine() == 'arm64', "skipped due to #1892") + @unittest.skipIf( + MACOS and platform.machine() == 'arm64', "skipped due to #1892" + ) def test_cpu_freq(self): freq = psutil.cpu_freq() self.assertEqual( diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index e59de3fb2..68b720a4b 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -104,12 +104,14 @@ class TestFetchAllProcesses(PsutilTestCase): """ def setUp(self): + psutil._set_debug(False) # Using a pool in a CI env may result in deadlock, see: # https://github.com/giampaolo/psutil/issues/2104 if USE_PROC_POOL: self.pool = multiprocessing.Pool() def tearDown(self): + psutil._set_debug(True) if USE_PROC_POOL: self.pool.terminate() self.pool.join() @@ -459,6 +461,79 @@ def environ(self, ret, info): self.assertIsInstance(v, str) +class TestPidsRange(PsutilTestCase): + """Given pid_exists() return value for a range of PIDs which may or + may not exist, make sure that psutil.Process() and psutil.pids() + agree with pid_exists(). This guarantees that the 3 APIs are all + consistent with each other. See: + https://github.com/giampaolo/psutil/issues/2359 + + XXX - Note about Windows: it turns out there are some "hidden" PIDs + which are not returned by psutil.pids() and are also not revealed + by taskmgr.exe and ProcessHacker, still they can be instantiated by + psutil.Process() and queried. One of such PIDs is "conhost.exe". + Running as_dict() for it reveals that some Process() APIs + erroneously raise NoSuchProcess, so we know we have problem there. + Let's ignore this for now, since it's quite a corner case (who even + imagined hidden PIDs existed on Windows?). + """ + + def setUp(self): + psutil._set_debug(False) + + def tearDown(self): + psutil._set_debug(True) + + def test_it(self): + def is_linux_tid(pid): + try: + f = open("/proc/%s/status" % pid, "rb") + except FileNotFoundError: + return False + else: + with f: + for line in f: + if line.startswith(b"Tgid:"): + tgid = int(line.split()[1]) + # If tgid and pid are different then we're + # dealing with a process TID. + return tgid != pid + raise ValueError("'Tgid' line not found") + + def check(pid): + # In case of failure retry up to 3 times in order to avoid + # race conditions, especially when running in a CI + # environment where PIDs may appear and disappear at any + # time. + x = 3 + while True: + exists = psutil.pid_exists(pid) + try: + if exists: + psutil.Process(pid) + if not WINDOWS: # see docstring + self.assertIn(pid, psutil.pids()) + else: + with self.assertRaises(psutil.NoSuchProcess): + psutil.Process(pid) + if not WINDOWS: # see docstring + self.assertNotIn(pid, psutil.pids()) + except (psutil.Error, AssertionError) as err: + x -= 1 + if x == 0: + raise + else: + return + + for pid in range(1, 3000): + if LINUX and is_linux_tid(pid): + # On Linux a TID (thread ID) can be passed to the + # Process class and is querable like a PID (process + # ID). Skip it. + continue + check(pid) + + if __name__ == '__main__': from psutil.tests.runner import run_from_name From 841902c1c342121ee8d07d4b061c23de43de050a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 6 Apr 2024 21:51:42 +0200 Subject: [PATCH 28/46] OpenBSD: pid_exists() returns True for thread IDs (TIDs) (#2395) --- HISTORY.rst | 2 ++ psutil/_psbsd.py | 17 ++++++++++++++--- psutil/tests/test_process_all.py | 11 ++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0ae9a74e0..89207a63a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ **Bug fixes** +- 2395_, [OpenBSD]: `pid_exists()`_ erroneously return True if the argument is + a thread ID (TID) instead of a PID (process ID). - 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) - 2272_: Add pickle support to psutil Exceptions. - 2359_, [Windows], [CRITICAL]: `pid_exists()`_ disagrees with `Process`_ on diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index da68f5efd..4e67a9d85 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -561,10 +561,9 @@ def pids(): return ret -if OPENBSD or NETBSD: +if NETBSD: def pid_exists(pid): - """Return True if pid exists.""" exists = _psposix.pid_exists(pid) if not exists: # We do this because _psposix.pid_exists() lies in case of @@ -573,7 +572,19 @@ def pid_exists(pid): else: return True -else: +elif OPENBSD: + + def pid_exists(pid): + exists = _psposix.pid_exists(pid) + if not exists: + return False + else: + # OpenBSD seems to be the only BSD platform where + # _psposix.pid_exists() returns True for thread IDs (tids), + # so we can't use it. + return pid in pids() + +else: # FreeBSD pid_exists = _psposix.pid_exists diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index 68b720a4b..700ffe078 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -514,8 +514,12 @@ def check(pid): if not WINDOWS: # see docstring self.assertIn(pid, psutil.pids()) else: - with self.assertRaises(psutil.NoSuchProcess): - psutil.Process(pid) + # On OpenBSD thread IDs can be instantiated, + # and oneshot() succeeds, but other APIs fail + # with EINVAL. + if not OPENBSD: + with self.assertRaises(psutil.NoSuchProcess): + psutil.Process(pid) if not WINDOWS: # see docstring self.assertNotIn(pid, psutil.pids()) except (psutil.Error, AssertionError) as err: @@ -531,7 +535,8 @@ def check(pid): # Process class and is querable like a PID (process # ID). Skip it. continue - check(pid) + with self.subTest(pid=pid): + check(pid) if __name__ == '__main__': From 5a3d56be329559e9aa06e44680da312986c1b9fa Mon Sep 17 00:00:00 2001 From: Andrea Blengino Date: Mon, 8 Apr 2024 19:36:03 +0200 Subject: [PATCH 29/46] Fix workflow visibility badges in README (#2399) Signed-off-by: Andrea Blengino In README file in .rst format, badges from Shields.io on GitHub workflows require a ".svg" in the path after the ".yml" --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0d14b99cd..586a2ca4e 100644 --- a/README.rst +++ b/README.rst @@ -18,11 +18,11 @@ :target: https://github.com/giampaolo/psutil/graphs/contributors :alt: Contributors -.. |github-actions-wheels| image:: https://img.shields.io/github/actions/workflow/status/giampaolo/psutil/.github/workflows/build.yml?label=Linux%2C%20macOS%2C%20Windows +.. |github-actions-wheels| image:: https://img.shields.io/github/actions/workflow/status/giampaolo/psutil/.github/workflows/build.yml.svg?label=Linux%2C%20macOS%2C%20Windows :target: https://github.com/giampaolo/psutil/actions?query=workflow%3Abuild :alt: Linux, macOS, Windows -.. |github-actions-bsd| image:: https://img.shields.io/github/actions/workflow/status/giampaolo/psutil/.github/workflows/bsd.yml?label=FreeBSD,%20NetBSD,%20OpenBSD +.. |github-actions-bsd| image:: https://img.shields.io/github/actions/workflow/status/giampaolo/psutil/.github/workflows/bsd.yml.svg?label=FreeBSD,%20NetBSD,%20OpenBSD :target: https://github.com/giampaolo/psutil/actions?query=workflow%3Absd-tests :alt: FreeBSD, NetBSD, OpenBSD From 7556e5d4bab656d4357204fb8772bdc9fe387e58 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 Apr 2024 17:01:29 +0200 Subject: [PATCH 30/46] Speedup process iter (don't check for PID reuse) (#2404) No longer make process_iter() check whether PID has been reused. This makes it around 20x times faster on Linux. Also changed Process.is_running() so that it will automatically remove the reused PID from process_iter() internal cache. In addition, also add a new process_iter.cache_clear() API. --- HISTORY.rst | 6 ++- docs/index.rst | 38 ++++++++----- psutil/__init__.py | 102 +++++++++++++++-------------------- psutil/tests/test_process.py | 11 ++++ psutil/tests/test_system.py | 50 ++++++++++++----- 5 files changed, 120 insertions(+), 87 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 89207a63a..981355127 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,12 +1,16 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.9.9 (IN DEVELOPMENT) +6.0.0 (IN DEVELOPMENT) ====================== **Enhancements** - 2366_, [Windows]: log debug message when using slower process APIs. - 2375_, [macOS]: provide arm64 wheels. (patch by Matthieu Darbois) +- 2396_: `process_iter()`_ no longer pre-emptively checks whether PIDs have + been reused. This makes `process_iter()`_ around 20x times faster. +- 2396_: a new ``psutil.process_iter.cache_clear()`` API can be used the clear + `process_iter()`_ internal cache. **Bug fixes** diff --git a/docs/index.rst b/docs/index.rst index 4959801f6..428e43111 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -928,12 +928,12 @@ Functions Return an iterator yielding a :class:`Process` class instance for all running processes on the local machine. - This should be preferred over :func:`psutil.pids()` to iterate over processes - as it's safe from race condition. + This should be preferred over :func:`psutil.pids()` to iterate over + processes, as retrieving info is safe from race conditions. Every :class:`Process` instance is only created once, and then cached for the next time :func:`psutil.process_iter()` is called (if PID is still alive). - Also it makes sure process PIDs are not reused. + Cache can optionally be cleared via ``process_iter.clear_cache()``. *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. If *attrs* is specified :meth:`Process.as_dict()` result will be stored as a @@ -963,9 +963,19 @@ Functions 3: {'name': 'ksoftirqd/0', 'username': 'root'}, ...} + Clear internal cache:: + + >>> psutil.process_iter.cache_clear() + .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. + .. versionchanged:: + 6.0.0 no longer checks whether each yielded process PID has been reused. + + .. versionchanged:: + 6.0.0 added ``psutil.process_iter.cache_clear()`` API. + .. function:: pid_exists(pid) Check whether the given PID exists in the current process list. This is @@ -1071,11 +1081,12 @@ Process class .. note:: - the way this class is bound to a process is uniquely via its **PID**. + the way this class is bound to a process is via its **PID**. That means that if the process terminates and the OS reuses its PID you may - end up interacting with another process. - The only exceptions for which process identity is preemptively checked - (via PID + creation time) is for the following methods: + inadvertently end up querying another process. To prevent this problem + you can use :meth:`is_running()` first. + The only methods which preemptively check whether PID has been reused + (via PID + creation time) are: :meth:`nice` (set), :meth:`ionice` (set), :meth:`cpu_affinity` (set), @@ -1087,13 +1098,8 @@ Process class :meth:`suspend` :meth:`resume`, :meth:`send_signal`, - :meth:`terminate` + :meth:`terminate` and :meth:`kill`. - To prevent this problem for all other methods you can use - :meth:`is_running()` before querying the process or - :func:`process_iter()` in case you're iterating over all processes. - It must be noted though that unless you deal with very "old" (inactive) - :class:`Process` instances this will hardly represent a problem. .. method:: oneshot() @@ -1979,11 +1985,17 @@ Process class This is reliable also in case the process is gone and its PID reused by another process, therefore it must be preferred over doing ``psutil.pid_exists(p.pid)``. + If PID has been reused this method will also remove the process from + :func:`process_iter()` internal cache. .. note:: this will return ``True`` also if the process is a zombie (``p.status() == psutil.STATUS_ZOMBIE``). + .. versionchanged:: 6.0.0 : automatically remove process from + :func:`process_iter()` internal cache if PID has been reused by another + process. + .. method:: send_signal(signal) Send a signal to process (see `signal module`_ constants) preemptively diff --git a/psutil/__init__.py b/psutil/__init__.py index 71ae4533a..934013819 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -213,7 +213,7 @@ AF_LINK = _psplatform.AF_LINK __author__ = "Giampaolo Rodola'" -__version__ = "5.9.9" +__version__ = "6.0.0" version_info = tuple([int(num) for num in __version__.split('.')]) _timer = getattr(time, 'monotonic', time.time) @@ -291,11 +291,9 @@ class Process(object): # noqa: UP004 If PID is omitted current process PID (os.getpid()) is used. Raise NoSuchProcess if PID does not exist. - Note that most of the methods of this class do not make sure - the PID of the process being queried has been reused over time. - That means you might end up retrieving an information referring - to another process in case the original one this instance - refers to is gone in the meantime. + Note that most of the methods of this class do not make sure that + the PID of the process being queried has been reused. That means + that you may end up retrieving information for another process. The only exceptions for which process identity is pre-emptively checked and guaranteed are: @@ -312,11 +310,8 @@ class Process(object): # noqa: UP004 - terminate() - kill() - To prevent this problem for all other methods you can: - - use is_running() before querying the process - - if you're continuously iterating over a set of Process - instances use process_iter() which pre-emptively checks - process identity for every yielded instance + To prevent this problem for all other methods you can use + is_running() before querying the process. """ def __init__(self, pid=None): @@ -384,19 +379,24 @@ def __str__(self): if self._name: info['name'] = self._name with self.oneshot(): - try: - info["name"] = self.name() - info["status"] = self.status() - except ZombieProcess: - info["status"] = "zombie" - except NoSuchProcess: - info["status"] = "terminated" - except AccessDenied: - pass + if self._pid_reused: + info["status"] = "terminated + PID reused" + else: + try: + info["name"] = self.name() + info["status"] = self.status() + except ZombieProcess: + info["status"] = "zombie" + except NoSuchProcess: + info["status"] = "terminated" + except AccessDenied: + pass + if self._exitcode not in (_SENTINEL, None): info["exitcode"] = self._exitcode if self._create_time is not None: info['started'] = _pprint_secs(self._create_time) + return "%s.%s(%s)" % ( self.__class__.__module__, self.__class__.__name__, @@ -436,7 +436,7 @@ def __hash__(self): def _raise_if_pid_reused(self): """Raises NoSuchProcess in case process PID has been reused.""" - if not self.is_running() and self._pid_reused: + if self._pid_reused or (not self.is_running() and self._pid_reused): # We may directly raise NSP in here already if PID is just # not running, but I prefer NSP to be raised naturally by # the actual Process API call. This way unit tests will tell @@ -599,19 +599,24 @@ def parents(self): def is_running(self): """Return whether this process is running. - It also checks if PID has been reused by another process in - which case return False. + + It also checks if PID has been reused by another process, in + which case it will remove the process from `process_iter()` + internal cache and return False. """ if self._gone or self._pid_reused: return False try: # Checking if PID is alive is not enough as the PID might - # have been reused by another process: we also want to - # verify process identity. - # Process identity / uniqueness over time is guaranteed by - # (PID + creation time) and that is verified in __eq__. + # have been reused by another process. Process identity / + # uniqueness over time is guaranteed by (PID + creation + # time) and that is verified in __eq__. self._pid_reused = self != Process(self.pid) - return not self._pid_reused + if self._pid_reused: + # remove this PID from `process_iter()` internal cache + _pmap.pop(self.pid, None) + raise NoSuchProcess(self.pid) + return True except ZombieProcess: # We should never get here as it's already handled in # Process.__init__; here just for extra safety. @@ -1463,10 +1468,7 @@ def process_iter(attrs=None, ad_value=None): Every new Process instance is only created once and then cached into an internal table which is updated every time this is used. - - Cached Process instances are checked for identity so that you're - safe in case a PID has been reused by another process, in which - case the cached instance is updated. + Cache can optionally be cleared via `process_iter.clear_cache()`. The sorting order in which processes are yielded is based on their PIDs. @@ -1482,8 +1484,6 @@ def process_iter(attrs=None, ad_value=None): def add(pid): proc = Process(pid) - if attrs is not None: - proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) pmap[proc.pid] = proc return proc @@ -1502,38 +1502,20 @@ def remove(pid): for pid, proc in ls: try: if proc is None: # new process - yield add(pid) - else: - # use is_running() to check whether PID has been - # reused by another process in which case yield a - # new Process instance - if proc.is_running(): - if attrs is not None: - proc.info = proc.as_dict( - attrs=attrs, ad_value=ad_value - ) - yield proc - else: - yield add(pid) + proc = add(pid) + if attrs is not None: + proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) + yield proc except NoSuchProcess: remove(pid) - except AccessDenied: - # Process creation time can't be determined hence there's - # no way to tell whether the pid of the cached process - # has been reused. Just return the cached version. - if proc is None and pid in pmap: - try: - yield pmap[pid] - except KeyError: - # If we get here it is likely that 2 threads were - # using process_iter(). - pass - else: - raise finally: _pmap = pmap +process_iter.cache_clear = lambda: _pmap.clear() # noqa +process_iter.cache_clear.__doc__ = "Clear process_iter() internal cache." + + def wait_procs(procs, timeout=None, callback=None): """Convenience function which waits for a list of processes to terminate. diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index bfba9f2d0..9613eac18 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1375,13 +1375,24 @@ def test_reused_pid(self): subp = self.spawn_testproc() p = psutil.Process(subp.pid) p._ident = (p.pid, p.create_time() + 100) + + list(psutil.process_iter()) + self.assertIn(p.pid, psutil._pmap) assert not p.is_running() + # make sure is_running() removed PID from process_iter() + # internal cache + self.assertNotIn(p.pid, psutil._pmap) + assert p != psutil.Process(subp.pid) msg = "process no longer exists and its PID has been reused" ns = process_namespace(p) for fun, name in ns.iter(ns.setters + ns.killers, clear_cache=False): with self.subTest(name=name): self.assertRaisesRegex(psutil.NoSuchProcess, msg, fun) + + self.assertIn("terminated + PID reused", str(p)) + self.assertIn("terminated + PID reused", repr(p)) + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.ppid) self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.parent) self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.parents) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index c1b88f595..0c4affd0f 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -61,8 +61,8 @@ # =================================================================== -class TestProcessAPIs(PsutilTestCase): - def test_process_iter(self): +class TestProcessIter(PsutilTestCase): + def test_pid_presence(self): self.assertIn(os.getpid(), [x.pid for x in psutil.process_iter()]) sproc = self.spawn_testproc() self.assertIn(sproc.pid, [x.pid for x in psutil.process_iter()]) @@ -71,24 +71,40 @@ def test_process_iter(self): p.wait() self.assertNotIn(sproc.pid, [x.pid for x in psutil.process_iter()]) - # assert there are no duplicates + def test_no_duplicates(self): ls = [x for x in psutil.process_iter()] self.assertEqual( sorted(ls, key=lambda x: x.pid), sorted(set(ls), key=lambda x: x.pid), ) - with mock.patch( - 'psutil.Process', side_effect=psutil.NoSuchProcess(os.getpid()) - ): - self.assertEqual(list(psutil.process_iter()), []) - with mock.patch( - 'psutil.Process', side_effect=psutil.AccessDenied(os.getpid()) - ): - with self.assertRaises(psutil.AccessDenied): - list(psutil.process_iter()) + def test_emulate_nsp(self): + list(psutil.process_iter()) # populate cache + for x in range(2): + with mock.patch( + 'psutil.Process.as_dict', + side_effect=psutil.NoSuchProcess(os.getpid()), + ): + self.assertEqual( + list(psutil.process_iter(attrs=["cpu_times"])), [] + ) + psutil.process_iter.cache_clear() # repeat test without cache - def test_process_iter_w_attrs(self): + def test_emulate_access_denied(self): + list(psutil.process_iter()) # populate cache + for x in range(2): + with mock.patch( + 'psutil.Process.as_dict', + side_effect=psutil.AccessDenied(os.getpid()), + ): + with self.assertRaises(psutil.AccessDenied): + list(psutil.process_iter(attrs=["cpu_times"])) + psutil.process_iter.cache_clear() # repeat test without cache + + def test_attrs(self): + for p in psutil.process_iter(attrs=['pid']): + self.assertEqual(list(p.info.keys()), ['pid']) + # yield again for p in psutil.process_iter(attrs=['pid']): self.assertEqual(list(p.info.keys()), ['pid']) with self.assertRaises(ValueError): @@ -113,6 +129,14 @@ def test_process_iter_w_attrs(self): self.assertGreaterEqual(p.info['pid'], 0) assert m.called + def test_cache_clear(self): + list(psutil.process_iter()) # populate cache + assert psutil._pmap + psutil.process_iter.cache_clear() + assert not psutil._pmap + + +class TestProcessAPIs(PsutilTestCase): @unittest.skipIf( PYPY and WINDOWS, "spawn_testproc() unreliable on PYPY + WINDOWS" ) From b6281c4674682eaa1d2e49517e6090c7f6361206 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 Apr 2024 17:52:40 +0200 Subject: [PATCH 31/46] Remove disk_partitions() `maxfile` and `maxpath` fields (#2405) --- HISTORY.rst | 14 ++++++++++++++ README.rst | 4 ++-- docs/DEVNOTES | 12 +++++------- docs/index.rst | 9 ++++----- psutil/__init__.py | 20 +------------------- psutil/_common.py | 3 +-- psutil/_psaix.py | 5 +---- psutil/_psbsd.py | 5 +---- psutil/_pslinux.py | 5 +---- psutil/_psosx.py | 5 +---- psutil/_pssunos.py | 5 +---- psutil/arch/windows/disk.c | 12 ++++-------- psutil/tests/test_contracts.py | 2 -- psutil/tests/test_system.py | 6 ------ 14 files changed, 36 insertions(+), 71 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 981355127..ca8e313a5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,9 @@ **Enhancements** +- 2109_: ``maxfile`` and ``maxpath`` fields were removed from the namedtuple + returned by `disk_partitions()`_. Reason: on network filesystems (NFS) this + can potentially take a very long time to complete. - 2366_, [Windows]: log debug message when using slower process APIs. - 2375_, [macOS]: provide arm64 wheels. (patch by Matthieu Darbois) - 2396_: `process_iter()`_ no longer pre-emptively checks whether PIDs have @@ -24,6 +27,17 @@ - 2362_, [macOS]: can't compile on macOS 10.11. (patch by Ryan Schmidt) - 2365_, [macOS]: can't compile on macOS < 10.9. (patch by Ryan Schmidt) +**Porting notes** + +Version 6.0.0 introduces some changes which affect backward compatibility: + +- 2109_: the namedtuple returned by `disk_partitions()`_' no longer has + ``maxfile`` and ``maxpath`` fields. +- 2396_: `process_iter()`_ no longer pre-emptively checks whether PIDs have + been reused. If you want to check for PID reusage you are supposed to use + `Process.is_running()`_ against the yielded `Process`_ instances. That will + also automatically remove reused PIDs from `process_iter()`_ internal cache. + 5.9.8 ===== diff --git a/README.rst b/README.rst index 586a2ca4e..9df03860b 100644 --- a/README.rst +++ b/README.rst @@ -228,8 +228,8 @@ Disks .. code-block:: python >>> psutil.disk_partitions() - [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid', maxfile=255, maxpath=4096), - sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext', opts='rw', maxfile=255, maxpath=4096)] + [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext', opts='rw')] >>> >>> psutil.disk_usage('/') sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) diff --git a/docs/DEVNOTES b/docs/DEVNOTES index fdd2bad18..49d449403 100644 --- a/docs/DEVNOTES +++ b/docs/DEVNOTES @@ -113,13 +113,11 @@ REJECTED IDEAS INCONSISTENCIES =============== -- disk_partitions(all=False) should have been "perdisk=False" instead: - * disk_io_counters(perdisk=False) - * net_io_counters(pernic=False) - * cpu_times_percent(percpu=False) - * cpu_times(percpu=False) - * cpu_freq(percpu=False) -- PROCFS_PATH should have been set_procfs_path() +- `Process.connections()` should have been `Process.net_connections()` for + consistency with `psutil.net_connections()`. +- PROCFS_PATH should have been set_procfs_path(). +- `virtual_memory()` should have been `memory_virtual()`. +- `swap_memory()` should have been `memory_swap()`. RESOURCES ========= diff --git a/docs/index.rst b/docs/index.rst index 428e43111..eb2b11388 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -450,17 +450,16 @@ Disks on Windows). * **opts**: a comma-separated string indicating different mount options for the drive/partition. Platform-dependent. - * **maxfile**: the maximum length a file name can have. - * **maxpath**: the maximum length a path name (directory name + base file - name) can have. >>> import psutil >>> psutil.disk_partitions() - [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro', maxfile=255, maxpath=4096), - sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw', maxfile=255, maxpath=4096)] + [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro'), + sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw')] .. versionchanged:: 5.7.4 added *maxfile* and *maxpath* fields + .. versionchanged:: 6.0.0 removed *maxfile* and *maxpath* fields + .. function:: disk_usage(path) Return disk usage statistics about the partition which contains the given diff --git a/psutil/__init__.py b/psutil/__init__.py index 934013819..f70ad5696 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2040,25 +2040,7 @@ def disk_partitions(all=False): If *all* parameter is False return physical devices only and ignore all others. """ - - def pathconf(path, name): - try: - return os.pathconf(path, name) - except (OSError, AttributeError): - pass - - ret = _psplatform.disk_partitions(all) - if POSIX: - new = [] - for item in ret: - nt = item._replace( - maxfile=pathconf(item.mountpoint, 'PC_NAME_MAX'), - maxpath=pathconf(item.mountpoint, 'PC_PATH_MAX'), - ) - new.append(nt) - return new - else: - return ret + return _psplatform.disk_partitions(all) def disk_io_counters(perdisk=False, nowrap=True): diff --git a/psutil/_common.py b/psutil/_common.py index f33f3e035..50179fb3c 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -189,8 +189,7 @@ class BatteryTime(enum.IntEnum): 'read_bytes', 'write_bytes', 'read_time', 'write_time']) # psutil.disk_partitions() -sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts', - 'maxfile', 'maxpath']) +sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts']) # psutil.net_io_counters() snetio = namedtuple('snetio', ['bytes_sent', 'bytes_recv', 'packets_sent', 'packets_recv', diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 7310ab6c3..204c1bcf2 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -191,10 +191,7 @@ def disk_partitions(all=False): # filter by filesystem having a total size > 0. if not disk_usage(mountpoint).total: continue - maxfile = maxpath = None # set later - ntuple = _common.sdiskpart( - device, mountpoint, fstype, opts, maxfile, maxpath - ) + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) return retlist diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 4e67a9d85..094a59f88 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -395,10 +395,7 @@ def disk_partitions(all=False): partitions = cext.disk_partitions() for partition in partitions: device, mountpoint, fstype, opts = partition - maxfile = maxpath = None # set later - ntuple = _common.sdiskpart( - device, mountpoint, fstype, opts, maxfile, maxpath - ) + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) return retlist diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 7c1a4f972..b0ca24777 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1363,10 +1363,7 @@ def disk_partitions(all=False): if not all: if not device or fstype not in fstypes: continue - maxfile = maxpath = None # set later - ntuple = _common.sdiskpart( - device, mountpoint, fstype, opts, maxfile, maxpath - ) + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) return retlist diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 673ac0db7..037e1d2b5 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -203,10 +203,7 @@ def disk_partitions(all=False): if not all: if not os.path.isabs(device) or not os.path.exists(device): continue - maxfile = maxpath = None # set later - ntuple = _common.sdiskpart( - device, mountpoint, fstype, opts, maxfile, maxpath - ) + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) return retlist diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 20987ecc8..8bb89961f 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -246,10 +246,7 @@ def disk_partitions(all=False): # https://github.com/giampaolo/psutil/issues/1674 debug("skipping %r: %s" % (mountpoint, err)) continue - maxfile = maxpath = None # set later - ntuple = _common.sdiskpart( - device, mountpoint, fstype, opts, maxfile, maxpath - ) + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) return retlist diff --git a/psutil/arch/windows/disk.c b/psutil/arch/windows/disk.c index 7cf0f20cc..004174561 100644 --- a/psutil/arch/windows/disk.c +++ b/psutil/arch/windows/disk.c @@ -318,13 +318,11 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strcat_s(mp_path, _countof(mp_path), mp_buf); py_tuple = Py_BuildValue( - "(ssssIi)", + "(ssss)", drive_letter, mp_path, fs_type, // typically "NTFS" - opts, - lpMaximumComponentLength, // max file length - MAX_PATH // max path length + opts ); if (!py_tuple || @@ -350,13 +348,11 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strcat_s(opts, _countof(opts), psutil_get_drive_type(type)); py_tuple = Py_BuildValue( - "(ssssIi)", + "(ssss)", drive_letter, drive_letter, fs_type, // either FAT, FAT32, NTFS, HPFS, CDFS, UDF or NWFS - opts, - lpMaximumComponentLength, // max file length - MAX_PATH // max path length + opts ); if (!py_tuple) goto error; diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index b2bfe29d6..e768e7d52 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -255,8 +255,6 @@ def test_disk_partitions(self): self.assertIsInstance(disk.mountpoint, str) self.assertIsInstance(disk.fstype, str) self.assertIsInstance(disk.opts, str) - self.assertIsInstance(disk.maxfile, (int, type(None))) - self.assertIsInstance(disk.maxpath, (int, type(None))) @unittest.skipIf(SKIP_SYSCONS, "requires root") def test_net_connections(self): diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 0c4affd0f..32ac62cbe 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -667,12 +667,6 @@ def check_ntuple(nt): self.assertIsInstance(nt.mountpoint, str) self.assertIsInstance(nt.fstype, str) self.assertIsInstance(nt.opts, str) - self.assertIsInstance(nt.maxfile, (int, type(None))) - self.assertIsInstance(nt.maxpath, (int, type(None))) - if nt.maxfile is not None and not GITHUB_ACTIONS: - self.assertGreater(nt.maxfile, 0) - if nt.maxpath is not None: - self.assertGreater(nt.maxpath, 0) # all = False ls = psutil.disk_partitions(all=False) From aefb59cdd14d1c33bf9b0b3d22887a756eb94022 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Apr 2024 09:23:21 +0200 Subject: [PATCH 32/46] Rename `Process.connections()` to `Process.net_connections()` (#2408) --- HISTORY.rst | 7 +++ Makefile | 2 +- README.rst | 2 +- docs/DEVNOTES | 4 +- docs/index.rst | 31 ++++++++----- psutil/__init__.py | 8 +++- psutil/_common.py | 8 ++-- psutil/_psaix.py | 2 +- psutil/_psbsd.py | 4 +- psutil/_pslinux.py | 14 +++--- psutil/_psosx.py | 6 +-- psutil/_pssunos.py | 2 +- psutil/_psutil_bsd.c | 2 +- psutil/_psutil_osx.c | 2 +- psutil/_pswindows.py | 2 +- psutil/arch/freebsd/proc_socks.c | 2 +- psutil/arch/freebsd/proc_socks.h | 2 +- psutil/arch/netbsd/proc.h | 2 +- psutil/arch/netbsd/socks.h | 2 +- psutil/arch/osx/proc.c | 2 +- psutil/arch/osx/proc.h | 2 +- psutil/tests/__init__.py | 17 ++++---- psutil/tests/test_connections.py | 74 ++++++++++++++++---------------- psutil/tests/test_linux.py | 4 +- psutil/tests/test_memleaks.py | 4 +- psutil/tests/test_process_all.py | 2 +- psutil/tests/test_testutils.py | 13 +++--- psutil/tests/test_unicode.py | 10 ++--- 28 files changed, 128 insertions(+), 104 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ca8e313a5..15333dbaf 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,9 @@ been reused. This makes `process_iter()`_ around 20x times faster. - 2396_: a new ``psutil.process_iter.cache_clear()`` API can be used the clear `process_iter()`_ internal cache. +- 2407_: `Process.connections()`_ was renamed to `Process.net_connections()`_. + The old name is still available, but it's deprecated (triggers a + ``DeprecationWarning``) and will be removed in the future. **Bug fixes** @@ -37,6 +40,9 @@ Version 6.0.0 introduces some changes which affect backward compatibility: been reused. If you want to check for PID reusage you are supposed to use `Process.is_running()`_ against the yielded `Process`_ instances. That will also automatically remove reused PIDs from `process_iter()`_ internal cache. +- 2407_: `Process.connections()`_ was renamed to `Process.net_connections()`_. + The old name is still available, but it's deprecated (triggers a + ``DeprecationWarning``) and will be removed in the future. 5.9.8 ===== @@ -2644,6 +2650,7 @@ In most cases accessing the old names will work but it will cause a .. _`Process.memory_maps()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_maps .. _`Process.memory_percent()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_percent .. _`Process.name()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.name +.. _`Process.net_connections()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.net_connections .. _`Process.nice()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.nice .. _`Process.num_ctx_switches()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.num_ctx_switches .. _`Process.num_fds()`: https://psutil.readthedocs.io/en/latest/#psutil.Process.num_fds diff --git a/Makefile b/Makefile index f09d83ec5..e123a93c0 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ test-contracts: ## APIs sanity tests. ${MAKE} build $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_contracts.py -test-connections: ## Test net_connections() and Process.connections(). +test-connections: ## Test psutil.net_connections() and Process.net_connections(). ${MAKE} build $(PYTHON_ENV_VARS) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_connections.py diff --git a/README.rst b/README.rst index 9df03860b..3fc6e601b 100644 --- a/README.rst +++ b/README.rst @@ -382,7 +382,7 @@ Process management [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768), popenfile(path='/var/log/monit.log', fd=4, position=235542, mode='a', flags=33793)] >>> - >>> p.connections(kind='tcp') + >>> p.net_connections(kind='tcp') [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING')] >>> diff --git a/docs/DEVNOTES b/docs/DEVNOTES index 49d449403..38ae74f82 100644 --- a/docs/DEVNOTES +++ b/docs/DEVNOTES @@ -39,7 +39,7 @@ FEATURES Example: ``` df -a -> psutil.disk_partitions - lsof -> psutil.Process.open_files() and psutil.Process.open_connections() + lsof -> psutil.Process.open_files() and psutil.Process.net_connections() killall-> (actual script) tty -> psutil.Process.terminal() who -> psutil.users() @@ -113,8 +113,6 @@ REJECTED IDEAS INCONSISTENCIES =============== -- `Process.connections()` should have been `Process.net_connections()` for - consistency with `psutil.net_connections()`. - PROCFS_PATH should have been set_procfs_path(). - `virtual_memory()` should have been `memory_virtual()`. - `swap_memory()` should have been `memory_swap()`. diff --git a/docs/index.rst b/docs/index.rst index eb2b11388..201333af4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -652,7 +652,7 @@ Network +----------------+-----------------------------------------------------+ On macOS and AIX this function requires root privileges. - To get per-process connections use :meth:`Process.connections`. + To get per-process connections use :meth:`Process.net_connections`. Also, see `netstat.py`_ example script. Example: @@ -1221,7 +1221,7 @@ Process class >>> import psutil >>> psutil.Process().environ() - {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'GNOME_DESKTOP_SESSION_ID': 'this-is-deprecated', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} + {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} .. note:: on macOS Big Sur this function returns something meaningful only for the @@ -1251,7 +1251,7 @@ Process class If *attrs* is specified it must be a list of strings reflecting available :class:`Process` class's attribute names. Here's a list of possible string values: - ``'cmdline'``, ``'connections'``, ``'cpu_affinity'``, ``'cpu_num'``, ``'cpu_percent'``, ``'cpu_times'``, ``'create_time'``, ``'cwd'``, ``'environ'``, ``'exe'``, ``'gids'``, ``'io_counters'``, ``'ionice'``, ``'memory_full_info'``, ``'memory_info'``, ``'memory_maps'``, ``'memory_percent'``, ``'name'``, ``'nice'``, ``'num_ctx_switches'``, ``'num_fds'``, ``'num_handles'``, ``'num_threads'``, ``'open_files'``, ``'pid'``, ``'ppid'``, ``'status'``, ``'terminal'``, ``'threads'``, ``'uids'``, ``'username'```. + ``'cmdline'``, ``'net_connections'``, ``'cpu_affinity'``, ``'cpu_num'``, ``'cpu_percent'``, ``'cpu_times'``, ``'create_time'``, ``'cwd'``, ``'environ'``, ``'exe'``, ``'gids'``, ``'io_counters'``, ``'ionice'``, ``'memory_full_info'``, ``'memory_info'``, ``'memory_maps'``, ``'memory_percent'``, ``'name'``, ``'nice'``, ``'num_ctx_switches'``, ``'num_fds'``, ``'num_handles'``, ``'num_threads'``, ``'open_files'``, ``'pid'``, ``'ppid'``, ``'status'``, ``'terminal'``, ``'threads'``, ``'uids'``, ``'username'```. If *attrs* argument is not passed all public read only attributes are assumed. *ad_value* is the value which gets assigned to a dict key in case @@ -1267,7 +1267,7 @@ Process class >>> >>> # get a list of valid attrs names >>> list(psutil.Process().as_dict().keys()) - ['status', 'cpu_num', 'num_ctx_switches', 'pid', 'memory_full_info', 'connections', 'cmdline', 'create_time', 'ionice', 'num_fds', 'memory_maps', 'cpu_percent', 'terminal', 'ppid', 'cwd', 'nice', 'username', 'cpu_times', 'io_counters', 'memory_info', 'threads', 'open_files', 'name', 'num_threads', 'exe', 'uids', 'gids', 'cpu_affinity', 'memory_percent', 'environ'] + ['cmdline', 'connections', 'cpu_affinity', 'cpu_num', 'cpu_percent', 'cpu_times', 'create_time', 'cwd', 'environ', 'exe', 'gids', 'io_counters', 'ionice', 'memory_full_info', 'memory_info', 'memory_maps', 'memory_percent', 'name', 'net_connections', 'nice', 'num_ctx_switches', 'num_fds', 'num_threads', 'open_files', 'pid', 'ppid', 'status', 'terminal', 'threads', 'uids', 'username'] .. versionchanged:: 3.0.0 *ad_value* is used also when incurring into @@ -1896,15 +1896,16 @@ Process class .. versionchanged:: 4.1.0 new *position*, *mode* and *flags* fields on Linux. - .. method:: connections(kind="inet") + .. method:: net_connections(kind="inet") Return socket connections opened by process as a list of named tuples. To get system-wide connections use :func:`psutil.net_connections()`. Every named tuple provides 6 attributes: - - **fd**: the socket file descriptor. This can be passed to `socket.fromfd`_ - to obtain a usable socket object. On Windows, FreeBSD and SunOS this is - always set to ``-1``. + - **fd**: the socket file descriptor. If the connection refers to the + current process this may be passed to `socket.fromfd`_ to obtain a usable + socket object. + On Windows, FreeBSD and SunOS this is always set to ``-1``. - **family**: the address family, either `AF_INET`_, `AF_INET6`_ or `AF_UNIX`_. - **type**: the address type, either `SOCK_STREAM`_, `SOCK_DGRAM`_ or @@ -1955,7 +1956,7 @@ Process class >>> p = psutil.Process(1694) >>> p.name() 'firefox' - >>> p.connections() + >>> p.net_connections() [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING'), pconn(fd=119, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status='ESTABLISHED'), @@ -1978,6 +1979,16 @@ Process class .. versionchanged:: 5.3.0 : *laddr* and *raddr* are named tuples. + .. versionchanged:: 6.0.0 : method renamed from `connections` to + `net_connections`. + + .. method:: connections() + + Same as :meth:`net_connections` (deprecated). + + .. warning:: + deprecated in version 6.0.0; use :meth:`net_connections` instead. + .. method:: is_running() Return whether the current process is running in the current process list. @@ -2386,7 +2397,7 @@ Connections constants .. data:: CONN_BOUND (Solaris) A set of strings representing the status of a TCP connection. - Returned by :meth:`psutil.Process.connections()` and + Returned by :meth:`psutil.Process.net_connections()` and :func:`psutil.net_connections` (`status` field). Hardware constants diff --git a/psutil/__init__.py b/psutil/__init__.py index f70ad5696..ba90d097a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1198,7 +1198,7 @@ def open_files(self): """ return self._proc.open_files() - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): """Return socket connections opened by process as a list of (fd, family, type, laddr, raddr, status) namedtuples. The *kind* parameter filters for connections that match the @@ -1220,7 +1220,11 @@ def connections(self, kind='inet'): | all | the sum of all the possible families and protocols | +------------+----------------------------------------------------+ """ - return self._proc.connections(kind) + return self._proc.net_connections(kind) + + @_common.deprecated_method(replacement="net_connections") + def connections(self, kind="inet"): + return self.net_connections(kind=kind) # --- signals diff --git a/psutil/_common.py b/psutil/_common.py index 50179fb3c..c1ff18d1f 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -120,7 +120,7 @@ STATUS_SUSPENDED = "suspended" # NetBSD STATUS_PARKED = "parked" # Linux -# Process.connections() and psutil.net_connections() +# Process.net_connections() and psutil.net_connections() CONN_ESTABLISHED = "ESTABLISHED" CONN_SYN_SENT = "SYN_SENT" CONN_SYN_RECV = "SYN_RECV" @@ -242,17 +242,17 @@ class BatteryTime(enum.IntEnum): pionice = namedtuple('pionice', ['ioclass', 'value']) # psutil.Process.ctx_switches() pctxsw = namedtuple('pctxsw', ['voluntary', 'involuntary']) -# psutil.Process.connections() +# psutil.Process.net_connections() pconn = namedtuple( 'pconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status'] ) -# psutil.connections() and psutil.Process.connections() +# psutil.net_connections() and psutil.Process.net_connections() addr = namedtuple('addr', ['ip', 'port']) # =================================================================== -# --- Process.connections() 'kind' parameter mapping +# --- Process.net_connections() 'kind' parameter mapping # =================================================================== diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 204c1bcf2..65ce3374f 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -448,7 +448,7 @@ def threads(self): return retlist @wrap_exceptions - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): ret = net_connections(kind, _pid=self.pid) # The underlying C implementation retrieves all OS connections # and filters them by PID. At this point we can't tell whether diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 094a59f88..b11b81c35 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -821,7 +821,7 @@ def threads(self): return retlist @wrap_exceptions - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): if kind not in conn_tmap: raise ValueError( "invalid %r kind argument; choose between %s" @@ -835,7 +835,7 @@ def connections(self, kind='inet'): elif OPENBSD: rawlist = cext.net_connections(self.pid, families, types) else: - rawlist = cext.proc_connections(self.pid, families, types) + rawlist = cext.proc_net_connections(self.pid, families, types) for item in rawlist: fd, fam, type, laddr, raddr, status = item[:6] diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b0ca24777..0ee134799 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -815,7 +815,7 @@ class _Ipv6UnsupportedError(Exception): pass -class Connections: +class NetConnections: """A wrapper on top of /proc/net/* files, retrieving per-process and system-wide open connections (TCP, UDP, UNIX) similarly to "netstat -an". @@ -981,8 +981,8 @@ def process_inet(file, family, type_, inodes, filter_pid=None): else: status = _common.CONN_NONE try: - laddr = Connections.decode_address(laddr, family) - raddr = Connections.decode_address(raddr, family) + laddr = NetConnections.decode_address(laddr, family) + raddr = NetConnections.decode_address(raddr, family) except _Ipv6UnsupportedError: continue yield (fd, family, type_, laddr, raddr, status, pid) @@ -1059,12 +1059,12 @@ def retrieve(self, kind, pid=None): return list(ret) -_connections = Connections() +_net_connections = NetConnections() def net_connections(kind='inet'): """Return system-wide open connections.""" - return _connections.retrieve(kind) + return _net_connections.retrieve(kind) def net_io_counters(): @@ -2344,8 +2344,8 @@ def open_files(self): return retlist @wrap_exceptions - def connections(self, kind='inet'): - ret = _connections.retrieve(kind, self.pid) + def net_connections(self, kind='inet'): + ret = _net_connections.retrieve(kind, self.pid) self._raise_if_not_alive() return ret diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 037e1d2b5..106709467 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -246,7 +246,7 @@ def net_connections(kind='inet'): ret = [] for pid in pids(): try: - cons = Process(pid).connections(kind) + cons = Process(pid).net_connections(kind) except NoSuchProcess: continue else: @@ -501,14 +501,14 @@ def open_files(self): return files @wrap_exceptions - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): if kind not in conn_tmap: raise ValueError( "invalid %r kind argument; choose between %s" % (kind, ', '.join([repr(x) for x in conn_tmap])) ) families, types = conn_tmap[kind] - rawlist = cext.proc_connections(self.pid, families, types) + rawlist = cext.proc_net_connections(self.pid, families, types) ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 8bb89961f..6112728b1 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -666,7 +666,7 @@ def _get_unix_sockets(self, pid): yield (-1, socket.AF_UNIX, type, path, "", _common.CONN_NONE) @wrap_exceptions - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): ret = net_connections(kind, _pid=self.pid) # The underlying C implementation retrieves all OS connections # and filters them by PID. At this point we can't tell whether diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index fde3916d3..6517d5800 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -63,7 +63,7 @@ static PyMethodDef mod_methods[] = { {"proc_oneshot_info", psutil_proc_oneshot_info, METH_VARARGS}, {"proc_threads", psutil_proc_threads, METH_VARARGS}, #if defined(PSUTIL_FREEBSD) - {"proc_connections", psutil_proc_connections, METH_VARARGS}, + {"proc_net_connections", psutil_proc_net_connections, METH_VARARGS}, #endif {"proc_cwd", psutil_proc_cwd, METH_VARARGS}, #if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 369fbbfb4..4aa11d170 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -23,7 +23,7 @@ static PyMethodDef mod_methods[] = { // --- per-process functions {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS}, - {"proc_connections", psutil_proc_connections, METH_VARARGS}, + {"proc_net_connections", psutil_proc_net_connections, METH_VARARGS}, {"proc_cwd", psutil_proc_cwd, METH_VARARGS}, {"proc_environ", psutil_proc_environ, METH_VARARGS}, {"proc_exe", psutil_proc_exe, METH_VARARGS}, diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 3c60a949a..babb8e82e 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -1059,7 +1059,7 @@ def open_files(self): return list(ret) @wrap_exceptions - def connections(self, kind='inet'): + def net_connections(self, kind='inet'): return net_connections(kind, _pid=self.pid) @wrap_exceptions diff --git a/psutil/arch/freebsd/proc_socks.c b/psutil/arch/freebsd/proc_socks.c index 737467a8c..c84c3b04a 100644 --- a/psutil/arch/freebsd/proc_socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -178,7 +178,7 @@ psutil_search_tcplist(char *buf, struct kinfo_file *kif) { PyObject * -psutil_proc_connections(PyObject *self, PyObject *args) { +psutil_proc_net_connections(PyObject *self, PyObject *args) { // Return connections opened by process. pid_t pid; int i; diff --git a/psutil/arch/freebsd/proc_socks.h b/psutil/arch/freebsd/proc_socks.h index a7996b107..edc29b7aa 100644 --- a/psutil/arch/freebsd/proc_socks.h +++ b/psutil/arch/freebsd/proc_socks.h @@ -6,4 +6,4 @@ #include -PyObject* psutil_proc_connections(PyObject* self, PyObject* args); +PyObject* psutil_proc_net_connections(PyObject* self, PyObject* args); diff --git a/psutil/arch/netbsd/proc.h b/psutil/arch/netbsd/proc.h index 8c51914d7..138ff6ebd 100644 --- a/psutil/arch/netbsd/proc.h +++ b/psutil/arch/netbsd/proc.h @@ -15,8 +15,8 @@ int psutil_get_proc_list(kinfo_proc **procList, size_t *procCount); char *psutil_get_cmd_args(pid_t pid, size_t *argsize); PyObject *psutil_proc_cmdline(PyObject *self, PyObject *args); -PyObject *psutil_proc_connections(PyObject *self, PyObject *args); PyObject *psutil_proc_cwd(PyObject *self, PyObject *args); +PyObject *psutil_proc_net_connections(PyObject *self, PyObject *args); PyObject *psutil_proc_num_fds(PyObject *self, PyObject *args); PyObject *psutil_proc_threads(PyObject *self, PyObject *args); PyObject* psutil_proc_exe(PyObject* self, PyObject* args); diff --git a/psutil/arch/netbsd/socks.h b/psutil/arch/netbsd/socks.h index 9e6a97c0a..9c2a87d4d 100644 --- a/psutil/arch/netbsd/socks.h +++ b/psutil/arch/netbsd/socks.h @@ -6,5 +6,5 @@ * found in the LICENSE file. */ -PyObject *psutil_proc_connections(PyObject *, PyObject *); PyObject *psutil_net_connections(PyObject *, PyObject *); +PyObject *psutil_proc_net_connections(PyObject *, PyObject *); diff --git a/psutil/arch/osx/proc.c b/psutil/arch/osx/proc.c index 6f66c8613..2cdb9911c 100644 --- a/psutil/arch/osx/proc.c +++ b/psutil/arch/osx/proc.c @@ -857,7 +857,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { * - /usr/include/sys/proc_info.h */ PyObject * -psutil_proc_connections(PyObject *self, PyObject *args) { +psutil_proc_net_connections(PyObject *self, PyObject *args) { pid_t pid; int num_fds; int i; diff --git a/psutil/arch/osx/proc.h b/psutil/arch/osx/proc.h index 63f16ccdd..f18f5f1fd 100644 --- a/psutil/arch/osx/proc.h +++ b/psutil/arch/osx/proc.h @@ -8,13 +8,13 @@ PyObject *psutil_pids(PyObject *self, PyObject *args); PyObject *psutil_proc_cmdline(PyObject *self, PyObject *args); -PyObject *psutil_proc_connections(PyObject *self, PyObject *args); PyObject *psutil_proc_cwd(PyObject *self, PyObject *args); PyObject *psutil_proc_environ(PyObject *self, PyObject *args); PyObject *psutil_proc_exe(PyObject *self, PyObject *args); PyObject *psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args); PyObject *psutil_proc_memory_uss(PyObject *self, PyObject *args); PyObject *psutil_proc_name(PyObject *self, PyObject *args); +PyObject *psutil_proc_net_connections(PyObject *self, PyObject *args); PyObject *psutil_proc_num_fds(PyObject *self, PyObject *args); PyObject *psutil_proc_open_files(PyObject *self, PyObject *args); PyObject *psutil_proc_pidtaskinfo_oneshot(PyObject *self, PyObject *args); diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5e50e1787..b18b74239 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -86,7 +86,7 @@ "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", - "HAS_SENSORS_TEMPERATURES", "MACOS_11PLUS", + "HAS_SENSORS_TEMPERATURES", "HAS_NET_CONNECTIONS_UNIX", "MACOS_11PLUS", "MACOS_12PLUS", "COVERAGE", # subprocesses 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', @@ -105,7 +105,7 @@ # sync primitives 'call_until', 'wait_for_pid', 'wait_for_file', # network - 'check_net_address', 'filter_proc_connections', + 'check_net_address', 'filter_proc_net_connections', 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', 'unix_socketpair', 'create_sockets', # compat @@ -205,13 +205,13 @@ def macos_version(): # --- support -HAS_CONNECTIONS_UNIX = POSIX and not SUNOS HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") -HAS_GETLOADAVG = hasattr(psutil, "getloadavg") HAS_ENVIRON = hasattr(psutil.Process, "environ") +HAS_GETLOADAVG = hasattr(psutil, "getloadavg") HAS_IONICE = hasattr(psutil.Process, "ionice") HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") +HAS_NET_CONNECTIONS_UNIX = POSIX and not SUNOS HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") @@ -1399,8 +1399,9 @@ class process_namespace: ignored = [ ('as_dict', (), {}), ('children', (), {'recursive': True}), + ('connections', (), {}), # deprecated ('is_running', (), {}), - ('memory_info_ex', (), {}), + ('memory_info_ex', (), {}), # deprecated ('oneshot', (), {}), ('parent', (), {}), ('parents', (), {}), @@ -1410,7 +1411,6 @@ class process_namespace: getters = [ ('cmdline', (), {}), - ('connections', (), {'kind': 'all'}), ('cpu_times', (), {}), ('create_time', (), {}), ('cwd', (), {}), @@ -1418,6 +1418,7 @@ class process_namespace: ('memory_full_info', (), {}), ('memory_info', (), {}), ('name', (), {}), + ('net_connections', (), {'kind': 'all'}), ('nice', (), {}), ('num_ctx_switches', (), {}), ('num_threads', (), {}), @@ -1758,7 +1759,7 @@ def create_sockets(): if supports_ipv6(): socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) - if POSIX and HAS_CONNECTIONS_UNIX: + if POSIX and HAS_NET_CONNECTIONS_UNIX: fname1 = get_testfn() fname2 = get_testfn() s1, s2 = unix_socketpair(fname1) @@ -1883,7 +1884,7 @@ def check_status(conn): check_status(conn) -def filter_proc_connections(cons): +def filter_proc_net_connections(cons): """Our process may start with some open UNIX sockets which are not initialized by us, invalidating unit tests. """ diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index e261fc0f9..de3ae59df 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Tests for net_connections() and Process.connections() APIs.""" +"""Tests for psutil.net_connections() and Process.net_connections() APIs.""" import os import socket @@ -28,14 +28,14 @@ from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import AF_UNIX -from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import HAS_NET_CONNECTIONS_UNIX from psutil.tests import SKIP_SYSCONS from psutil.tests import PsutilTestCase from psutil.tests import bind_socket from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import create_sockets -from psutil.tests import filter_proc_connections +from psutil.tests import filter_proc_net_connections from psutil.tests import reap_children from psutil.tests import retry_on_failure from psutil.tests import serialrun @@ -48,21 +48,21 @@ SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) -def this_proc_connections(kind): - cons = psutil.Process().connections(kind=kind) +def this_proc_net_connections(kind): + cons = psutil.Process().net_connections(kind=kind) if kind in ("all", "unix"): - return filter_proc_connections(cons) + return filter_proc_net_connections(cons) return cons @serialrun class ConnectionTestCase(PsutilTestCase): def setUp(self): - self.assertEqual(this_proc_connections(kind='all'), []) + self.assertEqual(this_proc_net_connections(kind='all'), []) def tearDown(self): # Make sure we closed all resources. - self.assertEqual(this_proc_connections(kind='all'), []) + self.assertEqual(this_proc_net_connections(kind='all'), []) def compare_procsys_connections(self, pid, proc_cons, kind='all'): """Given a process PID and its list of connections compare @@ -94,11 +94,11 @@ def test_system(self): def test_process(self): with create_sockets(): - for conn in this_proc_connections(kind='all'): + for conn in this_proc_net_connections(kind='all'): check_connection_ntuple(conn) def test_invalid_kind(self): - self.assertRaises(ValueError, this_proc_connections, kind='???') + self.assertRaises(ValueError, this_proc_net_connections, kind='???') self.assertRaises(ValueError, psutil.net_connections, kind='???') @@ -107,7 +107,7 @@ class TestUnconnectedSockets(ConnectionTestCase): """Tests sockets which are open but not connected to anything.""" def get_conn_from_sock(self, sock): - cons = this_proc_connections(kind='all') + cons = this_proc_net_connections(kind='all') smap = dict([(c.fd, c) for c in cons]) if NETBSD or FREEBSD: # NetBSD opens a UNIX socket to /var/log/run @@ -146,8 +146,8 @@ def check_socket(self, sock): self.assertEqual(conn.laddr, laddr) # XXX Solaris can't retrieve system-wide UNIX sockets - if sock.family == AF_UNIX and HAS_CONNECTIONS_UNIX: - cons = this_proc_connections(kind='all') + if sock.family == AF_UNIX and HAS_NET_CONNECTIONS_UNIX: + cons = this_proc_net_connections(kind='all') self.compare_procsys_connections(os.getpid(), cons, kind='all') return conn @@ -209,17 +209,17 @@ class TestConnectedSocket(ConnectionTestCase): @unittest.skipIf(SUNOS, "unreliable on SUONS") def test_tcp(self): addr = ("127.0.0.1", 0) - self.assertEqual(this_proc_connections(kind='tcp4'), []) + self.assertEqual(this_proc_net_connections(kind='tcp4'), []) server, client = tcp_socketpair(AF_INET, addr=addr) try: - cons = this_proc_connections(kind='tcp4') + cons = this_proc_net_connections(kind='tcp4') self.assertEqual(len(cons), 2) self.assertEqual(cons[0].status, psutil.CONN_ESTABLISHED) self.assertEqual(cons[1].status, psutil.CONN_ESTABLISHED) # May not be fast enough to change state so it stays # commenteed. # client.close() - # cons = this_proc_connections(kind='all') + # cons = this_proc_net_connections(kind='all') # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) finally: @@ -231,7 +231,7 @@ def test_unix(self): testfn = self.get_testfn() server, client = unix_socketpair(testfn) try: - cons = this_proc_connections(kind='unix') + cons = this_proc_net_connections(kind='unix') assert not (cons[0].laddr and cons[0].raddr), cons assert not (cons[1].laddr and cons[1].raddr), cons if NETBSD or FREEBSD: @@ -257,7 +257,7 @@ def test_unix(self): class TestFilters(ConnectionTestCase): def test_filters(self): def check(kind, families, types): - for conn in this_proc_connections(kind=kind): + for conn in this_proc_net_connections(kind=kind): self.assertIn(conn.family, families) self.assertIn(conn.type, types) if not SKIP_SYSCONS: @@ -279,7 +279,7 @@ def check(kind, families, types): check('udp', [AF_INET, AF_INET6], [SOCK_DGRAM]) check('udp4', [AF_INET], [SOCK_DGRAM]) check('udp6', [AF_INET6], [SOCK_DGRAM]) - if HAS_CONNECTIONS_UNIX: + if HAS_NET_CONNECTIONS_UNIX: check( 'unix', [AF_UNIX], @@ -310,7 +310,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): self.assertEqual(conn.raddr, raddr) self.assertEqual(conn.status, status) for kind in all_kinds: - cons = proc.connections(kind=kind) + cons = proc.net_connections(kind=kind) if kind in kinds: self.assertNotEqual(cons, []) else: @@ -318,7 +318,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # compare against system-wide connections # XXX Solaris can't retrieve system-wide UNIX # sockets. - if HAS_CONNECTIONS_UNIX: + if HAS_NET_CONNECTIONS_UNIX: self.compare_procsys_connections(proc.pid, [conn]) tcp_template = textwrap.dedent(""" @@ -373,7 +373,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): udp6_addr = None for p in psutil.Process().children(): - cons = p.connections() + cons = p.net_connections() self.assertEqual(len(cons), 1) for conn in cons: # TCP v4 @@ -428,56 +428,56 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_count(self): with create_sockets(): # tcp - cons = this_proc_connections(kind='tcp') + cons = this_proc_net_connections(kind='tcp') self.assertEqual(len(cons), 2 if supports_ipv6() else 1) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertEqual(conn.type, SOCK_STREAM) # tcp4 - cons = this_proc_connections(kind='tcp4') + cons = this_proc_net_connections(kind='tcp4') self.assertEqual(len(cons), 1) self.assertEqual(cons[0].family, AF_INET) self.assertEqual(cons[0].type, SOCK_STREAM) # tcp6 if supports_ipv6(): - cons = this_proc_connections(kind='tcp6') + cons = this_proc_net_connections(kind='tcp6') self.assertEqual(len(cons), 1) self.assertEqual(cons[0].family, AF_INET6) self.assertEqual(cons[0].type, SOCK_STREAM) # udp - cons = this_proc_connections(kind='udp') + cons = this_proc_net_connections(kind='udp') self.assertEqual(len(cons), 2 if supports_ipv6() else 1) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertEqual(conn.type, SOCK_DGRAM) # udp4 - cons = this_proc_connections(kind='udp4') + cons = this_proc_net_connections(kind='udp4') self.assertEqual(len(cons), 1) self.assertEqual(cons[0].family, AF_INET) self.assertEqual(cons[0].type, SOCK_DGRAM) # udp6 if supports_ipv6(): - cons = this_proc_connections(kind='udp6') + cons = this_proc_net_connections(kind='udp6') self.assertEqual(len(cons), 1) self.assertEqual(cons[0].family, AF_INET6) self.assertEqual(cons[0].type, SOCK_DGRAM) # inet - cons = this_proc_connections(kind='inet') + cons = this_proc_net_connections(kind='inet') self.assertEqual(len(cons), 4 if supports_ipv6() else 2) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # inet6 if supports_ipv6(): - cons = this_proc_connections(kind='inet6') + cons = this_proc_net_connections(kind='inet6') self.assertEqual(len(cons), 2) for conn in cons: self.assertEqual(conn.family, AF_INET6) self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # Skipped on BSD becayse by default the Python process # creates a UNIX socket to '/var/run/log'. - if HAS_CONNECTIONS_UNIX and not (FREEBSD or NETBSD): - cons = this_proc_connections(kind='unix') + if HAS_NET_CONNECTIONS_UNIX and not (FREEBSD or NETBSD): + cons = this_proc_net_connections(kind='unix') self.assertEqual(len(cons), 3) for conn in cons: self.assertEqual(conn.family, AF_UNIX) @@ -501,7 +501,7 @@ def check(cons, families, types_): for kind, groups in conn_tmap.items(): # XXX: SunOS does not retrieve UNIX sockets. - if kind == 'unix' and not HAS_CONNECTIONS_UNIX: + if kind == 'unix' and not HAS_NET_CONNECTIONS_UNIX: continue families, types_ = groups cons = psutil.net_connections(kind) @@ -511,8 +511,8 @@ def check(cons, families, types_): @retry_on_failure() def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different - # sockets. For each process check that proc.connections() - # and net_connections() return the same results. + # sockets. For each process check that proc.net_connections() + # and psutil.net_connections() return the same results. # This is done mainly to check whether net_connections()'s # pid is properly set, see: # https://github.com/giampaolo/psutil/issues/1013 @@ -547,11 +547,11 @@ def test_multi_sockets_procs(self): len([x for x in syscons if x.pid == pid]), expected ) p = psutil.Process(pid) - self.assertEqual(len(p.connections('all')), expected) + self.assertEqual(len(p.net_connections('all')), expected) class TestMisc(PsutilTestCase): - def test_connection_constants(self): + def test_net_connection_constants(self): ints = [] strs = [] for name in dir(psutil): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 203b5d487..233584c84 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -2231,7 +2231,7 @@ def test_status_file_parsing(self): self.assertEqual(gids.saved, 1006) self.assertEqual(p._proc._get_eligible_cpus(), list(range(8))) - def test_connections_enametoolong(self): + def test_net_connections_enametoolong(self): # Simulate a case where /proc/{pid}/fd/{fd} symlink points to # a file with full path longer than PATH_MAX, see: # https://github.com/giampaolo/psutil/issues/1940 @@ -2241,7 +2241,7 @@ def test_connections_enametoolong(self): ) as m: p = psutil.Process() with mock.patch("psutil._pslinux.debug"): - self.assertEqual(p.connections(), []) + self.assertEqual(p.net_connections(), []) assert m.called diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index 8a5b7f7f9..8232b18ad 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -250,13 +250,13 @@ def test_rlimit_set(self): # Windows implementation is based on a single system-wide # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") - def test_connections(self): + def test_net_connections(self): # TODO: UNIX sockets are temporarily implemented by parsing # 'pfiles' cmd output; we don't want that part of the code to # be executed. with create_sockets(): kind = 'inet' if SUNOS else 'all' - self.execute(lambda: self.proc.connections(kind)) + self.execute(lambda: self.proc.net_connections(kind)) @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index 700ffe078..7c6ce7808 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -356,7 +356,7 @@ def num_fds(self, ret, info): self.assertIsInstance(ret, int) self.assertGreaterEqual(ret, 0) - def connections(self, ret, info): + def net_connections(self, ret, info): with create_sockets(): self.assertEqual(len(ret), len(set(ret))) for conn in ret: diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py index a93f9f09f..17cc9eb08 100755 --- a/psutil/tests/test_testutils.py +++ b/psutil/tests/test_testutils.py @@ -26,7 +26,7 @@ from psutil._common import supports_ipv6 from psutil.tests import CI_TESTING from psutil.tests import COVERAGE -from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import HAS_NET_CONNECTIONS_UNIX from psutil.tests import PYTHON_EXE from psutil.tests import PYTHON_EXE_ENV from psutil.tests import PsutilTestCase @@ -36,7 +36,7 @@ from psutil.tests import call_until from psutil.tests import chdir from psutil.tests import create_sockets -from psutil.tests import filter_proc_connections +from psutil.tests import filter_proc_net_connections from psutil.tests import get_free_port from psutil.tests import is_namedtuple from psutil.tests import mock @@ -320,7 +320,7 @@ def test_unix_socketpair(self): p = psutil.Process() num_fds = p.num_fds() self.assertEqual( - filter_proc_connections(p.connections(kind='unix')), [] + filter_proc_net_connections(p.net_connections(kind='unix')), [] ) name = self.get_testfn() server, client = unix_socketpair(name) @@ -329,7 +329,10 @@ def test_unix_socketpair(self): assert stat.S_ISSOCK(os.stat(name).st_mode) self.assertEqual(p.num_fds() - num_fds, 2) self.assertEqual( - len(filter_proc_connections(p.connections(kind='unix'))), 2 + len( + filter_proc_net_connections(p.net_connections(kind='unix')) + ), + 2, ) self.assertEqual(server.getsockname(), name) self.assertEqual(client.getpeername(), name) @@ -348,7 +351,7 @@ def test_create_sockets(self): self.assertGreaterEqual(fams[socket.AF_INET], 2) if supports_ipv6(): self.assertGreaterEqual(fams[socket.AF_INET6], 2) - if POSIX and HAS_CONNECTIONS_UNIX: + if POSIX and HAS_NET_CONNECTIONS_UNIX: self.assertGreaterEqual(fams[socket.AF_UNIX], 2) self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index cb4bccf7f..7fb2ef199 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -37,12 +37,12 @@ ('not tested' means they are not tested to deal with non-ASCII strings): * Process.cmdline() -* Process.connections('unix') * Process.cwd() * Process.environ() * Process.exe() * Process.memory_maps() * Process.name() +* Process.net_connections('unix') * Process.open_files() * Process.username() (not tested) @@ -88,9 +88,9 @@ from psutil.tests import APPVEYOR from psutil.tests import ASCII_FS from psutil.tests import CI_TESTING -from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_NET_CONNECTIONS_UNIX from psutil.tests import INVALID_UNICODE_SUFFIX from psutil.tests import PYPY from psutil.tests import TESTFN_PREFIX @@ -253,7 +253,7 @@ def test_proc_open_files(self): ) @unittest.skipIf(not POSIX, "POSIX only") - def test_proc_connections(self): + def test_proc_net_connections(self): name = self.get_testfn(suffix=self.funky_suffix) try: sock = bind_unix_socket(name) @@ -263,12 +263,12 @@ def test_proc_connections(self): else: raise unittest.SkipTest("not supported") with closing(sock): - conn = psutil.Process().connections('unix')[0] + conn = psutil.Process().net_connections('unix')[0] self.assertIsInstance(conn.laddr, str) self.assertEqual(conn.laddr, name) @unittest.skipIf(not POSIX, "POSIX only") - @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") + @unittest.skipIf(not HAS_NET_CONNECTIONS_UNIX, "can't list UNIX sockets") @skip_on_access_denied() def test_net_connections(self): def find_sock(cons): From 20ed8f9b3d53847408e08442857704590d4f8b4c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Apr 2024 11:23:09 +0200 Subject: [PATCH 33/46] #2408: ignore old "connections" name into as_dict() --- psutil/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index ba90d097a..e1e2b7d5d 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1341,7 +1341,7 @@ def wait(self, timeout=None): [x for x in dir(Process) if not x.startswith('_') and x not in {'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', - 'memory_info_ex', 'oneshot'}]) + 'memory_info_ex', 'connections', 'oneshot'}]) # fmt: on From 8bfecf0459216f67b1ec089ced03dd162fca75de Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Apr 2024 14:00:29 +0200 Subject: [PATCH 34/46] #2408: fix some tests which were still failing due to the old name Signed-off-by: Giampaolo Rodola --- psutil/tests/test_process.py | 6 +++--- scripts/procinfo.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 9613eac18..44a5e8901 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1207,9 +1207,9 @@ def test_as_dict(self): self.assertEqual(sorted(d.keys()), ['exe', 'name']) p = psutil.Process(min(psutil.pids())) - d = p.as_dict(attrs=['connections'], ad_value='foo') - if not isinstance(d['connections'], list): - self.assertEqual(d['connections'], 'foo') + d = p.as_dict(attrs=['net_connections'], ad_value='foo') + if not isinstance(d['net_connections'], list): + self.assertEqual(d['net_connections'], 'foo') # Test ad_value is set on AccessDenied. with mock.patch( diff --git a/scripts/procinfo.py b/scripts/procinfo.py index 95c99d442..24004a960 100755 --- a/scripts/procinfo.py +++ b/scripts/procinfo.py @@ -241,13 +241,13 @@ def run(pid, verbose=False): else: print_('open-files', '') - if pinfo['connections']: + if pinfo['net_connections']: template = '%-5s %-25s %-25s %s' print_( 'connections', template % ('PROTO', 'LOCAL ADDR', 'REMOTE ADDR', 'STATUS'), ) - for conn in pinfo['connections']: + for conn in pinfo['net_connections']: if conn.type == socket.SOCK_STREAM: type = 'TCP' elif conn.type == socket.SOCK_DGRAM: From 0b0ea8e55fb5eeb00074d25f9292a04cc70c82ef Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Apr 2024 18:10:27 +0200 Subject: [PATCH 35/46] skip flaky test on netbsd --- psutil/tests/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 44a5e8901..307d3dfa9 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1468,6 +1468,7 @@ def clean_dict(d): MACOS_11PLUS, "macOS 11+ can't get another process environment, issue #2084", ) + @unittest.skipIf(NETBSD, "sometimes fails on `assert is_running()`") def test_weird_environ(self): # environment variables can contain values without an equals sign code = textwrap.dedent(""" From 3c518a384dda6dd11223499fc26b5bacf08bb8d5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 15 May 2024 19:35:05 +0200 Subject: [PATCH 36/46] fix ruff errs --- psutil/__init__.py | 4 ++-- pyproject.toml | 5 ++++- scripts/internal/print_announce.py | 10 +++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index e1e2b7d5d..30f45987e 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1601,14 +1601,14 @@ def check_gone(proc, timeout): check_gone(proc, timeout) else: check_gone(proc, max_timeout) - alive = alive - gone + alive = alive - gone # noqa PLR6104 if alive: # Last attempt over processes survived so far. # timeout == 0 won't make this function wait any further. for proc in alive: check_gone(proc, 0) - alive = alive - gone + alive = alive - gone # noqa: PLR6104 return (list(gone), list(alive)) diff --git a/pyproject.toml b/pyproject.toml index 94266a7bf..6f5bab2da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,10 +46,13 @@ ignore = [ "FIX", # Line contains TODO / XXX / ..., consider resolving the issue "FLY", # flynt (PYTHON2.7 COMPAT) "FURB101", # `open` and `read` should be replaced by `Path(src).read_text()` + "FURB103", # `open` and `write` should be replaced by `Path(...).write_text(...)` "FURB113", # Use `x.extend(('a', 'b', 'c'))` instead of repeatedly calling `x.append()` + "FURB116", # [*] Replace `hex` call with `f"{start:x}"` "FURB118", # [*] Use `operator.add` instead of defining a lambda "FURB140", # [*] Use `itertools.starmap` instead of the generator "FURB145", # [*] Prefer `copy` method over slicing (PYTHON2.7 COMPAT) + "FURB192", # [*] Prefer `min` over `sorted()` to compute the minimum value in a sequence "INP", # flake8-no-pep420 "N801", # Class name `async_chat` should use CapWords convention (ASYNCORE COMPAT) "N802", # Function name X should be lowercase. @@ -108,7 +111,7 @@ ignore = [ "psutil/tests/*" = ["EM101", "TRY003"] "psutil/tests/runner.py" = ["T201", "T203"] "scripts/*" = ["T201", "T203"] -"scripts/internal/*" = ["T201", "T203"] +"scripts/internal/*" = ["EM101", "T201", "T203", "TRY003"] "setup.py" = ["T201", "T203"] [tool.ruff.lint.isort] diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 68201d7fc..d1a7d297f 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -86,13 +86,15 @@ def get_changes(): block = [] # eliminate the part preceding the first block - for line in lines: + while lines: line = lines.pop(0) if line.startswith('===='): break - lines.pop(0) + else: + raise ValueError("something wrong") - for line in lines: + lines.pop(0) + while lines: line = lines.pop(0) line = line.rstrip() if re.match(r"^- \d+_", line): @@ -101,6 +103,8 @@ def get_changes(): if line.startswith('===='): break block.append(line) + else: + raise ValueError("something wrong") # eliminate bottom empty lines block.pop(-1) From 1c7cb0aaf24f135e4ffded5a39387514f862c92e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 18 May 2024 12:43:20 -0400 Subject: [PATCH 37/46] Don't build with limited API for 3.13 free-threaded build (#2402) The `--disable-gil` configuration of CPython 3.13 does not currently support the limited API. Signed-off-by: Sam Gross --- HISTORY.rst | 1 + setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 15333dbaf..a53590555 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ been reused. This makes `process_iter()`_ around 20x times faster. - 2396_: a new ``psutil.process_iter.cache_clear()`` API can be used the clear `process_iter()`_ internal cache. +- 2401_, Support building with free-threaded CPython 3.13. - 2407_: `Process.connections()`_ was renamed to `Process.net_connections()`_. The old name is still available, but it's deprecated (triggers a ``DeprecationWarning``) and will be removed in the future. diff --git a/setup.py b/setup.py index 7c59f5645..3c7900669 100755 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ import struct import subprocess import sys +import sysconfig import tempfile import warnings @@ -65,6 +66,7 @@ PY37_PLUS = sys.version_info[:2] >= (3, 7) CP36_PLUS = PY36_PLUS and sys.implementation.name == "cpython" CP37_PLUS = PY37_PLUS and sys.implementation.name == "cpython" +Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") macros = [] if POSIX: @@ -118,10 +120,10 @@ def get_version(): # Py_LIMITED_API lets us create a single wheel which works with multiple # python versions, including unreleased ones. -if bdist_wheel and CP36_PLUS and (MACOS or LINUX): +if bdist_wheel and CP36_PLUS and (MACOS or LINUX) and not Py_GIL_DISABLED: py_limited_api = {"py_limited_api": True} macros.append(('Py_LIMITED_API', '0x03060000')) -elif bdist_wheel and CP37_PLUS and WINDOWS: +elif bdist_wheel and CP37_PLUS and WINDOWS and not Py_GIL_DISABLED: # PyErr_SetFromWindowsErr / PyErr_SetFromWindowsErrWithFilename are # part of the stable API/ABI starting with CPython 3.7 py_limited_api = {"py_limited_api": True} From 553098524de5367c04a3e057b88208dc4b86aba4 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Sat, 18 May 2024 20:04:11 +0200 Subject: [PATCH 38/46] chore(ci): update actions (#2417) --- .github/workflows/build.yml | 46 ++++++++++++++++++++++++++---------- .github/workflows/issues.yml | 2 +- Makefile | 2 +- pyproject.toml | 6 ++++- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bd907fc6..c4e9c745e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,20 +37,30 @@ jobs: archs: "x86" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + + # see https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 + - name: "Install python 3.8 universal2 on macOS arm64" + if: runner.os == 'macOS' && runner.arch == 'ARM64' + uses: actions/setup-python@v5 + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + with: + python-version: 3.8 + + - uses: actions/setup-python@v5 with: python-version: 3.11 - name: Create wheels + run tests - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.18.0 env: CIBW_ARCHS: "${{ matrix.archs }}" CIBW_PRERELEASE_PYTHONS: True - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'ubuntu') && 'all' || matrix.archs }} path: wheelhouse - name: Generate .tar.gz @@ -78,7 +88,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -86,9 +96,9 @@ jobs: uses: pypa/cibuildwheel@v1.12.0 - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-py2-${{ matrix.os }} path: wheelhouse - name: Generate .tar.gz @@ -103,24 +113,36 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - name: 'Run linters' run: | - python3 -m pip install ruff==0.3.4 black rstcheck toml-sort sphinx + python3 -m pip install ruff==0.4.4 black rstcheck toml-sort sphinx make lint-all + # upload weels as a single artefact + upload-wheels: + needs: [py2, py3] + runs-on: ubuntu-latest + steps: + - uses: actions/upload-artifact/merge@v4 + with: + name: wheels + pattern: wheels-* + separate-directories: false + delete-merged: true + # Check sanity of .tar.gz + wheel files check-dist: - needs: [py2, py3] + needs: [upload-wheels] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: wheels path: wheelhouse diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml index 3d3adbf83..a9f665eb6 100644 --- a/.github/workflows/issues.yml +++ b/.github/workflows/issues.yml @@ -15,7 +15,7 @@ jobs: # install python - uses: actions/checkout@v4 - name: Install Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' diff --git a/Makefile b/Makefile index e123a93c0..0406fb6da 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ PY3_DEPS = \ pypinfo \ requests \ rstcheck \ - ruff==0.3.4 \ + ruff==0.4.4 \ setuptools \ sphinx_rtd_theme \ teyit \ diff --git a/pyproject.toml b/pyproject.toml index 6f5bab2da..99a9b44d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,11 @@ spaces_indent_inline_array = 4 trailing_comma_inline_array = true [tool.cibuildwheel] -skip = ["*-musllinux*", "pp*"] +skip = [ + "*-musllinux*", + "cp313-win*", # pywin32 is not available on cp313 yet + "pp*", +] test-command = [ "env PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 PSUTIL_SCRIPTS_DIR={project}/scripts python {project}/psutil/tests/runner.py", "env PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 PSUTIL_SCRIPTS_DIR={project}/scripts python {project}/psutil/tests/test_memleaks.py", From 20be5ae95a4a01f734ba0692bc38c35df4e434d3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 3 Jun 2024 01:36:06 +0200 Subject: [PATCH 39/46] ruff: enable and fix 'unused variable' rule --- Makefile | 2 +- psutil/_common.py | 2 +- psutil/_psaix.py | 2 +- psutil/_psbsd.py | 6 +++--- psutil/_pslinux.py | 22 +++++++++++----------- psutil/_psosx.py | 2 +- psutil/_pssunos.py | 2 +- psutil/_pswindows.py | 8 ++++---- psutil/tests/runner.py | 2 +- psutil/tests/test_bsd.py | 10 +++++----- psutil/tests/test_process.py | 4 ++-- psutil/tests/test_process_all.py | 2 +- psutil/tests/test_system.py | 2 +- psutil/tests/test_testutils.py | 2 +- psutil/tests/test_windows.py | 2 +- pyproject.toml | 1 - setup.py | 2 +- 17 files changed, 36 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 0406fb6da..60d82336e 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ fix-black: @git ls-files '*.py' | xargs $(PYTHON) -m black fix-ruff: - @git ls-files '*.py' | xargs $(PYTHON) -m ruff check --no-cache --fix + @git ls-files '*.py' | xargs $(PYTHON) -m ruff check --no-cache --fix $(ARGS) fix-unittests: ## Fix unittest idioms. @git ls-files '*test_*.py' | xargs $(PYTHON) -m teyit --show-stats diff --git a/psutil/_common.py b/psutil/_common.py index c1ff18d1f..9fd7b0cfb 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -980,7 +980,7 @@ def debug(msg): if PSUTIL_DEBUG: import inspect - fname, lineno, _, lines, index = inspect.getframeinfo( + fname, lineno, _, _lines, _index = inspect.getframeinfo( inspect.currentframe().f_back ) if isinstance(msg, Exception): diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 65ce3374f..f48425eb8 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -105,7 +105,7 @@ def virtual_memory(): - total, avail, free, pinned, inuse = cext.virtual_mem() + total, avail, free, _pinned, inuse = cext.virtual_mem() percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, inuse, free) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index b11b81c35..4d49cf98b 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -321,7 +321,7 @@ def cpu_stats(): if FREEBSD: # Note: the C ext is returning some metrics we are not exposing: # traps. - ctxsw, intrs, soft_intrs, syscalls, traps = cext.cpu_stats() + ctxsw, intrs, soft_intrs, syscalls, _traps = cext.cpu_stats() elif NETBSD: # XXX # Note about intrs: the C extension returns 0. intrs @@ -332,7 +332,7 @@ def cpu_stats(): # # Note: the C ext is returning some metrics we are not exposing: # traps, faults and forks. - ctxsw, intrs, soft_intrs, syscalls, traps, faults, forks = ( + ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = ( cext.cpu_stats() ) with open('/proc/stat', 'rb') as f: @@ -342,7 +342,7 @@ def cpu_stats(): elif OPENBSD: # Note: the C ext is returning some metrics we are not exposing: # traps, faults and forks. - ctxsw, intrs, soft_intrs, syscalls, traps, faults, forks = ( + ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = ( cext.cpu_stats() ) return _common.scpustats(ctxsw, intrs, soft_intrs, syscalls) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0ee134799..167183881 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1080,25 +1080,25 @@ def net_io_counters(): name = line[:colon].strip() fields = line[colon + 1 :].strip().split() - # in ( + # in bytes_recv, packets_recv, errin, dropin, - fifoin, # unused - framein, # unused - compressedin, # unused - multicastin, # unused + _fifoin, # unused + _framein, # unused + _compressedin, # unused + _multicastin, # unused # out bytes_sent, packets_sent, errout, dropout, - fifoout, # unused - collisionsout, # unused - carrierout, # unused - compressedout, + _fifoout, # unused + _collisionsout, # unused + _carrierout, # unused + _compressedout, # unused ) = map(int, fields) retdict[name] = ( @@ -2091,9 +2091,9 @@ def get_blocks(lines, current_block): for header, data in get_blocks(lines, current_block): hfields = header.split(None, 5) try: - addr, perms, offset, dev, inode, path = hfields + addr, perms, _offset, _dev, _inode, path = hfields except ValueError: - addr, perms, offset, dev, inode, path = hfields + [''] + addr, perms, _offset, _dev, _inode, path = hfields + [''] if not path: path = '[anon]' else: diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 106709467..41263fd73 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -165,7 +165,7 @@ def cpu_count_cores(): def cpu_stats(): - ctx_switches, interrupts, soft_interrupts, syscalls, traps = ( + ctx_switches, interrupts, soft_interrupts, syscalls, _traps = ( cext.cpu_stats() ) return _common.scpustats( diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 6112728b1..1c0b96e9e 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -209,7 +209,7 @@ def cpu_count_cores(): def cpu_stats(): """Return various CPU stats as a named tuple.""" - ctx_switches, interrupts, syscalls, traps = cext.cpu_stats() + ctx_switches, interrupts, syscalls, _traps = cext.cpu_stats() soft_interrupts = 0 return _common.scpustats( ctx_switches, interrupts, soft_interrupts, syscalls diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index babb8e82e..0ba511b90 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -238,7 +238,7 @@ def getpagesize(): def virtual_memory(): """System virtual memory as a namedtuple.""" mem = cext.virtual_mem() - totphys, availphys, totsys, availsys = mem + totphys, availphys, _totsys, _availsys = mem total = totphys avail = availphys free = availphys @@ -337,7 +337,7 @@ def cpu_count_cores(): def cpu_stats(): """Return CPU statistics.""" - ctx_switches, interrupts, dpcs, syscalls = cext.cpu_stats() + ctx_switches, interrupts, _dpcs, syscalls = cext.cpu_stats() soft_interrupts = 0 return _common.scpustats( ctx_switches, interrupts, soft_interrupts, syscalls @@ -986,7 +986,7 @@ def create_time(self): # Note: proc_times() not put under oneshot() 'cause create_time() # is already cached by the main Process class. try: - user, system, created = cext.proc_times(self.pid) + _user, _system, created = cext.proc_times(self.pid) return created except OSError as err: if is_permission_err(err): @@ -1010,7 +1010,7 @@ def threads(self): @wrap_exceptions def cpu_times(self): try: - user, system, created = cext.proc_times(self.pid) + user, system, _created = cext.proc_times(self.pid) except OSError as err: if not is_permission_err(err): raise diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py index a054e4817..3b28b64f1 100755 --- a/psutil/tests/runner.py +++ b/psutil/tests/runner.py @@ -256,7 +256,7 @@ def run(self, suite): # At this point we should have N zombies (the workers), which # will disappear with wait(). orphans = psutil.Process().children() - gone, alive = psutil.wait_procs(orphans, timeout=1) + _gone, alive = psutil.wait_procs(orphans, timeout=1) if alive: cprint("alive processes %s" % alive, "red") reap_children() diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index a714632dc..8512b4f9e 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -171,7 +171,7 @@ def test_memory_maps(self): while lines: line = lines.pop() fields = line.split() - _, start, stop, perms, res = fields[:5] + _, start, stop, _perms, res = fields[:5] map = maps.pop() self.assertEqual("%s-%s" % (start, stop), map.addr) self.assertEqual(int(res), map.rss) @@ -416,19 +416,19 @@ def test_cpu_stats_syscalls(self): # --- swap memory def test_swapmem_free(self): - total, used, free = self.parse_swapinfo() + _total, _used, free = self.parse_swapinfo() self.assertAlmostEqual( psutil.swap_memory().free, free, delta=TOLERANCE_SYS_MEM ) def test_swapmem_used(self): - total, used, free = self.parse_swapinfo() + _total, used, _free = self.parse_swapinfo() self.assertAlmostEqual( psutil.swap_memory().used, used, delta=TOLERANCE_SYS_MEM ) def test_swapmem_total(self): - total, used, free = self.parse_swapinfo() + total, _used, _free = self.parse_swapinfo() self.assertAlmostEqual( psutil.swap_memory().total, total, delta=TOLERANCE_SYS_MEM ) @@ -447,7 +447,7 @@ def test_boot_time(self): @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors_battery(self): def secs2hours(secs): - m, s = divmod(secs, 60) + m, _s = divmod(secs, 60) h, m = divmod(m, 60) return "%d:%02d" % (h, m) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 307d3dfa9..0cae26d7e 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -808,7 +808,7 @@ def test_prog_w_funky_name(self): @unittest.skipIf(not POSIX, 'POSIX only') def test_uids(self): p = psutil.Process() - real, effective, saved = p.uids() + real, effective, _saved = p.uids() # os.getuid() refers to "real" uid self.assertEqual(real, os.getuid()) # os.geteuid() refers to "effective" uid @@ -822,7 +822,7 @@ def test_uids(self): @unittest.skipIf(not POSIX, 'POSIX only') def test_gids(self): p = psutil.Process() - real, effective, saved = p.gids() + real, effective, _saved = p.gids() # os.getuid() refers to "real" uid self.assertEqual(real, os.getgid()) # os.geteuid() refers to "effective" uid diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index 7c6ce7808..d1f476bb5 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -522,7 +522,7 @@ def check(pid): psutil.Process(pid) if not WINDOWS: # see docstring self.assertNotIn(pid, psutil.pids()) - except (psutil.Error, AssertionError) as err: + except (psutil.Error, AssertionError): x -= 1 if x == 0: raise diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 32ac62cbe..554fbffb2 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -845,7 +845,7 @@ def test_net_if_addrs(self): 0, socket.AI_PASSIVE, )[0] - af, socktype, proto, canonname, sa = info + af, socktype, proto, _canonname, sa = info s = socket.socket(af, socktype, proto) with contextlib.closing(s): s.bind(sa) diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py index 17cc9eb08..1a18b65a8 100755 --- a/psutil/tests/test_testutils.py +++ b/psutil/tests/test_testutils.py @@ -240,7 +240,7 @@ def test_spawn_children_pair(self): @unittest.skipIf(not POSIX, "POSIX only") def test_spawn_zombie(self): - parent, zombie = self.spawn_zombie() + _parent, zombie = self.spawn_zombie() self.assertEqual(zombie.status(), psutil.STATUS_ZOMBIE) def test_terminate(self): diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 5983af70a..7778cf4a6 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -407,7 +407,7 @@ def test_special_pid(self): p.username() self.assertGreaterEqual(p.create_time(), 0.0) try: - rss, vms = p.memory_info()[:2] + rss, _vms = p.memory_info()[:2] except psutil.AccessDenied: # expected on Windows Vista and Windows 7 if platform.uname()[1] not in ('vista', 'win-7', 'win7'): diff --git a/pyproject.toml b/pyproject.toml index 99a9b44d5..b03392071 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ ignore = [ "D", # pydocstyle "DTZ", # flake8-datetimez "ERA001", # Found commented-out code - "F841", # Local variable `parent` is assigned to but never used "FBT", # flake8-boolean-trap (makes zero sense) "FIX", # Line contains TODO / XXX / ..., consider resolving the issue "FLY", # flynt (PYTHON2.7 COMPAT) diff --git a/setup.py b/setup.py index 3c7900669..e3375004b 100755 --- a/setup.py +++ b/setup.py @@ -431,7 +431,7 @@ def get_sunos_update(): class bdist_wheel_abi3(bdist_wheel): def get_tag(self): - python, abi, plat = bdist_wheel.get_tag(self) + python, _abi, plat = bdist_wheel.get_tag(self) return python, "abi3", plat cmdclass["bdist_wheel"] = bdist_wheel_abi3 From 4b1a05419df6101c43ce327ae97ea9154f8aa0f9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 6 Jun 2024 21:30:17 +0200 Subject: [PATCH 40/46] Fix #2250 / NetBSD / cmdline: retry on EBUSY. (#2421) `Process.cmdline()` sometimes fail with EBUSY. It usually happens for long cmdlines with lots of arguments. In this case retry getting the cmdline for up to 50 times, and return an empty list as last resort. --- HISTORY.rst | 10 +++++++--- psutil/arch/netbsd/proc.c | 32 ++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a53590555..bc183170d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,15 +21,19 @@ **Bug fixes** -- 2395_, [OpenBSD]: `pid_exists()`_ erroneously return True if the argument is - a thread ID (TID) instead of a PID (process ID). -- 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch by Shade Gladden) +- 2250_, [NetBSD]: `Process.cmdline()`_ sometimes fail with EBUSY. It usually + happens for long cmdlines with lots of arguments. In this case retry getting + the cmdline for up to 50 times, and return an empty list as last resort. +- 2254_, [Linux]: offline cpus raise NotImplementedError in cpu_freq() (patch + by Shade Gladden) - 2272_: Add pickle support to psutil Exceptions. - 2359_, [Windows], [CRITICAL]: `pid_exists()`_ disagrees with `Process`_ on whether a pid exists when ERROR_ACCESS_DENIED. - 2360_, [macOS]: can't compile on macOS < 10.13. (patch by Ryan Schmidt) - 2362_, [macOS]: can't compile on macOS 10.11. (patch by Ryan Schmidt) - 2365_, [macOS]: can't compile on macOS < 10.9. (patch by Ryan Schmidt) +- 2395_, [OpenBSD]: `pid_exists()`_ erroneously return True if the argument is + a thread ID (TID) instead of a PID (process ID). **Porting notes** diff --git a/psutil/arch/netbsd/proc.c b/psutil/arch/netbsd/proc.c index c645f301e..4cd43c4c7 100644 --- a/psutil/arch/netbsd/proc.c +++ b/psutil/arch/netbsd/proc.c @@ -332,6 +332,8 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { pid_t pid; int mib[4]; int st; + int attempt; + int max_attempts = 50; size_t len = 0; size_t pos = 0; char *procargs = NULL; @@ -359,10 +361,32 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { PyErr_NoMemory(); goto error; } - st = sysctl(mib, __arraycount(mib), procargs, &len, NULL, 0); - if (st == -1) { - PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ARGV)"); - goto error; + + while (1) { + st = sysctl(mib, __arraycount(mib), procargs, &len, NULL, 0); + if (st == -1) { + if (errno == EBUSY) { + // Usually happens with TestProcess.test_long_cmdline. See: + // https://github.com/giampaolo/psutil/issues/2250 + attempt += 1; + if (attempt < max_attempts) { + psutil_debug("proc %zu cmdline(): retry on EBUSY", pid); + continue; + } + else { + psutil_debug( + "proc %zu cmdline(): return [] due to EBUSY", pid + ); + free(procargs); + return py_retlist; + } + } + else { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ARGV)"); + goto error; + } + } + break; } if (len > 0) { From 9421bf8e81994b511d39c70f3bf68ccf69cf6567 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 6 Jun 2024 22:03:05 +0200 Subject: [PATCH 41/46] openbsd: skip test if cmdline() returns [] due to EBUSY --- psutil/tests/test_process.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 0cae26d7e..9ddaf9cfe 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -758,9 +758,15 @@ def test_long_cmdline(self): self.assertEqual(p.cmdline(), cmdline) except psutil.ZombieProcess: raise unittest.SkipTest("OPENBSD: process turned into zombie") - else: + elif NETBSD: + ret = p.cmdline() + if ret == []: + # https://github.com/giampaolo/psutil/issues/2250 + raise unittest.SkipTest("OPENBSD: returned EBUSY") self.assertEqual(p.cmdline(), cmdline) + self.assertEqual(p.cmdline(), cmdline) + def test_name(self): p = self.spawn_psproc() name = p.name().lower() From 89b6096f2a026ec85f2a188920877371d4515b60 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 9 Jun 2024 23:06:40 +0200 Subject: [PATCH 42/46] process_iter(): use another global var to keep track of reused PIDs --- psutil/__init__.py | 9 +++++++-- psutil/tests/test_linux.py | 4 ++-- psutil/tests/test_misc.py | 1 + psutil/tests/test_process.py | 14 +++++++++++++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 30f45987e..3a503503c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -85,6 +85,7 @@ from ._common import NoSuchProcess from ._common import TimeoutExpired from ._common import ZombieProcess +from ._common import debug from ._common import memoize_when_activated from ._common import wrap_numbers as _wrap_numbers from ._compat import PY3 as _PY3 @@ -613,8 +614,7 @@ def is_running(self): # time) and that is verified in __eq__. self._pid_reused = self != Process(self.pid) if self._pid_reused: - # remove this PID from `process_iter()` internal cache - _pmap.pop(self.pid, None) + _pids_reused.add(self.pid) raise NoSuchProcess(self.pid) return True except ZombieProcess: @@ -1464,6 +1464,7 @@ def pid_exists(pid): _pmap = {} +_pids_reused = set() def process_iter(attrs=None, ad_value=None): @@ -1501,6 +1502,10 @@ def remove(pid): gone_pids = b - a for pid in gone_pids: remove(pid) + while _pids_reused: + pid = _pids_reused.pop() + debug("refreshing Process instance for reused PID %s" % pid) + remove(pid) try: ls = sorted(list(pmap.items()) + list(dict.fromkeys(new_pids).items())) for pid, proc in ls: diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 233584c84..be264ae1d 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1120,10 +1120,10 @@ def ifconfig(nic): except RuntimeError: continue self.assertAlmostEqual( - stats.bytes_recv, ifconfig_ret['bytes_recv'], delta=1024 * 5 + stats.bytes_recv, ifconfig_ret['bytes_recv'], delta=1024 * 10 ) self.assertAlmostEqual( - stats.bytes_sent, ifconfig_ret['bytes_sent'], delta=1024 * 5 + stats.bytes_sent, ifconfig_ret['bytes_sent'], delta=1024 * 10 ) self.assertAlmostEqual( stats.packets_recv, ifconfig_ret['packets_recv'], delta=1024 diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 93204fa06..59416592e 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -219,6 +219,7 @@ def test__all__(self): dir_psutil = dir(psutil) for name in dir_psutil: if name in ( + 'debug', 'long', 'tests', 'test', diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 9ddaf9cfe..4f80277b7 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -36,6 +36,7 @@ from psutil._compat import PY3 from psutil._compat import FileNotFoundError from psutil._compat import long +from psutil._compat import redirect_stderr from psutil._compat import super from psutil.tests import APPVEYOR from psutil.tests import CI_TESTING @@ -763,7 +764,6 @@ def test_long_cmdline(self): if ret == []: # https://github.com/giampaolo/psutil/issues/2250 raise unittest.SkipTest("OPENBSD: returned EBUSY") - self.assertEqual(p.cmdline(), cmdline) self.assertEqual(p.cmdline(), cmdline) @@ -1378,6 +1378,11 @@ def test_zombie_process_status_w_exc(self): def test_reused_pid(self): # Emulate a case where PID has been reused by another process. + if PY3: + from io import StringIO + else: + from StringIO import StringIO + subp = self.spawn_testproc() p = psutil.Process(subp.pid) p._ident = (p.pid, p.create_time() + 100) @@ -1385,8 +1390,15 @@ def test_reused_pid(self): list(psutil.process_iter()) self.assertIn(p.pid, psutil._pmap) assert not p.is_running() + # make sure is_running() removed PID from process_iter() # internal cache + with redirect_stderr(StringIO()) as f: + list(psutil.process_iter()) + self.assertIn( + "refreshing Process instance for reused PID %s" % p.pid, + f.getvalue(), + ) self.assertNotIn(p.pid, psutil._pmap) assert p != psutil.Process(subp.pid) From 5f80c123d2497af404639669049213a48531e2fe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 11 Jun 2024 23:51:52 +0200 Subject: [PATCH 43/46] Fix #2412, [macOS]: can't compile on macOS 10.4 PowerPC due to missing `MNT_` constants. --- HISTORY.rst | 2 ++ psutil/arch/osx/disk.c | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bc183170d..ae664900f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,8 @@ - 2365_, [macOS]: can't compile on macOS < 10.9. (patch by Ryan Schmidt) - 2395_, [OpenBSD]: `pid_exists()`_ erroneously return True if the argument is a thread ID (TID) instead of a PID (process ID). +- 2412_, [macOS]: can't compile on macOS 10.4 PowerPC due to missing `MNT_` + constants. **Porting notes** diff --git a/psutil/arch/osx/disk.c b/psutil/arch/osx/disk.c index 961fc42a4..d02cf794d 100644 --- a/psutil/arch/osx/disk.c +++ b/psutil/arch/osx/disk.c @@ -86,8 +86,6 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",async", sizeof(opts)); if (flags & MNT_EXPORTED) strlcat(opts, ",exported", sizeof(opts)); - if (flags & MNT_QUARANTINE) - strlcat(opts, ",quarantine", sizeof(opts)); if (flags & MNT_LOCAL) strlcat(opts, ",local", sizeof(opts)); if (flags & MNT_QUOTA) @@ -108,10 +106,6 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",nouserxattr", sizeof(opts)); if (flags & MNT_DEFWRITE) strlcat(opts, ",defwrite", sizeof(opts)); - if (flags & MNT_MULTILABEL) - strlcat(opts, ",multilabel", sizeof(opts)); - if (flags & MNT_NOATIME) - strlcat(opts, ",noatime", sizeof(opts)); if (flags & MNT_UPDATE) strlcat(opts, ",update", sizeof(opts)); if (flags & MNT_RELOAD) @@ -120,7 +114,19 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",force", sizeof(opts)); if (flags & MNT_CMDFLAGS) strlcat(opts, ",cmdflags", sizeof(opts)); - + // requires macOS >= 10.5 +#ifdef MNT_QUARANTINE + if (flags & MNT_QUARANTINE) + strlcat(opts, ",quarantine", sizeof(opts)); +#endif +#ifdef MNT_MULTILABEL + if (flags & MNT_MULTILABEL) + strlcat(opts, ",multilabel", sizeof(opts)); +#endif +#ifdef MNT_NOATIME + if (flags & MNT_NOATIME) + strlcat(opts, ",noatime", sizeof(opts)); +#endif py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); if (! py_dev) goto error; From 1d092e728abddb628738d2f0a110b74152969771 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 12 Jun 2024 00:15:21 +0200 Subject: [PATCH 44/46] test subprocesses: sleep() with an interval of 0.1 to make the test process more 'killable' --- psutil/tests/__init__.py | 2 +- psutil/tests/test_connections.py | 6 ++-- psutil/tests/test_process.py | 52 ++++++++++++++++++++++++++------ psutil/tests/test_testutils.py | 6 +++- psutil/tests/test_unicode.py | 18 +++++++++-- 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index b18b74239..684c68ae2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -408,7 +408,7 @@ def spawn_children_pair(): s += "f = open('%s', 'w');" s += "f.write(str(os.getpid()));" s += "f.close();" - s += "time.sleep(60);" + s += "[time.sleep(0.1) for x in range(100 * 6)];" p = subprocess.Popen([r'%s', '-c', s]) p.wait() """ % (os.path.basename(testfn), PYTHON_EXE)) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index de3ae59df..4a0674d62 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -328,7 +328,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): s.listen(5) with open('{testfn}', 'w') as f: f.write(str(s.getsockname()[:2])) - time.sleep(60) + [time.sleep(0.1) for x in range(100)] """) udp_template = textwrap.dedent(""" @@ -337,7 +337,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): s.bind(('{addr}', 0)) with open('{testfn}', 'w') as f: f.write(str(s.getsockname()[:2])) - time.sleep(60) + [time.sleep(0.1) for x in range(100)] """) # must be relative on Windows @@ -530,7 +530,7 @@ def test_multi_sockets_procs(self): with create_sockets(): with open(r'%s', 'w') as f: f.write("hello") - time.sleep(60) + [time.sleep(0.1) for x in range(100)] """ % fname) sproc = self.pyrun(src) pids.append(sproc.pid) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 4f80277b7..1ee5393c2 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -255,7 +255,8 @@ def test_cpu_percent_numcpus_none(self): def test_cpu_times(self): times = psutil.Process().cpu_times() - assert (times.user > 0.0) or (times.system > 0.0), times + assert times.user >= 0.0, times + assert times.system >= 0.0, times assert times.children_user >= 0.0, times assert times.children_system >= 0.0, times if LINUX: @@ -727,8 +728,17 @@ def test_exe(self): self.assertEqual(out, 'hey') def test_cmdline(self): - cmdline = [PYTHON_EXE, "-c", "import time; time.sleep(60)"] + cmdline = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] p = self.spawn_psproc(cmdline) + + if NETBSD and p.cmdline() == []: + # https://github.com/giampaolo/psutil/issues/2250 + raise unittest.SkipTest("OPENBSD: returned EBUSY") + # XXX - most of the times the underlying sysctl() call on Net # and Open BSD returns a truncated string. # Also /proc/pid/cmdline behaves the same so it looks @@ -750,7 +760,9 @@ def test_cmdline(self): def test_long_cmdline(self): cmdline = [PYTHON_EXE] cmdline.extend(["-v"] * 50) - cmdline.extend(["-c", "import time; time.sleep(10)"]) + cmdline.extend( + ["-c", "import time; [time.sleep(0.1) for x in range(100)]"] + ) p = self.spawn_psproc(cmdline) if OPENBSD: # XXX: for some reason the test process may turn into a @@ -776,7 +788,11 @@ def test_name(self): @unittest.skipIf(PYPY, "unreliable on PYPY") def test_long_name(self): pyexe = create_py_exe(self.get_testfn(suffix="0123456789" * 2)) - cmdline = [pyexe, "-c", "import time; time.sleep(10)"] + cmdline = [ + pyexe, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] p = self.spawn_psproc(cmdline) if OPENBSD: # XXX: for some reason the test process may turn into a @@ -805,7 +821,11 @@ def test_prog_w_funky_name(self): # with funky chars such as spaces and ")", see: # https://github.com/giampaolo/psutil/issues/628 pyexe = create_py_exe(self.get_testfn(suffix='foo bar )')) - cmdline = [pyexe, "-c", "import time; time.sleep(10)"] + cmdline = [ + pyexe, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] p = self.spawn_psproc(cmdline) self.assertEqual(p.cmdline(), cmdline) self.assertEqual(p.name(), os.path.basename(pyexe)) @@ -931,7 +951,10 @@ def test_cwd_2(self): cmd = [ PYTHON_EXE, "-c", - "import os, time; os.chdir('..'); time.sleep(60)", + ( + "import os, time; os.chdir('..'); [time.sleep(0.1) for x in" + " range(100)]" + ), ] p = self.spawn_psproc(cmd) call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") @@ -1031,7 +1054,10 @@ def test_open_files(self): assert os.path.isfile(file.path), file # another process - cmdline = "import time; f = open(r'%s', 'r'); time.sleep(60);" % testfn + cmdline = ( + "import time; f = open(r'%s', 'r'); [time.sleep(0.1) for x in" + " range(100)];" % testfn + ) p = self.spawn_psproc([PYTHON_EXE, "-c", cmdline]) for x in range(100): @@ -1599,7 +1625,11 @@ def test_misc(self): # XXX this test causes a ResourceWarning on Python 3 because # psutil.__subproc instance doesn't get properly freed. # Not sure what to do though. - cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] with psutil.Popen( cmd, stdout=subprocess.PIPE, @@ -1635,7 +1665,11 @@ def test_kill_terminate(self): # subprocess.Popen()'s terminate(), kill() and send_signal() do # not raise exception after the process is gone. psutil.Popen # diverges from that. - cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] with psutil.Popen( cmd, stdout=subprocess.PIPE, diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py index 1a18b65a8..ef98a3abe 100755 --- a/psutil/tests/test_testutils.py +++ b/psutil/tests/test_testutils.py @@ -255,7 +255,11 @@ def test_terminate(self): self.assertPidGone(p.pid) terminate(p) # by psutil.Popen - cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] p = psutil.Popen( cmd, stdout=subprocess.PIPE, diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7fb2ef199..45aeb4e92 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -197,7 +197,11 @@ def expect_exact_path_match(self): # --- def test_proc_exe(self): - cmd = [self.funky_name, "-c", "import time; time.sleep(10)"] + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] subp = self.spawn_testproc(cmd) p = psutil.Process(subp.pid) exe = p.exe() @@ -208,7 +212,11 @@ def test_proc_exe(self): ) def test_proc_name(self): - cmd = [self.funky_name, "-c", "import time; time.sleep(10)"] + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] subp = self.spawn_testproc(cmd) name = psutil.Process(subp.pid).name() self.assertIsInstance(name, str) @@ -216,7 +224,11 @@ def test_proc_name(self): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): - cmd = [self.funky_name, "-c", "import time; time.sleep(10)"] + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] subp = self.spawn_testproc(cmd) p = psutil.Process(subp.pid) cmdline = p.cmdline() From 5b30ef4796f7d0a1c4cd31a3b74b7405105331d0 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Tue, 18 Jun 2024 22:32:26 +0200 Subject: [PATCH 45/46] Add aarch64 manylinux wheels (#2425) --- .github/workflows/build.yml | 29 +++++++++++++++-------------- HISTORY.rst | 1 + psutil/tests/__init__.py | 8 +++++++- psutil/tests/test_contracts.py | 2 ++ psutil/tests/test_linux.py | 23 ++++++++++++++++++++--- psutil/tests/test_memleaks.py | 9 +++++++++ psutil/tests/test_misc.py | 5 +++++ psutil/tests/test_posix.py | 7 ++++++- psutil/tests/test_process.py | 24 +++++++++++++++++++----- psutil/tests/test_process_all.py | 4 ++++ psutil/tests/test_system.py | 3 +++ pyproject.toml | 4 ++++ 12 files changed, 95 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c4e9c745e..d23bd179c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,23 +18,20 @@ concurrency: jobs: # Linux + macOS + Windows Python 3 py3: - name: py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'ubuntu') && 'all' || matrix.archs }} + name: "py3-${{ matrix.os }}-${{ matrix.arch }}" runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: include: - - os: ubuntu-latest - archs: "x86_64 i686" - - os: macos-12 - archs: "x86_64" - - os: macos-14 - archs: "arm64" - - os: windows-2019 - archs: "AMD64" - - os: windows-2019 - archs: "x86" + - {os: ubuntu-latest, arch: x86_64} + - {os: ubuntu-latest, arch: i686} + - {os: ubuntu-latest, arch: aarch64} + - {os: macos-12, arch: x86_64} + - {os: macos-14, arch: arm64} + - {os: windows-2019, arch: AMD64} + - {os: windows-2019, arch: x86} steps: - uses: actions/checkout@v4 @@ -51,16 +48,20 @@ jobs: with: python-version: 3.11 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + if: matrix.arch == 'aarch64' + - name: Create wheels + run tests - uses: pypa/cibuildwheel@v2.18.0 + uses: pypa/cibuildwheel@v2.19.1 env: - CIBW_ARCHS: "${{ matrix.archs }}" + CIBW_ARCHS: "${{ matrix.arch }}" CIBW_PRERELEASE_PYTHONS: True - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'ubuntu') && 'all' || matrix.archs }} + name: wheels-py3-${{ matrix.os }}-${{ matrix.arch }} path: wheelhouse - name: Generate .tar.gz diff --git a/HISTORY.rst b/HISTORY.rst index ae664900f..5107b1345 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ - 2407_: `Process.connections()`_ was renamed to `Process.net_connections()`_. The old name is still available, but it's deprecated (triggers a ``DeprecationWarning``) and will be removed in the future. +- 2425_: [Linux]: provide aarch64 wheels. (patch by Matthieu Darbois / Ben Raz) **Bug fixes** diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 684c68ae2..dccb9e4b2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -87,7 +87,7 @@ "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_NET_CONNECTIONS_UNIX", "MACOS_11PLUS", - "MACOS_12PLUS", "COVERAGE", + "MACOS_12PLUS", "COVERAGE", 'AARCH64', "QEMU_USER", # subprocesses 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', 'spawn_children_pair', @@ -128,8 +128,14 @@ GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ CI_TESTING = APPVEYOR or GITHUB_ACTIONS COVERAGE = 'COVERAGE_RUN' in os.environ +if LINUX and GITHUB_ACTIONS: + with open('/proc/1/cmdline') as f: + QEMU_USER = "/bin/qemu-" in f.read() +else: + QEMU_USER = False # are we a 64 bit process? IS_64BIT = sys.maxsize > 2**32 +AARCH64 = platform.machine() == "aarch64" @memoize diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index e768e7d52..a5469ac8a 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -30,6 +30,7 @@ from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import SKIP_SYSCONS from psutil.tests import PsutilTestCase from psutil.tests import create_sockets @@ -277,6 +278,7 @@ def test_net_if_addrs(self): self.assertIsInstance(addr.netmask, (str, type(None))) self.assertIsInstance(addr.broadcast, (str, type(None))) + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_stats(self): # Duplicate of test_system.py. Keep it anyway. for ifname, info in psutil.net_if_stats().items(): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index be264ae1d..d8faac968 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -28,6 +28,7 @@ from psutil._compat import PY3 from psutil._compat import FileNotFoundError from psutil._compat import basestring +from psutil.tests import AARCH64 from psutil.tests import GITHUB_ACTIONS from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_BATTERY @@ -35,6 +36,7 @@ from psutil.tests import HAS_GETLOADAVG from psutil.tests import HAS_RLIMIT from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import TOLERANCE_DISK_USAGE from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import PsutilTestCase @@ -277,8 +279,14 @@ def test_used(self): # This got changed in: # https://gitlab.com/procps-ng/procps/commit/ # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + # Newer versions of procps are using yet another way to compute used + # memory. + # https://gitlab.com/procps-ng/procps/commit/ + # 2184e90d2e7cdb582f9a5b706b47015e56707e4d if get_free_version_info() < (3, 3, 12): - raise unittest.SkipTest("old free version") + raise unittest.SkipTest("free version too old") + if get_free_version_info() >= (4, 0, 0): + raise unittest.SkipTest("free version too recent") cli_value = free_physmem().used psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( @@ -341,8 +349,14 @@ def test_used(self): # This got changed in: # https://gitlab.com/procps-ng/procps/commit/ # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + # Newer versions of procps are using yet another way to compute used + # memory. + # https://gitlab.com/procps-ng/procps/commit/ + # 2184e90d2e7cdb582f9a5b706b47015e56707e4d if get_free_version_info() < (3, 3, 12): - raise unittest.SkipTest("old free version") + raise unittest.SkipTest("free version too old") + if get_free_version_info() >= (4, 0, 0): + raise unittest.SkipTest("free version too recent") vmstat_value = vmstat('used memory') * 1024 psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( @@ -830,6 +844,7 @@ def path_exists_mock(path): assert psutil.cpu_freq() @unittest.skipIf(not HAS_CPU_FREQ, "not supported") + @unittest.skipIf(AARCH64, "aarch64 does not report mhz in /proc/cpuinfo") def test_emulate_use_cpuinfo(self): # Emulate a case where /sys/devices/system/cpu/cpufreq* does not # exist and /proc/cpuinfo is used instead. @@ -1037,6 +1052,7 @@ def test_ips(self): @unittest.skipIf(not LINUX, "LINUX only") +@unittest.skipIf(QEMU_USER, "QEMU user not supported") class TestSystemNetIfStats(PsutilTestCase): @unittest.skipIf(not which("ifconfig"), "ifconfig utility not available") def test_against_ifconfig(self): @@ -1596,7 +1612,7 @@ def test_issue_687(self): with ThreadTask(): p = psutil.Process() threads = p.threads() - self.assertEqual(len(threads), 2) + self.assertEqual(len(threads), 3 if QEMU_USER else 2) tid = sorted(threads, key=lambda x: x.id)[1].id self.assertNotEqual(p.pid, tid) pt = psutil.Process(tid) @@ -2276,6 +2292,7 @@ def test_name(self): value = self.read_status_file("Name:") self.assertEqual(self.proc.name(), value) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_status(self): value = self.read_status_file("State:") value = value[value.find('(') + 1 : value.rfind(')')] diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index 8232b18ad..6506497c9 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -19,6 +19,7 @@ import functools import os import platform +import sys import unittest import psutil @@ -43,6 +44,7 @@ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import QEMU_USER from psutil.tests import TestMemoryLeak from psutil.tests import create_sockets from psutil.tests import get_testfn @@ -398,6 +400,7 @@ def test_disk_usage(self): times = FEW_TIMES if POSIX else self.times self.execute(lambda: psutil.disk_usage('.'), times=times) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_disk_partitions(self): self.execute(psutil.disk_partitions) @@ -435,6 +438,7 @@ def test_net_if_addrs(self): tolerance = 80 * 1024 if WINDOWS else self.tolerance self.execute(psutil.net_if_addrs, tolerance=tolerance) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_net_if_stats(self): self.execute(psutil.net_if_stats) @@ -491,6 +495,11 @@ def test_win_service_get_description(self): if __name__ == '__main__': + from psutil.tests.runner import cprint from psutil.tests.runner import run_from_name + if QEMU_USER: + cprint("skipping %s tests under QEMU_USER" % __file__, "brown") + sys.exit(0) + run_from_name(__file__) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 59416592e..5c05d1764 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -43,6 +43,7 @@ from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import PYTHON_EXE from psutil.tests import PYTHON_EXE_ENV +from psutil.tests import QEMU_USER from psutil.tests import SCRIPTS_DIR from psutil.tests import PsutilTestCase from psutil.tests import mock @@ -288,6 +289,9 @@ def check(ret): for fun, name in ns.iter(ns.getters): if name in {"win_service_iter", "win_service_get"}: continue + if QEMU_USER and name == "net_if_stats": + # OSError: [Errno 38] ioctl(SIOCETHTOOL) not implemented + continue with self.subTest(name=name): try: ret = fun() @@ -1008,6 +1012,7 @@ def test_pstree(self): def test_netstat(self): self.assert_stdout('netstat.py') + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_ifconfig(self): self.assert_stdout('ifconfig.py') diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 5203c2707..941f0fac1 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -25,6 +25,7 @@ from psutil import SUNOS from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import PYTHON_EXE +from psutil.tests import QEMU_USER from psutil.tests import PsutilTestCase from psutil.tests import mock from psutil.tests import retry_on_failure @@ -102,7 +103,11 @@ def ps_name(pid): field = "command" if SUNOS: field = "comm" - return ps(field, pid).split()[0] + command = ps(field, pid).split() + if QEMU_USER: + assert "/bin/qemu-" in command[0] + return command[1] + return command[0] def ps_args(pid): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 1ee5393c2..363474c78 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -54,6 +54,7 @@ from psutil.tests import PYPY from psutil.tests import PYTHON_EXE from psutil.tests import PYTHON_EXE_ENV +from psutil.tests import QEMU_USER from psutil.tests import PsutilTestCase from psutil.tests import ThreadTask from psutil.tests import call_until @@ -253,6 +254,7 @@ def test_cpu_percent_numcpus_none(self): psutil.Process().cpu_percent() assert m.called + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_cpu_times(self): times = psutil.Process().cpu_times() assert times.user >= 0.0, times @@ -265,6 +267,7 @@ def test_cpu_times(self): for name in times._fields: time.strftime("%H:%M:%S", time.localtime(getattr(times, name))) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_cpu_times_2(self): user_time, kernel_time = psutil.Process().cpu_times()[:2] utime, ktime = os.times()[:2] @@ -633,6 +636,8 @@ def test_memory_maps(self): for nt in maps: if not nt.path.startswith('['): + if QEMU_USER and "/bin/qemu-" in nt.path: + continue assert os.path.isabs(nt.path), nt.path if POSIX: try: @@ -698,6 +703,7 @@ def test_is_running(self): assert not p.is_running() assert not p.is_running() + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_exe(self): p = self.spawn_psproc() exe = p.exe() @@ -754,6 +760,9 @@ def test_cmdline(self): ' '.join(p.cmdline()[1:]), ' '.join(cmdline[1:]) ) return + if QEMU_USER: + self.assertEqual(' '.join(p.cmdline()[2:]), ' '.join(cmdline)) + return self.assertEqual(' '.join(p.cmdline()), ' '.join(cmdline)) @unittest.skipIf(PYPY, "broken on PYPY") @@ -771,13 +780,14 @@ def test_long_cmdline(self): self.assertEqual(p.cmdline(), cmdline) except psutil.ZombieProcess: raise unittest.SkipTest("OPENBSD: process turned into zombie") - elif NETBSD: + elif QEMU_USER: + self.assertEqual(p.cmdline()[2:], cmdline) + else: ret = p.cmdline() - if ret == []: + if NETBSD and ret == []: # https://github.com/giampaolo/psutil/issues/2250 raise unittest.SkipTest("OPENBSD: returned EBUSY") - - self.assertEqual(p.cmdline(), cmdline) + self.assertEqual(ret, cmdline) def test_name(self): p = self.spawn_psproc() @@ -785,7 +795,8 @@ def test_name(self): pyexe = os.path.basename(os.path.realpath(sys.executable)).lower() assert pyexe.startswith(name), (pyexe, name) - @unittest.skipIf(PYPY, "unreliable on PYPY") + @unittest.skipIf(PYPY or QEMU_USER, "unreliable on PYPY") + @unittest.skipIf(QEMU_USER, "unreliable on QEMU user") def test_long_name(self): pyexe = create_py_exe(self.get_testfn(suffix="0123456789" * 2)) cmdline = [ @@ -816,6 +827,7 @@ def test_long_name(self): @unittest.skipIf(SUNOS, "broken on SUNOS") @unittest.skipIf(AIX, "broken on AIX") @unittest.skipIf(PYPY, "broken on PYPY") + @unittest.skipIf(QEMU_USER, "broken on QEMU user") def test_prog_w_funky_name(self): # Test that name(), exe() and cmdline() correctly handle programs # with funky chars such as spaces and ")", see: @@ -922,6 +934,7 @@ def cleanup(init): except psutil.AccessDenied: pass + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_status(self): p = psutil.Process() self.assertEqual(p.status(), psutil.STATUS_RUNNING) @@ -1149,6 +1162,7 @@ def test_parent_multi(self): self.assertEqual(grandchild.parent(), child) self.assertEqual(child.parent(), parent) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") @retry_on_failure() def test_parents(self): parent = psutil.Process() diff --git a/psutil/tests/test_process_all.py b/psutil/tests/test_process_all.py index d1f476bb5..48833a105 100755 --- a/psutil/tests/test_process_all.py +++ b/psutil/tests/test_process_all.py @@ -32,6 +32,7 @@ from psutil._compat import long from psutil._compat import unicode from psutil.tests import CI_TESTING +from psutil.tests import QEMU_USER from psutil.tests import VALID_PROC_STATUSES from psutil.tests import PsutilTestCase from psutil.tests import check_connection_ntuple @@ -235,6 +236,9 @@ def username(self, ret, info): def status(self, ret, info): self.assertIsInstance(ret, str) assert ret, ret + if QEMU_USER: + # status does not work under qemu user + return self.assertNotEqual(ret, '?') # XXX self.assertIn(ret, VALID_PROC_STATUSES) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 554fbffb2..e228f6d32 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -48,6 +48,7 @@ from psutil.tests import IS_64BIT from psutil.tests import MACOS_12PLUS from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import UNICODE_SUFFIX from psutil.tests import PsutilTestCase from psutil.tests import check_net_address @@ -806,6 +807,7 @@ def test_net_io_counters_no_nics(self): self.assertEqual(psutil.net_io_counters(pernic=True), {}) assert m.called + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_addrs(self): nics = psutil.net_if_addrs() assert nics, nics @@ -893,6 +895,7 @@ def test_net_if_addrs_mac_null_bytes(self): else: self.assertEqual(addr.address, '06-3d-29-00-00-00') + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_stats(self): nics = psutil.net_if_stats() assert nics, nics diff --git a/pyproject.toml b/pyproject.toml index b03392071..ba13f2829 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -207,6 +207,7 @@ trailing_comma_inline_array = true skip = [ "*-musllinux*", "cp313-win*", # pywin32 is not available on cp313 yet + "cp3{7,8,9,10,11,12}-*linux_{aarch64,ppc64le,s390x}", # Only test cp36/cp313 on qemu tested architectures "pp*", ] test-command = [ @@ -218,6 +219,9 @@ test-extras = "test" [tool.cibuildwheel.macos] archs = ["arm64", "x86_64"] +[tool.cibuildwheel.linux] +before-all = "yum install -y net-tools" + [build-system] build-backend = "setuptools.build_meta" requires = ["setuptools>=43", "wheel"] From 3d5522aafb6c9cbcbcf4edfa11348f755af97d96 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 18 Jun 2024 23:20:42 +0200 Subject: [PATCH 46/46] release --- HISTORY.rst | 4 ++-- Makefile | 4 ++-- docs/index.rst | 4 ++++ scripts/internal/print_dist.py | 8 +++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5107b1345..d290221f2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,7 +1,7 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -6.0.0 (IN DEVELOPMENT) -====================== +6.0.0 2024-06-18 +================ **Enhancements** diff --git a/Makefile b/Makefile index 60d82336e..f54354d35 100644 --- a/Makefile +++ b/Makefile @@ -309,8 +309,8 @@ pre-release: ## Check if we're ready to produce a new release. release: ## Upload a new release. ${MAKE} check-sdist ${MAKE} check-wheels - $(PYTHON) -m twine upload dist/*.tar.gz - $(PYTHON) -m twine upload dist/*.whl + $(PYTHON) -m twine upload --verbose dist/*.tar.gz + $(PYTHON) -m twine upload --verbose dist/*.whl ${MAKE} git-tag-release generate-manifest: ## Generates MANIFEST.in file. diff --git a/docs/index.rst b/docs/index.rst index 201333af4..d9fe1c03c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2672,6 +2672,10 @@ PyPy3. Timeline ======== +- 2024-06-18: + `6.0.0 `__ - + `what's new `__ - + `diff `__ - 2024-01-19: `5.9.8 `__ - `what's new `__ - diff --git a/scripts/internal/print_dist.py b/scripts/internal/print_dist.py index 5b6531608..1fb4b3f3a 100755 --- a/scripts/internal/print_dist.py +++ b/scripts/internal/print_dist.py @@ -58,11 +58,13 @@ def platform(self): def arch(self): if self.name.endswith(('x86_64.whl', 'amd64.whl')): - return '64' + return '64-bit' if self.name.endswith(("i686.whl", "win32.whl")): - return '32' + return '32-bit' if self.name.endswith("arm64.whl"): return 'arm64' + if self.name.endswith("aarch64.whl"): + return 'aarch64' return '?' def pyver(self): @@ -109,7 +111,7 @@ def main(): tot_files = 0 tot_size = 0 - templ = "%-120s %7s %7s %7s" + templ = "%-120s %7s %8s %7s" for platf, pkgs in groups.items(): ppn = "%s (%s)" % (platf, len(pkgs)) s = templ % (ppn, "size", "arch", "pyver")