From d10f74157276db126806bcfa85ec7b3630957547 Mon Sep 17 00:00:00 2001 From: Panos Mavrogiorgos Date: Mon, 27 Jun 2022 14:59:34 +0300 Subject: [PATCH] multi: Fallback to `os.cpu_count()` on non UNIX OSes `os.sched_getaffinity()` is the only method in the StdLib that correctly handles `taskset` restrictions etc. Nevertheless, it is only defined on Linux. Therefore, on different OSes we fall back to `cpu_count()` and if that is not available, we set `MAX_AVAILABLE_PROCESSES = 1`. Further info: https://stackoverflow.com/questions/1006289/how-to-find-out-the-number-of-cpus-using-python Fixes #35 --- poetry.lock | 34 +++++++++++++++---------------- requirements/requirements-dev.txt | 10 ++++----- searvey/multi.py | 7 ++++++- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index 07d4ce4..a1cd94e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -431,7 +431,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.11.4" +version = "4.12.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -443,7 +443,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -750,11 +750,11 @@ python-versions = ">=3.5" [[package]] name = "nodeenv" -version = "1.6.0" +version = "1.7.0" description = "Node.js virtual environment builder" category = "dev" optional = false -python-versions = "*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" [[package]] name = "numpy" @@ -880,7 +880,7 @@ virtualenv = ">=20.0.8" [[package]] name = "prompt-toolkit" -version = "3.0.29" +version = "3.0.30" description = "Library for building powerful interactive command lines in Python" category = "dev" optional = false @@ -1533,7 +1533,7 @@ documentation = ["m2r2", "sphinx", "sphinx-rtd-theme"] [[package]] name = "types-requests" -version = "2.27.31" +version = "2.28.0" description = "Typing stubs for requests" category = "dev" optional = false @@ -1587,7 +1587,7 @@ yarl = {version = "*", markers = "python_version >= \"3.6\""} [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.15.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1947,8 +1947,8 @@ imagesize = [ {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, - {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -2251,8 +2251,8 @@ nest-asyncio = [ {file = "nest_asyncio-1.5.5.tar.gz", hash = "sha256:e442291cd942698be619823a17a86a5759eabe1f8613084790de189fe9e16d65"}, ] nodeenv = [ - {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, - {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] numpy = [ {file = "numpy-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58bfd40eb478f54ff7a5710dd61c8097e169bc36cc68333d00a9bcd8def53b38"}, @@ -2334,8 +2334,8 @@ pre-commit = [ {file = "pre_commit-2.19.0.tar.gz", hash = "sha256:4233a1e38621c87d9dda9808c6606d7e7ba0e087cd56d3fe03202a01d2919615"}, ] prompt-toolkit = [ - {file = "prompt_toolkit-3.0.29-py3-none-any.whl", hash = "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752"}, - {file = "prompt_toolkit-3.0.29.tar.gz", hash = "sha256:bd640f60e8cecd74f0dc249713d433ace2ddc62b65ee07f96d358e0b152b6ea7"}, + {file = "prompt_toolkit-3.0.30-py3-none-any.whl", hash = "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"}, + {file = "prompt_toolkit-3.0.30.tar.gz", hash = "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0"}, ] prospector = [ {file = "prospector-1.7.7-py3-none-any.whl", hash = "sha256:2dec5dac06f136880a3710996c0886dcc99e739007bbc05afc32884973f5c058"}, @@ -2792,8 +2792,8 @@ typepigeon = [ {file = "typepigeon-1.0.14.tar.gz", hash = "sha256:8385dfee5a6f4dccebfe1355f40569c10233a57948b3463234b46cf5c792f021"}, ] types-requests = [ - {file = "types-requests-2.27.31.tar.gz", hash = "sha256:6fab97b99fea52b9c7b466a4dd93e06bb325bc7e7420475e87831026a8dd35cc"}, - {file = "types_requests-2.27.31-py3-none-any.whl", hash = "sha256:1b6cf6a2bf57fd8018c1b636b69762900466fafddfb62e1330e092f3d4b0966a"}, + {file = "types-requests-2.28.0.tar.gz", hash = "sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3"}, + {file = "types_requests-2.28.0-py3-none-any.whl", hash = "sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab"}, ] types-urllib3 = [ {file = "types-urllib3-1.26.15.tar.gz", hash = "sha256:c89283541ef92e344b7f59f83ea9b5a295b16366ceee3f25ecfc5593c79f794e"}, @@ -2812,8 +2812,8 @@ vcrpy = [ {file = "vcrpy-4.1.1.tar.gz", hash = "sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599"}, ] virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, + {file = "virtualenv-20.15.0-py2.py3-none-any.whl", hash = "sha256:804cce4de5b8a322f099897e308eecc8f6e2951f1a8e7e2b3598dff865f01336"}, + {file = "virtualenv-20.15.0.tar.gz", hash = "sha256:4c44b1d77ca81f8368e2d7414f9b20c428ad16b343ac6d226206c5b84e2b4fcc"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 028ffd3..46e90cd 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -37,7 +37,7 @@ html5lib==1.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (p identify==2.5.1; python_version >= "3.7" idna==3.3; python_version >= "3.7" and python_version < "4" imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -importlib-metadata==4.11.4; python_version < "3.10" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +importlib-metadata==4.12.0; python_version < "3.10" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") iniconfig==1.1.1; python_version >= "3.7" ipykernel==6.15.0; python_version >= "3.7" ipython==8.4.0; python_version >= "3.8" @@ -60,7 +60,7 @@ munch==2.5.0; python_version >= "3.7" mypy-extensions==0.4.3; python_version >= "3.6" mypy==0.950; python_version >= "3.6" nest-asyncio==1.5.5; python_version >= "3.7" -nodeenv==1.6.0; python_version >= "3.7" +nodeenv==1.7.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0" and python_version >= "3.7" numpy==1.23.0 packaging==21.3; python_version >= "3.8" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") pandas==1.4.3; python_version >= "3.8" @@ -71,7 +71,7 @@ pickleshare==0.7.5; python_version >= "3.8" platformdirs==2.5.2; python_full_version >= "3.7.2" and python_version >= "3.7" and python_version < "4.0" pluggy==1.0.0; python_version >= "3.7" pre-commit==2.19.0; python_version >= "3.7" -prompt-toolkit==3.0.29; python_full_version >= "3.6.2" and python_version >= "3.8" +prompt-toolkit==3.0.30; python_full_version >= "3.6.2" and python_version >= "3.8" prospector==1.7.7; python_full_version >= "3.6.2" and python_version < "4.0" psutil==5.9.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7" ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.8" @@ -123,12 +123,12 @@ tornado==6.1; python_version >= "3.7" tqdm==4.64.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") traitlets==5.3.0; python_version >= "3.8" typepigeon==1.0.14; python_version >= "3.6" and python_version < "4.0" -types-requests==2.27.31 +types-requests==2.28.0 types-urllib3==1.26.15 typing-extensions==4.2.0; python_version >= "3.7" and python_full_version >= "3.7.2" and python_version < "3.10" urllib3==1.26.9; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" vcrpy==4.1.1; python_version >= "3.5" -virtualenv==20.14.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" +virtualenv==20.15.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.8" webencodings==0.5.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" wrapt==1.14.1; python_full_version >= "3.7.2" and python_version >= "3.7" and python_version < "4.0" diff --git a/searvey/multi.py b/searvey/multi.py index 09ddfdf..59a7240 100644 --- a/searvey/multi.py +++ b/searvey/multi.py @@ -28,7 +28,12 @@ # https://docs.python.org/3/library/os.html#os.cpu_count -MAX_AVAILABLE_PROCESSES = len(os.sched_getaffinity(0)) +try: + MAX_AVAILABLE_PROCESSES = len(os.sched_getaffinity(0)) +except AttributeError: + MAX_AVAILABLE_PROCESSES = os.cpu_count() + if MAX_AVAILABLE_PROCESSES is None: + MAX_AVAILABLE_PROCESSES = 1 class FutureResult(pydantic.BaseModel):