Gentoo Archives: gentoo-portage-dev

From: robbat2@g.o
To: gentoo-portage-dev@l.g.o
Cc: "Robin H. Johnson" <robbat2@g.o>
Subject: [gentoo-portage-dev] [PATCH 2/2] Replace multiprocessing.cpu_count with portage.util.cpuinfo.get_cpu_count
Date: Sat, 16 Feb 2019 06:21:56
Message-Id: 20190216062137.689-2-robbat2@gentoo.org
In Reply to: [gentoo-portage-dev] [PATCH 1/2] cpuinfo: use better available CPU calculation by robbat2@gentoo.org
1 From: "Robin H. Johnson" <robbat2@g.o>
2
3 portage.util.cpuinfo.get_cpu_count was only used in one spot before, and
4 other call-sites just used multiprocessing.cpu_count() directly.
5
6 Replace all multiprocessing.cpu_count() calls with get_cpu_count() in
7 portage.util.cpuinfo, to ensure consistency in CPU calculation.
8
9 Signed-off-by: Robin H. Johnson <robbat2@g.o>
10 ---
11 lib/portage/dbapi/porttree.py | 4 ++--
12 lib/portage/util/futures/executor/fork.py | 4 ++--
13 lib/portage/util/futures/iter_completed.py | 18 +++++++++---------
14 3 files changed, 13 insertions(+), 13 deletions(-)
15
16 diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
17 index 2ff3e1b34..64a5f3681 100644
18 --- a/lib/portage/dbapi/porttree.py
19 +++ b/lib/portage/dbapi/porttree.py
20 @@ -1471,11 +1471,11 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None,
21 @param cpv_list: list of ebuild cpv values for a Manifest
22 @type cpv_list: list
23 @param max_jobs: max number of futures to process concurrently (default
24 - is multiprocessing.cpu_count())
25 + is portage.util.cpuinfo.get_cpu_count())
26 @type max_jobs: int
27 @param max_load: max load allowed when scheduling a new future,
28 otherwise schedule no more than 1 future at a time (default
29 - is multiprocessing.cpu_count())
30 + is portage.util.cpuinfo.get_cpu_count())
31 @type max_load: int or float
32 @param loop: event loop
33 @type loop: EventLoop
34 diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py
35 index 72844403c..add7b3c9e 100644
36 --- a/lib/portage/util/futures/executor/fork.py
37 +++ b/lib/portage/util/futures/executor/fork.py
38 @@ -7,13 +7,13 @@ __all__ = (
39
40 import collections
41 import functools
42 -import multiprocessing
43 import os
44 import sys
45 import traceback
46
47 from portage.util._async.AsyncFunction import AsyncFunction
48 from portage.util.futures import asyncio
49 +from portage.util.cpuinfo import get_cpu_count
50
51
52 class ForkExecutor(object):
53 @@ -24,7 +24,7 @@ class ForkExecutor(object):
54 This is entirely driven by an event loop.
55 """
56 def __init__(self, max_workers=None, loop=None):
57 - self._max_workers = max_workers or multiprocessing.cpu_count()
58 + self._max_workers = max_workers or get_cpu_count()
59 self._loop = asyncio._wrap_loop(loop)
60 self._submit_queue = collections.deque()
61 self._running_tasks = {}
62 diff --git a/lib/portage/util/futures/iter_completed.py b/lib/portage/util/futures/iter_completed.py
63 index 31b5e0c78..4c48ea0fe 100644
64 --- a/lib/portage/util/futures/iter_completed.py
65 +++ b/lib/portage/util/futures/iter_completed.py
66 @@ -2,11 +2,11 @@
67 # Distributed under the terms of the GNU General Public License v2
68
69 import functools
70 -import multiprocessing
71
72 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
73 from portage.util._async.TaskScheduler import TaskScheduler
74 from portage.util.futures import asyncio
75 +from portage.util.cpuinfo import get_cpu_count
76
77
78 def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
79 @@ -18,11 +18,11 @@ def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
80 @param futures: iterator of asyncio.Future (or compatible)
81 @type futures: iterator
82 @param max_jobs: max number of futures to process concurrently (default
83 - is multiprocessing.cpu_count())
84 + is portage.util.cpuinfo.get_cpu_count())
85 @type max_jobs: int
86 @param max_load: max load allowed when scheduling a new future,
87 otherwise schedule no more than 1 future at a time (default
88 - is multiprocessing.cpu_count())
89 + is portage.util.cpuinfo.get_cpu_count())
90 @type max_load: int or float
91 @param loop: event loop
92 @type loop: EventLoop
93 @@ -47,11 +47,11 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
94 @param futures: iterator of asyncio.Future (or compatible)
95 @type futures: iterator
96 @param max_jobs: max number of futures to process concurrently (default
97 - is multiprocessing.cpu_count())
98 + is portage.util.cpuinfo.get_cpu_count())
99 @type max_jobs: int
100 @param max_load: max load allowed when scheduling a new future,
101 otherwise schedule no more than 1 future at a time (default
102 - is multiprocessing.cpu_count())
103 + is portage.util.cpuinfo.get_cpu_count())
104 @type max_load: int or float
105 @param loop: event loop
106 @type loop: EventLoop
107 @@ -61,8 +61,8 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
108 """
109 loop = asyncio._wrap_loop(loop)
110
111 - max_jobs = max_jobs or multiprocessing.cpu_count()
112 - max_load = max_load or multiprocessing.cpu_count()
113 + max_jobs = max_jobs or portage.util.cpuinfo.get_cpu_count()
114 + max_load = max_load or portage.util.cpuinfo.get_cpu_count()
115
116 future_map = {}
117 def task_generator():
118 @@ -120,11 +120,11 @@ def iter_gather(futures, max_jobs=None, max_load=None, loop=None):
119 @param futures: iterator of asyncio.Future (or compatible)
120 @type futures: iterator
121 @param max_jobs: max number of futures to process concurrently (default
122 - is multiprocessing.cpu_count())
123 + is portage.util.cpuinfo.get_cpu_count())
124 @type max_jobs: int
125 @param max_load: max load allowed when scheduling a new future,
126 otherwise schedule no more than 1 future at a time (default
127 - is multiprocessing.cpu_count())
128 + is portage.util.cpuinfo.get_cpu_count())
129 @type max_load: int or float
130 @param loop: event loop
131 @type loop: EventLoop
132 --
133 2.18.0

Replies