-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathconcurrent.html
186 lines (170 loc) · 14.4 KB
/
concurrent.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
<!DOCTYPE html>
<html lang="vi">
<head>
<title>Tin tức Python PyMI.vn</title>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta charset="utf-8" />
<link href="https://n.pymi.vn/feeds/all.atom.xml" type="application/atom+xml" rel="alternate" title="Tin tức Python PyMI.vn Full Atom Feed" />
<!-- twitter card metadata -->
<meta name="twitter:site" content="">
<meta name="twitter:title" content="Giới thiệu concurrent.futures trong Python 3">
<meta name="twitter:description" content="tăng tốc độ xử lý với concurent.futures">
<!-- OG Tags -->
<meta property="og:url" content="./concurrent.html"/>
<meta property="og:title" content="Giới thiệu concurrent.futures trong Python 3 | Tin tức Python PyMI.vn" />
<meta property="og:description" content="tăng tốc độ xử lý với concurent.futures" />
<!-- favicon -->
<!-- moment.js for date formatting -->
<script src="./theme/js/moment.js"></script>
<!-- css -->
<link rel="stylesheet" type="text/css" href="./theme/css/main.css" />
<script>
/*! grunt-grunticon Stylesheet Loader - v2.1.2 | https://github.com/filamentgroup/grunticon | (c) 2015 Scott Jehl, Filament Group, Inc. | MIT license. */
(function(e){function t(t,n,r,o){"use strict";function a(){for(var e,n=0;u.length>n;n++)u[n].href&&u[n].href.indexOf(t)>-1&&(e=!0);e?i.media=r||"all":setTimeout(a)}var i=e.document.createElement("link"),l=n||e.document.getElementsByTagName("script")[0],u=e.document.styleSheets;return i.rel="stylesheet",i.href=t,i.media="only x",i.onload=o||null,l.parentNode.insertBefore(i,l),a(),i}var n=function(r,o){"use strict";if(r&&3===r.length){var a=e.navigator,i=e.Image,l=!(!document.createElementNS||!document.createElementNS("http://www.w3.org/2000/svg","svg").createSVGRect||!document.implementation.hasFeature("http://www.w3.org/TR/SVG11/feature#Image","1.1")||e.opera&&-1===a.userAgent.indexOf("Chrome")||-1!==a.userAgent.indexOf("Series40")),u=new i;u.onerror=function(){n.method="png",n.href=r[2],t(r[2])},u.onload=function(){var e=1===u.width&&1===u.height,a=r[e&&l?0:e?1:2];n.method=e&&l?"svg":e?"datapng":"png",n.href=a,t(a,null,null,o)},u.src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==",document.documentElement.className+=" grunticon"}};n.loadCSS=t,e.grunticon=n})(this);(function(e,t){"use strict";var n=t.document,r="grunticon:",o=function(e){if(n.attachEvent?"complete"===n.readyState:"loading"!==n.readyState)e();else{var t=!1;n.addEventListener("readystatechange",function(){t||(t=!0,e())},!1)}},a=function(e){return t.document.querySelector('link[href$="'+e+'"]')},c=function(e){var t,n,o,a,c,i,u={};if(t=e.sheet,!t)return u;n=t.cssRules?t.cssRules:t.rules;for(var l=0;n.length>l;l++)o=n[l].cssText,a=r+n[l].selectorText,c=o.split(");")[0].match(/US\-ASCII\,([^"']+)/),c&&c[1]&&(i=decodeURIComponent(c[1]),u[a]=i);return u},i=function(e){var t,o,a;o="data-grunticon-embed";for(var c in e)if(a=c.slice(r.length),t=n.querySelectorAll(a+"["+o+"]"),t.length)for(var i=0;t.length>i;i++)t[i].innerHTML=e[c],t[i].style.backgroundImage="none",t[i].removeAttribute(o);return t},u=function(t){"svg"===e.method&&o(function(){i(c(a(e.href))),"function"==typeof t&&t()})};e.embedIcons=i,e.getCSS=a,e.getIcons=c,e.ready=o,e.svgLoadedCallback=u,e.embedSVG=u})(grunticon,this);
grunticon(["./theme/css/icons.data.svg.css", "./theme/css/icons.data.png.css", "./theme/css/icons.fallback.css"]);
</script>
<noscript><link href="./theme/css/icons.fallback.css" rel="stylesheet"></noscript>
<!-- menu toggle javascript -->
<script type="text/javascript">
document.addEventListener("DOMContentLoaded", initMenu);
function initMenu(){
var menu = document.getElementById("menu");
var menulink = document.getElementById("menu-link");
menulink.addEventListener("click", function toggleMenu(){
window.event.preventDefault();
menulink.classList.toggle('active');
menu.classList.toggle('active');
});
};
</script>
<meta name="description" content="tăng tốc độ xử lý với concurent.futures" />
<meta name="tags" content="concurrent.futures" />
<meta name="tags" content="concurrent" />
<meta name="tags" content="threading" />
<meta name="tags" content="multiprocesses" />
</head>
<body>
<div role="banner" id="masthead">
<header>
<h1><a href="/">Pymiers's Blog</a></h1>
<a href="#menu" id="menu-link">more stuff</a>
<nav id="menu">
<ul>
<li><a href="./category/features.html">features</a></li>
<li class="active"><a href="./category/news.html">news</a></li>
<li><a href="./category/pymivn.html">pymi.vn</a></li>
</ul>
</nav>
</header>
</div>
<div class="page" role="main">
<div class="article" role="article">
<article>
<footer>
<a name="top"></a>
<p>
<time datetime=" 2022-05-22 00:00:00+07:00">
<script>document.write(moment('2022-05-22 00:00:00+07:00').format('LL'));</script>
</time>
</p>
</footer>
<header>
<h2>
Giới thiệu concurrent.futures trong Python 3
</h2>
<center>
<h4>
by Pymier0
</h4>
</center>
</header>
<div class="content">
<h2>Concurrency</h2>
<p>Concurrency là khái niệm chương trình thực hiện nhiều công việc cùng một
lúc (dễ thấy ở các chương trình có giao diện đồ họa: vừa hiển thị giao diện,
vừa kết nối đến trang web, hay các chương trình server phục vụ nhiều người dùng
cùng lúc).</p>
<p>Python (cũng như nhiều ngôn ngữ lập trình khác) từ xưa đã có hai cách làm phổ
biến để viết concurrent code: dùng <a href="https://docs.python.org/3/library/threading.html">threading</a> hoặc <a href="https://docs.python.org/3/library/multiprocessing.html">multiprocessing</a>.</p>
<p>Python 3 giới thiệu thư viện “bậc cao” dễ dùng hơn có tên <a href="https://docs.python.org/3/library/concurrent.futures.html">concurrent.futures</a>.</p>
<p><img alt="img" src="https://images.unsplash.com/photo-1511229577011-6b24bfc30871?crop=entropy&cs=tinysrgb&fm=jpg&ixlib=rb-1.2.1&q=80&raw_url=true&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=600"></p>
<h2><span class="caps">CPU</span> bound <span class="amp">&</span> <span class="caps">IO</span> bound</h2>
<p><span class="dquo">“</span>bound” ở đây hiểu theo nghĩa: chương trình tốn hầu hết thời gian <strong>thực hiện</strong>
tính toán (<span class="caps">CPU</span>) hay <strong>chờ</strong> (hệ điều hành) đọc ghi dữ liệu, bao gồm cả kết nối
mạng (<span class="caps">IO</span>).</p>
<p>CPython có một giới hạn về thiết kế khiến cho khi dùng threading, chỉ 1 thread
được chạy (dùng <span class="caps">CPU</span>) 1 lúc (<a href="https://docs.python.org/3/glossary.html#term-global-interpreter-lock">global interpreter lock</a>) , muốn dùng nhiều <span class="caps">CPU</span> phải chuyển qua dùng multiprocessing.
Thread nhẹ hơn process, máy tính bình thường có thể có hàng chục hay trăm ngàn
thread nhưng
không đủ (<span class="caps">RAM</span>) để tạo 10_000 process. Trong Python, khi chương trình <span class="caps">IO</span> bound,
có thể dùng thread, khi chương trình <span class="caps">CPU</span> bound thì dùng multiprocessing mới có
thể tăng tốc.</p>
<p>concurrent.futures cho phép chuyển đổi giữa threading hay multiprocessing một cách
đơn giản.</p>
<h2>Ví dụ</h2>
<p>Tính tổng các số từ 1 đến 30 triệu, 4 lần.</p>
<p>Việc dùng concurrent.futures chỉ gồm 2 bước:</p>
<ul>
<li>tạo Thread/Process Pool Executor</li>
<li>chạy executor.map với 2 argument: function sẽ được chạy ở thread/Process, và
list chứa argument cho mỗi lần gọi function.</li>
</ul>
<p>Trên máy có 8 <span class="caps">CPU</span>, kết qủa thấy dùng
ProcessPoolExecutor cho việc tính tóan <span class="caps">CPU</span> bound này nhanh gấp gần 4 lần
so với dùng ThreadPoolExecutor.</p>
<div class="highlight"><pre><span></span><code><span class="kn">from</span> <span class="nn">concurrent.futures</span> <span class="kn">import</span> <span class="n">ThreadPoolExecutor</span><span class="p">,</span> <span class="n">ProcessPoolExecutor</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="k">def</span> <span class="nf">sumto</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
<span class="n">r</span> <span class="o">=</span> <span class="mi">1</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
<span class="n">r</span> <span class="o">=</span> <span class="n">r</span> <span class="o">+</span> <span class="n">i</span>
<span class="k">return</span> <span class="n">r</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ThreadPoolExecutor: max_workers=</span><span class="si">{</span><span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
<span class="n">executor</span> <span class="o">=</span> <span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">())</span>
<span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">sumto</span><span class="p">,</span> <span class="p">[</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">]):</span>
<span class="nb">print</span><span class="p">(</span><span class="n">r</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="n">executor</span> <span class="o">=</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">())</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ProcessPoolExecutor: max_workers=</span><span class="si">{</span><span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
<span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">sumto</span><span class="p">,</span> <span class="p">[</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">,</span><span class="mi">30_000_000</span><span class="p">]):</span>
<span class="nb">print</span><span class="p">(</span><span class="n">r</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
</code></pre></div>
<p>Kết quả</p>
<div class="highlight"><pre><span></span><code>$ python concurrent.py
ThreadPoolExecutor: <span class="nv">max_workers</span><span class="o">=</span><span class="m">8</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">5</span>.539357423782349
ProcessPoolExecutor: <span class="nv">max_workers</span><span class="o">=</span><span class="m">8</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">450000015000001</span>
<span class="m">1</span>.5521023273468018
</code></pre></div>
<p>Hết</p>
<h2>Tham khảo</h2>
<p>Đăng ký ngay tại <a href="https://pymi.vn">PyMI.vn</a> để học Python tại Hà Nội <span class="caps">TP</span> <span class="caps">HCM</span> (Sài Gòn),
trở thành lập trình viên #python chuyên nghiệp ngay sau khóa học.</p>
</div>
<div class="back-to-top">
<a href="#top">back to top</a>
</div>
</article>
</div>
<!-- end article -->
<footer>
<div class="icons">
<a href="https://github.com/pymivn" target="_blank"><div class="icon-github icon"></div></a>
</div>
<p>© <script>document.write(moment().format('YYYY'));</script> Pymiers</p>
</footer>
</div>
</body>
</html>