diff --git a/CPU_bound_task.py b/CPU_bound_task.py new file mode 100644 index 0000000..1cbc6a7 --- /dev/null +++ b/CPU_bound_task.py @@ -0,0 +1,21 @@ +import concurrent.futures +from hashlib import md5 +from random import choice + + +def generate_coin(c): + while True: + s = "".join([choice("0123456789") for i in range(50)]) + h = md5(s.encode('utf8')).hexdigest() + if h.endswith("00000"): + return f"{s} {h}" + + +def main(): + with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor: + for coin in zip(executor.map(generate_coin, [0, 0, 0, 0])): + print(coin) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/IO-bound-task.py b/IO-bound-task.py new file mode 100644 index 0000000..d786bf3 --- /dev/null +++ b/IO-bound-task.py @@ -0,0 +1,22 @@ +import concurrent.futures +import urllib +import urllib.request + +links = open('urls.txt', encoding='utf8').read().split('\n') + + +def load_url(url, timeout): + with urllib.request.urlopen(url, timeout=timeout) as conn: + return conn.read() + + +with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: + future_to_url = {executor.submit(load_url, url, 60): url for url in links} + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + try: + data = future.result() + except Exception as exc: + print(url, exc) + else: + print(200) diff --git a/README.md b/README.md index 9bc87a8..5071030 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# Отчет в файле Recording.md + # Параллелизм и асинхронность Мы затронем только самые общие аспекты работы с потоками и процессами. Задачи, которые мы будем рассматривать обладают свойством [чрезвычайная параллельности](https://ru.wikipedia.org/wiki/%D0%A7%D1%80%D0%B5%D0%B7%D0%B2%D1%8B%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D0%BF%D0%B0%D1%80%D0%B0%D0%BB%D0%BB%D0%B5%D0%BB%D1%8C%D0%BD%D0%BE%D1%81%D1%82%D1%8C). @@ -116,7 +118,7 @@ from tqdm import tqdm url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0' -res = open('res.txt', 'w', encoding='utf8') +res = open('urals.txt', 'w', encoding='utf8') for i in tqdm(range(100)): html = urlopen(url).read().decode('utf8') @@ -135,13 +137,13 @@ for i in tqdm(range(100)): from urllib.request import Request, urlopen from urllib.parse import unquote -links = open('res.txt', encoding='utf8').read().split('\n') +links = open('urals.txt', encoding='utf8').read().split('\n') for url in links: try: request = Request( url, - headers={'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, + headers={'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, ) resp = urlopen(request, timeout=5) code = resp.code diff --git a/Recording.md b/Recording.md new file mode 100644 index 0000000..49d283e --- /dev/null +++ b/Recording.md @@ -0,0 +1,36 @@ +#Параллеизм и асинхронность + +##Проверяем ссылки на страницах +Полученные ссылки извлекаем в файл `urls.txt` + +Время синхронной проверки ссылок +![img_1.png](scrins/img_1.png) + +Время работа 'IO-bound-task' c 5 воркерами: +![img.png](scrins/img_2.png) + +Время работа 'IO-bound-task' c 10 воркерами: +![img.png](scrins/img_3.png) + +Время работа 'IO-bound-task' c 100 воркерами: +![img.png](scrins/img_4.png) + +Изменение количества воркеров существенно не влияет на загрузку ресурсов. Однако увеличение количества воркеров приводит к уменьшению времени работы программы и увелечению активности отправки и получения. + +##CPU-bound. Генерируем монетки +Генерация 4 монеток на 1 ядре: +![img.png](scrins/img_5.png) +Генерация 4 монеток c 2 воркерами: +![img.png](scrins/img_6.png) +Генерация 4 монеток с 4 воркерами: +![img.png](scrins/img_11.png) +Генерация 4 монеток с 5 воркерами: +![img.png](scrins/img_12.png) +Генерация 4 монеток с 10 воркерами: +![img.png](scrins/img_9.png) +Генерация 4 монеток с 61 воркерами: +![img.png](scrins/img_10.png) + +Изменение количества воркеров влияет на загрузку ресурсов. Максимальное количество воркеров - 61 из-за ограничения ОС. Увеличение скорости генерации монет будет происходить, пока не будут использоваться все ресурсы процессора. + + diff --git a/get-links-Task.py b/get-links-Task.py new file mode 100644 index 0000000..28f916a --- /dev/null +++ b/get-links-Task.py @@ -0,0 +1,18 @@ +from urllib.request import urlopen +from urllib.parse import unquote +from bs4 import BeautifulSoup +from tqdm import tqdm + +url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0' + +res = open('urls.txt', 'w', encoding='utf8') +#Количество страниц было уменьшено, так как занимало большое количество времени +for i in tqdm(range(20)): + html = urlopen(url).read().decode('utf8') + soup = BeautifulSoup(html, 'html.parser') + links = soup.find_all('a') + + for l in links: + href = l.get('href') + if href and href.startswith('http') and 'wiki' not in href: + print(href, file=res) \ No newline at end of file diff --git a/scrins/img_1.png b/scrins/img_1.png new file mode 100644 index 0000000..d080818 Binary files /dev/null and b/scrins/img_1.png differ diff --git a/scrins/img_10.png b/scrins/img_10.png new file mode 100644 index 0000000..0cd32b8 Binary files /dev/null and b/scrins/img_10.png differ diff --git a/scrins/img_11.png b/scrins/img_11.png new file mode 100644 index 0000000..164d219 Binary files /dev/null and b/scrins/img_11.png differ diff --git a/scrins/img_12.png b/scrins/img_12.png new file mode 100644 index 0000000..1845eaf Binary files /dev/null and b/scrins/img_12.png differ diff --git a/scrins/img_2.png b/scrins/img_2.png new file mode 100644 index 0000000..e422dd9 Binary files /dev/null and b/scrins/img_2.png differ diff --git a/scrins/img_3.png b/scrins/img_3.png new file mode 100644 index 0000000..2ecb5d2 Binary files /dev/null and b/scrins/img_3.png differ diff --git a/scrins/img_4.png b/scrins/img_4.png new file mode 100644 index 0000000..61e6d96 Binary files /dev/null and b/scrins/img_4.png differ diff --git a/scrins/img_5.png b/scrins/img_5.png new file mode 100644 index 0000000..d64347c Binary files /dev/null and b/scrins/img_5.png differ diff --git a/scrins/img_6.png b/scrins/img_6.png new file mode 100644 index 0000000..922d2f9 Binary files /dev/null and b/scrins/img_6.png differ diff --git a/scrins/img_8.png b/scrins/img_8.png new file mode 100644 index 0000000..b4ef410 Binary files /dev/null and b/scrins/img_8.png differ diff --git a/scrins/img_9.png b/scrins/img_9.png new file mode 100644 index 0000000..4c8d801 Binary files /dev/null and b/scrins/img_9.png differ diff --git a/urls.txt b/urls.txt new file mode 100644 index 0000000..60c6674 --- /dev/null +++ b/urls.txt @@ -0,0 +1,158 @@ +https://wuni15.sportresult.com/fisu/en/-120/Participant/ParticipantInfo/c4a302ac-4cf0-44e2-84ec-af3cfbb84d0c +https://www.curlingzone.com/player.php?playerid=23220 +https://results.worldcurling.org/Person/Details/7552 +https://www.teamusa.org/Athletes/KA/Kyle-Kakela +https://web.archive.org/web/20110722151716/http://www.lzuu.lt/file.doc?id=17317 +http://www.lzuu.lt/file.doc?id=17317 +https://web.archive.org/web/20110722151716/http://www.lzuu.lt/file.doc?id=17317 +http://www.aidai.us/index.php?view=article&catid=51%3A7203&id=158%3Apo&tmpl=component&print=1&page=&option=com_content&Itemid=60 +http://rkka.ru/ihandbook.htm +https://www.webcitation.org/65OTPMdKn?url=http://rkka.ru/ihandbook.htm +http://minchanin.esmasoft.com/maps/ussr1939/index.html +http://mplotnikova.jimdo.com +http://smallflower_ru.livejournal.com/profile +http://smallflower_ru.livejournal.com/ +https://facebook.com/smallflower.ru +http://mplotnikova.jimdo.com/ +http://magazines.russ.ru/authors/p/plotnikova/ +https://web.archive.org/web/20140504140204/http://bp01.ru/authors.php?new=603 +http://www.youblisher.com/p/766426-%D0%9F%D0%BE%D1%8D%D0%B7%D0%B8%D1%8F-%D0%B6%D0%B5%D0%BD%D1%81%D0%BA%D0%BE%D0%B3%D0%BE-%D1%80%D0%BE%D0%B4%D0%B0/ +http://www.piiter.ru/view.php?cid=11&aid=164 +http://magazines.russ.ru/authors/p/plotnikova/ +https://web.archive.org/web/20140504140002/http://resbash.ru/stat/2/3442/ +https://web.archive.org/web/20140504121318/http://bp01.ru/public.php?public=2868&sphrase_id=13543 +https://web.archive.org/web/20150919031300/http://ufa.rusplt.ru/index/poetessa-plotnikova-s-receptom-izbavleniya-ot-plohih-stihov-14355.html +https://youtube.com/watch?v=Mc4Uw19v0pM +https://youtube.com/watch?v=0GpYUaL2yiU +https://youtube.com/watch?v=_iN4M8mXqd0 +https://youtube.com/watch?v=GObWQhRxlIA +https://youtube.com/watch?v=GdVIpE19eXM +https://youtube.com/watch?v=RlGxjoPQqtE +https://www.youtube.com/watch?v=Eo80PzURSLU/ +https://www.youtube.com/watch?v=4AVsVRq3BnE#t=33/ +http://www.salavatlife.ru/monospektakl-snizu-vverx-sostoyalsya-v-salavate/ +https://web.archive.org/web/20140504121438/http://www.salavatlife.ru/monospektakl-snizu-vverx-sostoyalsya-v-salavate/ +http://www.bp01.ru/public.php?public=2682 +https://web.archive.org/web/20160305014817/http://www.bp01.ru/public.php?public=2682 +http://hk-dynamo-1995.my1.ru/index/istorija_shkoly/0-53 +http://www.dynamo-history.ru/person/players/cart/p/payusov/ +https://www.eliteprospects.com/player.php?player=422617&lang=en +https://Eliteprospects.com +http://simbad.u-strasbg.fr/simbad/sim-id?Ident=NGC%203468 +http://spider.seds.org/ngc/ngc.cgi?3468 +http://spider.seds.org/ngc/ngc_fr.cgi?3468 +http://spider.seds.org/ngc/revngcic.cgi?NGC3468 +http://vizier.u-strasbg.fr/viz-bin/VizieR-S?NGC+3468 +http://nedwww.ipac.caltech.edu/cgi-bin/nph-objsearch?objname=NGC+3468 +http://adsabs.harvard.edu/cgi-bin/basic_connect?qsearch=%22NGC+3468%22&version=1 +https://www.worldcat.org/issn/0004-6256 +https://www.worldcat.org/issn/1538-3881 +https://dx.doi.org/10.1088/0004-6256/149/5/171 +http://arxiv.org/abs/1503.03134 +https://web.archive.org/web/20080306075246/http://www.caroline-martin-musique.com/artistes/jude/jude.html +http://www.pianobleu.com/jude.html +https://icd.who.int/browse10/2019/en#/K70 +http://www.icd9data.com/getICD9Code.ashx?icd9=571.1 +https://medlineplus.gov/ency/article/000281.htm +https://meshb.nlm.nih.gov/record/ui?ui=D008108 +https://web.archive.org/web/20141021124119/http://www.aasld.org/practiceguidelines/Documents/Bookmarked%20Practice%20Guidelines/AlcoholicLiverDisease1-2010.pdf +https://doi.org/10.1002%2Fhep.23258 +http://www.aasld.org/practiceguidelines/Documents/Bookmarked%20Practice%20Guidelines/AlcoholicLiverDisease1-2010.pdf +https://doi.org/10.4254%2Fwjh.v3.i5.108 +https://doi.org/10.3748%2Fwjg.v20.i36.12934 +https://doi.org/10.1016%2Fj.cca.2006.08.035 +https://cyberleninka.ru/article/n/alkogolnoe-porazhenie-pecheni/viewer +https://www.openstreetmap.org/?mlat=60.23045&mlon=29.46491&zoom=12 +https://www.openstreetmap.org/?mlat=60.23045&mlon=29.46491&zoom=12 +https://classinform.ru/okato/search.php?str=41215864025 +https://classinform.ru/oktmo/search.php?str=41615464211 +http://msu.lenobl.ru/media/content/docs/188/2_-_soderzhanie.pdf +http://www.consultant.ru/document/cons_doc_LAW_114656/b2707989c276b5a188e63bc41e7bcbcc18723de8/ +http://reglib.ru/Files/file/administrativno-territorialnoe%20delenie%20leningradskoi%20oblasti_%202007%20g_8.pdf +http://www.webcitation.org/6Y4hwXc9U +http://petrostat.gks.ru/wps/wcm/connect/rosstat_ts/petrostat/resources/b281da004d2c553abe51bef5661033e3/Ленинградская+область.rar +http://www.webcitation.org/6RioXaUNx +https://apiv3.iucnredlist.org/api/v3/taxonredirect/22680380 +https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=175137 +https://eol.org/pages/1048481 +http://www.birdlife.org/action/science/species/global_species_programme/whats_new.html +https://web.archive.org/web/20080914020126/http://www.birdlife.org/action/science/species/global_species_programme/whats_new.html +http://www.birdlife.org/news/news/2006/11/madagascar_pochard_redisc.html +http://www.bbc.co.uk/news/world-europe-jersey-17609801 +http://www.bbc.co.uk/nature/17616488 +http://www.wwt.org.uk/conservation/our-work/saving-the-madgascar-pochard/ +http://www.iucnredlist.org/details/22680380/0 +http://all-ht.ru/inf/pc/cp_struct.html +http://www.nsc.ru/win/elbib/data/show_page.dhtml?77+739 +https://www.studytonight.com/computer-architecture/risc-cisc-processors +http://curlie.org/World/Russian/Компьютеры/Аппаратное_обеспечение/Процессоры/CISC/ +http://toll-charge.hu/ +https://web.archive.org/web/20180612053945/http://nif.hu/ +http://toll-charge.hu/files/document/document/171/Руководство%20по%20венгерской%20электронной%20виньете​.pdf +https://books.google.az/books?id=tgDsAAAAMAAJ&q=%D0%9A%D0%B8%D0%B5%D0%B2%D1%81%D0%BA%D0%B0%D1%8F+%D0%BE%D0%B1%D0%BB%D0%B0%D1%81%D1%82%D1%8C+%D0%B1%D1%8B%D0%BB%D0%B0+%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B0+27+%D1%84%D0%B5%D0%B2%D1%80%D0%B0%D0%BB%D1%8F+1932+%D0%B3%D0%BE%D0%B4%D0%B0&dq=%D0%9A%D0%B8%D0%B5%D0%B2%D1%81%D0%BA%D0%B0%D1%8F+%D0%BE%D0%B1%D0%BB%D0%B0%D1%81%D1%82%D1%8C+%D0%B1%D1%8B%D0%BB%D0%B0+%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B0+27+%D1%84%D0%B5%D0%B2%D1%80%D0%B0%D0%BB%D1%8F+1932+%D0%B3%D0%BE%D0%B4%D0%B0&hl=ru&sa=X&ved=0ahUKEwjV1LPS5NjeAhVEh6YKHcBaDdoQ6AEINDAD +http://src-h.slav.hokudai.ac.jp/ukrregions/data/158.html +https://www.president.gov.ua/ru/news/golovne-zavdannya-ce-zahist-lyudej-v-tomu-chisli-i-socialnij-50682 +http://kievvlast.com.ua/base/dose-_majbozhenko_vladimir_vladimirovich40130 +http://politrada.com/dossier/Nikolaj-Viktorovich-Babenko/ +http://politrada.com/dossier/Anna-Vitalevna-Starikova/ +http://p-ch.jp/ +http://www.destructoid.com/persona-5-uses-atlus-internal-engine-210273.phtml +http://www.escapistmagazine.com/news/view/132479-Atlus-USA-Confirms-Localization-of-Persona-5-Spinoffs +http://www.vg247.com/2014/09/01/persona-5-ps4-ps3/ +http://www.siliconera.com/2015/09/17/persona-5-will-release-in-japan-in-summer-2016/ +http://persona5.jp/ +https://www.imdb.com/title/tt3944082 +https://www.behindthevoiceactors.com/video-games/Persona-5 +https://gamefaqs.gamespot.com/-/835628- +https://gamefaqs.gamespot.com/-/641809- +https://www.giantbomb.com/wd/3030-30486/ +https://www.mobygames.com/game/persona-5 +https://www.mobygames.com/game/persona-5-steelbook-launch-edition +https://www.mobygames.com/game/persona-5-take-your-heart-premium-edition +https://www.mobygames.com/game/persona-5-orpheus-orpheus-picaro-set +https://www.mobygames.com/game/persona-5-catherine-costume-bgm-special-set +https://www.mobygames.com/game/persona-5-phantom-thieves-logo-morgana-car-sticker +https://www.mobygames.com/game/persona-5-kaguya-kaguya-picaro-set +https://www.mobygames.com/game/persona-5-ultimate-edition +https://vgmdb.net/product/4134 +https://www.openstreetmap.org/?mlat=47.75&mlon=4.666667&zoom=12 +https://www.openstreetmap.org/?mlat=47.75&mlon=4.666667&zoom=12 +https://www.insee.fr/fr/statistiques/2011101?geo=COM-21549 +http://www.insee.fr/fr/themes/tableau_local.asp?ref_id=POP&millesime=2010&nivgeo=COM&codgeo=21549 +http://www.insee.fr/fr/ppp/bases-de-donnees/donnees-detaillees/rp2010/chiffres-cles/base-cc-emploi-pop-active-2010/base-cc-emploi-pop-active-2010.zip +http://www.insee.fr/fr/methodes/nomenclatures/cog/fichecommunale.asp?codedep=21&codecom=549 +http://www.goldensite.ru/text.aspx?id=GS-Archive_2007_Final +http://web.archive.org/web/20140108005556/http://www.goldensite.ru/text.aspx?id=GS-Archive_2007_Final +http://www.goldensite.ru/ +https://web.archive.org/web/20120714030507/http://www.webplanet.ru/company/life/2006/12/22/goldensite.html +http://ria.ru/video/20110317/354779654.html +http://www.ul.aif.ru/society/news/54941 +http://www.lenta.ru/news/2005/07/15/goldensite/ +http://www.exler.ru/reviews/26-12-06.htm#4 +http://telegraf.by/2010/03/sajt_gorodka_viigral_konkurs_zolotoj_kibermaster +http://www.aif.ru/society/news/46820 +https://www.imdb.com/title/tt0061735/ +http://www.boxofficemojo.com/movies/?id=guesswhoscomingtodinner.htm +https://www.imdb.com/title/tt0061735/ +http://www.eslnotes.com/movies/pdf/guess-whos-coming-to-dinne.pdf +http://www.script-o-rama.com/movie_scripts/g/guess-whos-coming-to-dinner-script.html +https://movies.nytimes.com/movie/21044/Guess-Who-s-Coming-to-Dinner/awards +https://www.allmovie.com/movie/v21044 +https://www.allocine.fr/film/fichefilm_gen_cfilm=12693.html +https://www.boxofficemojo.com/movies/?id=guesswhoscomingtodinner.htm +https://www.csfd.cz/film/5303 +https://www.filmaffinity.com/en/film420852.html +https://www.imdb.com/title/tt0061735 +https://www.metacritic.com/movie/guess-whos-coming-to-dinner +https://www.rottentomatoes.com/m/guess_whos_coming_to_dinner +https://catalog.afi.com/Catalog/moviedetails/19359 +https://www.britannica.com/topic/Guess-Whos-Coming-to-Dinner +http://ask.bibsys.no/ask/action/result?cmd=&kilde=biblio&cql=bs.autid+%3D+8034165&feltselect=bs.autid +http://cantic.bnc.cat/registres/CUCId/a1202658x +http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX3743875 +https://catalogue.bnf.fr/ark:/12148/cb164576184 +https://id.loc.gov/authorities/no2016120430 +https://www.idref.fr/224289195 +https://viaf.org/viaf/316752081 +https://www.worldcat.org/identities/containsVIAFID/316752081 +https://www.britannica.com/topic/Triballi