diff --git a/CPU-bound-task.py b/CPU-bound-task.py new file mode 100644 index 0000000..27887a3 --- /dev/null +++ b/CPU-bound-task.py @@ -0,0 +1,23 @@ +from hashlib import md5 +from random import choice +import concurrent.futures + + +def generate_coin(c): + while True: + s = "".join([choice("0123456789") for i in range(50)]) + h = md5(s.encode('utf8')).hexdigest() + if h.endswith("00000"): + c = f"{s} {h}" + break + return c + + +def main(): + with concurrent.futures.ProcessPoolExecutor(max_workers=61) as executor: + for coin in zip(executor.map(generate_coin, [0, 0, 0, 0])): + print(coin) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/IO-bound-task.py b/IO-bound-task.py new file mode 100644 index 0000000..f61d568 --- /dev/null +++ b/IO-bound-task.py @@ -0,0 +1,22 @@ +import concurrent.futures +import urllib +import urllib.request + + +def load_url(url, timeout): + with urllib.request.urlopen(url, timeout=timeout) as conn: + return conn.read() + + +links = open('links.txt', encoding='utf8').read().split('\n') + +with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: + future_to_url = {executor.submit(load_url, url, 60): url for url in links} + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + try: + data = future.result() + except Exception as exc: + print(url, exc) + else: + print(200) diff --git a/README.md b/README.md index 9bc87a8..3f06f61 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# *Отчет в файле `Report.md`* + # Параллелизм и асинхронность Мы затронем только самые общие аспекты работы с потоками и процессами. Задачи, которые мы будем рассматривать обладают свойством [чрезвычайная параллельности](https://ru.wikipedia.org/wiki/%D0%A7%D1%80%D0%B5%D0%B7%D0%B2%D1%8B%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D0%BF%D0%B0%D1%80%D0%B0%D0%BB%D0%BB%D0%B5%D0%BB%D1%8C%D0%BD%D0%BE%D1%81%D1%82%D1%8C). @@ -116,7 +118,7 @@ from tqdm import tqdm url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0' -res = open('res.txt', 'w', encoding='utf8') +res = open('links.txt', 'w', encoding='utf8') for i in tqdm(range(100)): html = urlopen(url).read().decode('utf8') @@ -135,13 +137,13 @@ for i in tqdm(range(100)): from urllib.request import Request, urlopen from urllib.parse import unquote -links = open('res.txt', encoding='utf8').read().split('\n') +links = open('links.txt', encoding='utf8').read().split('\n') for url in links: try: request = Request( url, - headers={'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, + headers={'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, ) resp = urlopen(request, timeout=5) code = resp.code diff --git a/Report.md b/Report.md new file mode 100644 index 0000000..40b28b6 --- /dev/null +++ b/Report.md @@ -0,0 +1,44 @@ +# Параллелизм и асинхронность + +## IO-bound. Проверяем ссылки на страницах Википедии + +Из полученных страниц извлекаем все ссылки в файл `links.txt`. + +*Время, затраченное на синхронную проверку ссылок:* +![img.png](scrins/scrin1.png) + +*Время работы 'IO-bound-task' с 5 воркерами:* +![img.png](scrins/scrin2.png) + +*Время работы 'IO-bound-task' с 10 воркерами:* +![img.png](scrins/scrin3.png) + +*Время работы 'IO-bound-task' с 100 воркерами:* +![img.png](scrins/scrin4.png) + +Изменение количества воркеров не вызывает существенного изменения загрузки ресурсов пк. Увеличение количества воркеров +увеличивает быстродействие программы и приводит к увеличению активности работы с сетью + +## CPU-bound. Генерируем монетки + +*Время генерации 4 монеток на 1 ядре:* +![img.png](scrins/scrin5.png) + +*Время генерации 4 монеток с 2 воркерами:* +![img.png](scrins/scrin6.png) + +*Время генерации 4 монеток с 4 воркерами:* +![img.png](scrins/scrin7.png) + +*Время генерации 4 монеток с 5 воркерами:* +![img.png](scrins/scrin8.png) + +*Время генерации 4 монеток с 10 воркерами:* +![img.png](scrins/scrin9.png) + +*Время генерации 4 монеток с 61 воркерами:* +![img.png](scrins/scrin10.png) + +Изменение количества воркеров существенно увеличивает загрузку памяти и процессора. Запустить на 100 воркерах нельзя +из-за ограничения в 61. Увеличение скорости генерации происходит до тех пор, пока не будут использованы все ресурсы +процессора. \ No newline at end of file diff --git a/get-links.py b/get-links.py new file mode 100644 index 0000000..8979733 --- /dev/null +++ b/get-links.py @@ -0,0 +1,19 @@ +from urllib.request import urlopen +from urllib.parse import unquote +from bs4 import BeautifulSoup +from tqdm import tqdm + +url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87' \ + '%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0 ' + +res = open('links.txt', 'w', encoding='utf8') +# Количество страниц уменьшено до 20, иначе время на выполнение превышало 30 минут +for i in tqdm(range(20)): + html = urlopen(url).read().decode('utf8') + soup = BeautifulSoup(html, 'html.parser') + links = soup.find_all('a') + + for l in links: + href = l.get('href') + if href and href.startswith('http') and 'wiki' not in href: + print(href, file=res) diff --git a/links.txt b/links.txt new file mode 100644 index 0000000..8030e4d --- /dev/null +++ b/links.txt @@ -0,0 +1,162 @@ +https://www.openstreetmap.org/?mlat=46.26306&mlon=33.50556&zoom=13 +https://www.openstreetmap.org/?mlat=46.26306&mlon=33.50556&zoom=13 +http://w1.c1.rada.gov.ua/pls/z7503/A005?rdat1=31.07.2011&rf7571=34910 +http://www.etomesto.ru/map-rkka_l-36-b/?x=33.506465&y=46.264056 +https://www.openstreetmap.org/?mlat=58.42278&mlon=54.20333&zoom=12 +https://www.openstreetmap.org/?mlat=58.31778&mlon=53.88694&zoom=15 +https://www.openstreetmap.org/?mlat=58.42278&mlon=54.20333&zoom=12 +http://textual.ru/gvr/index.php?card=181219 +https://www.webcitation.org/69HHN5YrP?url=http://textual.ru/gvr/index.php?card=181219 +https://www.openstreetmap.org/?mlat=52.994633&mlon=37.229065&zoom=12 +https://www.openstreetmap.org/?mlat=52.994633&mlon=37.229065&zoom=12 +https://classinform.ru/okato/search.php?str=54243834002 +https://classinform.ru/oktmo/search.php?str=54643434116 +http://orel.gks.ru/wps/wcm/connect/rosstat_ts/orel/resources/be7830004129485cb23df7367ccd0f13/pub-01-07.pdf +http://www.webcitation.org/6N3pNF33v +http://web.archive.org/web/20051102122701/http://www.auditorium.ru/books/437/keppen_p_13.htm +https://web.archive.org/web/20160304233201/https://m.slovari.yandex.ru/article.xml?book=litenc&title=%D0%91%D0%B0%D1%80%D1%82%D0%B0 +https://www.openstreetmap.org/?mlat=54.6839528&mlon=25.2849389&zoom=17 +https://www.openstreetmap.org/?mlat=54.6839528&mlon=25.2849389&zoom=17 +http://kvr.kpd.lt/#/static-heritage-detail/9f58e1e8-d141-4e3c-a784-ba1c602f5fc4 +http://vienuolynai.mch.mii.lt/V8-46/Vilbonifrat.htm +http://www.xxiamzius.lt/numeriai/2004/10/22/kbend_01.html +http://www.vilnius-tourism.lt/index.php/en/45971/ +https://web.archive.org/web/20070928063502/http://www.vilnius.skynet.lt/hramy13.html +http://www.vilnius-tourism.lt/index.php/en/45971/ +http://vilnius.lcn.lt/parapijos/kryziaus/ +http://vienuolynai.mch.mii.lt/V8-46/Vilbonifrat.htm +https://www.openstreetmap.org/?mlat=49.604999999999997&mlon=37.440555555555555&zoom=13 +https://www.openstreetmap.org/?mlat=49.604999999999997&mlon=37.440555555555555&zoom=13 +http://gska2.rada.gov.ua/pls/z7503/A005?rdat1=09.06.2009&rf7571=34214 +http://rusgothic.ru/interview/xymox.htm +https://archive.today/20130113165742/http://rusgothic.ru/interview/xymox.htm +http://www.treffpunkt-schwarz.de/interview-mit-clan-xymox +https://www.webcitation.org/6542CMmZ6?url=http://www.treffpunkt-schwarz.de/interview-mit-clan-xymox +http://www.musicfolio.com/modernrock/xymox.html +https://www.webcitation.org/66Z3n3bh8?url=http://www.musicfolio.com/modernrock/xymox.html +http://www.clanofxymox.com/albumreview.html#Caitlin +https://www.webcitation.org/6542Bh4Sy?url=http://www.clanofxymox.com/albumreview.html#Caitlin +http://www.discogs.com/Clan-Of-Xymox-Hidden-Faces/master/30343 +http://www.bbc.com/russian/rolling_news/2015/07/150731_rn_olympics_beijing +https://web.archive.org/web/20061205035200/http://ea-spouse.livejournal.com/ +http://planeta42.com +http://www.indiegamemag.com/what-is-an-indie-game/ +https://www.webcitation.org/6Jpkafpgl?url=http://www.indiegamemag.com/what-is-an-indie-game/ +http://uk.gamespot.com/news/2677654.html +https://archive.is/20120716071934/http://uk.gamespot.com/news/2677654.html +https://news.google.com/newspapers?id=nwsdAAAAIBAJ&sjid=QX8EAAAAIBAJ&pg=3635%2C1989311 +http://www.turtlebeach.com/ +http://www.artofmanliness.com/2010/09/29/so-you-want-my-job-video-game-producer/ +http://ea-spouse.livejournal.com/274.html +https://web.archive.org/web/20160215195334/http://ea-spouse.livejournal.com/274.html +http://www.bigfishgames.com/blog/video-gaming-industry-numbers-by-region/ +http://www.gamecareerguide.com/features/416/the_game_industry_salary_survey_.php?page=1 +http://www.gamasutra.com/view/news/221533/Game_Developer_Salary_Survey_2014_The_results_are_in.php +http://www.igda.org/articles/erobinson_crunch.php +http://web.archive.org/web/20090302202706/http://www.igda.org/articles/erobinson_crunch.php +http://push.cx/2009/the-game-industry +http://news.cnet.com/8301-10784_3-5449296-7.html +http://www.gamasutra.com/php-bin/news_index.php?story=4543 +http://www.rockband.com/zine/crunchtime +https://web.archive.org/web/20100716125134/http://www.rockband.com/zine/crunchtime +https://archive.is/20100304063510/http://archives.igda.org/breakingin/ +http://sloperama.com/advice/idea.htm +http://sloperama.com/advice/designprep.htm +http://gameqol.org +https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=17224 +https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=32101 +https://eol.org/pages/597748 +https://npgsweb.ars-grin.gov/gringlobal/taxonomydetail.aspx?id=312948 +http://www.ipni.org/ipni/idPlantNameSearch.do?id=17210060-1 +http://www.theplantlist.org/tpl1.1/record/tro-26600295 +https://www.biodiversitylibrary.org/item/196378#page/13/mode/1up +http://www.plantarium.ru/page/view/item/30693.html +http://bioaltai-sayan.ru/regnum/species_all.php?right=box-spec-p/pteridium.php&left=go.php&species=pteridium_aquilinum +http://www.theplantlist.org/tpl1.1/record/tro-26600295 +http://www.theplantlist.org/ +http://herba.msu.ru/shipunov/school/books/flora_sssr1934_1.djvu +http://herba.msu.ru/shipunov/school/books/gubanov2002_illustr_opred_rast_sred_rossii.djvu +http://www.tropicos.org/Name/26600295 +http://www.plantarium.ru/page/view/item/30693.html +http://oopt.aari.ru/bio/22418 +https://www.enciclopedia.cat/enciclopèdies/gran-enciclopèdia-catalana/EC-GEC-0182998.xml +https://www.britannica.com/plant/bracken +https://eol.org/pages/597748 +https://www.gbif.org/species/5275012 +https://www.inaturalist.org/taxa/52681 +https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=32101 +https://www.irmng.org/aphia.php?p=taxdetails&id=11130842 +https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=17224 +https://academic.microsoft.com/#/detail/2779413664 +http://id.ndl.go.jp/auth/ndlna/00574146 +http://newswe.com/index.php?go=Pages&id=5694&in=view +http://eao.memo27reg.org/pamat-1/bezvinyosuzdennaa +https://web.archive.org/web/20141105104426/http://eao.memo27reg.org/pamat-1/bezvinyosuzdennaa +http://eaomedia.ru/news/birobidjan/12.07.2012/216019/vnuchka-pervogo-predsedatelya-oblispolkoma-eao-quot-peredala-privet-quot-zhitelyam.html +https://web.archive.org/web/20101120071358/http://www.knowbysight.info/SSS/03490.asp +http://data-rulers.ru/BelSSR.html +https://www.openstreetmap.org/?mlat=-17.65000&mlon=168.13333&zoom=15 +https://www.openstreetmap.org/?mlat=-17.65000&mlon=168.13333&zoom=15 +http://vu.geoview.info/eretoka_island,2137077 +https://geographic.org/geographic_names/name.php?uni=177578&fid=4284&c=vanuatu +https://books.google.ru/books?id=t6CFD3Gt7R8C&pg=PA30&dq=Eretoka+island+vanuatu&hl=en&sa=X&ved=0ahUKEwjsiZPB49vcAhVOyqYKHf8pC0sQ6AEIPTAD#v=onepage&q=Eretoka%20island%20vanuatu&f=false +http://whc.unesco.org/en/list/1280/ +https://www.radionz.co.nz/international/pacific-news/178159/world-heritage-status-set-to-ensure-protection-of-vanuatu's-roi-mata-domain +http://www.lki.ru/text.php?id=2330 +https://www.openstreetmap.org/?mlat=58.951709&mlon=47.095805&zoom=12 +https://www.openstreetmap.org/?mlat=58.951709&mlon=47.095805&zoom=12 +https://classinform.ru/okato/search.php?str=34228816004 +https://classinform.ru/oktmo/search.php?str=34628416111 +http://docs.cntd.ru/document/412301869 +http://www.webcitation.org/6WvWqzDZj +http://bogovarovo.adm44.ru/i/u/matpoobosnovaniyu1.rar +http://www.consultant.ru/document/cons_doc_LAW_114656/b2707989c276b5a188e63bc41e7bcbcc18723de8/ +http://docs.cntd.ru/document/460161770 +http://www.webcitation.org/6WXQ4l8Fd +http://std.gmcrosstata.ru/webapi/opendatabase?id=VPN2002_2010L +http://lingvarium.org/russia/BD/02c_Kostromskaja.xls +http://www.trait-carre.org/Afficher.aspx?supersection=256&langue=fr +https://web.archive.org/web/20070706112510/http://www.trait-carre.org/Afficher.aspx?supersection=256&langue=fr +http://www.ville.quebec.qc.ca/apropos/vie_democratique/participation_citoyenne/conseils_quartier/charlesbourg/desjesuites/index.aspx +http://eglisesdequebec.org/ToutesLesEglises/SaintJerome/SaintJerome.html +http://web.archive.org/web/20090601203219/http://eglisesdequebec.org/ToutesLesEglises/SaintJerome/SaintJerome.html +https://web.archive.org/web/20090307042752/http://www.ville.quebec.qc.ca/apropos/portrait/arrondissements/charlesbourg/plan.aspx +http://www.leseglisesdemonquartier.com/862.html +https://www.imdb.com/name/nm0002170/ +http://www.hydeparkentertainment.com/about.html +http://www.wimbledon.com/en_GB/roll_of_honour/boys-singles.html +https://web.archive.org/web/20141208094938/http://www.wimbledon.com/en_GB/roll_of_honour/boys-singles.html +http://www.variety.com/article/VR1118018390?refCatId=13 +https://www.webcitation.org/6B2zM57zD?url=http://www.variety.com/article/VR1118018390?refCatId=13 +https://www.imdb.com/name/nm2170/ +https://web.archive.org/web/20120905154217/http://www.wtt.com/Pictures/MG2011_F_online.pdf +http://www.wtt.com/Pictures/MG2011_F_online.pdf +https://web.archive.org/web/20120905154217/http://www.wtt.com/Pictures/MG2011_F_online.pdf +http://gulfnews.com/life-style/people/racquets-to-riches-1.24718 +https://www.webcitation.org/6B2zMqiiq?url=http://gulfnews.com/life-style/people/racquets-to-riches-1.24718 +http://www.thewrap.com/movies/article/national-geographic-films-being-folded-hyde-park-joint-venture-33876 +https://books.google.com/books?id=e07vBwAAQBAJ&pg=PR17 +http://www.hydeparkentertainment.com/aboutus.html +https://web.archive.org/web/20120625213421/http://www.hydeparkentertainment.com/aboutus.html +https://www.imdb.com/name/nm2170/ +https://web.archive.org/web/20120625213421/http://www.hydeparkentertainment.com/aboutus.html +https://twitter.com/AshokAmritraj +https://www.atptour.com/en/players/-/A059/overview +https://www.itftennis.com/en/players/ashok-amritraj/800175325/ind +https://www.allmovie.com/artist/p79541 +https://www.allocine.fr/personne/fichepersonne_gen_cpersonne=498657.html +https://www.csfd.cz/tvurce/209679 +https://www.dfi.dk/viden-om-film/filmdatabasen/person/39462 +https://www.filmportal.de/8eebd0fc920b43b689f569a4a787d34c +https://www.imdb.com/name/nm0002170 +https://www.kinopoisk.ru/name/162/ +http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1259590 +https://d-nb.info/gnd/1061940780 +http://isni-url.oclc.nl/isni/0000000063023091 +https://id.loc.gov/authorities/no2003110512 +https://viaf.org/viaf/51386949 +https://www.worldcat.org/identities/containsVIAFID/51386949 +http://www.sugababes.com/ +https://www.discogs.com/artist/746399 +http://www.telegraph.co.uk/motoring/main.jhtml?xml=/motoring/2003/06/13/emfjen14.xml +https://web.archive.org/web/20080612134413/http://www.telegraph.co.uk/motoring/main.jhtml?xml=%2Fmotoring%2F2003%2F06%2F13%2Femfjen14.xml diff --git a/scrins/scrin1.png b/scrins/scrin1.png new file mode 100644 index 0000000..1ff68dc Binary files /dev/null and b/scrins/scrin1.png differ diff --git a/scrins/scrin10.png b/scrins/scrin10.png new file mode 100644 index 0000000..cecd3be Binary files /dev/null and b/scrins/scrin10.png differ diff --git a/scrins/scrin2.png b/scrins/scrin2.png new file mode 100644 index 0000000..4efb11f Binary files /dev/null and b/scrins/scrin2.png differ diff --git a/scrins/scrin3.png b/scrins/scrin3.png new file mode 100644 index 0000000..4698672 Binary files /dev/null and b/scrins/scrin3.png differ diff --git a/scrins/scrin4.png b/scrins/scrin4.png new file mode 100644 index 0000000..cecddc8 Binary files /dev/null and b/scrins/scrin4.png differ diff --git a/scrins/scrin5.png b/scrins/scrin5.png new file mode 100644 index 0000000..e2484d6 Binary files /dev/null and b/scrins/scrin5.png differ diff --git a/scrins/scrin6.png b/scrins/scrin6.png new file mode 100644 index 0000000..b8307e3 Binary files /dev/null and b/scrins/scrin6.png differ diff --git a/scrins/scrin7.png b/scrins/scrin7.png new file mode 100644 index 0000000..c2fac68 Binary files /dev/null and b/scrins/scrin7.png differ diff --git a/scrins/scrin8.png b/scrins/scrin8.png new file mode 100644 index 0000000..b72c587 Binary files /dev/null and b/scrins/scrin8.png differ diff --git a/scrins/scrin9.png b/scrins/scrin9.png new file mode 100644 index 0000000..ecbfe15 Binary files /dev/null and b/scrins/scrin9.png differ