diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..9482969 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..fd589d7 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/multi.iml b/.idea/multi.iml new file mode 100644 index 0000000..33f358f --- /dev/null +++ b/.idea/multi.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/LinkTestsResult b/LinkTestsResult new file mode 100644 index 0000000..be3e82c --- /dev/null +++ b/LinkTestsResult @@ -0,0 +1,15 @@ +Синхронный тест + Время: 127084647810 нс + Загрузка cpu: большую часть времени около 0%, иногда скачет от 0.5 до 11% + +Асинхронный тест (5) + Свреднее время за 5 раз: 27953558220 нс + Загрузка cpu: в среднем около 3%, не доходило до 5% + +Асинхронный тест (10) + Свреднее время за 5 раз: 14401374040 нс + Загрузка cpu: в среднем около 5%, не доходило до 7% + +Асинхронный тест (100) + Свреднее время за 5 раз: 5179337140 нс + Загрузка cpu: в среднем около 15%, не доходило до 22% \ No newline at end of file diff --git a/TokenTestResult b/TokenTestResult new file mode 100644 index 0000000..2bfbbf5 --- /dev/null +++ b/TokenTestResult @@ -0,0 +1,25 @@ +10 монет + +Синхронный тест + Время: 336110559700 нс + Загрузка cpu: в среднем 32% + +Асинхронный тест (2) + Время: 129867795480 нс + Загрузка cpu: в среднем 58% + +Асинхронный тест (4) + Время: 63891222500 нс + Загрузка cpu: в среднем 93% + +Асинхронный тест (5) + Время: 100395945220 нс + Загрузка cpu: в среднем 100% + +Асинхронный тест (10) + Время: 111814434300 нс + Загрузка cpu: в среднем 100% + +Асинхронный тест (61) + Время: 116016165900 нс + Загрузка cpu: в среднем 100% \ No newline at end of file diff --git a/async_IO.py b/async_IO.py new file mode 100644 index 0000000..8626efd --- /dev/null +++ b/async_IO.py @@ -0,0 +1,19 @@ +import concurrent.futures +import url_supplies + +links = url_supplies.get_links() + + +def test(max_workers): + result = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_url = {executor.submit(url_supplies.load_url, url, 2): url for url in links} + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + try: + data = future.result() + except Exception as exc: + result.append('%r generated an exception: %s' % (url, exc)) + else: + result.append('%r page is %d bytes' % (url, data)) + return result \ No newline at end of file diff --git a/async_token_gen.py b/async_token_gen.py new file mode 100644 index 0000000..769aa95 --- /dev/null +++ b/async_token_gen.py @@ -0,0 +1,12 @@ +import concurrent.futures + +import tokens + + +def test(tokens_num, max_workers): + result = [] + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(tokens.get_token) for _ in range(0, tokens_num)] + for future in concurrent.futures.as_completed(futures): + result += future.result() + return result diff --git a/main.py b/main.py new file mode 100644 index 0000000..0c11deb --- /dev/null +++ b/main.py @@ -0,0 +1,51 @@ +import time + +import async_IO +import sync_IO +import async_token_gen +import sync_token_gen + + +def do_tests(test, repeats, *arg): + start_time = time.perf_counter_ns() + for i in range(repeats): + test(*arg) + end_time = time.perf_counter_ns() + return (end_time-start_time)/repeats + + +def link_tests(): + #print(f'Sync time {do_tests(sync_IO.test, 1)}') + + workers = 5 + #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}') + + workers = 10 + #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}') + + workers = 100 + #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}') + + +def token_test(): + #print(f'Sync time {do_tests(sync_token_gen.test, 1, 10)}') + + workers = 2 + #print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}') + + workers = 4 + print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}') + + workers = 5 + print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}') + + workers = 10 + print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}') + + workers = 61 + print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}') + + +if __name__ == '__main__': + link_tests() + token_test() diff --git a/res.txt b/res.txt new file mode 100644 index 0000000..d922570 --- /dev/null +++ b/res.txt @@ -0,0 +1,133 @@ +https://web.archive.org/web/20160304081805/http://www.knowbysight.info/1_ukra/04629.asp +https://www.openstreetmap.org/?mlat=51.12500&mlon=71.47222&zoom=12 +https://www.openstreetmap.org/?mlat=51.12500&mlon=71.47222&zoom=12 +http://www.muslim.kz +http://m.megapolis.kz/art/V_klassicheskom_islamskom_stile +http://www.bestnews.kz/home/novosti-sajta/item/mechet-hazret-sultan-fotoreportazh.html +https://web.archive.org/web/20161111233523/http://www.bestnews.kz/home/novosti-sajta/item/mechet-hazret-sultan-fotoreportazh.html +https://mustvisit.world/place/kazahstan-astana-mechet-hazret-sultan +http://www.zakon.kz/4467415-sobornaja-mechet-gorit-v-astane.html +http://www.zakon.kz/kazakhstan/4468273-pozhar-ne-prichinil-osobogo-vreda.html +http://www.voxpopuli.kz/post/view/id/760 +https://www.facebook.com/www.muslim.kz +https://apiv3.iucnredlist.org/api/v3/taxonredirect/22823 +https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=180542 +https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=29073 +https://eol.org/pages/46559143 +http://fossilworks.org/bridge.pl?a=taxonInfo&taxon_no=52609 +http://www.sevin.ru/redbooksevin/content/381.html +http://programmes.putin.kremlin.ru/bear/animal +http://lenta.ru/news/2011/07/08/irish/ +http://elementy.ru/news?newsid=431819 +https://www.nytimes.com/2012/07/24/science/brown-bears-and-polar-bears-split-up-but-continued-coupling.html +https://dx.doi.org/10.2307%2F3503828 +https://www.jstor.org/stable/3503828 +https://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf +https://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf +https://web.archive.org/web/20090510095701/http://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf +https://archive.org/details/polarbears00stir +https://animalreader.ru/morzh-i-belyiy-medved-vzaimootnosheniya-dvuh-severyan.html +https://www.youtube.com/watch?v=bKWWygkM-wY#t=06m50s +http://www.telegraph.co.uk/earth/earthnews/8280904/Polar-bear-tracked-during-nine-day-swim.html +https://dela.ru/news/254648/ +http://static.iea.ras.ru/neotlozhka/Oparin_Umanskaya_220_2010.pdf +http://programmes.putin.kremlin.ru/bear/history +http://www.rgo.ru/ru/proekty/sohranenie-redkih-vidov-belyy-medved +http://pbsg.npolar.no/en/status/status-table.html +http://gazetazp.ru/2009/140/5 +https://web.archive.org/web/20091123081554/http://gazetazp.ru/2009/140/5 +http://ria.ru/earth/20131204/981873396.html +http://www.wwf.ru/about/what_we_do/species/polarbear/faq?root=531 +http://www.bundabergrum.com.au/explore/distillery/chapter/03 +https://web.archive.org/web/20150923195147/http://www.bundabergrum.com.au/explore/distillery/chapter/03 +http://www.quarterdesigns.com/proposed/alaska/ak01.jpg +http://www.muenzeoesterreich.at/eng/produkte/arctic-adventure2 +http://www.muenzeoesterreich.at/eng/produkte/arctic-adventure +http://sochi2014.com/sochi-live/news/38688 +http://www.wwf.ru/resources/news/article/10877 +http://www.departments.bucknell.edu/biology/resources/msw3/browse.asp?id=14000987 +http://www.worldcat.org/oclc/62265494 +https://web.archive.org/web/20120123121143/http://www.nat-geo.ru/article/635/ +http://www.sevin.ru/vertebrates/index.html?Mammals/217.html +http://www.moscowzoo.ru/animals/khishchnye/belyy-medved/ +https://web.archive.org/web/20140606070025/http://myttk.ru/media/webcam/zoo_polar_bears/ +https://web.archive.org/web/20140529084432/http://myttk.ru/media/webcam/zoo_polar_bears2/ +http://loveopium.ru/zhivotnye-2/belye-medvedi-3.html +http://www.marinespecies.org/aphia.php?p=taxdetails&id=137085 +https://bigenc.ru/text/1856686 +https://www.britannica.com/animal/polar-bear +https://www.universalis.fr/encyclopedie/ours-blanc-ours-polaire/ +https://eol.org/pages/46559143 +https://www.gbif.org/species/2433451 +https://www.inaturalist.org/taxa/41644 +https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=29073 +https://www.irmng.org/aphia.php?p=taxdetails&id=11061491 +https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=180542 +https://www.sealifebase.ca/Summary/SpeciesSummary.php?id=69192 +https://www.marinespecies.org/aphia.php?p=taxdetails&id=137085 +https://catalogue.bnf.fr/ark:/12148/cb12008666z +https://d-nb.info/gnd/4151473-7 +https://id.loc.gov/authorities/sh85104130 +https://viaf.org/processed/LNB%7CLNC10-000075321 +http://id.ndl.go.jp/auth/ndlna/01006575 +https://www.imdb.com/event/ev0000123/2004 +http://www.bafta.org/awards/academy-fellows,125,BA.html +http://www.bafta.org/awards-database.html?year=2003&category=Film&award=false +http://irbis-nbuv.gov.ua/everlib/item/er-0002146 +https://elib.nlu.org.ua/view.html?&id=1443Бібліотека +https://en.calameo.com/read/004761641c03ecde51392 +http://feb-web.ru/feb/masanov/man/09/man17558.htm +https://elib.nlu.org.ua/view.html?id=1353 +http://esu.com.ua/search_articles.php?id=37270 +http://holocaustmuseum.kharkov.ua/didgest-e/didgest-2003/12-2003/borovich.html +http://esu.com.ua/search_articles.php?id=37270 +https://web.archive.org/web/20130208025017/http://www.goteborg2013.com/competition/timetable/ +http://www.claymath.org/publications/Harmonic_Analysis +http://www.numdam.org/item?id=SB_1970-1971__13__123_0 +http://www.msri.org/communications/books/Book35/index.html +http://www.jmilne.org/math/articles/2005aX.pdf +https://web.archive.org/web/20120921090302/http://vpered.org.ru/index.php?id=164&category=10 +http://openleft.ru/?p=8467 +http://vpered.org.ru/index.php?id=174&category=10 +https://web.archive.org/web/20111019050013/http://vpered.org.ru/index.php?id=171&category=10 +http://vpered.org.ru/index.php?id=145&category=8 +https://web.archive.org/web/20111019050005/http://vpered.org.ru/index.php?id=166&category=10 +https://web.archive.org/web/20081205163921/http://www.marksizm.info/content/view/3493/60/ +http://commons.com.ua/uk/roman-rozdolskij/ +http://ernestmandel.org/ +https://marxists.org/ +http://www.trotskyana.net/Trotskyists/Ernest_Mandel/ernest_mandel.html +https://www.dailymotion.com/video/k24MxCAxVKRaZs5QdM?start=5577#from=embed +https://music.apple.com/ru/artist/433080266 +https://www.imdb.com/name/nm2663203 +https://www.enciclopedia.cat/enciclopèdies/gran-enciclopèdia-catalana/EC-GEC-0039702.xml +https://snl.no/Ernest_Mandel +https://www.vle.lt/straipsnis/ernest-esra-mandel +https://www.britannica.com/biography/Ernest-Mandel +https://brockhaus.de/ecs/enzy/article/mandel-ernest +https://www.universalis.fr/encyclopedie/ernest-mandel/ +http://ask.bibsys.no/ask/action/result?cmd=&kilde=biblio&cql=bs.autid+%3D+90105620&feltselect=bs.autid +http://cantic.bnc.cat/registres/CUCId/a1011791x +http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1100713 +https://catalogue.bnf.fr/ark:/12148/cb11914218n +https://ci.nii.ac.jp/author/DA00556846 +https://viaf.org/processed/EGAXA%7Cvtlsvtls000062075 +https://d-nb.info/gnd/118576941 +http://data.beeldengeluid.nl/gtaa/127805 +http://isni-url.oclc.nl/isni/0000000368602473 +http://isni-url.oclc.nl/isni/0000000368602465 +http://isni-url.oclc.nl/isni/000000011613583X +https://id.loc.gov/authorities/n80067110 +http://id.ndl.go.jp/auth/ndlna/00448654 +http://aut.nkp.cz/skuk0003900 +https://nla.gov.au/anbd.aut-an35325132 +https://nlg.okfn.gr/resource/authority/record85033 +http://mak.bn.org.pl/cgi-bin/KHW/makwww.exe?BM=01&IM=04&NU=01&WI=A10802514 +https://viaf.org/processed/NSK%7C000002155 +https://data.bibliotheken.nl/id/thes/p068378017 +https://viaf.org/processed/NUKAT%7Cn98080683 +https://viaf.org/processed/PTBNP%7C75920 +https://libris.kb.se/katalogisering/khw03x734mhz4vv +https://www.idref.fr/031288448 +https://viaf.org/viaf/29537419 +https://www.worldcat.org/identities/containsVIAFID/29537419 \ No newline at end of file diff --git a/sync_IO.py b/sync_IO.py new file mode 100644 index 0000000..3c599fe --- /dev/null +++ b/sync_IO.py @@ -0,0 +1,13 @@ +import url_supplies + +links = url_supplies.get_links() + + +def test(): + result = [] + for url in links: + try: + result.append(url_supplies.load_url(url, 2)) + except Exception as e: + result.append((url, e)) + return result diff --git a/sync_token_gen.py b/sync_token_gen.py new file mode 100644 index 0000000..3dcc8c3 --- /dev/null +++ b/sync_token_gen.py @@ -0,0 +1,5 @@ +import tokens + + +def test(tokens_num): + return [tokens.get_token() for _ in range(0, tokens_num)] diff --git a/tokens.py b/tokens.py new file mode 100644 index 0000000..85357f3 --- /dev/null +++ b/tokens.py @@ -0,0 +1,11 @@ +from hashlib import md5 +from random import choice + + +def get_token(): + while True: + s = "".join([choice("0123456789") for i in range(50)]) + h = md5(s.encode('utf8')).hexdigest() + + if h.endswith("00000"): + return s, h diff --git a/url_supplies.py b/url_supplies.py new file mode 100644 index 0000000..3840b77 --- /dev/null +++ b/url_supplies.py @@ -0,0 +1,39 @@ +import os +from urllib.request import Request, urlopen +from bs4 import BeautifulSoup +from tqdm import tqdm + +url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0' + +if not os.path.exists('res.txt'): + res = open('res.txt', 'w', encoding='utf8') + links_count = 0 + for i in tqdm(range(100)): + if links_count > 100: + continue + html = urlopen(url).read().decode('utf8') + soup = BeautifulSoup(html, 'html.parser') + links = soup.find_all('a') + + for l in links: + href = l.get('href') + if href and href.startswith('http') and 'wiki' not in href: + links_count += 1 + print(href, file=res) + + res.close() + + +def get_links(): + with open('res.txt', 'r', encoding='utf-8') as f: + return f.read().split('\n') + + +def load_url(url, timeout): + request = Request( + url, + headers={ + 'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, + ) + resp = urlopen(request, timeout=timeout) + return resp.code \ No newline at end of file