Анецких Алексей АТ-13

LinkTestsResult

@@ -0,0 +1,15 @@
+Синхронный тест
+    Время: 127084647810 нс
+    Загрузка cpu: большую часть времени около 0%, иногда скачет от 0.5 до 11%
+
+Асинхронный тест (5)
+    Свреднее время за 5 раз: 27953558220 нс
+    Загрузка cpu: в среднем около 3%, не доходило до 5%
+
+Асинхронный тест (10)
+    Свреднее время за 5 раз: 14401374040 нс
+    Загрузка cpu: в среднем около 5%, не доходило до 7%
+
+Асинхронный тест (100)
+    Свреднее время за 5 раз: 5179337140 нс
+    Загрузка cpu: в среднем около 15%, не доходило до 22%

TokenTestResult

@@ -0,0 +1,25 @@
+10 монет
+
+Синхронный тест
+    Время: 336110559700 нс
+    Загрузка cpu: в среднем 32%
+
+Асинхронный тест (2)
+    Время: 129867795480 нс
+    Загрузка cpu: в среднем 58%
+
+Асинхронный тест (4)
+    Время: 63891222500 нс
+    Загрузка cpu: в среднем 93%
+
+Асинхронный тест (5)
+    Время: 100395945220 нс
+    Загрузка cpu: в среднем 100%
+
+Асинхронный тест (10)
+    Время: 111814434300 нс
+    Загрузка cpu: в среднем 100%
+
+Асинхронный тест (61)
+    Время: 116016165900 нс
+    Загрузка cpu: в среднем 100%

async_IO.py

@@ -0,0 +1,19 @@
+import concurrent.futures
+import url_supplies
+
+links = url_supplies.get_links()
+
+
+def test(max_workers):
+    result = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_url = {executor.submit(url_supplies.load_url, url, 2): url for url in links}
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                data = future.result()
+            except Exception as exc:
+                result.append('%r generated an exception: %s' % (url, exc))
+            else:
+                result.append('%r page is %d bytes' % (url, data))
+    return result

async_token_gen.py

@@ -0,0 +1,12 @@
+import concurrent.futures
+
+import tokens
+
+
+def test(tokens_num, max_workers):
+    result = []
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(tokens.get_token) for _ in range(0, tokens_num)]
+        for future in concurrent.futures.as_completed(futures):
+            result += future.result()
+    return result

main.py

@@ -0,0 +1,51 @@
+import time
+
+import async_IO
+import sync_IO
+import async_token_gen
+import sync_token_gen
+
+
+def do_tests(test, repeats, *arg):
+    start_time = time.perf_counter_ns()
+    for i in range(repeats):
+        test(*arg)
+    end_time = time.perf_counter_ns()
+    return (end_time-start_time)/repeats
+
+
+def link_tests():
+    #print(f'Sync time {do_tests(sync_IO.test, 1)}')
+
+    workers = 5
+    #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}')
+
+    workers = 10
+    #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}')
+
+    workers = 100
+    #print(f'Async with {workers} max_workers time {do_tests(async_IO.test, 5, workers)}')
+
+
+def token_test():
+    #print(f'Sync time {do_tests(sync_token_gen.test, 1, 10)}')
+
+    workers = 2
+    #print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}')
+
+    workers = 4
+    print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}')
+
+    workers = 5
+    print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}')
+
+    workers = 10
+    print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}')
+
+    workers = 61
+    print(f'Async with {workers} max_workers time {do_tests(async_token_gen.test, 5, 10, workers)}')
+
+
+if __name__ == '__main__':
+    link_tests()
+    token_test()

res.txt

@@ -0,0 +1,133 @@
+https://web.archive.org/web/20160304081805/http://www.knowbysight.info/1_ukra/04629.asp
+https://www.openstreetmap.org/?mlat=51.12500&mlon=71.47222&zoom=12
+https://www.openstreetmap.org/?mlat=51.12500&mlon=71.47222&zoom=12
+http://www.muslim.kz
+http://m.megapolis.kz/art/V_klassicheskom_islamskom_stile
+http://www.bestnews.kz/home/novosti-sajta/item/mechet-hazret-sultan-fotoreportazh.html
+https://web.archive.org/web/20161111233523/http://www.bestnews.kz/home/novosti-sajta/item/mechet-hazret-sultan-fotoreportazh.html
+https://mustvisit.world/place/kazahstan-astana-mechet-hazret-sultan
+http://www.zakon.kz/4467415-sobornaja-mechet-gorit-v-astane.html
+http://www.zakon.kz/kazakhstan/4468273-pozhar-ne-prichinil-osobogo-vreda.html
+http://www.voxpopuli.kz/post/view/id/760
+https://www.facebook.com/www.muslim.kz
+https://apiv3.iucnredlist.org/api/v3/taxonredirect/22823
+https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=180542
+https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=29073
+https://eol.org/pages/46559143
+http://fossilworks.org/bridge.pl?a=taxonInfo&taxon_no=52609
+http://www.sevin.ru/redbooksevin/content/381.html
+http://programmes.putin.kremlin.ru/bear/animal
+http://lenta.ru/news/2011/07/08/irish/
+http://elementy.ru/news?newsid=431819
+https://www.nytimes.com/2012/07/24/science/brown-bears-and-polar-bears-split-up-but-continued-coupling.html
+https://dx.doi.org/10.2307%2F3503828
+https://www.jstor.org/stable/3503828
+https://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf
+https://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf
+https://web.archive.org/web/20090510095701/http://alaska.fws.gov/fisheries/mmm/polarbear/pdf/Polar_Bear_%20Status_Assessment.pdf
+https://archive.org/details/polarbears00stir
+https://animalreader.ru/morzh-i-belyiy-medved-vzaimootnosheniya-dvuh-severyan.html
+https://www.youtube.com/watch?v=bKWWygkM-wY#t=06m50s
+http://www.telegraph.co.uk/earth/earthnews/8280904/Polar-bear-tracked-during-nine-day-swim.html
+https://dela.ru/news/254648/
+http://static.iea.ras.ru/neotlozhka/Oparin_Umanskaya_220_2010.pdf
+http://programmes.putin.kremlin.ru/bear/history
+http://www.rgo.ru/ru/proekty/sohranenie-redkih-vidov-belyy-medved
+http://pbsg.npolar.no/en/status/status-table.html
+http://gazetazp.ru/2009/140/5
+https://web.archive.org/web/20091123081554/http://gazetazp.ru/2009/140/5
+http://ria.ru/earth/20131204/981873396.html
+http://www.wwf.ru/about/what_we_do/species/polarbear/faq?root=531
+http://www.bundabergrum.com.au/explore/distillery/chapter/03
+https://web.archive.org/web/20150923195147/http://www.bundabergrum.com.au/explore/distillery/chapter/03
+http://www.quarterdesigns.com/proposed/alaska/ak01.jpg
+http://www.muenzeoesterreich.at/eng/produkte/arctic-adventure2
+http://www.muenzeoesterreich.at/eng/produkte/arctic-adventure
+http://sochi2014.com/sochi-live/news/38688
+http://www.wwf.ru/resources/news/article/10877
+http://www.departments.bucknell.edu/biology/resources/msw3/browse.asp?id=14000987
+http://www.worldcat.org/oclc/62265494
+https://web.archive.org/web/20120123121143/http://www.nat-geo.ru/article/635/
+http://www.sevin.ru/vertebrates/index.html?Mammals/217.html
+http://www.moscowzoo.ru/animals/khishchnye/belyy-medved/
+https://web.archive.org/web/20140606070025/http://myttk.ru/media/webcam/zoo_polar_bears/
+https://web.archive.org/web/20140529084432/http://myttk.ru/media/webcam/zoo_polar_bears2/
+http://loveopium.ru/zhivotnye-2/belye-medvedi-3.html
+http://www.marinespecies.org/aphia.php?p=taxdetails&id=137085
+https://bigenc.ru/text/1856686
+https://www.britannica.com/animal/polar-bear
+https://www.universalis.fr/encyclopedie/ours-blanc-ours-polaire/
+https://eol.org/pages/46559143
+https://www.gbif.org/species/2433451
+https://www.inaturalist.org/taxa/41644
+https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=29073
+https://www.irmng.org/aphia.php?p=taxdetails&id=11061491
+https://www.itis.gov/servlet/SingleRpt/SingleRpt?search_topic=TSN&search_value=180542
+https://www.sealifebase.ca/Summary/SpeciesSummary.php?id=69192
+https://www.marinespecies.org/aphia.php?p=taxdetails&id=137085
+https://catalogue.bnf.fr/ark:/12148/cb12008666z
+https://d-nb.info/gnd/4151473-7
+https://id.loc.gov/authorities/sh85104130
+https://viaf.org/processed/LNB%7CLNC10-000075321
+http://id.ndl.go.jp/auth/ndlna/01006575
+https://www.imdb.com/event/ev0000123/2004
+http://www.bafta.org/awards/academy-fellows,125,BA.html
+http://www.bafta.org/awards-database.html?year=2003&category=Film&award=false
+http://irbis-nbuv.gov.ua/everlib/item/er-0002146
+https://elib.nlu.org.ua/view.html?&id=1443Бібліотека
+https://en.calameo.com/read/004761641c03ecde51392
+http://feb-web.ru/feb/masanov/man/09/man17558.htm
+https://elib.nlu.org.ua/view.html?id=1353
+http://esu.com.ua/search_articles.php?id=37270
+http://holocaustmuseum.kharkov.ua/didgest-e/didgest-2003/12-2003/borovich.html
+http://esu.com.ua/search_articles.php?id=37270
+https://web.archive.org/web/20130208025017/http://www.goteborg2013.com/competition/timetable/
+http://www.claymath.org/publications/Harmonic_Analysis
+http://www.numdam.org/item?id=SB_1970-1971__13__123_0
+http://www.msri.org/communications/books/Book35/index.html
+http://www.jmilne.org/math/articles/2005aX.pdf
+https://web.archive.org/web/20120921090302/http://vpered.org.ru/index.php?id=164&category=10
+http://openleft.ru/?p=8467
+http://vpered.org.ru/index.php?id=174&category=10
+https://web.archive.org/web/20111019050013/http://vpered.org.ru/index.php?id=171&category=10
+http://vpered.org.ru/index.php?id=145&category=8
+https://web.archive.org/web/20111019050005/http://vpered.org.ru/index.php?id=166&category=10
+https://web.archive.org/web/20081205163921/http://www.marksizm.info/content/view/3493/60/
+http://commons.com.ua/uk/roman-rozdolskij/
+http://ernestmandel.org/
+https://marxists.org/
+http://www.trotskyana.net/Trotskyists/Ernest_Mandel/ernest_mandel.html
+https://www.dailymotion.com/video/k24MxCAxVKRaZs5QdM?start=5577#from=embed
+https://music.apple.com/ru/artist/433080266
+https://www.imdb.com/name/nm2663203
+https://www.enciclopedia.cat/enciclopèdies/gran-enciclopèdia-catalana/EC-GEC-0039702.xml
+https://snl.no/Ernest_Mandel
+https://www.vle.lt/straipsnis/ernest-esra-mandel
+https://www.britannica.com/biography/Ernest-Mandel
+https://brockhaus.de/ecs/enzy/article/mandel-ernest
+https://www.universalis.fr/encyclopedie/ernest-mandel/
+http://ask.bibsys.no/ask/action/result?cmd=&kilde=biblio&cql=bs.autid+%3D+90105620&feltselect=bs.autid
+http://cantic.bnc.cat/registres/CUCId/a1011791x
+http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1100713
+https://catalogue.bnf.fr/ark:/12148/cb11914218n
+https://ci.nii.ac.jp/author/DA00556846
+https://viaf.org/processed/EGAXA%7Cvtlsvtls000062075
+https://d-nb.info/gnd/118576941
+http://data.beeldengeluid.nl/gtaa/127805
+http://isni-url.oclc.nl/isni/0000000368602473
+http://isni-url.oclc.nl/isni/0000000368602465
+http://isni-url.oclc.nl/isni/000000011613583X
+https://id.loc.gov/authorities/n80067110
+http://id.ndl.go.jp/auth/ndlna/00448654
+http://aut.nkp.cz/skuk0003900
+https://nla.gov.au/anbd.aut-an35325132
+https://nlg.okfn.gr/resource/authority/record85033
+http://mak.bn.org.pl/cgi-bin/KHW/makwww.exe?BM=01&IM=04&NU=01&WI=A10802514
+https://viaf.org/processed/NSK%7C000002155
+https://data.bibliotheken.nl/id/thes/p068378017
+https://viaf.org/processed/NUKAT%7Cn98080683
+https://viaf.org/processed/PTBNP%7C75920
+https://libris.kb.se/katalogisering/khw03x734mhz4vv
+https://www.idref.fr/031288448
+https://viaf.org/viaf/29537419
+https://www.worldcat.org/identities/containsVIAFID/29537419

sync_IO.py

@@ -0,0 +1,13 @@
+import url_supplies
+
+links = url_supplies.get_links()
+
+
+def test():
+    result = []
+    for url in links:
+        try:
+            result.append(url_supplies.load_url(url, 2))
+        except Exception as e:
+            result.append((url, e))
+    return result

sync_token_gen.py

@@ -0,0 +1,5 @@
+import tokens
+
+
+def test(tokens_num):
+    return [tokens.get_token() for _ in range(0, tokens_num)]

tokens.py

@@ -0,0 +1,11 @@
+from hashlib import md5
+from random import choice
+
+
+def get_token():
+    while True:
+        s = "".join([choice("0123456789") for i in range(50)])
+        h = md5(s.encode('utf8')).hexdigest()
+
+        if h.endswith("00000"):
+            return s, h

url_supplies.py

@@ -0,0 +1,39 @@
+import os
+from urllib.request import Request, urlopen
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0'
+
+if not os.path.exists('res.txt'):
+    res = open('res.txt', 'w', encoding='utf8')
+    links_count = 0
+    for i in tqdm(range(100)):
+        if links_count > 100:
+            continue
+        html = urlopen(url).read().decode('utf8')
+        soup = BeautifulSoup(html, 'html.parser')
+        links = soup.find_all('a')
+
+        for l in links:
+            href = l.get('href')
+            if href and href.startswith('http') and 'wiki' not in href:
+                links_count += 1
+                print(href, file=res)
+
+    res.close()
+
+
+def get_links():
+    with open('res.txt', 'r', encoding='utf-8') as f:
+        return f.read().split('\n')
+
+
+def load_url(url, timeout):
+    request = Request(
+            url,
+            headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'},
+            )
+    resp = urlopen(request, timeout=timeout)
+    return resp.code