From d1fd5edfcf17b89cd218fba176e1f1cac3f87593 Mon Sep 17 00:00:00 2001 From: Steve Anderson / Steve Ronuken Date: Tue, 28 Jun 2022 17:14:43 +0100 Subject: [PATCH 1/2] Update aggloader-esi.py --- scripts/aggloader-esi.py | 372 +++++++++++++++++++++++++++------------ 1 file changed, 260 insertions(+), 112 deletions(-) diff --git a/scripts/aggloader-esi.py b/scripts/aggloader-esi.py index 35f8215..7e7b1cc 100644 --- a/scripts/aggloader-esi.py +++ b/scripts/aggloader-esi.py @@ -19,7 +19,9 @@ import base64 import gzip -from StringIO import StringIO +from io import StringIO +from six.moves import configparser +import traceback import logging @@ -27,26 +29,9 @@ -def RateLimited(maxPerSecond): - minInterval = 1.0 / float(maxPerSecond) - def decorate(func): - lastTimeCalled = [0.0] - def rateLimitedFunction(*args,**kargs): - elapsed = time.clock() - lastTimeCalled[0] - leftToWait = minInterval - elapsed - if leftToWait>0: - time.sleep(leftToWait) - ret = func(*args,**kargs) - lastTimeCalled[0] = time.clock() - return ret - return rateLimitedFunction - return decorate - - - def processData(result,orderwriter,ordersetid,connection,orderTable): - + try: resp=result.result() regionid=result.region @@ -75,19 +60,31 @@ def processData(result,orderwriter,ordersetid,connection,orderTable): ) if len(orders)>0: - nextpage=result.url + if int(result.page) < int(resp.headers['X-Pages']): + logging.info('{}'.format(resp.headers['X-Pages'])) + nextpage=result.url + else: + nextpage=None else: nextpage=None logging.info('{}: next page {}'.format(result.url,nextpage)) return {'retry':0,'url':nextpage,'region':result.region,'page':result.page+1,'structure':result.structure} - except: + except Exception as inst: logging.error("URL: {} could not be parsed".format(result.url)) + logging.error("{} {} {} {}".format(type(inst),inst.args,inst,traceback.format_exc())) file = open("logs/{}-{}.txt".format(result.region,result.page),"wb") file.write(resp.content) file.close() elif resp.status_code==403: logging.error("403 status. {} returned {}".format(resp.url,resp.status_code)) return {'retry':4} + elif resp.status_code==404: + logging.error("404 status. {} returned {}".format(resp.url,resp.status_code)) + return {'retry':4} + elif resp.status_code==420: + logging.error("420 status. sleeping for 60. {} returned {} on retry {}".format(resp.url,resp.status_code,result.retry)) + sleep(60) + return {'retry':result.retry+1,'url':result.url,'region':result.region,'page':result.page,'structure':result.structure} else: logging.error("Non 200 status. {} returned {} on retry {}".format(resp.url,resp.status_code,result.retry)) return {'retry':result.retry+1,'url':result.url,'region':result.region,'page':result.page,'structure':result.structure} @@ -95,12 +92,11 @@ def processData(result,orderwriter,ordersetid,connection,orderTable): logging.error(e) return {'retry':result.retry+1,'url':result.url,'region':result.region,'page':result.page,'structure':result.structure} return {'retry':result.retry+1,'url':result.url,'region':result.region,'page':result.page,'structure':result.structure} - - - -@RateLimited(150) + + + def getData(requestsConnection,url,retry,page,region,structure): future=requestsConnection.get(url+str(page)) logging.info('getting {}#{}#{}#{}'.format(retry,page,region,url+str(page))) @@ -114,11 +110,10 @@ def getData(requestsConnection,url,retry,page,region,structure): if __name__ == "__main__": - import ConfigParser, os fileLocation = os.path.dirname(os.path.realpath(__file__)) inifile=fileLocation+'/esi.cfg' - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read(inifile) clientid=config.get('oauth','clientid') @@ -133,7 +128,7 @@ def getData(requestsConnection,url,retry,page,region,structure): engine = create_engine(connectionstring, echo=False) metadata = MetaData() connection = engine.connect() - + session = FuturesSession(max_workers=reqs_num_workers) session.headers.update({'UserAgent':useragent}); @@ -153,7 +148,7 @@ def getData(requestsConnection,url,retry,page,region,structure): Column('region',Integer), Column('orderSet',BigInteger) ) - + Index("orders_1",orderTable.c.typeID) Index("orders_2",orderTable.c.typeID,orderTable.c.buy) Index("orders_5",orderTable.c.region,orderTable.c.typeID,orderTable.c.buy) @@ -170,8 +165,8 @@ def getData(requestsConnection,url,retry,page,region,structure): #metadata.create_all(engine,checkfirst=True) urls=[] - - for regionid in xrange(10000001,10000070): + + for regionid in range(10000001,10000070): if regionid not in (10000024,10000026): urls.append({'url':"https://esi.evetech.net/latest/markets/{}/orders/?order_type=all&datasource=tranquility&page=".format(regionid),'retry':0,'page':1,'region':regionid,'structure':0}) @@ -185,12 +180,12 @@ def getData(requestsConnection,url,retry,page,region,structure): - with open('/tmp/orderset-{}.csv'.format(ordersetid), 'wb') as csvfile: + with open('/tmp/orderset-{}.csv'.format(ordersetid), 'w') as csvfile: orderwriter = csv.writer(csvfile,quoting=csv.QUOTE_MINIMAL,delimiter="\t") # Loop through the urls in batches while len(urls)>0: futures=[] - logging.warn("Loop restarting {}".format(ordersetid)); + logging.warning("Loop restarting {}".format(ordersetid)); for url in urls: logging.info('URL:{} Retry:{} page:{}'.format(url['url'],url['retry'],url['page'])); futures.append(getData(session,url['url'],url['retry'],url['page'],url['region'],url['structure'])) @@ -203,29 +198,40 @@ def getData(requestsConnection,url,retry,page,region,structure): if presult['retry'] == 0 and presult['url'] is not None: logging.info('{} has more pages. {}'.format(result.url,presult['retry'])) urls.append(presult) - # Get authorization - headers = {'Authorization':'Basic '+ base64.b64encode(clientid+':'+secret),'User-Agent':useragent} + token=clientid+':'+secret + message_bytes = token.encode('ascii') + base64_bytes = base64.b64encode(message_bytes) + base64_message = base64_bytes.decode('ascii') + headers = {'Authorization':'Basic '+ base64_message,'User-Agent':useragent,"Content-Type": "application/x-www-form-urlencoded"} query = {'grant_type':'refresh_token','refresh_token':refreshtoken} - r = requests.post('https://login.eveonline.com/oauth/token',params=query,headers=headers) + r = requests.post('https://login.eveonline.com/v2/oauth/token',data=query,headers=headers) response = r.json() accesstoken = response['access_token'] - logging.warn("Access Token {}".format(accesstoken)) + refreshtokennew = response['refresh_token'] + if refreshtokennew != refreshtoken: + cfgfile = open(inifile,'w') + config.set('oauth','refreshtoken',refreshtokennew) + config.write(cfgfile) + cfgfile.close() + + logging.warning("Access Token {}".format(accesstoken)) + logging.warning("refresh Token {}".format(refreshtoken)) session.headers.update({'Authorization':'Bearer '+accesstoken}); - + results=connection.execute('select "stationID",mss."regionID" from evesde."staStations" sta join evesde."mapSolarSystems" mss on mss."solarSystemID"=sta."solarSystemID" where "stationID">100000000').fetchall() for result in results: urls.append({'url':"https://esi.evetech.net/latest/markets/structures/{}/?&datasource=tranquility&page=".format(result[0]),'retry':0,'page':1,'region':result[1],'structure':1}) - - + + while len(urls)>0: futures=[] - logging.warn("Loop restarting {}".format(ordersetid)); + logging.warning("Loop restarting {}".format(ordersetid)); for url in urls: logging.info('URL:{} Retry:{} page:{}'.format(url['url'],url['retry'],url['page'])); futures.append(getData(session,url['url'],url['retry'],url['page'],url['region'],url['structure'])) @@ -239,22 +245,22 @@ def getData(requestsConnection,url,retry,page,region,structure): logging.info('{} has more pages. {}'.format(result.url,presult['retry'])) urls.append(presult) - logging.warn("Loading Data File {}".format(ordersetid)); - connection.execute("""copy orders_{}("orderID","typeID",issued,buy,volume,"volumeEntered","minVolume",price,"stationID",range,duration,region,"orderSet") from '/tmp/orderset-{}.csv'""".format((int(ordersetid)/100)%10,ordersetid)) - logging.warn("Complete load {}".format(ordersetid)); + logging.warning("Loading Data File {}".format(ordersetid)); + connection.execute("""copy orders_{}("orderID","typeID",issued,buy,volume,"volumeEntered","minVolume",price,"stationID",range,duration,region,"orderSet") from '/tmp/orderset-{}.csv'""".format(int((int(ordersetid)/100)%10),ordersetid)) + logging.warning("Complete load {}".format(ordersetid)); trans.commit() - - logging.warn("Pandas populating sell {}".format(ordersetid)); - + + logging.warning("Pandas populating sell {}".format(ordersetid)); + sell=pandas.read_sql_query("""select region||'|'||"typeID"||'|'||buy as what,price,sum(volume) volume from orders where "orderSet"={} and buy=False group by region,"typeID",buy,price order by region,"typeID",price asc""".format(ordersetid),connection); - logging.warn("Pandas populating buy {}".format(ordersetid)); + logging.warning("Pandas populating buy {}".format(ordersetid)); buy=pandas.read_sql_query("""select region||'|'||"typeID"||'|'||buy as what,price,sum(volume) volume from orders where "orderSet"={} and buy=True group by region,"typeID",buy,price order by region,"typeID",price desc""".format(ordersetid),connection); - logging.warn("Pandas populated {}".format(ordersetid)); + logging.warning("Pandas populated {}".format(ordersetid)); - logging.warn("Sell Math running {}".format(ordersetid)); + logging.warning("Sell Math running {}".format(ordersetid)); sell['min']=sell.groupby('what')['price'].transform('min') sell['volume']=sell.apply(lambda x: 0 if x['price']>x['min']*100 else x['volume'],axis=1) sell['cumsum']=sell.groupby('what')['volume'].apply(lambda x: x.cumsum()) @@ -264,7 +270,7 @@ def getData(requestsConnection,url,retry,page,region,structure): sell['applies']=sell.apply(lambda x: x['volume'] if x['cumsum']<=x['fivepercent'] else x['fivepercent']-x['lastsum'],axis=1) num = sell._get_numeric_data() num[num < 0] = 0 - logging.warn("Buy Math running {}".format(ordersetid)); + logging.warning("Buy Math running {}".format(ordersetid)); buy['max']=buy.groupby('what')['price'].transform('max') buy['volume']=buy.apply(lambda x: 0 if x['price']x['min']*100 else x['volume'],axis=1) sell['cumsum']=sell.groupby('what')['volume'].apply(lambda x: x.cumsum()) @@ -340,7 +362,7 @@ def getData(requestsConnection,url,retry,page,region,structure): sell['applies']=sell.apply(lambda x: x['volume'] if x['cumsum']<=x['fivepercent'] else x['fivepercent']-x['lastsum'],axis=1) num = sell._get_numeric_data() num[num < 0] = 0 - logging.warn("Buy Math running {}".format(ordersetid)); + logging.warning("Buy Math running {}".format(ordersetid)); buy['max']=buy.groupby('what')['price'].transform('max') buy['volume']=buy.apply(lambda x: 0 if x['price']x['min']*100 else x['volume'],axis=1) + sell['cumsum']=sell.groupby('what')['volume'].apply(lambda x: x.cumsum()) + sell['fivepercent']=sell.groupby('what')['volume'].transform('sum')/20 + sell['lastsum']=sell.groupby('what')['cumsum'].shift(1) + sell.fillna(0,inplace=True) + sell['applies']=sell.apply(lambda x: x['volume'] if x['cumsum']<=x['fivepercent'] else x['fivepercent']-x['lastsum'],axis=1) + num = sell._get_numeric_data() + num[num < 0] = 0 + logging.warning("Buy Math running {}".format(ordersetid)); + buy['max']=buy.groupby('what')['price'].transform('max') + buy['volume']=buy.apply(lambda x: 0 if x['price']1000: + count=0 + pipe.execute() + pipe.execute() + except ZeroDivisionError: + logging.warning("bah!") + + logging.warning("Universe Aggregates {}".format(ordersetid)); + + logging.warning("Pandas populating sell {}".format(ordersetid)); + + sell=pandas.read_sql_query("""select '0|'||"typeID"||'|'||buy as what,price,sum(volume) volume from orders where "orderSet"={} and buy=False group by "typeID",buy,price order by "typeID",price asc""".format(ordersetid),connection); + logging.warning("Pandas populating buy {}".format(ordersetid)); + buy=pandas.read_sql_query("""select '0|'||"typeID"||'|'||buy as what,price,sum(volume) volume from orders where "orderSet"={} and buy=True group by "typeID",buy,price order by "typeID",price desc""".format(ordersetid),connection); + logging.warning("Pandas populated {}".format(ordersetid)); + + + logging.warning("Sell Math running {}".format(ordersetid)); sell['min']=sell.groupby('what')['price'].transform('min') sell['volume']=sell.apply(lambda x: 0 if x['price']>x['min']*100 else x['volume'],axis=1) sell['cumsum']=sell.groupby('what')['volume'].apply(lambda x: x.cumsum()) @@ -411,7 +541,7 @@ def getData(requestsConnection,url,retry,page,region,structure): sell['applies']=sell.apply(lambda x: x['volume'] if x['cumsum']<=x['fivepercent'] else x['fivepercent']-x['lastsum'],axis=1) num = sell._get_numeric_data() num[num < 0] = 0 - logging.warn("Buy Math running {}".format(ordersetid)); + logging.warning("Buy Math running {}".format(ordersetid)); buy['max']=buy.groupby('what')['price'].transform('max') buy['volume']=buy.apply(lambda x: 0 if x['price'] Date: Fri, 30 Sep 2022 19:43:42 +0000 Subject: [PATCH 2/2] Bump twig/twig from 1.24.1 to 1.42.5 Bumps [twig/twig](https://github.com/twigphp/Twig) from 1.24.1 to 1.42.5. - [Release notes](https://github.com/twigphp/Twig/releases) - [Changelog](https://github.com/twigphp/Twig/blob/v1.42.5/CHANGELOG) - [Commits](https://github.com/twigphp/Twig/compare/v1.24.1...v1.42.5) --- updated-dependencies: - dependency-name: twig/twig dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- composer.lock | 144 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 115 insertions(+), 29 deletions(-) diff --git a/composer.lock b/composer.lock index 8de02e3..4dc2718 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,9 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "hash": "ee23e1d29a6e0e22691181cef14e89d7", "content-hash": "ee6796f33afffa39b5e2ebf9322b42d3", "packages": [ { @@ -32,7 +31,7 @@ "MIT" ], "description": "Promoting the interoperability of container objects (DIC, SL, etc.)", - "time": "2014-12-30 15:22:37" + "time": "2014-12-30T15:22:37+00:00" }, { "name": "monolog/monolog", @@ -110,7 +109,7 @@ "logging", "psr-3" ], - "time": "2016-04-12 18:29:35" + "time": "2016-04-12T18:29:35+00:00" }, { "name": "nikic/fast-route", @@ -153,7 +152,7 @@ "router", "routing" ], - "time": "2016-06-12 19:08:51" + "time": "2016-06-12T19:08:51+00:00" }, { "name": "pimple/pimple", @@ -199,7 +198,7 @@ "container", "dependency injection" ], - "time": "2015-09-11 15:10:35" + "time": "2015-09-11T15:10:35+00:00" }, { "name": "predis/predis", @@ -249,7 +248,7 @@ "predis", "redis" ], - "time": "2016-06-16 16:22:20" + "time": "2016-06-16T16:22:20+00:00" }, { "name": "psr/http-message", @@ -298,7 +297,7 @@ "request", "response" ], - "time": "2015-05-04 20:22:00" + "time": "2015-05-04T20:22:00+00:00" }, { "name": "psr/log", @@ -336,7 +335,7 @@ "psr", "psr-3" ], - "time": "2012-12-21 11:40:51" + "time": "2012-12-21T11:40:51+00:00" }, { "name": "slim/http-cache", @@ -385,7 +384,7 @@ "middleware", "slim" ], - "time": "2015-08-13 13:31:25" + "time": "2015-08-13T13:31:25+00:00" }, { "name": "slim/php-view", @@ -434,7 +433,7 @@ "template", "view" ], - "time": "2016-03-04 09:48:50" + "time": "2016-03-04T09:48:50+00:00" }, { "name": "slim/slim", @@ -504,7 +503,7 @@ "micro", "router" ], - "time": "2016-05-25 11:23:38" + "time": "2016-05-25T11:23:38+00:00" }, { "name": "slim/twig-view", @@ -554,38 +553,121 @@ "twig", "view" ], - "time": "2016-03-13 20:58:41" + "time": "2016-03-13T20:58:41+00:00" + }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.19.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "aed596913b70fae57be53d86faa2e9ef85a2297b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/aed596913b70fae57be53d86faa2e9ef85a2297b", + "reference": "aed596913b70fae57be53d86faa2e9ef85a2297b", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.19-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.19.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2020-10-23T09:01:57+00:00" }, { "name": "twig/twig", - "version": "v1.24.1", + "version": "v1.42.5", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "3566d311a92aae4deec6e48682dc5a4528c4a512" + "reference": "87b2ea9d8f6fd014d0621ca089bb1b3769ea3f8e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/3566d311a92aae4deec6e48682dc5a4528c4a512", - "reference": "3566d311a92aae4deec6e48682dc5a4528c4a512", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/87b2ea9d8f6fd014d0621ca089bb1b3769ea3f8e", + "reference": "87b2ea9d8f6fd014d0621ca089bb1b3769ea3f8e", "shasum": "" }, "require": { - "php": ">=5.2.7" + "php": ">=5.5.0", + "symfony/polyfill-ctype": "^1.8" }, "require-dev": { - "symfony/debug": "~2.7", - "symfony/phpunit-bridge": "~2.7" + "psr/container": "^1.0", + "symfony/phpunit-bridge": "^4.4|^5.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.24-dev" + "dev-master": "1.42-dev" } }, "autoload": { "psr-0": { "Twig_": "lib/" + }, + "psr-4": { + "Twig\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -599,23 +681,26 @@ "homepage": "http://fabien.potencier.org", "role": "Lead Developer" }, + { + "name": "Twig Team", + "role": "Contributors" + }, { "name": "Armin Ronacher", "email": "armin.ronacher@active-4.com", "role": "Project Founder" - }, - { - "name": "Twig Team", - "homepage": "http://twig.sensiolabs.org/contributors", - "role": "Contributors" } ], "description": "Twig, the flexible, fast, and secure template language for PHP", - "homepage": "http://twig.sensiolabs.org", + "homepage": "https://twig.symfony.com", "keywords": [ "templating" ], - "time": "2016-05-30 09:11:59" + "support": { + "issues": "https://github.com/twigphp/Twig/issues", + "source": "https://github.com/twigphp/Twig/tree/1.x" + }, + "time": "2020-02-11T05:59:23+00:00" } ], "packages-dev": [], @@ -627,5 +712,6 @@ "platform": { "php": ">=5.5.0" }, - "platform-dev": [] + "platform-dev": [], + "plugin-api-version": "2.3.0" }