From b8dda02d83bb4bc1bef9b4b37bbca2c0062d7123 Mon Sep 17 00:00:00 2001 From: backtrader Date: Mon, 30 Jul 2018 23:43:42 +0200 Subject: [PATCH 01/37] proposed fix for compression only resampling --- backtrader/resamplerfilter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backtrader/resamplerfilter.py b/backtrader/resamplerfilter.py index e94a5aa54..c4cec0d21 100644 --- a/backtrader/resamplerfilter.py +++ b/backtrader/resamplerfilter.py @@ -111,7 +111,7 @@ class _BaseResampler(with_metaclass(metabase.MetaParams, object)): def __init__(self, data): self.subdays = TimeFrame.Ticks < self.p.timeframe < TimeFrame.Days self.subweeks = self.p.timeframe < TimeFrame.Weeks - self.componly = (self.subdays and + self.componly = (not self.subdays and data._timeframe == self.p.timeframe and not (self.p.compression % data._compression) ) @@ -524,7 +524,6 @@ def __call__(self, data, fromcheck=False, forcedata=None): if consumed: self.bar.bupdate(data) # update new or existing bar - data.backwards() # remove used bar # if self.bar.isopen and (onedge or (docheckover and checkbarover)) cond = self.bar.isopen() @@ -533,6 +532,10 @@ def __call__(self, data, fromcheck=False, forcedata=None): if docheckover: cond = self._checkbarover(data, fromcheck=fromcheck, forcedata=forcedata) + + if consumed: + data.backwards() # remove used bar + if cond: dodeliver = False if forcedata is not None: From cc2751a5f53166f68c5340eb876579f1a5590bf5 Mon Sep 17 00:00:00 2001 From: backtrader Date: Fri, 17 Aug 2018 06:09:25 +0200 Subject: [PATCH 02/37] Fix regression introduced with 8f537a1c2c271eb5cfc592b373697732597d26d6 --- backtrader/analyzers/tradeanalyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/analyzers/tradeanalyzer.py b/backtrader/analyzers/tradeanalyzer.py index 8d768318b..118a072b5 100644 --- a/backtrader/analyzers/tradeanalyzer.py +++ b/backtrader/analyzers/tradeanalyzer.py @@ -86,7 +86,7 @@ def notify_trade(self, trade): # Trade just closed won = res.won = int(trade.pnlcomm >= 0.0) - lost = res.lost = int(won) + lost = res.lost = int(not won) tlong = res.tlong = trade.long tshort = res.tshort = not trade.long From 32cdac10b8e0cb19dbe87db58b2cfc53cf12065b Mon Sep 17 00:00:00 2001 From: backtrader Date: Fri, 17 Aug 2018 13:20:27 +0200 Subject: [PATCH 03/37] Allow rollover to distinguish between no values temporarily (with None) and no values permanently (with False) --- backtrader/feeds/rollover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/feeds/rollover.py b/backtrader/feeds/rollover.py index bd00e108e..2d64b0822 100644 --- a/backtrader/feeds/rollover.py +++ b/backtrader/feeds/rollover.py @@ -152,7 +152,7 @@ def _checkcondition(self, d0, d1): def _load(self): while self._d is not None: - if not self._d.next(): # no values from current data source + if self._d.next() is not False: # no values from current data src if self._ds: self._d = self._ds.pop(0) self._dts.pop(0) From bf4052539e580845b0d59417903ff76f368b0973 Mon Sep 17 00:00:00 2001 From: backtrader Date: Sat, 25 Aug 2018 17:16:55 +0200 Subject: [PATCH 04/37] Avoid math domain error for negative returns in logarithmic calculations --- backtrader/analyzers/returns.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/backtrader/analyzers/returns.py b/backtrader/analyzers/returns.py index 7f426a376..3f9348672 100644 --- a/backtrader/analyzers/returns.py +++ b/backtrader/analyzers/returns.py @@ -124,8 +124,17 @@ def stop(self): self._value_end = self.strategy.broker.fundvalue # Compound return - self.rets['rtot'] = rtot = ( - math.log(self._value_end / self._value_start)) + try: + nlrtot = self._value_end / self._value_start + except ZeroDivisionError: + rtot = float('-inf') + else: + if nlrtot < 0.0: + rtot = float('-inf') + else: + rtot = math.log(rtot) + + self.rets['rtot'] = rtot # Average return self.rets['ravg'] = ravg = rtot / self._tcount @@ -135,7 +144,11 @@ def stop(self): if tann is None: tann = self._TANN.get(self.data._timeframe, 1.0) # assign default - self.rets['rnorm'] = rnorm = math.expm1(ravg * tann) + if ravg > float('-inf'): + self.rets['rnorm'] = rnorm = math.expm1(ravg * tann) + else: + self.rets['rnorm'] = rnorm = ravg + self.rets['rnorm100'] = rnorm * 100.0 # human readable % def _on_dt_over(self): From 57545348763060e1077654f9a94ca71c090f0d2c Mon Sep 17 00:00:00 2001 From: backtrader Date: Sat, 25 Aug 2018 17:21:01 +0200 Subject: [PATCH 05/37] Fix local variable declaration for compound returns --- backtrader/analyzers/returns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/analyzers/returns.py b/backtrader/analyzers/returns.py index 3f9348672..074cb4d61 100644 --- a/backtrader/analyzers/returns.py +++ b/backtrader/analyzers/returns.py @@ -132,7 +132,7 @@ def stop(self): if nlrtot < 0.0: rtot = float('-inf') else: - rtot = math.log(rtot) + rtot = math.log(nlrtot) self.rets['rtot'] = rtot From f087a5558601529903b71ba290d6e2cd272d78b3 Mon Sep 17 00:00:00 2001 From: backtrader Date: Thu, 6 Sep 2018 11:59:13 +0200 Subject: [PATCH 06/37] Fix typo in date2num tz conversion which shows up in direct usage --- backtrader/utils/dateintern.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/utils/dateintern.py b/backtrader/utils/dateintern.py index cfcd30be5..bcced1f53 100644 --- a/backtrader/utils/dateintern.py +++ b/backtrader/utils/dateintern.py @@ -206,7 +206,7 @@ def date2num(dt, tz=None): is a :func:`float`. """ if tz is not None: - dt = tz.localize(dwhen) + dt = tz.localize(tz) if hasattr(dt, 'tzinfo') and dt.tzinfo is not None: delta = dt.tzinfo.utcoffset(dt) From 0ae1abd9aed797f54dbb658a179a953dd69aedc8 Mon Sep 17 00:00:00 2001 From: backtrader Date: Thu, 6 Sep 2018 12:01:32 +0200 Subject: [PATCH 07/37] Release 1.9.66.122 --- backtrader/version.py | 2 +- changelog.txt | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backtrader/version.py b/backtrader/version.py index 982541b94..a0a22dd3e 100644 --- a/backtrader/version.py +++ b/backtrader/version.py @@ -22,6 +22,6 @@ unicode_literals) -__version__ = '1.9.65.122' +__version__ = '1.9.66.122' __btversion__ = tuple(int(x) for x in __version__.split('.')) diff --git a/changelog.txt b/changelog.txt index 57692b4b2..fe5b591df 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,12 @@ +1.9.66.122 + - Fix regression introduced with 8f537a1c2c271eb5cfc592b373697732597d26d6 + which voids the count of lost trades + - Allow rollover to distinguish between no values temporarily (with None) + and no values permanently (with False) + - Avoid math domain error for negative returns in logarithmic calculations + - Fix local variable declaration for compound returns + - Fix typo in date2num tz conversion which shows up in direct usage + 1.9.65.122 - Fix commission info assigment and orderref seeking in OandaStore (PR#367) - Add strategy type to OptReturn (PR#364) From e621a35fff514ddbeedf8f63d508718f643964a2 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 28 Oct 2017 18:45:18 +0300 Subject: [PATCH 08/37] Implement CCXT feed and broker This is a draft implementation - to be continued Signed-off-by: Ed Bartosh --- backtrader/brokers/__init__.py | 5 + backtrader/brokers/ccxtbroker.py | 107 +++++++++++++++ backtrader/feeds/__init__.py | 5 + backtrader/feeds/ccxt.py | 220 +++++++++++++++++++++++++++++++ 4 files changed, 337 insertions(+) create mode 100644 backtrader/brokers/ccxtbroker.py create mode 100644 backtrader/feeds/ccxt.py diff --git a/backtrader/brokers/__init__.py b/backtrader/brokers/__init__.py index df6e0dff9..b638c0d29 100644 --- a/backtrader/brokers/__init__.py +++ b/backtrader/brokers/__init__.py @@ -40,3 +40,8 @@ from .oandabroker import OandaBroker except ImportError as e: pass # The user may not have something installed + +try: + from .ccxtbroker import CCXTBroker +except ImportError as e: + pass # The user may not have something installed diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py new file mode 100644 index 000000000..1007ba1bc --- /dev/null +++ b/backtrader/brokers/ccxtbroker.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8; py-indent-offset:4 -*- +############################################################################### +# +# Copyright (C) 2015, 2016, 2017 Daniel Rodriguez +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import collections +from copy import copy +from datetime import date, datetime, timedelta +import threading +import time +import uuid + +import ccxt + +from backtrader import BrokerBase, OrderBase, Order +from backtrader.utils.py3 import with_metaclass, queue, MAXFLOAT +from backtrader.metabase import MetaParams + +class CCXTOrder(OrderBase): + def __init__(self, owner, data, ccxt_order): + self.owner = owner + self.data = data + self.ccxt_order = ccxt_order + self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell + self.size = float(ccxt_order['info']['original_amount']) + + super(CCXTOrder, self).__init__() + +class CCXTBroker(BrokerBase): + '''Broker implementation for CCXT cryptocurrency trading library. + + This class maps the orders/positions from CCXT to the + internal API of ``backtrader``. + ''' + + order_types = {Order.Market: 'market', + Order.Limit: 'limit', + Order.Stop: 'stop', + Order.StopLimit: 'stop limit'} + + def __init__(self, exchange, currency, config): + super(CCXTBroker, self).__init__() + + self.exchange = getattr(ccxt, exchange)(config) + self.currency = currency + + self.notifs = queue.Queue() # holds orders which are notified + + def getcash(self): + return self.exchange.fetch_balance()['free'][self.currency] + + def getvalue(self): + return self.exchange.fetch_balance()['total'][self.currency] + + def get_notification(self): + try: + return self.notifs.get(False) + except queue.Empty: + return None + + def notify(self, order): + self.notifs.put(order) + + def getposition(self, data): + currency = data.symbol.split('/')[0] + return self.exchange.fetch_balance()['total'][currency] + + def _submit(self, owner, data, exectype, side, amount, price, params): + order_type = self.order_types.get(exectype) + ccxt_order = self.exchange.create_order(symbol=data.symbol, type=order_type, side=side, + amount=amount, price=price, params=params) + order = CCXTOrder(owner, data, ccxt_order) + self.notify(order) + return order + + def buy(self, owner, data, size, price=None, plimit=None, + exectype=None, valid=None, tradeid=0, oco=None, + trailamount=None, trailpercent=None, + **kwargs): + return self._submit(owner, data, exectype, 'buy', size, price, kwargs) + + def sell(self, owner, data, size, price=None, plimit=None, + exectype=None, valid=None, tradeid=0, oco=None, + trailamount=None, trailpercent=None, + **kwargs): + return self._submit(owner, data, exectype, 'sell', size, price, kwargs) + + def cancel(self, order): + return self.exchange.cancel_order(self, order['id']) diff --git a/backtrader/feeds/__init__.py b/backtrader/feeds/__init__.py index e39af5633..5c0da9550 100644 --- a/backtrader/feeds/__init__.py +++ b/backtrader/feeds/__init__.py @@ -52,3 +52,8 @@ from .rollover import RollOver from .chainer import Chainer + +try: + from .ccxt import CCXT +except ImportError: + pass # The user may not have something installed diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py new file mode 100644 index 000000000..5c7488a59 --- /dev/null +++ b/backtrader/feeds/ccxt.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python +# -*- coding: utf-8; py-indent-offset:4 -*- +############################################################################### +# +# Copyright (C) 2015, 2016, 2017 Daniel Rodriguez +# Copyright (C) 2017 Ed Bartosh +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from collections import deque +from datetime import datetime +from time import sleep + +import backtrader as bt +from backtrader.feed import DataBase + +import ccxt + +class CCXT(DataBase): + """ + CryptoCurrency eXchange Trading Library Data Feed. + + Params: + + - ``historical`` (default: ``False``) + + If set to ``True`` the data feed will stop after doing the first + download of data. + + The standard data feed parameters ``fromdate`` and ``todate`` will be + used as reference. + + - ``backfill_start`` (default: ``True``) + + Perform backfilling at the start. The maximum possible historical data + will be fetched in a single request. + """ + + params = ( + ('historical', False), # only historical download + ('backfill_start', False), # do backfilling at the start + ) + + # Supported granularities + _GRANULARITIES = { + (bt.TimeFrame.Minutes, 1): '1m', + (bt.TimeFrame.Minutes, 3): '3m', + (bt.TimeFrame.Minutes, 5): '5m', + (bt.TimeFrame.Minutes, 15): '15m', + (bt.TimeFrame.Minutes, 30): '30m', + (bt.TimeFrame.Minutes, 60): '1h', + (bt.TimeFrame.Minutes, 90): '90m', + (bt.TimeFrame.Minutes, 120): '2h', + (bt.TimeFrame.Minutes, 240): '4h', + (bt.TimeFrame.Minutes, 360): '6h', + (bt.TimeFrame.Minutes, 480): '8h', + (bt.TimeFrame.Minutes, 720): '12h', + (bt.TimeFrame.Days, 1): '1d', + (bt.TimeFrame.Days, 3): '3d', + (bt.TimeFrame.Weeks, 1): '1w', + (bt.TimeFrame.Weeks, 2): '2w', + (bt.TimeFrame.Months, 1): '1M', + (bt.TimeFrame.Months, 3): '3M', + (bt.TimeFrame.Months, 6): '6M', + (bt.TimeFrame.Years, 1): '1y', + } + + # States for the Finite State Machine in _load + _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) + + def __init__(self, exchange, symbol, ohlcv_limit=10): + self.exchange = getattr(ccxt, exchange)() + self.symbol = symbol + self.ohlcv_limit = ohlcv_limit + + self._data = deque() # data queue for price data + self._last_id = None # last processed data id (trade id or timestamp for ohlcv) + + def start(self, ): + super(CCXT, self).start() + + if self.p.fromdate: + self._state = self._ST_HISTORBACK + self.put_notification(self.DELAYED) + + self._fetch_ohlcv(self.p.fromdate) + else: + self._state = self._ST_LIVE + self.put_notification(self.LIVE) + + def _load(self): + if self._state == self._ST_OVER: + return False + + while True: + if self._state == self._ST_LIVE: + if self._timeframe == bt.TimeFrame.Ticks: + return self._load_ticks() + elif self.exchange.hasFetchOHLCV: + self._fetch_ohlcv() + return self._load_ohlcv() + else: + raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ + self.exchange.name) + elif self._state == self._ST_HISTORBACK: + ret = self._load_ohlcv() + if ret: + return ret + else: + # End of historical data + if self.p.historical: # only historical + self.put_notification(self.DISCONNECTED) + self._state = self._ST_OVER + return False # end of historical + else: + self._state = self._ST_LIVE + self.put_notification(self.LIVE) + continue + + def _fetch_ohlcv(self, fromdate=None): + """Fetch OHLCV data into self._data queue""" + granularity = self._GRANULARITIES.get((self._timeframe, self._compression)) + if granularity is None: + raise ValueError("'%s' exchange doesn't support fetching OHLCV data for " + "time frame %s, comression %s" % \ + (self.exchange.name, bt.TimeFrame.getname(self._timeframe), + self._compression)) + + if fromdate: + since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000) + limit = None + else: + since = None + limit = self.ohlcv_limit + + sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds + + for ohlcv in self.exchange.fetch_ohlcv(self.symbol, timeframe=granularity, + since=since, limit=limit)[::-1]: + tstamp = ohlcv[0] + if tstamp > self._last_id: + self._data.append(ohlcv) + self._last_id = tstamp + + def _load_ticks(self): + sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds + if self._last_id is None: + # first time get the latest trade only + trades = [self.exchange.fetch_trades(self.symbol)[-1]] + else: + trades = self.exchange.fetch_trades(self.symbol) + + for trade in trades: + trade_id = trade['id'] + + if trade_id > self._last_id: + trade_time = datetime.strptime(trade['datetime'], '%Y-%m-%dT%H:%M:%S.%fZ') + self._data.append((trade_time, float(trade['price']), float(trade['amount']))) + self._last_id = trade_id + + try: + trade = self._data.popleft() + except IndexError: + return # no data in the queue + + trade_time, price, size = trade + + self.lines.datetime[0] = bt.date2num(trade_time) + self.lines.open[0] = price + self.lines.high[0] = price + self.lines.low[0] = price + self.lines.close[0] = price + self.lines.volume[0] = size + + print("%s: loaded tick: time: %s, price: %s, size: %s" % (self._name, trade_time, price, size)) + + return True + + def _load_ohlcv(self): + try: + ohlcv = self._data.popleft() + except IndexError: + return # no data in the queue + + tstamp, open_, high, low, close, volume = ohlcv + + dtime = datetime.utcfromtimestamp(tstamp // 1000) + + self.lines.datetime[0] = bt.date2num(dtime) + self.lines.open[0] = open_ + self.lines.high[0] = high + self.lines.low[0] = low + self.lines.close[0] = close + self.lines.volume[0] = volume + + print("%s: loaded ohlcv: time: %s, open: %s, high: %s, low: %s, close: %s, volume: %s" % \ + (self._name, dtime.strftime('%Y-%m-%d %H:%M:%S'), open_, high, low, close, volume)) + + return True + + def haslivedata(self): + return self._state == self._ST_LIVE and self._data + + def islive(self): + return not self.p.historical From 68a6f8094b15fd2432380aa0f4d050c9d22db37c Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 27 Nov 2017 18:41:52 +0000 Subject: [PATCH 09/37] Made the code to work on Python3 Signed-off-by: Ed Bartosh --- backtrader/dataseries.py | 2 +- backtrader/feeds/ccxt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/dataseries.py b/backtrader/dataseries.py index 2770ffb27..a87df819c 100644 --- a/backtrader/dataseries.py +++ b/backtrader/dataseries.py @@ -40,7 +40,7 @@ class TimeFrame(object): names = Names # support old naming convention @classmethod - def getname(cls, tframe, compression=None): + def getname(cls, tframe, compression=1): tname = cls.Names[tframe] if compression > 1 or tname == cls.Names[-1]: return tname # for plural or 'NoTimeFrame' return plain entry diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 5c7488a59..c2781e0fe 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -89,7 +89,7 @@ def __init__(self, exchange, symbol, ohlcv_limit=10): self.ohlcv_limit = ohlcv_limit self._data = deque() # data queue for price data - self._last_id = None # last processed data id (trade id or timestamp for ohlcv) + self._last_id = '' # last processed data id (trade id or timestamp for ohlcv) def start(self, ): super(CCXT, self).start() From 0c5901b292286d3cf50260b9dd320488c39f24dd Mon Sep 17 00:00:00 2001 From: Daniel Drojanov Date: Sun, 3 Dec 2017 17:42:37 +0200 Subject: [PATCH 10/37] Added property to monitor last timestamp Instead of using last_id added new property _last_ts to track last data timestamp. This should fix a bug where no data was added when using a time based frame. --- backtrader/feeds/ccxt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index c2781e0fe..7988651b4 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -89,7 +89,8 @@ def __init__(self, exchange, symbol, ohlcv_limit=10): self.ohlcv_limit = ohlcv_limit self._data = deque() # data queue for price data - self._last_id = '' # last processed data id (trade id or timestamp for ohlcv) + self._last_id = '' # last processed trade id for ohlcv + self._last_ts = 0 # last processed timestamp for ohlcv def start(self, ): super(CCXT, self).start() @@ -153,9 +154,9 @@ def _fetch_ohlcv(self, fromdate=None): for ohlcv in self.exchange.fetch_ohlcv(self.symbol, timeframe=granularity, since=since, limit=limit)[::-1]: tstamp = ohlcv[0] - if tstamp > self._last_id: + if tstamp > self._last_ts: self._data.append(ohlcv) - self._last_id = tstamp + self._last_ts = tstamp def _load_ticks(self): sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds From 8f4aea4d56636e0f541dcca11f90f08575b21fb4 Mon Sep 17 00:00:00 2001 From: Daniel Drojanov Date: Wed, 13 Dec 2017 10:09:56 +0200 Subject: [PATCH 11/37] Move check to _fetch_ohlcv Check if exchange supports OHLCV fetching in _fetch_ohlcv method. This should prevent code duplication because this was not handled in the "start" method. --- backtrader/feeds/ccxt.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 7988651b4..b0dccb76a 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -112,12 +112,9 @@ def _load(self): if self._state == self._ST_LIVE: if self._timeframe == bt.TimeFrame.Ticks: return self._load_ticks() - elif self.exchange.hasFetchOHLCV: + else: self._fetch_ohlcv() return self._load_ohlcv() - else: - raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ - self.exchange.name) elif self._state == self._ST_HISTORBACK: ret = self._load_ohlcv() if ret: @@ -135,6 +132,10 @@ def _load(self): def _fetch_ohlcv(self, fromdate=None): """Fetch OHLCV data into self._data queue""" + if not self.exchange.hasFetchOHLCV: + raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ + self.exchange.name) + granularity = self._GRANULARITIES.get((self._timeframe, self._compression)) if granularity is None: raise ValueError("'%s' exchange doesn't support fetching OHLCV data for " From 79a1f951239ad48380ea3841902e05f2b318659c Mon Sep 17 00:00:00 2001 From: Daniel Drojanov Date: Wed, 13 Dec 2017 10:13:05 +0200 Subject: [PATCH 12/37] Fetch the ohlcv data only since the last timestamp Fetched data since the last timestamp instead of fetching the whole data feed which made it more slower. This makes it more efficient to feed the live data, so for each iteration it will fetch only the newer data. --- backtrader/feeds/ccxt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index b0dccb76a..2c0df0564 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -147,7 +147,11 @@ def _fetch_ohlcv(self, fromdate=None): since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000) limit = None else: - since = None + if 0 < self._last_ts: + since = self._last_ts + else: + since = None + limit = self.ohlcv_limit sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds From bdb93b176d404e7a06e92223574cae84af0de78f Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sun, 17 Dec 2017 12:29:47 +0000 Subject: [PATCH 13/37] Fetch OHLCV data in chunks Fetched OHLCV data in chunks of ohlcv_limit bars. Do it while there is new data available. This should fix hangups and crashes when a lot of history data requested. Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 2c0df0564..ab67ccbca 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -83,7 +83,7 @@ class CCXT(DataBase): # States for the Finite State Machine in _load _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) - def __init__(self, exchange, symbol, ohlcv_limit=10): + def __init__(self, exchange, symbol, ohlcv_limit=450): self.exchange = getattr(ccxt, exchange)() self.symbol = symbol self.ohlcv_limit = ohlcv_limit @@ -145,23 +145,28 @@ def _fetch_ohlcv(self, fromdate=None): if fromdate: since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000) - limit = None else: if 0 < self._last_ts: since = self._last_ts else: since = None - limit = self.ohlcv_limit + limit = self.ohlcv_limit - sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds - - for ohlcv in self.exchange.fetch_ohlcv(self.symbol, timeframe=granularity, - since=since, limit=limit)[::-1]: - tstamp = ohlcv[0] - if tstamp > self._last_ts: - self._data.append(ohlcv) - self._last_ts = tstamp + while True: + sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds + + dlen = len(self._data) + for ohlcv in self.exchange.fetch_ohlcv(self.symbol, timeframe=granularity, + since=since, limit=limit)[::-1]: + tstamp = ohlcv[0] + if tstamp > self._last_ts: + self._data.append(ohlcv) + self._last_ts = tstamp + since = tstamp + 1 + + if dlen == len(self._data): + break def _load_ticks(self): sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds From 93c1ae7d27cb7dce9c52fed5dc4be09e7ff91c52 Mon Sep 17 00:00:00 2001 From: Michael Korbakov Date: Wed, 20 Dec 2017 20:28:56 +0200 Subject: [PATCH 14/37] Add config to CCXT feed parameters closes #3 --- backtrader/feeds/ccxt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index ab67ccbca..90f4408d9 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -83,8 +83,8 @@ class CCXT(DataBase): # States for the Finite State Machine in _load _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) - def __init__(self, exchange, symbol, ohlcv_limit=450): - self.exchange = getattr(ccxt, exchange)() + def __init__(self, exchange, symbol, ohlcv_limit=450, config={}): + self.exchange = getattr(ccxt, exchange)(config) self.symbol = symbol self.ohlcv_limit = ohlcv_limit From 846c8f6bef09badb729cd0fa41b6eb246b4f4c29 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 23 Dec 2017 16:33:24 +0000 Subject: [PATCH 15/37] Move ccxt API calls to ccxtstore We'll need to handle request timeouts and other things, so it makes sense to move all ccxt-related functionality to the separate module to be able to call it from broker and feed classes. Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 24 ++++---- backtrader/feeds/ccxt.py | 55 +++--------------- backtrader/stores/ccxtstore.py | 96 ++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 58 deletions(-) create mode 100644 backtrader/stores/ccxtstore.py diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 1007ba1bc..662530429 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -28,11 +28,10 @@ import time import uuid -import ccxt - from backtrader import BrokerBase, OrderBase, Order from backtrader.utils.py3 import with_metaclass, queue, MAXFLOAT from backtrader.metabase import MetaParams +from backtrader.stores.ccxtstore import CCXTStore class CCXTOrder(OrderBase): def __init__(self, owner, data, ccxt_order): @@ -59,16 +58,17 @@ class CCXTBroker(BrokerBase): def __init__(self, exchange, currency, config): super(CCXTBroker, self).__init__() - self.exchange = getattr(ccxt, exchange)(config) + self.store = CCXTStore(exchange, config) + self.currency = currency self.notifs = queue.Queue() # holds orders which are notified def getcash(self): - return self.exchange.fetch_balance()['free'][self.currency] + return self.store.getcash(self.currency) - def getvalue(self): - return self.exchange.fetch_balance()['total'][self.currency] + def getvalue(self, datas=None): + return self.store.getvalue(self.currency) def get_notification(self): try: @@ -81,15 +81,15 @@ def notify(self, order): def getposition(self, data): currency = data.symbol.split('/')[0] - return self.exchange.fetch_balance()['total'][currency] + return self.store.getposition(currency) def _submit(self, owner, data, exectype, side, amount, price, params): order_type = self.order_types.get(exectype) - ccxt_order = self.exchange.create_order(symbol=data.symbol, type=order_type, side=side, - amount=amount, price=price, params=params) - order = CCXTOrder(owner, data, ccxt_order) + _order = self.store.create_order(symbol=data.symbol, order_type=order_type, side=side, + amount=amount, price=price, params=params) + order = CCXTOrder(owner, data, _order) self.notify(order) - return order + return order def buy(self, owner, data, size, price=None, plimit=None, exectype=None, valid=None, tradeid=0, oco=None, @@ -104,4 +104,4 @@ def sell(self, owner, data, size, price=None, plimit=None, return self._submit(owner, data, exectype, 'sell', size, price, kwargs) def cancel(self, order): - return self.exchange.cancel_order(self, order['id']) + return self.store.cancel_order(order['id']) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 90f4408d9..9dd9b649d 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -24,12 +24,10 @@ from collections import deque from datetime import datetime -from time import sleep import backtrader as bt from backtrader.feed import DataBase - -import ccxt +from backtrader.stores.ccxtstore import CCXTStore class CCXT(DataBase): """ @@ -56,44 +54,21 @@ class CCXT(DataBase): ('backfill_start', False), # do backfilling at the start ) - # Supported granularities - _GRANULARITIES = { - (bt.TimeFrame.Minutes, 1): '1m', - (bt.TimeFrame.Minutes, 3): '3m', - (bt.TimeFrame.Minutes, 5): '5m', - (bt.TimeFrame.Minutes, 15): '15m', - (bt.TimeFrame.Minutes, 30): '30m', - (bt.TimeFrame.Minutes, 60): '1h', - (bt.TimeFrame.Minutes, 90): '90m', - (bt.TimeFrame.Minutes, 120): '2h', - (bt.TimeFrame.Minutes, 240): '4h', - (bt.TimeFrame.Minutes, 360): '6h', - (bt.TimeFrame.Minutes, 480): '8h', - (bt.TimeFrame.Minutes, 720): '12h', - (bt.TimeFrame.Days, 1): '1d', - (bt.TimeFrame.Days, 3): '3d', - (bt.TimeFrame.Weeks, 1): '1w', - (bt.TimeFrame.Weeks, 2): '2w', - (bt.TimeFrame.Months, 1): '1M', - (bt.TimeFrame.Months, 3): '3M', - (bt.TimeFrame.Months, 6): '6M', - (bt.TimeFrame.Years, 1): '1y', - } - # States for the Finite State Machine in _load _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) def __init__(self, exchange, symbol, ohlcv_limit=450, config={}): - self.exchange = getattr(ccxt, exchange)(config) self.symbol = symbol self.ohlcv_limit = ohlcv_limit + self.store = CCXTStore(exchange, config) + self._data = deque() # data queue for price data self._last_id = '' # last processed trade id for ohlcv self._last_ts = 0 # last processed timestamp for ohlcv def start(self, ): - super(CCXT, self).start() + DataBase.start(self) if self.p.fromdate: self._state = self._ST_HISTORBACK @@ -132,16 +107,7 @@ def _load(self): def _fetch_ohlcv(self, fromdate=None): """Fetch OHLCV data into self._data queue""" - if not self.exchange.hasFetchOHLCV: - raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ - self.exchange.name) - - granularity = self._GRANULARITIES.get((self._timeframe, self._compression)) - if granularity is None: - raise ValueError("'%s' exchange doesn't support fetching OHLCV data for " - "time frame %s, comression %s" % \ - (self.exchange.name, bt.TimeFrame.getname(self._timeframe), - self._compression)) + granularity = self.store.get_granularity(self._timeframe, self._compression) if fromdate: since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000) @@ -154,11 +120,9 @@ def _fetch_ohlcv(self, fromdate=None): limit = self.ohlcv_limit while True: - sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds - dlen = len(self._data) - for ohlcv in self.exchange.fetch_ohlcv(self.symbol, timeframe=granularity, - since=since, limit=limit)[::-1]: + for ohlcv in self.store.fetch_ohlcv(self.symbol, timeframe=granularity, + since=since, limit=limit)[::-1]: tstamp = ohlcv[0] if tstamp > self._last_ts: self._data.append(ohlcv) @@ -169,12 +133,11 @@ def _fetch_ohlcv(self, fromdate=None): break def _load_ticks(self): - sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds if self._last_id is None: # first time get the latest trade only - trades = [self.exchange.fetch_trades(self.symbol)[-1]] + trades = [self.store.fetch_trades(self.symbol)[-1]] else: - trades = self.exchange.fetch_trades(self.symbol) + trades = self.store.fetch_trades(self.symbol) for trade in trades: trade_id = trade['id'] diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py new file mode 100644 index 000000000..5098de4b2 --- /dev/null +++ b/backtrader/stores/ccxtstore.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8; py-indent-offset:4 -*- +############################################################################### +# +# Copyright (C) 2017 Ed Bartosh +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import time + +import ccxt + +import backtrader as bt + + +class CCXTStore(object): + '''API provider for CCXT feed and broker classes.''' + + # Supported granularities + _GRANULARITIES = { + (bt.TimeFrame.Minutes, 1): '1m', + (bt.TimeFrame.Minutes, 3): '3m', + (bt.TimeFrame.Minutes, 5): '5m', + (bt.TimeFrame.Minutes, 15): '15m', + (bt.TimeFrame.Minutes, 30): '30m', + (bt.TimeFrame.Minutes, 60): '1h', + (bt.TimeFrame.Minutes, 90): '90m', + (bt.TimeFrame.Minutes, 120): '2h', + (bt.TimeFrame.Minutes, 240): '4h', + (bt.TimeFrame.Minutes, 360): '6h', + (bt.TimeFrame.Minutes, 480): '8h', + (bt.TimeFrame.Minutes, 720): '12h', + (bt.TimeFrame.Days, 1): '1d', + (bt.TimeFrame.Days, 3): '3d', + (bt.TimeFrame.Weeks, 1): '1w', + (bt.TimeFrame.Weeks, 2): '2w', + (bt.TimeFrame.Months, 1): '1M', + (bt.TimeFrame.Months, 3): '3M', + (bt.TimeFrame.Months, 6): '6M', + (bt.TimeFrame.Years, 1): '1y', + } + + def __init__(self, exchange, config): + self.exchange = getattr(ccxt, exchange)(config) + + def get_granularity(self, timeframe, compression): + if not self.exchange.hasFetchOHLCV: + raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ + self.exchange.name) + + granularity = self._GRANULARITIES.get((timeframe, compression)) + if granularity is None: + raise ValueError("'%s' exchange doesn't support fetching OHLCV data for " + "time frame %s, comression %s" % \ + (self.exchange.name, bt.TimeFrame.getname(timeframe), compression)) + + return granularity + + def getcash(self, currency): + return self.exchange.fetch_balance()['free'][currency] + + def getvalue(self, currency): + return self.exchange.fetch_balance()['total'][currency] + + def getposition(self, currency): + return self.getvalue(currency) + + def create_order(self, symbol, order_type, side, amount, price, params): + return self.exchange.create_order(symbol=symbol, type=order_type, side=side, + amount=amount, price=price, params=params) + + def cancel_order(self, order_id): + return self.exchange.cancel_order(order_id) + + def fetch_trades(self, symbol): + time.sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds + return self.exchange.fetch_trades(symbol) + + def fetch_ohlcv(self, symbol, timeframe, since, limit): + time.sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds + return self.exchange.fetch_ohlcv(symbol, timeframe=timeframe, since=since, limit=limit) From d635901c63e7a1b8def8e56f6c03390f79e3d60a Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 23 Dec 2017 16:50:34 +0000 Subject: [PATCH 16/37] Implement retrying queries Retry exchange queries when network error[s] occurs. Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 4 ++-- backtrader/feeds/ccxt.py | 4 ++-- backtrader/stores/ccxtstore.py | 29 +++++++++++++++++++++++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 662530429..4ec031e4e 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -55,10 +55,10 @@ class CCXTBroker(BrokerBase): Order.Stop: 'stop', Order.StopLimit: 'stop limit'} - def __init__(self, exchange, currency, config): + def __init__(self, exchange, currency, config, retries=5): super(CCXTBroker, self).__init__() - self.store = CCXTStore(exchange, config) + self.store = CCXTStore(exchange, config, retries) self.currency = currency diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 9dd9b649d..6a1515232 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -57,11 +57,11 @@ class CCXT(DataBase): # States for the Finite State Machine in _load _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) - def __init__(self, exchange, symbol, ohlcv_limit=450, config={}): + def __init__(self, exchange, symbol, ohlcv_limit=450, config={}, retries=5): self.symbol = symbol self.ohlcv_limit = ohlcv_limit - self.store = CCXTStore(exchange, config) + self.store = CCXTStore(exchange, config, retries) self._data = deque() # data queue for price data self._last_id = '' # last processed trade id for ohlcv diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 5098de4b2..f6d9fcadb 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -22,9 +22,11 @@ unicode_literals) import time - import ccxt +from functools import wraps +from ccxt.base.errors import NetworkError + import backtrader as bt @@ -55,8 +57,9 @@ class CCXTStore(object): (bt.TimeFrame.Years, 1): '1y', } - def __init__(self, exchange, config): + def __init__(self, exchange, config, retries): self.exchange = getattr(ccxt, exchange)(config) + self.retries = retries def get_granularity(self, timeframe, compression): if not self.exchange.hasFetchOHLCV: @@ -71,26 +74,44 @@ def get_granularity(self, timeframe, compression): return granularity + def retry(method): + @wraps(method) + def retry_method(self, *args, **kwargs): + for i in range(self.retries): + time.sleep(self.exchange.rateLimit / 1000) + try: + return method(self, *args, **kwargs) + except NetworkError: + if i == self.retries - 1: + raise + + return retry_method + + @retry def getcash(self, currency): return self.exchange.fetch_balance()['free'][currency] + @retry def getvalue(self, currency): return self.exchange.fetch_balance()['total'][currency] + @retry def getposition(self, currency): return self.getvalue(currency) + @retry def create_order(self, symbol, order_type, side, amount, price, params): return self.exchange.create_order(symbol=symbol, type=order_type, side=side, amount=amount, price=price, params=params) + @retry def cancel_order(self, order_id): return self.exchange.cancel_order(order_id) + @retry def fetch_trades(self, symbol): - time.sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds return self.exchange.fetch_trades(symbol) + @retry def fetch_ohlcv(self, symbol, timeframe, since, limit): - time.sleep(self.exchange.rateLimit / 1000) # time.sleep wants seconds return self.exchange.fetch_ohlcv(symbol, timeframe=timeframe, since=since, limit=limit) From 0f95cc18eb724c8dc32956e797ce436db4ea2dbc Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 23 Dec 2017 17:02:30 +0000 Subject: [PATCH 17/37] Fix Pylint warnings&errors Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 10 +--------- backtrader/feeds/ccxt.py | 11 +++-------- backtrader/stores/ccxtstore.py | 4 ++-- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 4ec031e4e..7f6df1d2f 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -21,16 +21,8 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import collections -from copy import copy -from datetime import date, datetime, timedelta -import threading -import time -import uuid - from backtrader import BrokerBase, OrderBase, Order -from backtrader.utils.py3 import with_metaclass, queue, MAXFLOAT -from backtrader.metabase import MetaParams +from backtrader.utils.py3 import queue from backtrader.stores.ccxtstore import CCXTStore class CCXTOrder(OrderBase): diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 6a1515232..94b0ec8e6 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -112,7 +112,7 @@ def _fetch_ohlcv(self, fromdate=None): if fromdate: since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000) else: - if 0 < self._last_ts: + if self._last_ts > 0: since = self._last_ts else: since = None @@ -150,7 +150,7 @@ def _load_ticks(self): try: trade = self._data.popleft() except IndexError: - return # no data in the queue + return False # no data in the queue trade_time, price, size = trade @@ -161,15 +161,13 @@ def _load_ticks(self): self.lines.close[0] = price self.lines.volume[0] = size - print("%s: loaded tick: time: %s, price: %s, size: %s" % (self._name, trade_time, price, size)) - return True def _load_ohlcv(self): try: ohlcv = self._data.popleft() except IndexError: - return # no data in the queue + return False # no data in the queue tstamp, open_, high, low, close, volume = ohlcv @@ -182,9 +180,6 @@ def _load_ohlcv(self): self.lines.close[0] = close self.lines.volume[0] = volume - print("%s: loaded ohlcv: time: %s, open: %s, high: %s, low: %s, close: %s, volume: %s" % \ - (self._name, dtime.strftime('%Y-%m-%d %H:%M:%S'), open_, high, low, close, volume)) - return True def haslivedata(self): diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index f6d9fcadb..13d001ffc 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -22,9 +22,9 @@ unicode_literals) import time -import ccxt - from functools import wraps + +import ccxt from ccxt.base.errors import NetworkError import backtrader as bt From e85744ddd66647ceef619e9a38f8dae975a3fef7 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sun, 24 Dec 2017 12:17:58 +0000 Subject: [PATCH 18/37] Check if exchange supports requested time frame Checked if requested timeframe/compression is supported by the exchange. Signed-off-by: Ed Bartosh --- backtrader/stores/ccxtstore.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 13d001ffc..449d6a0d7 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -68,9 +68,13 @@ def get_granularity(self, timeframe, compression): granularity = self._GRANULARITIES.get((timeframe, compression)) if granularity is None: + raise ValueError("backtrader CCXT module doesn't support fetching OHLCV " + "data for time frame %s, comression %s" % \ + (bt.TimeFrame.getname(timeframe), compression)) + + if self.exchange.timeframes and granularity not in self.exchange.timeframes: raise ValueError("'%s' exchange doesn't support fetching OHLCV data for " - "time frame %s, comression %s" % \ - (self.exchange.name, bt.TimeFrame.getname(timeframe), compression)) + "%s time frame" % (self.exchange.name, granularity)) return granularity From 918741e218b8d3369188b943822753ad6ed4e84a Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sun, 7 Jan 2018 23:07:51 +0000 Subject: [PATCH 19/37] return None when next bar is not available _load method of the feed should returned False when price data is not yet available. This caused cerebro to stop. It should return None in this case to indicate timeout getting the data. Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 94b0ec8e6..8db3cba29 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -150,7 +150,7 @@ def _load_ticks(self): try: trade = self._data.popleft() except IndexError: - return False # no data in the queue + return None # no data in the queue trade_time, price, size = trade @@ -167,7 +167,7 @@ def _load_ohlcv(self): try: ohlcv = self._data.popleft() except IndexError: - return False # no data in the queue + return None # no data in the queue tstamp, open_, high, low, close, volume = ohlcv From 1897cea029b662eecae70584a6f2fccefe2ba2e5 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 8 Jan 2018 00:26:02 +0000 Subject: [PATCH 20/37] skip incomplete ohlcv bars Don't process ohlcv bars that contain None prices Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 8db3cba29..d674e0377 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -123,6 +123,9 @@ def _fetch_ohlcv(self, fromdate=None): dlen = len(self._data) for ohlcv in self.store.fetch_ohlcv(self.symbol, timeframe=granularity, since=since, limit=limit)[::-1]: + if None in ohlcv: + continue + tstamp = ohlcv[0] if tstamp > self._last_ts: self._data.append(ohlcv) From 3d6acd57ac553640a79dece9a3353af543b8e239 Mon Sep 17 00:00:00 2001 From: napman Date: Sat, 13 Jan 2018 03:36:20 +0500 Subject: [PATCH 21/37] fix default ohlcv_limit Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index d674e0377..e12ced25b 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -57,7 +57,7 @@ class CCXT(DataBase): # States for the Finite State Machine in _load _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3) - def __init__(self, exchange, symbol, ohlcv_limit=450, config={}, retries=5): + def __init__(self, exchange, symbol, ohlcv_limit=None, config={}, retries=5): self.symbol = symbol self.ohlcv_limit = ohlcv_limit From 98e622c4f77bc2706e463dbc2098ff6e109a102d Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 16 Jan 2018 22:50:04 +0000 Subject: [PATCH 22/37] sorted fetch_ohlcv output to ensure order Default order of output historical bars is not the same for different exchanges. Explicitly specifying it with params={"reverse": False} seems not having any effect. Had to sort the output in the feed code to solve this. Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index e12ced25b..728fb5ad4 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -121,8 +121,8 @@ def _fetch_ohlcv(self, fromdate=None): while True: dlen = len(self._data) - for ohlcv in self.store.fetch_ohlcv(self.symbol, timeframe=granularity, - since=since, limit=limit)[::-1]: + for ohlcv in sorted(self.store.fetch_ohlcv(self.symbol, timeframe=granularity, + since=since, limit=limit)): if None in ohlcv: continue From 8782594ef29f2b7cb3e71fd0edd3bd75932ea1e1 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 17 Jan 2018 13:25:34 +0000 Subject: [PATCH 23/37] ccxtbroker: don't use unparsed order properties Used public order properties 'side' and 'amount' instead of original unparsed properties ['info']['side'] and ['info']['original_amount']. The latter are exchange-specific and may not be provided by all exchanges. This should fix crashes like this: Traceback (most recent call last): File "machine_engin.py", line 161, in cerebro.run() File "backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "backtrader/cerebro.py", line 1295, in runstrategies self._runnext(runstrats) File "backtrader/cerebro.py", line 1626, in _runnext strat._next() File "backtrader/strategy.py", line 325, in _next super(Strategy, self)._next() File "backtrader/lineiterator.py", line 266, in _next self.next() File "machine_engin_prod.py", line 138, in next price=self.data0.close[0]) File "backtrader/strategy.py", line 945, in sell **kwargs) File "backtrader/brokers/ccxtbroker.py", line 96, in sell return self._submit(owner, data, exectype, 'sell', size, price, kwargs) File "backtrader/brokers/ccxtbroker.py", line 82, in _submit order = CCXTOrder(owner, data, _order) File "backtrader/metabase.py", line 88, in __call__ _obj, args, kwargs = cls.doinit(_obj, *args, **kwargs) File "backtrader/metabase.py", line 78, in doinit _obj.__init__(*args, **kwargs) File "backtrader/brokers/ccxtbroker.py", line 34, in __init__ self.size = float(ccxt_order['info']['original_amount']) KeyError: 'original_amount' Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 7f6df1d2f..d9677e175 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -30,8 +30,8 @@ def __init__(self, owner, data, ccxt_order): self.owner = owner self.data = data self.ccxt_order = ccxt_order - self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell - self.size = float(ccxt_order['info']['original_amount']) + self.ordtype = self.Buy if ccxt_order['side'] == 'buy' else self.Sell + self.size = float(ccxt_order['amount']) super(CCXTOrder, self).__init__() From 0f044c8280fd1695fdf8c34cca3dc47af5040887 Mon Sep 17 00:00:00 2001 From: Kris Marsh Date: Wed, 24 Jan 2018 15:10:04 +0000 Subject: [PATCH 24/37] Fix for hasFetchOHLCV tag, so we work correctly with exchanges like Kraken that use the newer format --- backtrader/stores/ccxtstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 449d6a0d7..e550e5abf 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -62,7 +62,7 @@ def __init__(self, exchange, config, retries): self.retries = retries def get_granularity(self, timeframe, compression): - if not self.exchange.hasFetchOHLCV: + if not self.exchange.has['fetchOHLCV']: raise NotImplementedError("'%s' exchange doesn't support fetching OHLCV data" % \ self.exchange.name) From 10fb4c43af6811818f32aebb8dc84ac029417d47 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 3 Feb 2018 18:37:02 +0000 Subject: [PATCH 25/37] ccxtbroker: implement get_orders_open Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 3 +++ backtrader/stores/ccxtstore.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index d9677e175..c59f921b7 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -97,3 +97,6 @@ def sell(self, owner, data, size, price=None, plimit=None, def cancel(self, order): return self.store.cancel_order(order['id']) + + def get_orders_open(self, safe=False): + return self.store.fetch_open_orders() diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index e550e5abf..16f640627 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -119,3 +119,7 @@ def fetch_trades(self, symbol): @retry def fetch_ohlcv(self, symbol, timeframe, since, limit): return self.exchange.fetch_ohlcv(symbol, timeframe=timeframe, since=since, limit=limit) + + @retry + def fetch_open_orders(self): + return self.exchange.fetchOpenOrders() From 0af82201578788233cebdacd46c81a0b637f1522 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 27 Feb 2018 21:58:15 +0000 Subject: [PATCH 26/37] Retry when exchange errors occur ExchangeError occurs randomly from time to time on some exchanges. Added it to the list of exceptions in retry decorator to retry when it occurs. Signed-off-by: Ed Bartosh --- backtrader/stores/ccxtstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 16f640627..9075bc95a 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -25,7 +25,7 @@ from functools import wraps import ccxt -from ccxt.base.errors import NetworkError +from ccxt.base.errors import NetworkError, ExchangeError import backtrader as bt @@ -85,7 +85,7 @@ def retry_method(self, *args, **kwargs): time.sleep(self.exchange.rateLimit / 1000) try: return method(self, *args, **kwargs) - except NetworkError: + except (NetworkError, ExchangeError): if i == self.retries - 1: raise From bf2b59f2df19f5b942d7bb31c4d770bba40f9dbd Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 21 Apr 2018 17:57:47 +0100 Subject: [PATCH 27/37] Fix AttributeError: 'CCXTBroker' object has no attribute 'get_value' Added get_value and get_cash methods to the CCXTBroker to fix this error and similar error for get_cash: Traceback (most recent call last): strategy = cerebro.run()[0] File "backtrader-ccxt/backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "backtrader-ccxt/backtrader/cerebro.py", line 1290, in runstrategies self._runonce(runstrats) File "backtrader-ccxt/backtrader/cerebro.py", line 1691, in _runonce strat._oncepost(dt0) File "backtrader-ccxt/backtrader/strategy.py", line 293, in _oncepost self._next_analyzers(minperstatus, once=True) File "backtrader-ccxt/backtrader/strategy.py", line 364, in _next_analyzers analyzer._nextstart() # only called for the 1st value File "backtrader-ccxt/backtrader/analyzer.py", line 180, in _nextstart child._nextstart() File "backtrader-ccxt/backtrader/analyzer.py", line 182, in _nextstart self.nextstart() File "backtrader-ccxt/backtrader/analyzer.py", line 235, in nextstart self.next() File "backtrader-ccxt/backtrader/analyzers/positions.py", line 78, in next pvals = [self.strategy.broker.get_value([d]) for d in self.datas] File "backtrader-ccxt/backtrader/analyzers/positions.py", line 78, in pvals = [self.strategy.broker.get_value([d]) for d in self.datas] AttributeError: 'CCXTBroker' object has no attribute 'get_value' Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index c59f921b7..6d6fb3610 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -75,6 +75,12 @@ def getposition(self, data): currency = data.symbol.split('/')[0] return self.store.getposition(currency) + def get_value(self, datas=None, mkt=False, lever=False): + return self.store.getvalue(self.currency) + + def get_cash(self): + return self.store.getcash(self.currency) + def _submit(self, owner, data, exectype, side, amount, price, params): order_type = self.order_types.get(exectype) _order = self.store.create_order(symbol=data.symbol, order_type=order_type, side=side, From ead42b35ac1b43a93db68a015a7ddfe8fcf61686 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 16 Jun 2018 14:54:16 +0300 Subject: [PATCH 28/37] Set default value 0.0 for getcash and getvalue methods Signed-off-by: Ed Bartosh --- backtrader/stores/ccxtstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 9075bc95a..58d4fea36 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -93,11 +93,11 @@ def retry_method(self, *args, **kwargs): @retry def getcash(self, currency): - return self.exchange.fetch_balance()['free'][currency] + return self.exchange.fetch_balance()['free'].get(currency, 0.0) @retry def getvalue(self, currency): - return self.exchange.fetch_balance()['total'][currency] + return self.exchange.fetch_balance()['total'].get(currency, 0.0) @retry def getposition(self, currency): From d486460eb67cf8b7540dfaaaea91ce3639a247a5 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sun, 5 Aug 2018 12:32:38 +0300 Subject: [PATCH 29/37] fix possible KeyError exceptions Used ccxt_order['info'] data structure to fix this exception: File "backtrader/brokers/ccxtbroker.py", line 33, in __init__ self.ordtype = self.Buy if ccxt_order['side'] == 'buy' else self.Sell KeyError: 'side' Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 6d6fb3610..bae8300ed 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -30,8 +30,8 @@ def __init__(self, owner, data, ccxt_order): self.owner = owner self.data = data self.ccxt_order = ccxt_order - self.ordtype = self.Buy if ccxt_order['side'] == 'buy' else self.Sell - self.size = float(ccxt_order['amount']) + self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell + self.size = float(ccxt_order['info']['original_amount']) super(CCXTOrder, self).__init__() From 4ffbea70f7809eb8f762ed5dec40fdda5a7fc627 Mon Sep 17 00:00:00 2001 From: jacob hanouna Date: Tue, 7 Aug 2018 12:21:13 +0300 Subject: [PATCH 30/37] Moved CCXTOrder class and fix cancel method. --- backtrader/brokers/ccxtbroker.py | 16 +++------------- backtrader/stores/ccxtstore.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index bae8300ed..21a68802d 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -21,19 +21,9 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from backtrader import BrokerBase, OrderBase, Order +from backtrader import BrokerBase, Order from backtrader.utils.py3 import queue -from backtrader.stores.ccxtstore import CCXTStore - -class CCXTOrder(OrderBase): - def __init__(self, owner, data, ccxt_order): - self.owner = owner - self.data = data - self.ccxt_order = ccxt_order - self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell - self.size = float(ccxt_order['info']['original_amount']) - - super(CCXTOrder, self).__init__() +from backtrader.stores.ccxtstore import CCXTStore, CCXTOrder class CCXTBroker(BrokerBase): '''Broker implementation for CCXT cryptocurrency trading library. @@ -102,7 +92,7 @@ def sell(self, owner, data, size, price=None, plimit=None, return self._submit(owner, data, exectype, 'sell', size, price, kwargs) def cancel(self, order): - return self.store.cancel_order(order['id']) + return self.store.cancel_order(order) def get_orders_open(self, safe=False): return self.store.fetch_open_orders() diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 58d4fea36..673e9ee42 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -28,7 +28,17 @@ from ccxt.base.errors import NetworkError, ExchangeError import backtrader as bt +from backtrader import OrderBase +class CCXTOrder(OrderBase): + def __init__(self, owner, data, ccxt_order): + self.owner = owner + self.data = data + self.ccxt_order = ccxt_order + self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell + self.size = float(ccxt_order['info']['original_amount']) + + super(CCXTOrder, self).__init__() class CCXTStore(object): '''API provider for CCXT feed and broker classes.''' @@ -109,8 +119,8 @@ def create_order(self, symbol, order_type, side, amount, price, params): amount=amount, price=price, params=params) @retry - def cancel_order(self, order_id): - return self.exchange.cancel_order(order_id) + def cancel_order(self, order): + return self.exchange.cancel_order(order.ccxt_order['id']) @retry def fetch_trades(self, symbol): From cc4ba6778333391813d8301c3cbc45e9f4f79469 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 14 Aug 2018 21:39:07 +0300 Subject: [PATCH 31/37] parse order in create_order method This should make all order keys available on the first level (order["amount"] vs order["info"]["amount"]) for all supported exchanges. Signed-off-by: Ed Bartosh --- backtrader/stores/ccxtstore.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 673e9ee42..962d8baf8 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -35,8 +35,8 @@ def __init__(self, owner, data, ccxt_order): self.owner = owner self.data = data self.ccxt_order = ccxt_order - self.ordtype = self.Buy if ccxt_order['info']['side'] == 'buy' else self.Sell - self.size = float(ccxt_order['info']['original_amount']) + self.ordtype = self.Buy if ccxt_order['side'] == 'buy' else self.Sell + self.size = float(ccxt_order['amount']) super(CCXTOrder, self).__init__() @@ -115,8 +115,9 @@ def getposition(self, currency): @retry def create_order(self, symbol, order_type, side, amount, price, params): - return self.exchange.create_order(symbol=symbol, type=order_type, side=side, - amount=amount, price=price, params=params) + order = self.exchange.create_order(symbol=symbol, type=order_type, side=side, + amount=amount, price=price, params=params) + return self.exchange.parse_order(order['info']) @retry def cancel_order(self, order): From c836840499930faaa9a39767401c59a6eced6f42 Mon Sep 17 00:00:00 2001 From: JacobHanouna Date: Mon, 20 Aug 2018 12:19:31 +0300 Subject: [PATCH 32/37] minor fix on load_ticks self._last_id was set initially to ''. meaning this condition never resulted true ('if self._last_id is None:') --- backtrader/feeds/ccxt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 728fb5ad4..f5b5f90ba 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -64,7 +64,7 @@ def __init__(self, exchange, symbol, ohlcv_limit=None, config={}, retries=5): self.store = CCXTStore(exchange, config, retries) self._data = deque() # data queue for price data - self._last_id = '' # last processed trade id for ohlcv + self._last_id = 0 # last processed trade id for ohlcv self._last_ts = 0 # last processed timestamp for ohlcv def start(self, ): @@ -136,14 +136,14 @@ def _fetch_ohlcv(self, fromdate=None): break def _load_ticks(self): - if self._last_id is None: + if self._last_id == 0: # first time get the latest trade only trades = [self.store.fetch_trades(self.symbol)[-1]] else: trades = self.store.fetch_trades(self.symbol) for trade in trades: - trade_id = trade['id'] + trade_id = int(trade['id']) if trade_id > self._last_id: trade_time = datetime.strptime(trade['datetime'], '%Y-%m-%dT%H:%M:%S.%fZ') From 69f74a16f3d5e6a3772ebba5e3703bb4dfac283d Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 5 Sep 2018 22:17:05 +0300 Subject: [PATCH 33/37] Revert "minor fix on load_ticks" This commit caused ccxt feed code to crash with ValueError: invalid literal for int() with base 10 reverting This reverts commit bf5407c417dee1b1bab000aa2164992cfaeb7187. Fixes: #20 --- backtrader/feeds/ccxt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index f5b5f90ba..728fb5ad4 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -64,7 +64,7 @@ def __init__(self, exchange, symbol, ohlcv_limit=None, config={}, retries=5): self.store = CCXTStore(exchange, config, retries) self._data = deque() # data queue for price data - self._last_id = 0 # last processed trade id for ohlcv + self._last_id = '' # last processed trade id for ohlcv self._last_ts = 0 # last processed timestamp for ohlcv def start(self, ): @@ -136,14 +136,14 @@ def _fetch_ohlcv(self, fromdate=None): break def _load_ticks(self): - if self._last_id == 0: + if self._last_id is None: # first time get the latest trade only trades = [self.store.fetch_trades(self.symbol)[-1]] else: trades = self.store.fetch_trades(self.symbol) for trade in trades: - trade_id = int(trade['id']) + trade_id = trade['id'] if trade_id > self._last_id: trade_time = datetime.strptime(trade['datetime'], '%Y-%m-%dT%H:%M:%S.%fZ') From 85ad19f87653b4c23823a8e85a135c6f01a2abf9 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 5 Sep 2018 22:21:00 +0300 Subject: [PATCH 34/37] Fixed "if self._last_id is None" condition As self._last_id is set to empty string that condition was always false. Signed-off-by: Ed Bartosh --- backtrader/feeds/ccxt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index 728fb5ad4..c8fcd5e08 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -136,11 +136,11 @@ def _fetch_ohlcv(self, fromdate=None): break def _load_ticks(self): - if self._last_id is None: + if self._last_id: + trades = self.store.fetch_trades(self.symbol) + else: # first time get the latest trade only trades = [self.store.fetch_trades(self.symbol)[-1]] - else: - trades = self.store.fetch_trades(self.symbol) for trade in trades: trade_id = trade['id'] From 7ac035d36ec8a59d12b5dcf6daa85f324e1480ca Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 28 Aug 2018 23:01:13 +0300 Subject: [PATCH 35/37] do not create new story object for the same exchange Used the same store object for one exchange for any feed or broker that uses that exchange. Fixes: #18 Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 2 +- backtrader/feeds/ccxt.py | 2 +- backtrader/stores/ccxtstore.py | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 21a68802d..12d2cfdbc 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -40,7 +40,7 @@ class CCXTBroker(BrokerBase): def __init__(self, exchange, currency, config, retries=5): super(CCXTBroker, self).__init__() - self.store = CCXTStore(exchange, config, retries) + self.store = CCXTStore.get_store(exchange, config, retries) self.currency = currency diff --git a/backtrader/feeds/ccxt.py b/backtrader/feeds/ccxt.py index c8fcd5e08..516ccc00a 100644 --- a/backtrader/feeds/ccxt.py +++ b/backtrader/feeds/ccxt.py @@ -61,7 +61,7 @@ def __init__(self, exchange, symbol, ohlcv_limit=None, config={}, retries=5): self.symbol = symbol self.ohlcv_limit = ohlcv_limit - self.store = CCXTStore(exchange, config, retries) + self.store = CCXTStore.get_store(exchange, config, retries) self._data = deque() # data queue for price data self._last_id = '' # last processed trade id for ohlcv diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index 962d8baf8..c59834392 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -67,6 +67,27 @@ class CCXTStore(object): (bt.TimeFrame.Years, 1): '1y', } + # exchange -> store dictionary to track already initialized exchanges + stores = {} + # exchange -> config dictionry + configs = {} + + @classmethod + def get_store(cls, exchange, config, retries): + store = cls.stores.get(exchange) + if store: + store_conf = cls.configs[exchange] + if store_conf: + if not set(config.items()).issubset(set(store_conf.items())): + raise ValueError("%s exchange is already configured: %s" % \ + (exchange, store_conf)) + return store + + cls.configs[exchange] = config + cls.stores[exchange] = cls(exchange, config, retries) + + return cls.stores[exchange] + def __init__(self, exchange, config, retries): self.exchange = getattr(ccxt, exchange)(config) self.retries = retries From 3c32fc1d487d5a702972ea2f0d8673aa38fb72f2 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sun, 9 Sep 2018 12:11:05 +0300 Subject: [PATCH 36/37] ccxt: check if 'amount' field is set for the order Some exchanges don't set 'amount' for the order object. This causes crash in ccxt store code: File "backtrader\stores\ccxtstore.py", line 39, in __init__ self.size = float(ccxt_order['amount']) TypeError: float() argument must be a string or a number, not 'NoneType' Added check for 'amount' to be present in the order before checking it. Signed-off-by: Ed Bartosh --- backtrader/brokers/ccxtbroker.py | 2 +- backtrader/stores/ccxtstore.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backtrader/brokers/ccxtbroker.py b/backtrader/brokers/ccxtbroker.py index 12d2cfdbc..9d125fc3e 100644 --- a/backtrader/brokers/ccxtbroker.py +++ b/backtrader/brokers/ccxtbroker.py @@ -75,7 +75,7 @@ def _submit(self, owner, data, exectype, side, amount, price, params): order_type = self.order_types.get(exectype) _order = self.store.create_order(symbol=data.symbol, order_type=order_type, side=side, amount=amount, price=price, params=params) - order = CCXTOrder(owner, data, _order) + order = CCXTOrder(owner, data, amount, _order) self.notify(order) return order diff --git a/backtrader/stores/ccxtstore.py b/backtrader/stores/ccxtstore.py index c59834392..466305e7d 100644 --- a/backtrader/stores/ccxtstore.py +++ b/backtrader/stores/ccxtstore.py @@ -31,12 +31,16 @@ from backtrader import OrderBase class CCXTOrder(OrderBase): - def __init__(self, owner, data, ccxt_order): + def __init__(self, owner, data, size, ccxt_order): self.owner = owner self.data = data self.ccxt_order = ccxt_order self.ordtype = self.Buy if ccxt_order['side'] == 'buy' else self.Sell - self.size = float(ccxt_order['amount']) + amount = ccxt_order.get('amount') + if amount: + self.size = float(amount) + else: + self.size = size super(CCXTOrder, self).__init__() From 81a5fca114d77b8745672ae3cfa966544a66aa32 Mon Sep 17 00:00:00 2001 From: pofenglin079 <37055121+pofenglin079@users.noreply.github.com> Date: Sat, 11 Sep 2021 17:02:29 +0800 Subject: [PATCH 37/37] fix matplotlib.dates can't import warnings --- backtrader/plot/locator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backtrader/plot/locator.py b/backtrader/plot/locator.py index e9cf745b8..ccf070562 100644 --- a/backtrader/plot/locator.py +++ b/backtrader/plot/locator.py @@ -27,6 +27,7 @@ ''' import datetime +import warnings from matplotlib.dates import AutoDateLocator as ADLocator from matplotlib.dates import RRuleLocator as RRLocator @@ -36,7 +37,7 @@ MONTHS_PER_YEAR, DAYS_PER_WEEK, SEC_PER_HOUR, SEC_PER_DAY, num2date, rrulewrapper, YearLocator, - MicrosecondLocator, warnings) + MicrosecondLocator) from dateutil.relativedelta import relativedelta import numpy as np