Skip to content

add btc dominance indicator #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
scqsu wants to merge 3 commits into
base: master
Choose a base branch
from
Open

add btc dominance indicator #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3b8fb12
add btc dominance indicator
scqsu Apr 1, 2023
4742821
sync
scqsu Apr 3, 2024
d347431
Added functionality to check for mainstream cryptocurrencies.
scqsu Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added crawlers/indicators/spiders/btc_dominance/__init__.py
View file
Open in desktop
Empty file.
59 changes: 59 additions & 0 deletions crawlers/indicators/spiders/btc_dominance/btc_dominance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import json

import scrapy
import time
import datetime
from crawlers.utils import SpiderBase, rds
from crawlers.utils.group_alarm import catch_except
from jinja2 import Template
from crawlers.utils.humanize import humanize_float_en


class BTCDominanceAlert(SpiderBase):
name = 'btc_dominance_alert'
start_url = "https://api.coingecko.com/api/v3/global"

def start_requests(self):
yield scrapy.Request(url=self.start_url)

@catch_except
def parse(self, response):
data = response.json()['data']

pre_btc_dominance = rds.getex(self.name, 'btc_dominance')

btc_dominance = round(float(data['market_cap_percentage']['btc']), 2)

if pre_btc_dominance is not None :
btc_dominance_change = round(float(btc_dominance) - float(pre_btc_dominance), 4)
params = {
"btc_dominance": btc_dominance,
"btc_dominance_change": btc_dominance_change
}
print(Template(self.alert_cn_template()).render(params))
else :
pass

rds.setex(self.name, 'btc_dominance', str(btc_dominance), 60 * 60 * 25)


# must be declare
def alert_en_template(self):
return """
Today's BTC Dominance ratio is: {{btc_dominance}}%, the 24-hour dominance ratio{% if btc_dominance_change > 0 %} has risen by: {{btc_dominance}}%{% endif %}{% if btc_dominance_change < 0 %} has fallen by: {{btc_dominance}}%{% endif %}{% if btc_dominance_change == 0 %} remains unchanged. {% endif %}
"""

# must be declare
def alert_cn_template(self):
return """
据 KingData 数据监控：
今日 BTC 市值占比为：{{btc_dominance}}%，24h 市值占比{% if btc_dominance_change > 0 %}上涨：{{btc_dominance}}%{% endif %}{% if btc_dominance_change < 0 %}下降：{{btc_dominance}}%{% endif %}{% if btc_dominance_change == 0 %}不变。{% endif %}
"""



'''
BTC 市值占比可以用来衡量当前市场热度和交易标的，是趋势交易的关键指标之一。该指标的使用需要结合大盘价格走势并且关注当前指标处于高位 OR 低位进行判断。

BTC Dominance can be used to measure the current market heat and trading targets, and is one of the key indicators for trend trading. The use of this indicator requires combining the trend of the overall market prices and paying attention to whether the current indicator is at a high or low level for judgment.
'''
131 changes: 83 additions & 48 deletions crawlers/indicators/spiders/dex_swap_alert/dex_swap_alert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,35 @@
from jinja2 import Template
from crawlers.utils.group_alarm import catch_except
from crawlers.utils.humanize import humanize_float_en
from crawlers.utils.headers import common_headers

class DexSwapAlert(SpiderBase):
name = 'dex_swap_alert'
name = 'idx-dex-swap-tracker'

binance_symbol_list = {}

binance_url = 'https://api.binance.com/api/v3/ticker/price'

uniswap_v3_thegraph_chains = {
"Ethereum": "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v3",
"Polygon": "https://api.thegraph.com/subgraphs/name/ianlapham/uniswap-v3-polygon",
"Optimism": "https://api.thegraph.com/subgraphs/name/ianlapham/optimism-post-regenesis",
"Arbitrum": "https://api.thegraph.com/subgraphs/name/ianlapham/arbitrum-minimal",
"Celo": "https://api.thegraph.com/subgraphs/name/jesse-sawa/uniswap-celo",
"BNB Chain": "https://api.thegraph.com/subgraphs/name/ianlapham/uniswap-v3-bsc"
}

chains_scan_url = {
"Ethereum": "https://etherscan.io/tx/%s",
"Polygon": "https://polygonscan.com/tx/%s",
"Optimism": "https://optimistic.etherscan.io/tx/%s",
"Arbitrum": "https://arbiscan.io/tx/%s",
"Celo": "https://celoscan.io/tx/%s",
"BNB Chain": "https://bscscan.com/tx/%s"
}

uniswap_v3 = {
"project_name": 'uniswap_v3',
"query_url":'https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v3',
"query_url_dict": uniswap_v3_thegraph_chains,
"query_latest_block": '''
{
swaps(
Expand All @@ -25,7 +43,7 @@ class DexSwapAlert(SpiderBase):
where: {timestamp_gt: "%s"}
) {
transaction {
blockNumber
blockNumber
}
}
}''' % int(time.time()-600), # Subtract 600 seconds to ensure that the block is already confirm
Expand All @@ -38,6 +56,7 @@ class DexSwapAlert(SpiderBase):
blockNumber_lte: "%s"
}
}
first: 1000
orderBy: logIndex
orderDirection: asc
) {
Expand Down Expand Up @@ -68,16 +87,24 @@ def parse_binance_symbol(self, symbol_json): # 获取行情数据，最终输出
self.binance_symbol_list[symbol['symbol']] = symbol['price']

self.binance_symbol_list['WETH'] = self.binance_symbol_list['ETH']
self.binance_symbol_list['cbETH'] = self.binance_symbol_list['ETH']
self.binance_symbol_list['WBTC'] = self.binance_symbol_list['BTC']

self.binance_symbol_list['USDT'] = 1.0
# 一些异常交易对的白名单
self.binance_symbol_list['POLY'] = 0
self.binance_symbol_list['BTT'] = 0

@catch_except
def start_requests(self):
resp = requests.get(self.binance_url)
resp = requests.get(self.binance_url, headers=common_headers)
self.parse_binance_symbol(json.loads(resp.text)) # 获取行情数据

for dex in self.dex_projects :
yield scrapy.Request(url=dex['query_url'], method='POST',
query_dict = dex['query_url_dict']
for chain, url in query_dict.items():
dex['chain'] = chain
dex['query_url'] = url
yield scrapy.Request(url=url, method='POST',
body=json.dumps({"query": dex['query_latest_block']}),
callback=self.parse_blockNum,
cb_kwargs=dex)
Expand All @@ -86,24 +113,26 @@ def start_requests(self):
@catch_except
def parse_blockNum(self, response, **dex): # 获取最新 blockNum，结合本地存储 blockNum，不断获取新数据
current_blockNum = response.json()['data']['swaps'][0]['transaction']['blockNumber']
# pre_blockNum = rds.get(f"{self.name}:{dex['project_name']}:blockNum")
# rds.set(f"{self.name}:{dex['project_name']}:blockNum", current_blockNum, 60 * 60)
# # pre_blockNum = '16772669' #rds.getex(self.name + dex['project_name'], 'blockNum')
pre_blockNum = rds.get(f"{self.name}:{dex['project_name']}:{dex['chain']}:blockNum")
rds.set(f"{self.name}:{dex['project_name']}:{dex['chain']}:blockNum", current_blockNum, 60 * 60)

pre_blockNum = 16814041
current_blockNum = 16814042
# pre_blockNum = 16815569
# current_blockNum = 16815574

if not pre_blockNum:
return
print(f"开始爬取 {pre_blockNum} - {current_blockNum} 之间的 swap 数据")

print(f"开始爬取 {dex['chain']} {dex['project_name']} {pre_blockNum} - {current_blockNum} 之间的 swap 数据")

yield scrapy.Request(url=dex['query_url'], method='POST',
body=json.dumps({
"query": dex['query_swaps'] % (pre_blockNum, current_blockNum)
}),
headers=common_headers,
callback=self.parse_swaps,
cb_kwargs={"project_name": dex['project_name']})

cb_kwargs={"project_name": dex['project_name'],
"chain": dex['chain']})

@catch_except
def parse_swaps(self, response, **dex): # 解析 swaps 数据
Expand All @@ -116,8 +145,7 @@ def parse_swaps(self, response, **dex): # 解析 swaps 数据
for swap in swaps :
# 根据币种类型，处理交易 type，分为：alt_coin_trade 和
trade_type = 'alt_coin_trade'
mainstream_coin = ('WBTC', 'WETH', 'USDT', 'USDC', 'DAI', 'BUSD')
if (swap['token0']['symbol'] in mainstream_coin) and (swap['token1']['symbol'] in mainstream_coin):
if self.is_main_stream_token(swap['token0']['symbol']) and self.is_main_stream_token(swap['token1']['symbol']):
trade_type = 'mainstream_coin_trade'

# dex 里面到代币，名称没有换回来
Expand Down Expand Up @@ -150,24 +178,24 @@ def parse_swaps(self, response, **dex): # 解析 swaps 数据
'amount1': swap['amount1'],
'sender': swap['origin'],
'project_name': dex['project_name'],
'chain': dex['chain'],
'trade_type': trade_type,
'blockNumber': swap['transaction']['blockNumber']
}

filter_tag = swap['transaction']['blockNumber'] + '_' + swap['origin'] # 把同一个 blockNumber 里面，同一个 sender 执行了多笔交易的情况标记出来。这种不会是手动操作，肯定是调用了特殊合约 或者 批量发起
if filter_tag in swap_filter_dic.keys() : # 增加一个过滤的 dic
swap_filter_dic[filter_tag] += 1
else :
swap_filter_dic[filter_tag] = 0
filter_tag = swap['transaction']['blockNumber'] + '_' + swap['origin'] # 把同一个 blockNumber 里面，同一个 sender 执行了多笔交易的情况标记出来。这种不会是手动操作，肯定是调用了特殊合约 或者 批量发起

if filter_tag in swap_filter_dic.keys() : # 增加一个过滤的 dic
swap_filter_dic[filter_tag] += 1
else :
swap_filter_dic[filter_tag] = 0

# swap 处理好之后，计算每个 swap 的 value
for tx_id in swap_dic.keys() :
for tx_id in list(swap_dic.keys()):

filter_tag = swap_dic[tx_id]['blockNumber'] + '_' + swap_dic[tx_id]['sender']
if swap_filter_dic[filter_tag] > 0 : # 如果是同一个 block 多笔的情况，进行过滤
print("过滤："+tx_id)
swap_dic[tx_id] = 'pass_swap'
swap_dic.pop(tx_id)
continue

if swap_dic[tx_id]['symbol0'] in self.binance_symbol_list:
Expand All @@ -183,19 +211,27 @@ def parse_swaps(self, response, **dex): # 解析 swaps 数据
swap_dic[tx_id]['value1'] = 0


if swap_dic[tx_id]['value1'] > 0 and swap_dic[tx_id]['value0'] > 0 : # 如果两个 symbol 都有价值，则取最小值作为过滤条件
swap_dic[tx_id]['value'] = round(abs(swap_dic[tx_id]['value1']) if abs(swap_dic[tx_id]['value1']) < abs(swap_dic[tx_id]['value0']) else abs(swap_dic[tx_id]['value0']), 2)
if abs(swap_dic[tx_id]['value1']) > 0 and abs(swap_dic[tx_id]['value0']) > 0 : # 如果两个 symbol 都有价值，则取最小值作为过滤条件
swap_dic[tx_id]['value'] = round(min(abs(swap_dic[tx_id]['value1']), abs(swap_dic[tx_id]['value0'])), 2)
else : # 如果不是两个 value 都有价值，则取大值做过滤条件
swap_dic[tx_id]['value'] = round(abs(swap_dic[tx_id]['value1']) if abs(swap_dic[tx_id]['value1']) > abs(swap_dic[tx_id]['value0']) else abs(swap_dic[tx_id]['value0']), 2)
swap_dic[tx_id]['value'] = round(max(abs(swap_dic[tx_id]['value1']), abs(swap_dic[tx_id]['value0'])), 2)

self.alert_process(swap_dic)


def is_main_stream_token(self, token_name):
keywords = ['usd', 'btc', 'eth', 'pax']

lowercase_tokenname = token_name.lower()

for keyword in keywords:
if keyword in lowercase_tokenname:
return True

return False

def alert_process(self, swap_dic) :
for tx_id, swap in swap_dic.items():
if swap == 'pass_swap' :
continue

swap['tx_id'] = tx_id
# 交易 symbo，0 为买入 1 为卖出
if float(swap['amount0']) > 0 :
Expand All @@ -215,26 +251,22 @@ def alert_process(self, swap_dic) :
swap['amount0'] = abs(float(swap['amount0']))
swap['amount1'] = float(swap['amount1'])

swap['symbol'] = f"${swap['symbol1']} swap_to ${swap['symbol0']}"

tx_url = self.chains_scan_url[swap['chain']] % swap["tx_id"]

# 根据交易类型做过滤
if swap['trade_type'] == 'mainstream_coin_trade' and swap['value'] >= 2000000: # 3000000
if swap['trade_type'] == 'mainstream_coin_trade' and swap['value'] >= 10000000: # 10000000
self.foramt_swap(swap)
# Tools.multilingual_information_flow_push(tmp_name='DEX_SWAP_TRACKER', params= swap, origin_url=f'https://etherscan.io/tx/{swap["tx_id"]}')

print(Template(self.alert_en_template()).render(swap))
print(Template(self.alert_cn_template()).render(swap))

if swap['value'] >= 10000000 :
print('---- Global Notification ----')
elif swap['trade_type'] == 'alt_coin_trade' and swap['value'] >= 200000: #300000
elif swap['trade_type'] == 'alt_coin_trade' and swap['value'] >= 150000: #500000
self.foramt_swap(swap)
# Tools.multilingual_information_flow_push(tmp_name='DEX_SWAP_TRACKER', params= swap, origin_url=f'https://etherscan.io/tx/{swap["tx_id"]}')

print(Template(self.alert_en_template()).render(swap))
print(Template(self.alert_cn_template()).render(swap))

if swap['value'] >= 1000000 :
print('---- Global Notification ----')

def foramt_swap(self, params):
params['origin_value0'] = abs(params.get('value0') or 0)
params['amount0'] = humanize_float_en(float(params['amount0']))
Expand All @@ -244,20 +276,23 @@ def foramt_swap(self, params):
params['amount1'] = humanize_float_en(float(params['amount1']))
params['value1'] = humanize_float_en(abs(float(params.get('value1') or 0)))
params['symbol1_price'] = round(float(params.get('symbol1_price') or 0), 2) if params.get('symbol1_price') else None
params['account_tag'] = self.get_address_tags(params['sender'])
return params;

def alert_en_template(self):
return '''
According to KingData monitoring, {{amount1}} {{symbol1}}{% if origin_value1 > 1 %}(${{value1}}){% endif %} has just been swaped into {{amount0}} {{symbol0}}{% if origin_value0 > 1 %}(${{value0}}){% endif %}.
Sell/Quantity/Price: {{symbol1}} ｜ {{amount1}} | {% if symbol1_price %}${{symbol1_price}}{% else %}-{% endif %}
Buy/Quantity/Price: {{symbol0}} ｜ {{amount0}} | {% if symbol0_price %}${{symbol0_price}}{% else %}-{% endif %}
Account: {{sender}}
'''
Buy/Quantity/Price: {{symbol0}} ｜ {{amount0}} | {% if symbol0_price %}${{symbol0_price}}{% else %}-{% endif %}{% if account_tag.show_tag %}
TradeUser：{{account_tag.show_tag}}{% if account_tag.twitter %} (Twitter: @{{account_tag.twitter}}){% endif %}{% endif %}
Address: {{sender}}
'''

def alert_cn_template(self):
return '''
据 KingData 监控，刚刚 {{amount1}} {{symbol1}}{% if origin_value1 > 1 %}(${{value1}}){% endif %} 兑换成 {{amount0}} {{symbol0}}{% if origin_value0 > 1 %}(${{value0}}){% endif %}。
卖出币种/数量/价格：{{symbol1}} ｜ {{amount1}} | {% if symbol1_price %}${{symbol1_price}}{% else %}-{% endif %}
买入币种/数量/价格：{{symbol0}} ｜ {{amount0}} | {% if symbol0_price %}${{symbol0_price}}{% else %}-{% endif %}
交易用户：{{sender}}
'''
买入币种/数量/价格：{{symbol0}} ｜ {{amount0}} | {% if symbol0_price %}${{symbol0_price}}{% else %}-{% endif %}{% if account_tag.show_tag %}
用户：{{account_tag.show_tag}}{% if account_tag.twitter %} (Twitter: @{{account_tag.twitter}}){% endif %}{% endif %}
地址: {{sender}}
'''