From 2c1f6a7c383f8d84a130408d21e399560a8d2b18 Mon Sep 17 00:00:00 2001 From: ssi91 Date: Thu, 1 Jun 2023 21:26:44 -0400 Subject: [PATCH 1/8] implement TonScanAPIClient --- src/databases/ton_scan_service.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/databases/ton_scan_service.py diff --git a/src/databases/ton_scan_service.py b/src/databases/ton_scan_service.py new file mode 100644 index 0000000..81bafb2 --- /dev/null +++ b/src/databases/ton_scan_service.py @@ -0,0 +1,27 @@ +import requests + + +class TonAPIClientError(Exception): + pass + + +class TonAPIClient: + def __init__(self, base_url): + self.base_url = base_url + + def _request(self, address, params=None): + params = {} if params is None else params + end_point = self.base_url + address + response = requests.get(end_point, params) + if response.status_code != 200: + raise TonAPIClientError( + f"Ton Scan Service response with error code: {response.status_code}, error: {response.text}") + return response.json() + + def address_list(self, page=0, count=100): + addresses = self._request("/address_list/", {"page": page, "count": count}) + return addresses + + def address_details(self, address): + details = self._request(f"/address_details/{address}/") + return details From 5085e1e78ce5e6c322342b9ecbace4b047ba1aa1 Mon Sep 17 00:00:00 2001 From: ssi91 Date: Thu, 1 Jun 2023 21:29:26 -0400 Subject: [PATCH 2/8] move Ton Scan client to a suitable directory --- src/{databases => services}/ton_scan_service.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{databases => services}/ton_scan_service.py (100%) diff --git a/src/databases/ton_scan_service.py b/src/services/ton_scan_service.py similarity index 100% rename from src/databases/ton_scan_service.py rename to src/services/ton_scan_service.py From f7b5a17e66546202439b584485cd47b94c0d6334 Mon Sep 17 00:00:00 2001 From: Kirill Melcin Date: Thu, 8 Jun 2023 09:02:24 +0300 Subject: [PATCH 3/8] added tonapi --- src/services/tonapi.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/services/tonapi.py diff --git a/src/services/tonapi.py b/src/services/tonapi.py new file mode 100644 index 0000000..cb10ce1 --- /dev/null +++ b/src/services/tonapi.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from fastapi import FastAPI, HTTPException +import requests + +app = FastAPI() + + +def request_to_ton_api(url_request: str): + print(url_request) + response = requests.get(url_request) + data = response.json() + + return data + + +# get transactions by block: +@app.get("/api/v1/blockchain/blocks/{block_id}/transactions") +def read_item(block_id: str): + url_request = "https://tonapi.io/v2/blockchain/blocks/"+block_id+"/transactions" + return request_to_ton_api(url_request) + + +# get transactions by account: +@app.get("/api/v1/blockchain/accounts/{account_id}/transactions") +def read_item(account_id: str): + url_request = "https://tonapi.io/v2/blockchain/accounts/"+account_id+"/transactions" + return request_to_ton_api(url_request) + +@app.get("/api/v1/") +def welcome(): + return "Open TON API v1" + +@app.get("/api/v1/{request:path}") +def read_item(request: str): + print(request) + url_request = "https://tonapi.io/v2/" + request + return request_to_ton_api(url_request) \ No newline at end of file From e20abb13aa3240512e85698ba0e0fece2e01670e Mon Sep 17 00:00:00 2001 From: Kirill Melcin Date: Mon, 12 Jun 2023 01:42:26 +0300 Subject: [PATCH 4/8] Added tonapi.io service --- src/services/tonapi.py | 70 +++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/src/services/tonapi.py b/src/services/tonapi.py index cb10ce1..198e8ae 100644 --- a/src/services/tonapi.py +++ b/src/services/tonapi.py @@ -1,37 +1,57 @@ # -*- coding: utf-8 -*- -from fastapi import FastAPI, HTTPException import requests +import os -app = FastAPI() +TONAPI_KEY = os.getenv("TONAPI_KEY") or None +TONAPI_BASE_URL = "https://tonapi.io/v2" -def request_to_ton_api(url_request: str): - print(url_request) - response = requests.get(url_request) - data = response.json() +class TonAPIClientError(Exception): + pass - return data +class TonAPIClient: + def _request(self, address, params=None): + params = {} if params is None else params -# get transactions by block: -@app.get("/api/v1/blockchain/blocks/{block_id}/transactions") -def read_item(block_id: str): - url_request = "https://tonapi.io/v2/blockchain/blocks/"+block_id+"/transactions" - return request_to_ton_api(url_request) + # Making a request to tonapi.io + if TONAPI_KEY: + response = requests.get(TONAPI_BASE_URL + address, params, headers={"Authorization": f"Bearer {TONAPI_KEY}"}) + else: + response = requests.get(TONAPI_BASE_URL + address, params) + # Checking the response + if response.status_code != requests.codes.ok: + raise TonAPIClientError( + f"Tonapi response with error code: {response.status_code}, error: {response.text}") + return response.json() -# get transactions by account: -@app.get("/api/v1/blockchain/accounts/{account_id}/transactions") -def read_item(account_id: str): - url_request = "https://tonapi.io/v2/blockchain/accounts/"+account_id+"/transactions" - return request_to_ton_api(url_request) + def tx_by_block(self, block_id): + """ + Find transactions by block id. -@app.get("/api/v1/") -def welcome(): - return "Open TON API v1" + :param str block_id: Block id e.g. "(-1,8000000000000000,4234234)" + """ + request_url = f"/blockchain/blocks/{block_id}/transactions" + transactions = self._request(request_url) + return transactions -@app.get("/api/v1/{request:path}") -def read_item(request: str): - print(request) - url_request = "https://tonapi.io/v2/" + request - return request_to_ton_api(url_request) \ No newline at end of file + def tx_by_account(self, account_id): + """ + Find transactions by account id. + + :param str account_id: Account id in any format + """ + request_url = f"/blockchain/accounts/{account_id}/transactions" + transactions = self._request(request_url) + return transactions + + def raw_request(self, request_url, params): + """ + Make any request to tonapi.io + + :param request_url: Endpoint. e.g. "/rates?tokens=ton¤cies=rub" + :param params: Any parameters. e.g. {"tokens":"ton","currencies":"rub"} + """ + response = self._request(request_url, params) + return response From 008f583e4a2a59a6c12e2330a7579825f5d71b04 Mon Sep 17 00:00:00 2001 From: Kirill Melcin Date: Tue, 27 Jun 2023 21:28:43 +0300 Subject: [PATCH 5/8] Added rate limiter --- src/services/rate_limiter.py | 20 ++++++++++++++++++++ src/services/tonapi.py | 9 +++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 src/services/rate_limiter.py diff --git a/src/services/rate_limiter.py b/src/services/rate_limiter.py new file mode 100644 index 0000000..670504a --- /dev/null +++ b/src/services/rate_limiter.py @@ -0,0 +1,20 @@ +def rate_limiter(limit: int = 1): + def decorator_function(func): + import time + + last_request_time = 0 + + def wrapper(*args, **kwargs): + nonlocal last_request_time + + interval = 1.1 / limit + current_time = time.time() + time_since_last_request = current_time - last_request_time + if time_since_last_request < interval: + time.sleep(interval - time_since_last_request) + last_request_time = time.time() + return func(*args, **kwargs) + + return wrapper + + return decorator_function diff --git a/src/services/tonapi.py b/src/services/tonapi.py index 198e8ae..3db4854 100644 --- a/src/services/tonapi.py +++ b/src/services/tonapi.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import requests import os +from rate_limiter import rate_limiter TONAPI_KEY = os.getenv("TONAPI_KEY") or None TONAPI_BASE_URL = "https://tonapi.io/v2" @@ -26,6 +27,7 @@ def _request(self, address, params=None): f"Tonapi response with error code: {response.status_code}, error: {response.text}") return response.json() + @rate_limiter(limit=1) def tx_by_block(self, block_id): """ Find transactions by block id. @@ -36,6 +38,7 @@ def tx_by_block(self, block_id): transactions = self._request(request_url) return transactions + @rate_limiter(limit=1) def tx_by_account(self, account_id): """ Find transactions by account id. @@ -46,12 +49,14 @@ def tx_by_account(self, account_id): transactions = self._request(request_url) return transactions + @rate_limiter(limit=1) def raw_request(self, request_url, params): """ Make any request to tonapi.io - :param request_url: Endpoint. e.g. "/rates?tokens=ton¤cies=rub" - :param params: Any parameters. e.g. {"tokens":"ton","currencies":"rub"} + :param request_url: Endpoint. e.g. "/rates" + :param params: Any parameters. e.g. {"tokens": "ton", "currencies": "rub"} """ response = self._request(request_url, params) return response + From 257bbc3338663d36e60adc83b92334cbcc82552a Mon Sep 17 00:00:00 2001 From: ssi91 Date: Fri, 30 Jun 2023 13:45:03 +0300 Subject: [PATCH 6/8] a bit of refactoring over the rate-limiter --- src/services/rate_limiter.py | 7 +++++-- src/services/tonapi.py | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/services/rate_limiter.py b/src/services/rate_limiter.py index 670504a..c783079 100644 --- a/src/services/rate_limiter.py +++ b/src/services/rate_limiter.py @@ -1,4 +1,7 @@ -def rate_limiter(limit: int = 1): +from typing import Union + + +def rate_limiter(limit: Union[int, float] = 1): def decorator_function(func): import time @@ -7,7 +10,7 @@ def decorator_function(func): def wrapper(*args, **kwargs): nonlocal last_request_time - interval = 1.1 / limit + interval = 1 / limit current_time = time.time() time_since_last_request = current_time - last_request_time if time_since_last_request < interval: diff --git a/src/services/tonapi.py b/src/services/tonapi.py index 3db4854..6e4de78 100644 --- a/src/services/tonapi.py +++ b/src/services/tonapi.py @@ -27,7 +27,7 @@ def _request(self, address, params=None): f"Tonapi response with error code: {response.status_code}, error: {response.text}") return response.json() - @rate_limiter(limit=1) + @rate_limiter(limit=0.9) def tx_by_block(self, block_id): """ Find transactions by block id. @@ -38,7 +38,7 @@ def tx_by_block(self, block_id): transactions = self._request(request_url) return transactions - @rate_limiter(limit=1) + @rate_limiter(limit=0.9) def tx_by_account(self, account_id): """ Find transactions by account id. @@ -49,7 +49,7 @@ def tx_by_account(self, account_id): transactions = self._request(request_url) return transactions - @rate_limiter(limit=1) + @rate_limiter(limit=0.9) def raw_request(self, request_url, params): """ Make any request to tonapi.io @@ -59,4 +59,3 @@ def raw_request(self, request_url, params): """ response = self._request(request_url, params) return response - From 686dc873ecc5b23a7cfe5aa356d0c4536287d491 Mon Sep 17 00:00:00 2001 From: ssi91 Date: Sat, 1 Jul 2023 12:21:23 +0300 Subject: [PATCH 7/8] actualize connection settigs to the tests --- tests/test_redis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_redis.py b/tests/test_redis.py index 045cfa7..694b2a8 100644 --- a/tests/test_redis.py +++ b/tests/test_redis.py @@ -5,13 +5,14 @@ import pytest from redis import asyncio +from src.config import main_config from src.databases.redis import RedisRepo @pytest.fixture(scope='module') def redis_client(): """Test connection provider.""" - redis = asyncio.Redis() + redis = asyncio.Redis().from_url(main_config().cache.redis_url) # try: yield RedisRepo(redis) # finally: From dc78df3b88f18cdd5aac654dc97f9164eecc5905 Mon Sep 17 00:00:00 2001 From: ssi91 Date: Sat, 1 Jul 2023 17:29:08 +0300 Subject: [PATCH 8/8] add a draft of block loader worker --- src/block_loader.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/block_loader.py diff --git a/src/block_loader.py b/src/block_loader.py new file mode 100644 index 0000000..cb6738c --- /dev/null +++ b/src/block_loader.py @@ -0,0 +1,30 @@ +from arq.connections import RedisSettings + +from src.config import main_config +from src.services.tonapi import TonAPIClient + + +async def load_block(ctx, block_id): + api = TonAPIClient() + txs_data = api.tx_by_block(block_id) + # todo: save data to the DB + + +async def main(): + pass + + +async def startup(ctx): + pass + + +async def shutdown(ctx): + pass + + +class WorkerSettings: + redis_settings = RedisSettings.from_dsn(main_config().cache.redis_url) + functions = [load_block] + + on_startup = startup + on_shutdown = shutdown