From 296e74ef0645ad9f87c642e3b83d0afff35e38f0 Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Fri, 3 Jun 2022 14:36:43 +0200 Subject: [PATCH 1/6] lightweight indexing wip --- .../conseil/common/tezos/TezosOptics.scala | 30 +- .../src/main/resources/application.conf | 2 + .../resources/known_addresses/mainnet.csv | 261 ------------------ .../indexer/config/LorreConfiguration.scala | 3 +- .../tezos/TezosDatabaseConversions.scala | 16 +- .../tezos/TezosDatabaseOperations.scala | 21 +- .../tezos/TezosGovernanceOperations.scala | 2 +- .../conseil/indexer/tezos/TezosIndexer.scala | 120 ++++---- .../tezos/processing/BlocksProcessor.scala | 41 ++- 9 files changed, 138 insertions(+), 358 deletions(-) diff --git a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala index e3ad1e6c0..8d20fa799 100644 --- a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala +++ b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala @@ -299,12 +299,38 @@ object TezosOptics { * @return a Map holding for each group both external and internal operations' results */ def extractOperationsAlongWithInternalResults( - block: Block + block: Block, + knownAddresses: Option[List[Tables.KnownAddressesRow]] ): Map[OperationsGroup, (List[Operation], List[InternalOperationResults.InternalOperationResult])] = block.operationGroups.map { group => val updatedGroup = addIndexesToOperationGroup(group) - val internal = updatedGroup.contents.flatMap { op => + val internal = updatedGroup.contents.filter{ ff => + knownAddresses match { + case Some(ka) => + val addresses = ka.map(_.address) + ff match { + case Endorsement(level, metadata, blockOrder) => false + case EndorsementWithSlot(endorsement, metadata, blockOrder) => false + case Preendorsement(level, metadata, blockOrder) => false + case SeedNonceRevelation(level, nonce, metadata, blockOrder) => false + case ActivateAccount(pkh, secret, metadata, blockOrder) => addresses.contains(pkh.value) + case Reveal(counter, fee, gas_limit, storage_limit, public_key, source, metadata, blockOrder) => addresses.contains(public_key.value) || addresses.contains(source.value) + case Transaction(counter, amount, fee, gas_limit, storage_limit, source, destination, parameters, parameters_micheline, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(destination.id) + case Origination(counter, fee, source, balance, gas_limit, storage_limit, manager_pubkey, delegatable, delegate, spendable, script, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value) + case Delegation(counter, source, fee, gas_limit, storage_limit, delegate, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value) + case DoubleEndorsementEvidence(blockOrder) => false + case DoublePreendorsementEvidence(blockOrder) => false + case DoubleBakingEvidence(blockOrder) => false + case Proposals(source, period, proposals, blockOrder) => addresses.contains(source.getOrElse(ContractId("")).id) + case Ballot(ballot, proposal, source, period, blockOrder) => addresses.contains(source.getOrElse(ContractId("")).id) + case RegisterGlobalConstant(source, fee, counter, gas_limit, storage_limit, value, blockOrder, metadata) => addresses.contains(source.getOrElse(ContractId("")).id) + case SetDepositsLimit(source, fee, counter, gas_limit, storage_limit, limit, blockOrder, metadata) => addresses.contains(source.getOrElse(ContractId("")).id) + } + case None => true + } + + }.flatMap { op => op match { case r: Reveal => r.metadata.internal_operation_results.toList.flatten case t: Transaction => t.metadata.internal_operation_results.toList.flatten diff --git a/conseil-lorre/src/main/resources/application.conf b/conseil-lorre/src/main/resources/application.conf index 3bd45e8be..90a825bea 100644 --- a/conseil-lorre/src/main/resources/application.conf +++ b/conseil-lorre/src/main/resources/application.conf @@ -49,6 +49,8 @@ lorre { rights-processing-is-on: ${?CONSEIL_LORRE_RIGHTS_PROCESSING_ENABLED} baker-features-are-on: false baker-features-are-on: ${?CONSEIL_LORRE_BAKER_FEATURES_ENABLED} + lightweight-indexing: false + lightweight-indexing: ${?CONSEIL_LORRE_LIGHTWEIGHT_INDEXING_ENABLED} } # TODO Should we wrap configuration from below into 'Tezos' secion? The same in the *Configuration class? diff --git a/conseil-lorre/src/main/resources/known_addresses/mainnet.csv b/conseil-lorre/src/main/resources/known_addresses/mainnet.csv index 7f148cea3..5f2129324 100644 --- a/conseil-lorre/src/main/resources/known_addresses/mainnet.csv +++ b/conseil-lorre/src/main/resources/known_addresses/mainnet.csv @@ -1,264 +1,3 @@ address,alias tz1W1qbXXJrtLWwaaoXhe8LYGZqmNnpKaH9q,01No.de tz1WBfwbT66FC6BTLexc2BoyCCBM9LG7pnVW,888 XTZ -tz1iLbZZ9uoRuVJCrZ9ZwiJMpfzhy3c67mav,AirBie -tz1gk3TDbU7cJuiBRMhwQXVvgDnjsxuWhcEA,Airfoil -tz1QH3G2btaWc1vRLNsEfx2gHM7Ad81TeRit,Airfoil Payouts -tz1MJx9vhaNRSimcuXPK2rW4fLccQnDAnVKJ,AirGap -tz1LJycSuCT25AA5VJwNW1QYXVGyy7YLwZh9,AirGap Payout -KT1Vp4KUG1n1HrFEDxUNvNskwc1Msh7NRbhd,AirGap Payout KT -tz1UdeKoMJivgqRgRUNzroMwRJYmJY8BtaWe,AlphaTezos -tz1XhnCdVENzgko5x1MMswLHSoQbJ5NPwLZ6,Anonstake -tz3e75hU4EhDU3ukyJueh5v6UvEHzGwkg3yC,At James -tz1ZcTRk5uxD86EFEn1vvNffWWqJy7q5eVhc,ATEZA -tz1TqU5hwh6CtC8Ps5GYBi7YyawvtKLNCbvQ,ATEZA Payouts -KT1VG2WtYdSWz5E7chTeAdDPZNy2MpP8pTfL,Atomex -KT1RCAETpkm9z44MW7Cph3mrWshqVURPubT4,Bake n Rolls Payouts 2 -tz1ei4WtWEMEJekSv8qDnu9PExG6Q8HgRGr3,Bake Tz -tz1NRGxXV9h6SdNaZLcgmjuLx3hyy2f8YoGN,Bake ꜩ For Me -tz1NortRftucvAkD1J58L32EhSVrQEWJCEnB,Bake'n'Rolls -tz1Zrqm4TkJwqTxm5TiyVFh6taXG4Wrq7tko,Bake’n’Rolls Payouts -tz1aRhFErGMgL57DYHMT1vYwv7PzsJN1chrk,Baked Tezos -tz1KksC8RvjUWAbXYJuNrUbontHGor26Cztk,Bakednode -tz1Z1WwoqgRFbLE3YNdYRpCx44NSfiMJzeAG,Bakemon -tz1NkFRjmkqqcGkAhqe78fdgemDNKXvL7Bod,BakeMyStake -tz1WtkJEkKjHX3bMDYwJoDVC4gPksNPUa3v7,BakeMyStake Old -tz1YgNQBeLTgbwRb7QiyvHrx7hwN7AUbdqpB,Baker Avenue -KT1ChNsEFxwyCbJyWGSL3KdjeXE28AY1Kaog,Baker Registry -tz1iVCqjMRG7MfBoFnVjbLQyeLAeJQfLxXze,Bakery IL Payouts -tz1cYufsxHXJcvANhvS55h3aY32a9BAFB494,Bakery-IL -tz1Pv7ha8wzAyRHkR9RahWapdopK1tt6nVz1,Baking co-operative -tz1RV1MBbZMR68tacosb7Mwj6LkbPSUS1er1,Baking Tacos -tz1S8MNvuFEUsWgjHvi3AxibRBf388NhT1q2,Binance -tz1eogbwM5NdoojKEvvSnjYGcJapZgADRK3m,Binance Delegator -tz1XXayQohB8XRXN7kMoHbf2NFwNiH3oMRQQ,Bit Cat -tz1KtGwriE7VuLwT3LwuvU9Nv4wAxP7XZ57d,Bitfinex -tz1LmaFsWRkjr7QMCx5PtV6xTUz3AmEpKQiF,Blockpower -tz1d2EwsrmoCPmhNUjt7unCQ8a9RwVsVXEmd,BTFD -tz3bEQoFCZEEfZMskefZ8q8e4eiHH1pssRax,Ceibo XTZ -tz1Z7eWGw18LqUgRqmDqNZFQx7f8GEHXRfT8,Coinbase 1 -tz1bDhCGNZLQw1QXgf6MCzo6EtAVSGkqEB11,Coinbase 2 -tz1irJKkXS2DBWkU1NnmFQx1c1L7pbGg4yhk,Coinbase Custody -tz1NcoDFXMAfB26mpBhVrdSHmppyTeccT6Fi,Coinbase Pro -tz1dgoZ2xHX6Aou5KbFNFX5XiGJ6UpkqCyQb,CoinEconomy -tz1dbfppLAAxXZNtf2SDps7rch3qfUznKSoK,Coinhouse -tz1SYq214SCBy9naR6cvycQsYcUGpBqQAE8d,Coinone -tz1eEnQhbwf6trb8Q8mPb2RaPkNk2rN7BKi8,Cryptium Labs -tz1cNARmnRRrvZgspPr2rSTUWq5xtGTuKuHY,Cryptium Labs Payout -tz1Tnjaxk6tbAeC2TmMApPh8UsrEVQvhHvx5,Crypto Delegate -tz1MyXTZmeMCM4yFnrER9LNYDZ9t2rHYDvcH,Crypto Delegate Payouts -tz1NqKH9D5HdfvoEQssMS5Up13CVypVxLA1k,Cypher-Baker -tz1Lc87p6zRaDwzJs9kHvdpm7XzeWE8QTwVB,Dash Master -KT1A13N5XXJLgD9rQ6BF6xg71yhNhQhrCxia,Dashmaster Payouts -tz1ZRWFLgT9sz8iFi1VYWPfRYeUvUSFAaDao,Delegated Capital -tz1LsuPvP2u63nouVGAPi13AYqqB2VQjhoSF,Drunk Tortuga Bakery -tz1W1f1JrE7VsqgpUpj1iiDobqP5TixgZhDk,Elite Tezos -tz1PK8bNrfPbc73BtHH1A63jfU5UtbRFkvU4,EON -tz1MXFrtZoaXckE41bjUCSjAjAap3AFDSr3N,Everstake -tz1WcCGBxxPq61ddkp8X7vYMrWsBFMCxe5RD,FightCovid -tz1Scdr2HsZiQjc7bHMeBbmDRXYVvdhjJbBh,Figment -tz1ZQppA6UerMz5CJtGvZmmB6z8L9syq7ixu,First Block -KT1GJBSMRGApA2DnSivs73Fhau6dbfP1Sntd,Flippin Tacos Payouts 1 -tz1TacosCtikEQCJWYDZZ9MuA42sVpbbKZFF,Flippin Tacos Payouts 2 -tz1TzaNn7wSQSP5gYPXCnNzBCpyMiidCq1PX,Flippin' tacos -tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9,Foundation Baker 1 -tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5,Foundation Baker 2 -tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8,Foundation Baker 3 -tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk,Foundation Baker 4 -tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV,Foundation Baker 5 -tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r,Foundation Baker 6 -tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K,Foundation Baker 7 -tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN,Foundation Baker 8 -tz1P6kxtzRrJ8KfXE4kKqR1RBiSpEgimxN5o,Fractalide -tz1QLXqnfN51dkjeghXvKHkJfhvGiM5gK4tc,Fresh Tezos -tz1djRgXXWWJiY1rpMECCxr5d9ZBqWewuiU1,Galleon Support -tz1NpWrAyDL9k2Lmnyxcgr9xuJakbBxdq7FB,Gate.io -tz1WCd2jm4uSt4vntk4vSuUWoZQGhLcDuR9q,Happy Tezos -tz1acsihTQWHEnxxNz7EEsBDLMTztoZQE9SW,Happy Tezos Payout -tz1KzSC1J9aBxKp7u8TUnpN8L7S65PBRkgdF,HashQuark -tz1SohptP53wDPZhzTWzDUFAUcWF6DMBpaJV,Hayek Lab -tz1gg5bjopPcr9agjamyu9BbXKLibNc2rbAq,hodl.farm -tz1hodLFy8YDHtf1qJ3XH1gCyeXfwWEitkj9,hodl.farm Payouts -tz1Z3KCf8CLGAYfvVWPEr562jDDyWkwNF7sT,Hot Stake -tz1VHFxUuBhwopxC9YC9gm5s2MHBHLyCtvN1,Hyperblocks -tz1dp3tScJZy8Go3byPdioSujgDi5CdemiuZ,iBakeTezos -tz1ZePm7HyANf3P9hZfhmm8ShbrbCP1nYwT5,iBakeTezos Payouts 1 -tz1VQq41XksF8oJ4ntxbh99g5wrqcreVf7e1,iBakeTezos Payouts 2 -tz1Ryat7ZuRapjGUgsPym9DuMGrTYYyDBJoq,Illuminarean -tz1bRaSjKZrSrSeQHBDiCqjKXqtZYZM1t8FW,Imma Baker -tz1awXW7wuXy21c66vBudMXQVAPgRnqqwgTH,Infinity Stones -tz1ZTG13gkvouxSANka3HG3uys8C5gu3DPXZ,Just a Baker -tz1gfArv665EUkSg2ojMBzcbfwuPxAvqPvjo,Kraken -tz1bDXD6nNSrebqmAnnKKwnX1QdePSMCj4MX,Kraken 1 -tz1YpWsMwc4gSyjtxEF3JbmN6YrGiDidaSmg,Kraken 2 -tz1aqcYgG6NuViML5vdWhohHJBYxcDVLNUsE,LetzBake! -tz1MTAmuFmKu3MoxnkVsMj7xvSScj6d83hu5,LetzBake! Payouts -tz1VmiY38m3y95HqQLjMwqnMS7sdMfGomzKi,Lucid Mining -KT1THshzbGq3NJfL8w9j3ppbUSCfy5tQvWch,Lucid Mining Payouts 1 -tz1LsnEuv4fLsgxU9HJK21v8zHwcuBjb99Cd,Lucid Mining Payouts 2 -tz1MkjfzHNF3kk7PMUDVsaTrF8iRD6fpHftR,LuxBaker -tz1aEsrxrhTYNcg897HjSWnJ5yjR95pxVD1d,Mama Baker -tz1PWCDnz783NNGGQjEFFsHtrcK5yBW4E2rm,Melange -tz1cb8xcmJWcdVU7cNAd93MfEReorvP52P8x,Mint Capital -tz1NEKxGEHsFufk87CVZcrqWu8o22qh46GK6,Money Every 3 Days -tz1egbN6RK2bM5vt4aAZw6r9j4nL8z49bPdS,Money Every 3 Days -tz1c7pVR4w3KSQarJdBeh4NS2WxMUFBy1rHQ,MyBakerSpace -tz1d6Fx42mYgVFnHUW8T8A7WBfJ6nD9pVok8,MyTezosBaking -tz1Y2ufHak4AcisVazSS1ymGgv4HAmFGagh5,MyTezosBaking Payouts -tz1YdCPrYbksK7HCoYKDyzgfXwY16Fy9rrGa,Norn Delegate -tz1KtvGSYU5hdKD288a1koTBURWYuADJGrLE,Ø Crypto Pool -tz1UuQ4HWDu3ALNRgAq94dX9MhqhQhnuY3gC,OKEx -tz1X1fpAZtwQk94QXUgZwfgsvkQgyc2KHp9d,ownBLOCK -tz1hTFcQk2KJRPzZyHkCwbj7E1zY1xBkiHsk,ownBLOCK -tz1P2Po7YM526ughEsRbY4oR9zaUPDZjxFrb,P2P Validator -tz1Ldzz6k1BHdhuKvAtMRX7h5kJSMHESMHLC,PayTezos -tz1c9HEyZdAWXZCLE7h3KcZCyykjZr7VY3QM,PayTezos Payouts -KT1JjzGWesNFaHv9iyBwLBzLyTpTZ3HeBf7o,PayTezos Payouts KT -tz1Yju7jmmsaUiG9qQLoYv35v5pHgnWoLWbt,Polychain Labs -tz2TSvNTh2epDMhZHrw73nV9piBX7kLZ9K9m,Polychain Labs 2 -tz3TSvNTh2epDMhZHrw73nV9piBX7kLZ9K9m,Polychain Labs II -tz1Vyuu4EJ5Nym4JcrfRLnp3hpaq1DSEp1Ke,POS Bakerz -tz1V4qCyvPKZ5UeqdH14HN42rxvNPQfc9UZg,Shake 'n Bake -tz1ZuxNqUk7odhMC4Bfx2NXXeej9ReXKewa8,SNZ Holding -tz1MDKr36woXfVtrtXfV1ppPJERxPcm2wU6V,Stack Tezos -tz1V2FYediKBAEaTpXXJBSjuQpjkyCzrTSiE,Stake House -tz2FCNBrERXtaTtNX6iimR1UJ5JSDxvdHM93,Stake.fish -tz1MowoYvqgxhKToQMAmGgTtjfZcRfS733JY,StakeBake -tz1RCFbB9GpALpsZtu6J58sb74dm8qe6XBzv,Staked -tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194,StakeNow -tz1Ywgcavxq9D6hL32Q2AQWHAux9MrWqGoZC,StakeNow Payouts -KT1EctCuorV2NfVb1XTQgvzJ88MQtWP8cMMv,Staker -tz1go7f6mEQfT2xX2LuHAqgnRGN6c2zHPf5c,Stakery -tz1gs78wrxS34XhGPECmN5fABKpEuhUGwAXS,Stakery Payouts -tz1WpeqFaBG9Jm73Dmgqamy8eF8NWLz9JCoY,Staking Facilities -tz1STeamwbp68THcny9zk3LsbG3H36DMvbRK,Staking Team -tz1e2meErj7eEfXwqr7bDK6N1YatmLaugfMp,STB -tz1dNVDWPf3Q59SdJqnjdnu277iyvReiRS9M,Steak.and.Bake -tz1fvBC1cHJzPtUW3ZKd2GZDj7P4XVuLk9vW,Steaker -tz1LH4L6XYT2JNPhvWYC4Zq3XEiGgEwzNRvo,Stir Delegate -tz1Lhf4J9Qxoe3DZ2nfe8FGDnvVj7oKjnMY6,Tez Baker -KT1TUi7KGFcTLku6niB2iGqzkv49ZLYxLRy1,Tez Baker Payouts 1 -KT1Q6YAVkqcEmXAM7SUcQS9YSbLwbqMaVe3Z,Tez Baker Payouts 2 -KT1KcLH9Qz6QJucvajEdXQvcSHgxfwuwDjNe,Tez Baker Payouts 3 -KT1V86WP8CuxAtQ4xkpZ5fVtXv5XzK2F81mp,Tez Baker Payouts 4 -tz1NXPdPC2DfyA5ohitnS6ryqrDKbFJeu5Nn,Tez Boss -tz1hgjLqCZWCtarByUMYd81F4FKRY5t1V978,Tez Dispensery -tz1bakeKFwqmtLBzghw8CFnqFvRxLj849Vfg,Tez Milk -tz1UUgPwikRHW1mEyVZfGYy6QaxrY6Y7WaG5,Tez Patisserie -tz1abTjX2tjtMdaq5VCzkDtBnMSCFPW2oRPa,TEZ Rocket -tz1isXamBXpTUgbByQ6gXgZQg4GWNW7r6rKE,Tez Whale -tz1WHALEgzq8cUUkg1vBKrpQrg9mysEX9NYR,Tez Whale Payouts 1 -tz1WHALEP56kjCTuUFLWjcBkfFpyevC4xo3t,Tez Whale Payouts 2 -tz1LBEKXaxQbd5Gtzbc1ATCwc3pppu81aWGc,Tez-Baking -KT1BvVxWM6cjFuJNet4R9m64VDCN2iMvjuGE,Tez-Baking Token -tz1YhNsiRRU8aHNGg7NK3uuP6UDAyacJernB,TezBaguette -tz3LBtbMMvvguWQupgEmtfjtXy77cHgdr5TE,Tezcentral -tz1Yc6ATtfUJyDjHwJ8WoVL22sJueDenueke,TezDele Baker A -tz1SdwBHocSrcuMFNLPUg4LPRfx9eaqjVUEL,TezDele Baker B -tz1LnWa4AiFCjMezTgSNa65QFoo7DQGEWgEU,Tezeract -tz1VceyYUpq1gk5dtp6jXQRtCtY8hm5DKt72,Tezetetic -tz1MR42UxXgFmxsP4iVEF8e8a7q17TPLicpt,Tezetetic Payout -tz1W5VkdB5s7ENMESVBtwyt9kyvLqPcUczRT,Tezgate -tz1g4o4a2wxpzJ7EgG3onFM5TLaPyiRFjFhL,Tezgate Payouts -tz1MQJPGNMijnXnVoBENFz9rUhaPt3S7rWoz,Tezmania -tz1TaLYBeGZD3yKVHQGBM857CcNnFFNceLYh,Tezocracy -KT1CthjeJp2pLbSdo6xZR1VfgUD44QYDNmSZ,Tezocracy Payouts 1 -tz1hfXvfcbbfcA77U8RXn7xLbRn3NHU4ksU1,Tezocracy Payouts 2 -tz1RSWMYKGAykpizFteowByYMueCYv9TMn1L,Tezos Alliance -tz1PriNQyDC7d5ccPAD96ugujYy5YbdGLdQ5,Tezos Baker JP -tz1fZ767VDbqx4DeKiFswPSHh513f51mKEUZ,Tezos Bakery -tz1PYLN9TsKZHfn2GtrXnxkeGvahmYdBTG5v,Tezos Bakes 2 -tz1b2mCFQm8isfJRGUctgyxHhHcmTNTRPdps,Tezos Barcelona -tz1Z7bxn1M4ewEWUekPaZfiwtKQRFroezUjY,Tezos Bay -tz1Y42aKCk53vMbaJNpf1hBg1rznGdBxHJ5C,Tezos Berlin -tz1Z2jXfEXL7dXhs6bsLmyLFLfmAkXBzA9WE,Tezos Boutique -tz1aKxnrzx5PXZJe7unufEswVRCMU9yafmfb,Tezos Canada -tz3PdGc7U5tiyqPgTSgqCDct94qd6ovQwP6u,Tezos Capital -tz1TDSmoZXwVevLTEvKCTHWpomG76oC9S2fJ,Tezos Capital Legacy -tz1hThMBD8jQjFt78heuCnKxJnJtQo9Ao25X,Tezos Chef -tz1aCJuxfZPsXZ3PmfA6MvjsRTpGVVDMmb8X,Tezos Chef Payouts -tz1WJpSCYdQE9merfftanA8eFsqUPTpwAt8a,Tezos China -tz1YTyvABUyhE7JHpxMVBVqjZnZM4ofMrWKE,Tezos Delegate EU -tz1U638Z3xWRiXDDKx2S125MCCaAeGDdA896,Tezos France -tz1TAzNBtnq6vF6kg34ApectxXoxZVTyapq3,Tezos Gold -tz1WnfXMPaNTBmH7DBPwqCWs9cPDJdkGBTZ8,Tezos HODL -tz1bLwpPfr3xqy1gWBF4sGvv8bLJyPHR11kx,Tezos Hot Bakery -tz1Kn5Dyir12f7NafEztgooypotBkE92mKDr,Tezos Hot Bakery Payout -tz1Vm5cfHncKGBo7YvZfHc4mmudY4qpWzvSB,Tezos Istanbul -tz1MBERDzXMvMQPCMWBCEgxyKrdU71HgXaM8,Tezos Istanbul Payouts 1 -tz1Uf4Raxz7JzNdpAyM3893qLtqTRwNrdKVK,Tezos Istanbul Payouts 2 -tz1b7YSEeNRqgmjuX4d4aiai2sQTF4A7WBZf,Tezos Japan -tz1ijdEfmUoQXJfSiKCLRioeCfSrKP3sVsff,Tezos Japan 2 -tz1Nn14BBsDULrPXtkM9UQeXaE4iqJhmqmK5,Tezos Kitchen -tz1PPUo28B8BroqmVCMMNDudG4ShA2bzicrU,Tezos Korea -tz1S1Aew75hMrPUymqenKfHo8FspppXKpW7h,Tezos Mars -tz1TgvirXEwVJPg1cCrxT9zFubdzw7Ng6Ke9,Tezos Moon -tz1b3SaPHFSw51r92ARcV5mGyYbSSsdFd5Gz,Tezos MX -tz1djEKk6j1FqigTgbRsunbnY9BB7qsn1aAQ,Tezos Ninja -tz1eFXLaCUcKbhXkLBGUGMGNY9tKHjWGG25V,Tezos Node Spain -tz1SkU5S3dDRi6f8LHCvYeK3H3Mjq4MS8uHW,Tezos OTC -tz1PeZx7FXy7QRuMREGXGxeipb24RsMMzUNe,Tezos Panda -KT1PnRgUjx7dD4xrnXnpJHz9MfnCWoeT6UBr,Tezos Panda Payouts -tz1cZfFQpcYhwDp7y1njZXDsZqCrn2NqmVof,Tezos Rio -tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY,Tezos Seoul -tz1RAzdkp1x5ZDqms4ZUrSdfJuUYGH9JTPK2,Tezos Spanish -tz1hAYfexyzPGG6RhZZMpDvAHifubsbb6kgn,Tezos Suisse -tz1bkKTY9Y3rTsHbpr2fbGUCRm736LLquQfM,Tezos Tacos -tz1VYQpZvjVhv1CdcENuCNWJQXu1TWBJ8KTD,Tezos Tokyo -tz1bHzftcTKZMTZgLLtnrXydCm6UEqf4ivca,Tezos Vote -tz1ggJg924JdWdww6oLGFksRgKctfrDMUNfE,Tezos Vote Payouts -tz1WctSwL49BZtdWAQ75NY6vr4xsAx6oKdah,Tezos Wake n'Bake -tz1MkaTVYdM9QvXxKvYEwG9BPN2XxriTs7Jh,Tezos.bi23 -tz1fb7c66UwePkkfDXz4ajFaBP9hVNLdS7JJ,Tezos.nu -tz1gfipKzYrNRT14oSNQJMCdRRsUtcbZoKsd,Tezos.Rio_old -tz1YvG6poSGhPLTdzbzCmqShMndzoyhyVg7J,tezosbaking.club -tz1c3Wh8gNMMsYwZd67JndQpYxdaaPUV27E7,TezosBC -KT1E5rWDHRA679Q98Tks3DppCqvBLUzmnBEe,TezosBC Payouts -tz1QGZ3dD2YpRKZ4APeso6EXTeyCUUkw6MQC,TezosBC_old -tz1YKh8T79LAtWxX29N5VedCSmaZGw9LNVxQ,TezosBr -tz1TzHDLTNK1J1fCotGBLuE43DoogC5pHGyT,TezosHODL Payouts -tz1V3yg82mcrPJbegqVCPn6bC8w1CSTRp3f8,TezoShop -tz1hxtCEcD7idQJEDiJEq37vBkoRcwF6KC2X,TezoShop Payout -tz1b9MYGrbN1NAxphLEsPPNT9JC7aNFc5nA4,TezosRUs -tz1R6Ej25VSerE3MkSoEEeBjKHCDTFbpKuSX,TezosSEAsia -tz1Lh9jeLSWDHYy8AshvG2dpNQseDaHg7cms,TezosSpanish -tz1LLNkQK4UQV6QcFShiXJ2vT2ELw449MzAA,TezosTeam -tz1YrmJw6Lje27gWqZ94gU9mNavEjkHu1xGc,TezosTeam Payouts -tz1UJvHTgpVzcKWhTazGxVcn5wsHru5Gietg,Tezry -tz1Vtimi84kLh9RANfRVX2JvYtP4NPCT1aFm,TezTech Labs -tz1Zcxkfa5jKrRbBThG765GP29bUCU3C4ok5,Tezz City -tz1fUjvVhJrHLZCbhPNvDRckxApqbkievJHN,Tezzieland -tz1WntXgznbivRjyhE7Y5jEzoebAhMPB2iJa,Tezzieland Payouts -tz3adcvQaKXTCg12zbninqo3q8ptKKtDFTLv,Tezzigator -tz1iZEKy4LaAjnTmn2RuGDf2iqdAQKnRi8kY,Tezzigator Legacy -tz1dhwJY5B1jKdkF7zcyu1v6EBEQjpGmAp1E,Tezzz -KT1U1JZaXoG4u1EPnhHL4R4otzkWc1L34q3c,Trianon KYC Registrar -KT1RM4Nzwe9kgyiF3WpqPdUezJ2SS7ixUdHX,Trianon STO Issuer -KT1XHAmdRKugP1Q38CxDmpcRSxq143KpEiYx,Trianon STO Token -tz1Zhv3RkfU2pHrmaiDyxp7kFZpZrUCu1CiF,TZ Bake -tz1XQ7SRj4QQWjaeebNd8dFwuTrCot3GGDRF,Tz Baker -tz1cX93Q3KsiTADpCC4f12TBvAmS5tw7CW19,Tz Bakery -tz1bkg7rynMXVcjomoe3diB4URfv8GU2GAcw,Tz Bank -tz1PesW5khQNhy4revu2ETvMtWPtuVyH2XkZ,Tz Dutch -tz1iJ4qgGTzyhaYEzd1RnC6duEkLBd1nzexh,Tz Envoy -tz1Vd1rXpV8hTHbFXCXN3c3qzCsgcU5BZw1e,Tz Node -tz1Yh1nLXfwqVpP8btykuRguu61n3veVmADa,TzBakeOven -KT1LH2o12xVRwTpJMZ6QJG74Fox8gE9QieFd,TZBTC -KT1NCuMkcRrGDSiB5BX4xyBGEUtyyF7YDDYz,TZBTC 2 -tz1Pwgj6j55akKCyvTwwr9X4np1RskSXpQY4,Validators.com -tz1aiYKXmSRckyJ9EybKmpVry373rfyngJU8,View Nodes -tz1iMAHAVpkCVegF9FLGWUpQQeiAHh4ffdLQ,Wetez Wallet -tz1LesY3S4wfe15SNm1W3qJmQzWxLqVjTruH,Xtez -tz1TcH4Nb3aHNDJ7CGZhU7jgAK1BkSP4Lxds,XTZ Antipodes -tz1P9GwUWeAyPoT3ZwBc6gWCTP9exEPaVU3w,XTZ Bakery I -tz1TnYeuR4HhUTapQiXvzcjmcpvZYwR1B5S3,XTZ Bakery II -tz1Suih9uWEnubDeqCTCEfueSCGWYohjyaA5,XTZ Black -tz1Xek93iSXXckyQ6aYLVS5Rr2tge2en7ZxS,XTZ Delegate -tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj,XTZ Land -tz1KfEsrtDaA1sX7vdM4qmEPWuSytuqCDp5j,XTZ Master -tz1XTZM55hF7g98CzY88MXWhmK8QioGXHtuY,XTZ Master Payout -tz1XTZMBAARXu3zmFno6SMT88MvqSvbD5qrh,XTZ Master Payouts -tz1ZKi4VrDMEQpypn2NTn9pPbZL3xLRkETLA,XTZ-Baker -tz1Q8QkSBS63ZQnH3fBTiAMPes9R666Rn6Sc,YieldWallet -tz1eZwq8b5cvE2bPKokatLkVMzkxz24z3Don,You Loaf We Bake -tz1L3vFD8mFzBaS8yLHFsd7qDJY1t276Dh8i,Zednode \ No newline at end of file diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala index e99b8909d..de4ec4afa 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala @@ -110,7 +110,8 @@ final case class Features( metadataFetchingIsOn: Boolean, registeredTokensIsOn: Boolean, rightsProcessingIsOn: Boolean, - bakerFeaturesAreOn: Boolean + bakerFeaturesAreOn: Boolean, + lightweightIndexing: Boolean ) final case class TokenContracts( diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala index 35c681feb..64966f18b 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala @@ -823,32 +823,34 @@ private[tezos] object TezosDatabaseConversions { * To correctly create the relation on the db, we must first store the operations, get * each generated id, and pass it to the associated balance-updates */ - implicit val blockToOperationTablesData = new Conversion[List, Block, OperationTablesData] { + implicit val blockToOperationTablesData = new Conversion[List, (Block, Option[List[Tables.KnownAddressesRow]]), OperationTablesData] { import OperationBalances._ import SymbolSourceLabels.Show._ import tech.cryptonomic.conseil.common.util.Conversion.Syntax._ - override def convert(from: Block) = + override def convert(from: (Block, Option[List[Tables.KnownAddressesRow]])) = { + val (block, knownAddresses) = from TezosOptics.Blocks - .extractOperationsAlongWithInternalResults(from) + .extractOperationsAlongWithInternalResults(block, knownAddresses) .flatMap { case (group, (operations, internalResults)) => val mainOperationData = operations.map(op => - (from, group.hash, op).convertTo[Tables.OperationsRow] -> + (block, group.hash, op).convertTo[Tables.OperationsRow] -> BlockTagged - .fromBlockData(from.data, op) + .fromBlockData(block.data, op) .convertToA[List, Tables.BalanceUpdatesRow] ) val internalOperationData = internalResults.map { case oop => val op = oop.convertTo[Operation] - (from, group.hash, op) + (block, group.hash, op) .convertTo[Tables.OperationsRow] .copy(internal = true, nonce = Some(oop.nonce.toString)) -> BlockTagged - .fromBlockData(from.data, op) + .fromBlockData(block.data, op) .convertToA[List, Tables.BalanceUpdatesRow] } mainOperationData ++ internalOperationData } .toList + } } diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala index 60084a5d7..495b84f79 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala @@ -2,7 +2,6 @@ package tech.cryptonomic.conseil.indexer.tezos import java.sql.Timestamp import java.time.{Instant, ZoneOffset} - import cats.effect.Sync import cats.implicits._ import scribe._ @@ -13,13 +12,7 @@ import tech.cryptonomic.conseil.common.config.ChainEvent.AccountIdPattern import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query => _} import tech.cryptonomic.conseil.common.sql.CustomProfileExtension import tech.cryptonomic.conseil.common.tezos.Tables -import tech.cryptonomic.conseil.common.tezos.Tables.{ - AccountsRow, - GovernanceRow, - OperationsRow, - OriginatedAccountMapsRow, - RegisteredTokensRow -} +import tech.cryptonomic.conseil.common.tezos.Tables.{AccountsRow, GovernanceRow, KnownAddresses, OperationsRow, OriginatedAccountMapsRow, RegisteredTokensRow} import tech.cryptonomic.conseil.common.tezos.TezosTypes.Fee.AverageFees import tech.cryptonomic.conseil.common.tezos.TezosTypes._ import tech.cryptonomic.conseil.common.util.ConfigUtil @@ -35,7 +28,6 @@ import scala.concurrent.{ExecutionContext, Future} import scala.math import scala.util.{Failure, Success} import java.util.UUID - import slick.dbio.DBIOAction import tech.cryptonomic.conseil.indexer.tezos.RegisteredTokensFetcher.RegisteredToken @@ -103,7 +95,8 @@ object TezosDatabaseOperations extends ConseilLogSupport { */ def writeBlocks( blocks: List[Block], - tokenContracts: TokenContracts + tokenContracts: TokenContracts, + knownAddresses: Option[List[Tables.KnownAddressesRow]] )(implicit ec: ExecutionContext, tnsContracts: TNSContract): DBIO[Unit] = { // Kleisli is a Function with effects, Kleisli[F, A, B] ~= A => F[B] import TezosDatabaseConversions.OperationTablesData @@ -122,6 +115,7 @@ object TezosDatabaseOperations extends ConseilLogSupport { block.data.convertToA[List, BalanceUpdatesRow] } + val saveGroupsAction = Tables.OperationGroups ++= blocks.flatMap(_.convertToA[List, OperationGroupsRow]) //a function that takes a row to save and creates an action to do that, returning the new id @@ -149,7 +143,7 @@ object TezosDatabaseOperations extends ConseilLogSupport { saveBlocksAction, saveBlocksBalanceUpdatesAction, saveGroupsAction, - saveOperationsAndBalances.traverse(blocks.flatMap(_.convertToA[List, OperationTablesData])), + saveOperationsAndBalances.traverse(blocks.flatMap(xx => (xx->knownAddresses).convertToA[List, OperationTablesData])), saveBigMaps(blocks)(ec, tokenContracts, tnsContracts) ) @@ -398,7 +392,8 @@ object TezosDatabaseOperations extends ConseilLogSupport { */ def writeBlocksAndCheckpointAccounts( blocks: List[Block], - accountUpdates: List[BlockTagged[List[AccountId]]] + accountUpdates: List[BlockTagged[List[AccountId]]], + knownAddresses: Option[List[Tables.KnownAddressesRow]] )(implicit ec: ExecutionContext, tnsContracts: TNSContract): DBIO[Option[Int]] = { logger.info("Writing blocks and account checkpoints to the DB...") //sequence both operations in a single transaction @@ -409,7 +404,7 @@ object TezosDatabaseOperations extends ConseilLogSupport { ContractId(address) -> interfaces }.toList ) - (writeBlocks(blocks, tokens) andThen writeAccountsCheckpoint(accountUpdates.map(_.asTuple))).transactionally + (writeBlocks(blocks, tokens, knownAddresses) andThen writeAccountsCheckpoint(accountUpdates.map(_.asTuple))).transactionally } } diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosGovernanceOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosGovernanceOperations.scala index a948ce07b..599530395 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosGovernanceOperations.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosGovernanceOperations.scala @@ -373,7 +373,7 @@ object TezosGovernanceOperations extends ConseilLogSupport { //adapting database-level formats to the domain-level ones def downCastToInt(bd: BigDecimal) = Try(bd.toIntExact).toOption //should be the level we reached, before the current processing batch - val previousBatchHighLevel = blocks.map(_.data.header.level).min - 1 + val previousBatchHighLevel = Try(blocks.map(_.data.header.level).min.toLong).toOption.getOrElse(1L) - 1 TezosDatabaseOperations .getGovernanceForLevel(previousBatchHighLevel) diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala index f81773975..13e978a12 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala @@ -63,6 +63,7 @@ class TezosIndexer private ( forkHandler: ForkHandler[Future, TezosBlockHash], backtrackingForkProcessor: BacktracingForkProcessor, feeOperations: TezosFeeOperations, + knownAddresses: List[Tables.KnownAddressesRow], terminationSequence: () => Future[ShutdownComplete] )(implicit system: ActorSystem, @@ -348,6 +349,64 @@ object TezosIndexer extends ConseilLogSupport { lorreConf.headOffset ) + /* Reads csv resources to initialize db tables and smart contracts objects */ + def parseCSVConfigurations() = { + /* This will derive automatically all needed implicit arguments to convert csv rows into a proper table row + * Using generic representations for case classes, provided by the `shapeless` library, it will create the + * appropriate `kantan.csv.Decoder` for + * - headers, as extracted by the case class definition which, should match name and order of the csv header row + * - table row types, which can be converted to shapeless HLists, which so happens to be of any case class. + * + * What shapeless does, is provide an automatic conversion, from a case class to a typed generic list of values, + * where each element has a type corresponding to a field in the case class, and a "label" that is + * a sort of string extracted at compile-time via macros, to figure out the field names. + * Such special list, is a `shapeless.HList`, or heterogeneous list, sort of a dynamic tuple, + * to be built by adding/removing individual elements in the list, recursively. + * This allows generic libraries like kantan to define codecs for generic HLists and, + * using shapeless, to adapt any case class to his specific HList, at compile-time. + * + * For additional information, refer to the project wiki, and + * - http://nrinaudo.github.io/kantan.csv/ + * - http://www.shapeless.io/ + */ + import kantan.csv.generic._ + //we add any missing implicit decoder for column types, not provided by default from the library + import tech.cryptonomic.conseil.common.util.ConfigUtil.Csv._ + import cats.implicits._ + + /* Inits tables with values from CSV files */ + ( + TezosDb.initTableFromCsv(db, Tables.KnownAddresses, selectedNetwork), + TezosDb.initTableFromCsv(db, Tables.BakerRegistry, selectedNetwork), + TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) + ).mapN { case (ka, _, _) => + ka._1 + } + /* Here we want to initialize the registered tokens and additionally get the token data back + * since it's needed to process calls to the same token smart contracts as the chain evolves + */ + + } + + /* the shutdown sequence to free resources */ + val gracefulTermination = () => + for { + _ <- Future.successful(db.close()) + _ = materializer.shutdown() + _: Terminated <- system.terminate() + _: ShutdownComplete <- nodeOperator.node.shutdown() + } yield ShutdownComplete + + val knownAddresses = Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { + case Success(xx) => + logger.info("DB initialization successful") + xx + case Failure(exception) => + logger.error("DB initialization failed", exception) + gracefulTermination().map(_ => ()) + List.empty + } + /* provides operations to handle rights to bake and endorse blocks */ val rightsProcessor = new BakingAndEndorsingRightsProcessor( nodeOperator, @@ -397,7 +456,8 @@ object TezosIndexer extends ConseilLogSupport { tnsOperations, accountsProcessor, bakersProcessor, - lorreConf.enabledFeatures + lorreConf.enabledFeatures, + if(lorreConf.enabledFeatures.lightweightIndexing) Some(knownAddresses) else None ) val metadataProcessor = new MetadataProcessor( @@ -436,66 +496,9 @@ object TezosIndexer extends ConseilLogSupport { val feeOperations = new TezosFeeOperations(db) - /* the shutdown sequence to free resources */ - val gracefulTermination = () => - for { - _ <- Future.successful(db.close()) - _ = materializer.shutdown() - _: Terminated <- system.terminate() - _: ShutdownComplete <- nodeOperator.node.shutdown() - } yield ShutdownComplete - /* Used for fetching and handling registered tokens */ val registeredTokensFetcher = new RegisteredTokensFetcher(db, lorreConf.tokenContracts, gracefulTermination) - /* Reads csv resources to initialize db tables and smart contracts objects */ - def parseCSVConfigurations() = { - /* This will derive automatically all needed implicit arguments to convert csv rows into a proper table row - * Using generic representations for case classes, provided by the `shapeless` library, it will create the - * appropriate `kantan.csv.Decoder` for - * - headers, as extracted by the case class definition which, should match name and order of the csv header row - * - table row types, which can be converted to shapeless HLists, which so happens to be of any case class. - * - * What shapeless does, is provide an automatic conversion, from a case class to a typed generic list of values, - * where each element has a type corresponding to a field in the case class, and a "label" that is - * a sort of string extracted at compile-time via macros, to figure out the field names. - * Such special list, is a `shapeless.HList`, or heterogeneous list, sort of a dynamic tuple, - * to be built by adding/removing individual elements in the list, recursively. - * This allows generic libraries like kantan to define codecs for generic HLists and, - * using shapeless, to adapt any case class to his specific HList, at compile-time. - * - * For additional information, refer to the project wiki, and - * - http://nrinaudo.github.io/kantan.csv/ - * - http://www.shapeless.io/ - */ - import kantan.csv.generic._ - //we add any missing implicit decoder for column types, not provided by default from the library - import tech.cryptonomic.conseil.common.util.ConfigUtil.Csv._ - import cats.implicits._ - - /* Inits tables with values from CSV files */ - ( - TezosDb.initTableFromCsv(db, Tables.KnownAddresses, selectedNetwork), - TezosDb.initTableFromCsv(db, Tables.BakerRegistry, selectedNetwork), - TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) - ).mapN { case (_, _, _) => - () - } - /* Here we want to initialize the registered tokens and additionally get the token data back - * since it's needed to process calls to the same token smart contracts as the chain evolves - */ - - } - - Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { - case Success(_) => - () - logger.info("DB initialization successful") - case Failure(exception) => - logger.error("DB initialization failed", exception) - gracefulTermination().map(_ => ()) - } - new TezosIndexer( ignoreProcessFailures, lorreConf, @@ -511,6 +514,7 @@ object TezosIndexer extends ConseilLogSupport { forkHandler, backtracingForkProcessor, feeOperations, + knownAddresses, gracefulTermination ) } diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala index 26d6d4d9a..541ace69d 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala @@ -4,15 +4,9 @@ import scala.util.{Failure, Success, Try} import scala.concurrent.{ExecutionContext, Future} import slick.jdbc.PostgresProfile.api._ import cats.implicits._ -import tech.cryptonomic.conseil.indexer.tezos.{ - TezosGovernanceOperations, - TezosNamesOperations, - TezosNodeOperator, - Tzip16MetadataOperator, - TezosDatabaseOperations => TezosDb -} +import tech.cryptonomic.conseil.indexer.tezos.{TezosGovernanceOperations, TezosNamesOperations, TezosNodeOperator, Tzip16MetadataOperator, TezosDatabaseOperations => TezosDb} import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport -import tech.cryptonomic.conseil.common.tezos.TezosTypes +import tech.cryptonomic.conseil.common.tezos.{Tables, TezosTypes} import tech.cryptonomic.conseil.common.tezos.TezosTypes.{Block, InternalOperationResults, Voting} import tech.cryptonomic.conseil.common.tezos.TezosTypes.Syntax._ import tech.cryptonomic.conseil.indexer.config.Features @@ -38,10 +32,12 @@ class BlocksProcessor( tnsOperations: TezosNamesOperations, accountsProcessor: AccountsProcessor, bakersProcessor: BakersProcessor, - featureFlags: Features + featureFlags: Features, + knownAddresses: Option[List[Tables.KnownAddressesRow]] )(implicit tns: TNSContract) extends ConseilLogSupport { + println(knownAddresses) /* will store a single page of block results */ private[tezos] def processBlocksPage(results: nodeOperator.BlockFetchingResults)(implicit ec: ExecutionContext @@ -56,13 +52,28 @@ class BlocksProcessor( //ignore the account ids for storage, and prepare the checkpoint account data //we do this on a single sweep over the list, pairing the results and then unzipping the outcome - val (blocks, accountUpdates) = - results.map { case (block, accountIds) => - block -> accountIds.taggedWithBlockData(block.data) - }.unzip + val (blocks, accountUpdates) = { + knownAddresses match { + case Some(value) => + results.map { case (block, accountIds) => + val ids = accountIds.toSet.intersect(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet) + println(accountIds.toSet) + println(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet) + println(ids) + if(ids.nonEmpty) + Some(block -> accountIds.taggedWithBlockData(block.data)) + else None + }.filter(_.isDefined).map(_.get).unzip + case None => + results.map { case (block, accountIds) => + block -> accountIds.taggedWithBlockData(block.data) + }.unzip + } + + } - for { - _ <- db.run(TezosDb.writeBlocksAndCheckpointAccounts(blocks, accountUpdates)) andThen logBlockOutcome + for { + _ <- db.run(TezosDb.writeBlocksAndCheckpointAccounts(blocks, accountUpdates, knownAddresses)) andThen logBlockOutcome _ <- tnsOperations.processNamesRegistrations(blocks).flatMap(db.run) bakersCheckpoints <- accountsProcessor.processAccountsForBlocks(accountUpdates) // should this fail, we still recover data from the checkpoint _ <- bakersProcessor.processBakersForBlocks(bakersCheckpoints) From ade0c4949375009f83160821dbbded933ee9ee6f Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Wed, 15 Jun 2022 18:22:01 +0200 Subject: [PATCH 2/6] fixing issue when blocs weren't stored --- .../indexer/tezos/processing/BlocksProcessor.scala | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala index 541ace69d..638cf06ed 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala @@ -37,7 +37,6 @@ class BlocksProcessor( )(implicit tns: TNSContract) extends ConseilLogSupport { - println(knownAddresses) /* will store a single page of block results */ private[tezos] def processBlocksPage(results: nodeOperator.BlockFetchingResults)(implicit ec: ExecutionContext @@ -57,13 +56,8 @@ class BlocksProcessor( case Some(value) => results.map { case (block, accountIds) => val ids = accountIds.toSet.intersect(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet) - println(accountIds.toSet) - println(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet) - println(ids) - if(ids.nonEmpty) - Some(block -> accountIds.taggedWithBlockData(block.data)) - else None - }.filter(_.isDefined).map(_.get).unzip + block -> accountIds.taggedWithBlockData(block.data) + }.unzip case None => results.map { case (block, accountIds) => block -> accountIds.taggedWithBlockData(block.data) From c634a5da7cd4e893e8f09305cee5ec574b71f9d8 Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Thu, 30 Jun 2022 13:40:03 +0200 Subject: [PATCH 3/6] small fix when filtered account ids weren't used --- .../conseil/indexer/tezos/processing/BlocksProcessor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala index 638cf06ed..1c8d0494a 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala @@ -55,8 +55,8 @@ class BlocksProcessor( knownAddresses match { case Some(value) => results.map { case (block, accountIds) => - val ids = accountIds.toSet.intersect(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet) - block -> accountIds.taggedWithBlockData(block.data) + val ids = accountIds.toSet.intersect(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet).toList + block -> ids.taggedWithBlockData(block.data) }.unzip case None => results.map { case (block, accountIds) => From 28f4860d6541a063cfed5bb38aac06880681a26b Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Wed, 6 Jul 2022 19:04:12 +0200 Subject: [PATCH 4/6] fixed few last things in lightweight feature --- .../GenericPlatformDiscoveryOperations.scala | 8 +- .../conseil/common/tezos/TezosOptics.scala | 97 ++++++++++++++++--- .../conseil/common/tezos/TezosTypes.scala | 16 +-- .../tezos/TezosDatabaseConversions.scala | 53 +++++----- .../tezos/TezosDatabaseOperations.scala | 22 ++++- .../conseil/indexer/tezos/TezosIndexer.scala | 8 +- .../tezos/processing/BlocksProcessor.scala | 28 ++++-- 7 files changed, 164 insertions(+), 68 deletions(-) diff --git a/conseil-api/src/main/scala/tech/cryptonomic/conseil/api/routes/platform/discovery/GenericPlatformDiscoveryOperations.scala b/conseil-api/src/main/scala/tech/cryptonomic/conseil/api/routes/platform/discovery/GenericPlatformDiscoveryOperations.scala index 32284c217..bfe915373 100644 --- a/conseil-api/src/main/scala/tech/cryptonomic/conseil/api/routes/platform/discovery/GenericPlatformDiscoveryOperations.scala +++ b/conseil-api/src/main/scala/tech/cryptonomic/conseil/api/routes/platform/discovery/GenericPlatformDiscoveryOperations.scala @@ -153,11 +153,13 @@ class GenericPlatformDiscoveryOperations( * @param xs list of platform-network pairs for which we want to fetch attribute values * @return database action with attribute values to be cached */ - def initAttributeValuesCache(xs: List[(Platform, Network)]): Future[Boolean] = { + def initAttributeValuesCache(xs: List[(Platform, Network)]): Future[Boolean] = xs.map { case (platform, network) => IO.fromFuture(IO(dbRunners(platform.name, network.name).runQuery(preCacheAttributeValues(platform, network)))) - }.sequence.map(_.reduce(_ ++ _)).flatMap(caching.fillAttributeValuesCache) - }.unsafeToFuture() + }.sequence + .map(_.reduce(_ ++ _)) + .flatMap(caching.fillAttributeValuesCache) + .unsafeToFuture() /** Pre-caching attribute values from slick for specific platform * diff --git a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala index 21a615446..07e817211 100644 --- a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala +++ b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala @@ -389,32 +389,103 @@ object TezosOptics { block.operationGroups.map { group => val updatedGroup = addIndexesToOperationGroup(group) - val internal = updatedGroup.contents.filter{ ff => + val filteredOperations = updatedGroup.contents.filter { op => knownAddresses match { case Some(ka) => val addresses = ka.map(_.address) - ff match { + op match { case Endorsement(level, metadata, blockOrder) => false case EndorsementWithSlot(endorsement, metadata, blockOrder) => false case Preendorsement(level, metadata, blockOrder) => false case SeedNonceRevelation(level, nonce, metadata, blockOrder) => false case ActivateAccount(pkh, secret, metadata, blockOrder) => addresses.contains(pkh.value) - case Reveal(counter, fee, gas_limit, storage_limit, public_key, source, metadata, blockOrder) => addresses.contains(public_key.value) || addresses.contains(source.value) - case Transaction(counter, amount, fee, gas_limit, storage_limit, source, destination, parameters, parameters_micheline, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(destination.id) - case Origination(counter, fee, source, balance, gas_limit, storage_limit, manager_pubkey, delegatable, delegate, spendable, script, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value) - case Delegation(counter, source, fee, gas_limit, storage_limit, delegate, metadata, blockOrder) => addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value) + case Reveal(counter, fee, gas_limit, storage_limit, public_key, source, metadata, blockOrder) => + addresses.contains(public_key.value) || addresses.contains(source.value) + case Transaction( + counter, + amount, + fee, + gas_limit, + storage_limit, + source, + destination, + parameters, + parameters_micheline, + metadata, + blockOrder + ) => + addresses.contains(source.value) || addresses.contains(destination.id) + case Origination( + counter, + fee, + source, + balance, + gas_limit, + storage_limit, + manager_pubkey, + delegatable, + delegate, + spendable, + script, + metadata, + blockOrder + ) => + addresses.contains(source.value) || addresses.contains( + delegate.getOrElse(PublicKeyHash("")).value + ) || addresses.contains(manager_pubkey.getOrElse(PublicKeyHash("")).value) + case Delegation(counter, source, fee, gas_limit, storage_limit, delegate, metadata, blockOrder) => + addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value) case DoubleEndorsementEvidence(blockOrder) => false case DoublePreendorsementEvidence(blockOrder) => false case DoubleBakingEvidence(blockOrder) => false - case Proposals(source, period, proposals, blockOrder) => addresses.contains(source.getOrElse(ContractId("")).id) - case Ballot(ballot, proposal, source, period, blockOrder) => addresses.contains(source.getOrElse(ContractId("")).id) - case RegisterGlobalConstant(source, fee, counter, gas_limit, storage_limit, value, blockOrder, metadata) => addresses.contains(source.getOrElse(ContractId("")).id) - case SetDepositsLimit(source, fee, counter, gas_limit, storage_limit, limit, blockOrder, metadata) => addresses.contains(source.getOrElse(ContractId("")).id) + case Proposals(source, period, proposals, blockOrder) => + addresses.contains(source.getOrElse(ContractId("")).id) + case Ballot(ballot, proposal, source, period, blockOrder) => + addresses.contains(source.getOrElse(ContractId("")).id) + case RegisterGlobalConstant( + source, + fee, + counter, + gas_limit, + storage_limit, + value, + blockOrder, + metadata + ) => + addresses.contains(source.getOrElse(ContractId("")).id) + case SetDepositsLimit(source, fee, counter, gas_limit, storage_limit, limit, blockOrder, metadata) => + addresses.contains(source.getOrElse(ContractId("")).id) + case TxRollupCommit(counter, fee, source, gas_limit, storage_limit, rollup, metadata, blockOrder) => + addresses.contains(source.value) + case TxRollupFinalizeCommitment( + counter, + fee, + source, + gas_limit, + storage_limit, + rollup, + metadata, + blockOrder + ) => + addresses.contains(source.value) + case TxRollupOrigination(counter, fee, source, gas_limit, storage_limit, metadata, blockOrder) => + addresses.contains(source.value) + case TxRollupSubmitBatch( + counter, + fee, + source, + gas_limit, + storage_limit, + rollup, + metadata, + blockOrder + ) => + addresses.contains(source.value) } case None => true } - - }.flatMap { op => + } + val internal = filteredOperations.flatMap { op => op match { case r: Reveal => r.metadata.internal_operation_results.toList.flatten case t: Transaction => t.metadata.internal_operation_results.toList.flatten @@ -423,7 +494,7 @@ object TezosOptics { case _ => List.empty } } - updatedGroup -> (updatedGroup.contents, internal) + updatedGroup -> (filteredOperations, internal) }.toMap /** Extracts all operations, primary and internal, and pre-order traverse diff --git a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosTypes.scala b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosTypes.scala index e7be163ff..ad4770d60 100644 --- a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosTypes.scala +++ b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosTypes.scala @@ -353,14 +353,14 @@ object TezosTypes { ) extends Operation final case class TxRollupFinalizeCommitment( - counter: PositiveBigNumber, - fee: PositiveBigNumber, - source: PublicKeyHash, - gas_limit: PositiveBigNumber, - storage_limit: PositiveBigNumber, - rollup: PublicKeyHash, - metadata: ResultMetadata[OperationResult.Origination], - blockOrder: Option[Int] = None + counter: PositiveBigNumber, + fee: PositiveBigNumber, + source: PublicKeyHash, + gas_limit: PositiveBigNumber, + storage_limit: PositiveBigNumber, + rollup: PublicKeyHash, + metadata: ResultMetadata[OperationResult.Origination], + blockOrder: Option[Int] = None ) extends Operation final case class Delegation( diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala index 58ab9fb28..8d957abbc 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala @@ -827,36 +827,37 @@ private[tezos] object TezosDatabaseConversions { * To correctly create the relation on the db, we must first store the operations, get * each generated id, and pass it to the associated balance-updates */ - implicit val blockToOperationTablesData = new Conversion[List, (Block, Option[List[Tables.KnownAddressesRow]]), OperationTablesData] { - import OperationBalances._ - import SymbolSourceLabels.Show._ - import tech.cryptonomic.conseil.common.util.Conversion.Syntax._ - - override def convert(from: (Block, Option[List[Tables.KnownAddressesRow]])) = { - val (block, knownAddresses) = from - TezosOptics.Blocks - .extractOperationsAlongWithInternalResults(block, knownAddresses) - .flatMap { case (group, (operations, internalResults)) => - val mainOperationData = operations.map(op => - (block, group.hash, op).convertTo[Tables.OperationsRow] -> - BlockTagged + implicit val blockToOperationTablesData = + new Conversion[List, (Block, Option[List[Tables.KnownAddressesRow]]), OperationTablesData] { + import OperationBalances._ + import SymbolSourceLabels.Show._ + import tech.cryptonomic.conseil.common.util.Conversion.Syntax._ + + override def convert(from: (Block, Option[List[Tables.KnownAddressesRow]])) = { + val (block, knownAddresses) = from + TezosOptics.Blocks + .extractOperationsAlongWithInternalResults(block, knownAddresses) + .flatMap { case (group, (operations, internalResults)) => + val mainOperationData = operations.map(op => + (block, group.hash, op).convertTo[Tables.OperationsRow] -> + BlockTagged + .fromBlockData(block.data, op) + .convertToA[List, Tables.BalanceUpdatesRow] + ) + val internalOperationData = internalResults.map { case oop => + val op = oop.convertTo[Operation] + (block, group.hash, op) + .convertTo[Tables.OperationsRow] + .copy(internal = true, nonce = Some(oop.nonce.toString)) -> BlockTagged .fromBlockData(block.data, op) .convertToA[List, Tables.BalanceUpdatesRow] - ) - val internalOperationData = internalResults.map { case oop => - val op = oop.convertTo[Operation] - (block, group.hash, op) - .convertTo[Tables.OperationsRow] - .copy(internal = true, nonce = Some(oop.nonce.toString)) -> BlockTagged - .fromBlockData(block.data, op) - .convertToA[List, Tables.BalanceUpdatesRow] + } + mainOperationData ++ internalOperationData } - mainOperationData ++ internalOperationData - } - .toList - } + .toList + } - } + } implicit val blockAccountsAssociationToCheckpointRow = new Conversion[ diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala index 495b84f79..67496faac 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala @@ -12,7 +12,12 @@ import tech.cryptonomic.conseil.common.config.ChainEvent.AccountIdPattern import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query => _} import tech.cryptonomic.conseil.common.sql.CustomProfileExtension import tech.cryptonomic.conseil.common.tezos.Tables -import tech.cryptonomic.conseil.common.tezos.Tables.{AccountsRow, GovernanceRow, KnownAddresses, OperationsRow, OriginatedAccountMapsRow, RegisteredTokensRow} +import tech.cryptonomic.conseil.common.tezos.Tables.{ + GovernanceRow, + OperationsRow, + OriginatedAccountMapsRow, + RegisteredTokensRow +} import tech.cryptonomic.conseil.common.tezos.TezosTypes.Fee.AverageFees import tech.cryptonomic.conseil.common.tezos.TezosTypes._ import tech.cryptonomic.conseil.common.util.ConfigUtil @@ -113,9 +118,13 @@ object TezosDatabaseOperations extends ConseilLogSupport { val saveBlocksAction = Tables.Blocks ++= blocks.map(_.convertTo[BlocksRow]) val saveBlocksBalanceUpdatesAction = Tables.BalanceUpdates ++= blocks.flatMap { block => block.data.convertToA[List, BalanceUpdatesRow] + }.filter { updatesRow => + knownAddresses match { + case Some(value) => value.map(_.address).contains(updatesRow.accountId) + case None => true + } } - val saveGroupsAction = Tables.OperationGroups ++= blocks.flatMap(_.convertToA[List, OperationGroupsRow]) //a function that takes a row to save and creates an action to do that, returning the new id @@ -143,7 +152,9 @@ object TezosDatabaseOperations extends ConseilLogSupport { saveBlocksAction, saveBlocksBalanceUpdatesAction, saveGroupsAction, - saveOperationsAndBalances.traverse(blocks.flatMap(xx => (xx->knownAddresses).convertToA[List, OperationTablesData])), + saveOperationsAndBalances.traverse( + blocks.flatMap(block => (block -> knownAddresses).convertToA[List, OperationTablesData]) + ), saveBigMaps(blocks)(ec, tokenContracts, tnsContracts) ) @@ -404,7 +415,9 @@ object TezosDatabaseOperations extends ConseilLogSupport { ContractId(address) -> interfaces }.toList ) - (writeBlocks(blocks, tokens, knownAddresses) andThen writeAccountsCheckpoint(accountUpdates.map(_.asTuple))).transactionally + (writeBlocks(blocks, tokens, knownAddresses) andThen writeAccountsCheckpoint( + accountUpdates.map(_.asTuple) + )).transactionally } } @@ -859,7 +872,6 @@ object TezosDatabaseOperations extends ConseilLogSupport { import io.circe.parser.decode import RegisteredTokensFetcher.decoder import java.io.{BufferedReader, InputStreamReader} - import scala.io.Source val reader = new BufferedReader(new InputStreamReader(getClass.getResourceAsStream(s"/registered_tokens/$network.json"))) diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala index f20f3828f..287fb89d4 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala @@ -379,7 +379,7 @@ object TezosIndexer extends ConseilLogSupport { TezosDb.initTableFromCsv(db, Tables.KnownAddresses, selectedNetwork), TezosDb.initTableFromCsv(db, Tables.BakerRegistry, selectedNetwork), TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) - ).mapN { case (ka, _, _) => + ).mapN { case (ka, _, _) => ka._1 } /* Here we want to initialize the registered tokens and additionally get the token data back @@ -398,9 +398,9 @@ object TezosIndexer extends ConseilLogSupport { } yield ShutdownComplete val knownAddresses = Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { - case Success(xx) => + case Success(ka) => logger.info("DB initialization successful") - xx + ka case Failure(exception) => logger.error("DB initialization failed", exception) gracefulTermination().map(_ => ()) @@ -457,7 +457,7 @@ object TezosIndexer extends ConseilLogSupport { accountsProcessor, bakersProcessor, lorreConf.enabledFeatures, - if(lorreConf.enabledFeatures.lightweightIndexing) Some(knownAddresses) else None + if (lorreConf.enabledFeatures.lightweightIndexing) Some(knownAddresses) else None ) val metadataProcessor = new MetadataProcessor( diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala index 1c8d0494a..9f717e400 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/processing/BlocksProcessor.scala @@ -4,7 +4,13 @@ import scala.util.{Failure, Success, Try} import scala.concurrent.{ExecutionContext, Future} import slick.jdbc.PostgresProfile.api._ import cats.implicits._ -import tech.cryptonomic.conseil.indexer.tezos.{TezosGovernanceOperations, TezosNamesOperations, TezosNodeOperator, Tzip16MetadataOperator, TezosDatabaseOperations => TezosDb} +import tech.cryptonomic.conseil.indexer.tezos.{ + TezosGovernanceOperations, + TezosNamesOperations, + TezosNodeOperator, + Tzip16MetadataOperator, + TezosDatabaseOperations => TezosDb +} import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport import tech.cryptonomic.conseil.common.tezos.{Tables, TezosTypes} import tech.cryptonomic.conseil.common.tezos.TezosTypes.{Block, InternalOperationResults, Voting} @@ -56,7 +62,7 @@ class BlocksProcessor( case Some(value) => results.map { case (block, accountIds) => val ids = accountIds.toSet.intersect(value.map(x => TezosTypes.PublicKeyHash(x.address)).toSet).toList - block -> ids.taggedWithBlockData(block.data) + block -> ids.taggedWithBlockData(block.data) }.unzip case None => results.map { case (block, accountIds) => @@ -67,13 +73,17 @@ class BlocksProcessor( } for { - _ <- db.run(TezosDb.writeBlocksAndCheckpointAccounts(blocks, accountUpdates, knownAddresses)) andThen logBlockOutcome - _ <- tnsOperations.processNamesRegistrations(blocks).flatMap(db.run) - bakersCheckpoints <- accountsProcessor.processAccountsForBlocks(accountUpdates) // should this fail, we still recover data from the checkpoint - _ <- bakersProcessor.processBakersForBlocks(bakersCheckpoints) - _ <- bakersProcessor.updateBakersBalances(blocks) - rollsData <- nodeOperator.getBakerRollsForBlocks(blocks) - _ <- processBlocksForGovernance(rollsData.toMap) + _ <- db.run( + TezosDb.writeBlocksAndCheckpointAccounts(blocks, accountUpdates, knownAddresses) + ) andThen logBlockOutcome + _ <- tnsOperations.processNamesRegistrations(blocks).flatMap(db.run) + bakersCheckpoints <- accountsProcessor.processAccountsForBlocks( + accountUpdates + ) // should this fail, we still recover data from the checkpoint + _ <- bakersProcessor.processBakersForBlocks(bakersCheckpoints) + _ <- bakersProcessor.updateBakersBalances(blocks) + rollsData <- nodeOperator.getBakerRollsForBlocks(blocks) + _ <- processBlocksForGovernance(rollsData.toMap) } yield results.size } From e837665db1b31a6a35d5b0c3b1ecfec6beb47678 Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Wed, 6 Jul 2022 19:24:38 +0200 Subject: [PATCH 5/6] fixed merge problems --- .../conseil/common/tezos/TezosOptics.scala | 12 ++ .../registered_tokens/hangzhounet.csv | 0 .../registered_tokens/hangzhounet.json | 0 .../registered_tokens/ithacanet.json | 0 .../registered_tokens/jakartanet.json | 0 .../tezos/TezosDatabaseOperations.scala | 2 + .../conseil/indexer/tezos/TezosIndexer.scala | 145 ++++++------------ 7 files changed, 62 insertions(+), 97 deletions(-) rename conseil-lorre/src/main/resources/{ => tezos}/registered_tokens/hangzhounet.csv (100%) rename conseil-lorre/src/main/resources/{ => tezos}/registered_tokens/hangzhounet.json (100%) rename conseil-lorre/src/main/resources/{ => tezos}/registered_tokens/ithacanet.json (100%) rename conseil-lorre/src/main/resources/{ => tezos}/registered_tokens/jakartanet.json (100%) diff --git a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala index d8e548a2c..460bad6c4 100644 --- a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala +++ b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala @@ -507,6 +507,18 @@ object TezosOptics { blockOrder ) => addresses.contains(source.value) + case TxRollupDispatchTickets( + counter, + fee, + source, + gas_limit, + storage_limit, + tx_rollup, + metadata, + blockOrder + ) => + addresses.contains(source.value) + case DefaultOperation(kind, blockOrder) => true } case None => true } diff --git a/conseil-lorre/src/main/resources/registered_tokens/hangzhounet.csv b/conseil-lorre/src/main/resources/tezos/registered_tokens/hangzhounet.csv similarity index 100% rename from conseil-lorre/src/main/resources/registered_tokens/hangzhounet.csv rename to conseil-lorre/src/main/resources/tezos/registered_tokens/hangzhounet.csv diff --git a/conseil-lorre/src/main/resources/registered_tokens/hangzhounet.json b/conseil-lorre/src/main/resources/tezos/registered_tokens/hangzhounet.json similarity index 100% rename from conseil-lorre/src/main/resources/registered_tokens/hangzhounet.json rename to conseil-lorre/src/main/resources/tezos/registered_tokens/hangzhounet.json diff --git a/conseil-lorre/src/main/resources/registered_tokens/ithacanet.json b/conseil-lorre/src/main/resources/tezos/registered_tokens/ithacanet.json similarity index 100% rename from conseil-lorre/src/main/resources/registered_tokens/ithacanet.json rename to conseil-lorre/src/main/resources/tezos/registered_tokens/ithacanet.json diff --git a/conseil-lorre/src/main/resources/registered_tokens/jakartanet.json b/conseil-lorre/src/main/resources/tezos/registered_tokens/jakartanet.json similarity index 100% rename from conseil-lorre/src/main/resources/registered_tokens/jakartanet.json rename to conseil-lorre/src/main/resources/tezos/registered_tokens/jakartanet.json diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala index acd90985b..cdbb5159d 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala @@ -36,6 +36,8 @@ import java.util.UUID import slick.dbio.DBIOAction import tech.cryptonomic.conseil.indexer.tezos.RegisteredTokensFetcher.RegisteredToken +import scala.io.Source + /** * Functions for writing Tezos data to a database. */ diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala index f838f5ee5..7e0936a35 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala @@ -351,45 +351,6 @@ object TezosIndexer extends ConseilLogSupport { lorreConf.headOffset ) - /* Reads csv resources to initialize db tables and smart contracts objects */ - def parseCSVConfigurations() = { - /* This will derive automatically all needed implicit arguments to convert csv rows into a proper table row - * Using generic representations for case classes, provided by the `shapeless` library, it will create the - * appropriate `kantan.csv.Decoder` for - * - headers, as extracted by the case class definition which, should match name and order of the csv header row - * - table row types, which can be converted to shapeless HLists, which so happens to be of any case class. - * - * What shapeless does, is provide an automatic conversion, from a case class to a typed generic list of values, - * where each element has a type corresponding to a field in the case class, and a "label" that is - * a sort of string extracted at compile-time via macros, to figure out the field names. - * Such special list, is a `shapeless.HList`, or heterogeneous list, sort of a dynamic tuple, - * to be built by adding/removing individual elements in the list, recursively. - * This allows generic libraries like kantan to define codecs for generic HLists and, - * using shapeless, to adapt any case class to his specific HList, at compile-time. - * - * For additional information, refer to the project wiki, and - * - http://nrinaudo.github.io/kantan.csv/ - * - http://www.shapeless.io/ - */ - import kantan.csv.generic._ - //we add any missing implicit decoder for column types, not provided by default from the library - import tech.cryptonomic.conseil.common.util.ConfigUtil.Csv._ - import cats.implicits._ - - /* Inits tables with values from CSV files */ - ( - TezosDb.initTableFromCsv(db, Tables.KnownAddresses, selectedNetwork), - TezosDb.initTableFromCsv(db, Tables.BakerRegistry, selectedNetwork), - TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) - ).mapN { case (ka, _, _) => - ka._1 - } - /* Here we want to initialize the registered tokens and additionally get the token data back - * since it's needed to process calls to the same token smart contracts as the chain evolves - */ - - } - /* the shutdown sequence to free resources */ val gracefulTermination = () => for { @@ -399,16 +360,6 @@ object TezosIndexer extends ConseilLogSupport { _: ShutdownComplete <- nodeOperator.node.shutdown() } yield ShutdownComplete - val knownAddresses = Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { - case Success(ka) => - logger.info("DB initialization successful") - ka - case Failure(exception) => - logger.error("DB initialization failed", exception) - gracefulTermination().map(_ => ()) - List.empty - } - /* provides operations to handle rights to bake and endorse blocks */ val rightsProcessor = new BakingAndEndorsingRightsProcessor( nodeOperator, @@ -451,6 +402,54 @@ object TezosIndexer extends ConseilLogSupport { //build operations on tns based on the implicit contracts defined before val tnsOperations = new TezosNamesOperations(tns, nodeOperator) + /* Reads csv resources to initialize db tables and smart contracts objects */ + def parseCSVConfigurations() = { + /* This will derive automatically all needed implicit arguments to convert csv rows into a proper table row + * Using generic representations for case classes, provided by the `shapeless` library, it will create the + * appropriate `kantan.csv.Decoder` for + * - headers, as extracted by the case class definition which, should match name and order of the csv header row + * - table row types, which can be converted to shapeless HLists, which so happens to be of any case class. + * + * What shapeless does, is provide an automatic conversion, from a case class to a typed generic list of values, + * where each element has a type corresponding to a field in the case class, and a "label" that is + * a sort of string extracted at compile-time via macros, to figure out the field names. + * Such special list, is a `shapeless.HList`, or heterogeneous list, sort of a dynamic tuple, + * to be built by adding/removing individual elements in the list, recursively. + * This allows generic libraries like kantan to define codecs for generic HLists and, + * using shapeless, to adapt any case class to his specific HList, at compile-time. + * + * For additional information, refer to the project wiki, and + * - http://nrinaudo.github.io/kantan.csv/ + * - http://www.shapeless.io/ + */ + import kantan.csv.generic._ + //we add any missing implicit decoder for column types, not provided by default from the library + import tech.cryptonomic.conseil.common.util.ConfigUtil.Csv._ + import cats.implicits._ + + /* Inits tables with values from CSV files */ + ( + DefaultDatabaseOperations.initTableFromCsv(db, Tables.KnownAddresses, "tezos", selectedNetwork), + DefaultDatabaseOperations.initTableFromCsv(db, Tables.BakerRegistry, "tezos", selectedNetwork), + TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) + ).mapN { case (ka, _, _) => + ka._1 + } + /* Here we want to initialize the registered tokens and additionally get the token data back + * since it's needed to process calls to the same token smart contracts as the chain evolves + */ + } + + val knownAddresses = Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { + case Success(ka) => + logger.info("DB initialization successful") + ka + case Failure(exception) => + logger.error("DB initialization failed", exception) + gracefulTermination().map(_ => ()) + List.empty + } + /* this is the principal data processor, handling paginated blocks, and the correlated data within */ val blocksProcessor = new BlocksProcessor( nodeOperator, @@ -501,54 +500,6 @@ object TezosIndexer extends ConseilLogSupport { /* Used for fetching and handling registered tokens */ val registeredTokensFetcher = new RegisteredTokensFetcher(db, lorreConf.tokenContracts, gracefulTermination) - /* Reads csv resources to initialize db tables and smart contracts objects */ - def parseCSVConfigurations() = { - /* This will derive automatically all needed implicit arguments to convert csv rows into a proper table row - * Using generic representations for case classes, provided by the `shapeless` library, it will create the - * appropriate `kantan.csv.Decoder` for - * - headers, as extracted by the case class definition which, should match name and order of the csv header row - * - table row types, which can be converted to shapeless HLists, which so happens to be of any case class. - * - * What shapeless does, is provide an automatic conversion, from a case class to a typed generic list of values, - * where each element has a type corresponding to a field in the case class, and a "label" that is - * a sort of string extracted at compile-time via macros, to figure out the field names. - * Such special list, is a `shapeless.HList`, or heterogeneous list, sort of a dynamic tuple, - * to be built by adding/removing individual elements in the list, recursively. - * This allows generic libraries like kantan to define codecs for generic HLists and, - * using shapeless, to adapt any case class to his specific HList, at compile-time. - * - * For additional information, refer to the project wiki, and - * - http://nrinaudo.github.io/kantan.csv/ - * - http://www.shapeless.io/ - */ - import kantan.csv.generic._ - //we add any missing implicit decoder for column types, not provided by default from the library - import tech.cryptonomic.conseil.common.util.ConfigUtil.Csv._ - import cats.implicits._ - - /* Inits tables with values from CSV files */ - ( - DefaultDatabaseOperations.initTableFromCsv(db, Tables.KnownAddresses, "tezos", selectedNetwork), - DefaultDatabaseOperations.initTableFromCsv(db, Tables.BakerRegistry, "tezos", selectedNetwork), - TezosDb.initRegisteredTokensTableFromJson(db, selectedNetwork) - ).mapN { case (_, _, _) => - () - } - /* Here we want to initialize the registered tokens and additionally get the token data back - * since it's needed to process calls to the same token smart contracts as the chain evolves - */ - - } - - Try(Await.result(parseCSVConfigurations(), 5.seconds)) match { - case Success(_) => - () - logger.info("DB initialization successful") - case Failure(exception) => - logger.error("DB initialization failed", exception) - gracefulTermination().map(_ => ()) - } - new TezosIndexer( ignoreProcessFailures, lorreConf, From f6dd425fc63a0b21bb8041e736894b377259a2e1 Mon Sep 17 00:00:00 2001 From: piotrkosecki Date: Wed, 6 Jul 2022 19:26:45 +0200 Subject: [PATCH 6/6] reverted known addresses changes --- .../tezos/known_addresses/mainnet.csv | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/conseil-lorre/src/main/resources/tezos/known_addresses/mainnet.csv b/conseil-lorre/src/main/resources/tezos/known_addresses/mainnet.csv index 5f2129324..7f148cea3 100644 --- a/conseil-lorre/src/main/resources/tezos/known_addresses/mainnet.csv +++ b/conseil-lorre/src/main/resources/tezos/known_addresses/mainnet.csv @@ -1,3 +1,264 @@ address,alias tz1W1qbXXJrtLWwaaoXhe8LYGZqmNnpKaH9q,01No.de tz1WBfwbT66FC6BTLexc2BoyCCBM9LG7pnVW,888 XTZ +tz1iLbZZ9uoRuVJCrZ9ZwiJMpfzhy3c67mav,AirBie +tz1gk3TDbU7cJuiBRMhwQXVvgDnjsxuWhcEA,Airfoil +tz1QH3G2btaWc1vRLNsEfx2gHM7Ad81TeRit,Airfoil Payouts +tz1MJx9vhaNRSimcuXPK2rW4fLccQnDAnVKJ,AirGap +tz1LJycSuCT25AA5VJwNW1QYXVGyy7YLwZh9,AirGap Payout +KT1Vp4KUG1n1HrFEDxUNvNskwc1Msh7NRbhd,AirGap Payout KT +tz1UdeKoMJivgqRgRUNzroMwRJYmJY8BtaWe,AlphaTezos +tz1XhnCdVENzgko5x1MMswLHSoQbJ5NPwLZ6,Anonstake +tz3e75hU4EhDU3ukyJueh5v6UvEHzGwkg3yC,At James +tz1ZcTRk5uxD86EFEn1vvNffWWqJy7q5eVhc,ATEZA +tz1TqU5hwh6CtC8Ps5GYBi7YyawvtKLNCbvQ,ATEZA Payouts +KT1VG2WtYdSWz5E7chTeAdDPZNy2MpP8pTfL,Atomex +KT1RCAETpkm9z44MW7Cph3mrWshqVURPubT4,Bake n Rolls Payouts 2 +tz1ei4WtWEMEJekSv8qDnu9PExG6Q8HgRGr3,Bake Tz +tz1NRGxXV9h6SdNaZLcgmjuLx3hyy2f8YoGN,Bake ꜩ For Me +tz1NortRftucvAkD1J58L32EhSVrQEWJCEnB,Bake'n'Rolls +tz1Zrqm4TkJwqTxm5TiyVFh6taXG4Wrq7tko,Bake’n’Rolls Payouts +tz1aRhFErGMgL57DYHMT1vYwv7PzsJN1chrk,Baked Tezos +tz1KksC8RvjUWAbXYJuNrUbontHGor26Cztk,Bakednode +tz1Z1WwoqgRFbLE3YNdYRpCx44NSfiMJzeAG,Bakemon +tz1NkFRjmkqqcGkAhqe78fdgemDNKXvL7Bod,BakeMyStake +tz1WtkJEkKjHX3bMDYwJoDVC4gPksNPUa3v7,BakeMyStake Old +tz1YgNQBeLTgbwRb7QiyvHrx7hwN7AUbdqpB,Baker Avenue +KT1ChNsEFxwyCbJyWGSL3KdjeXE28AY1Kaog,Baker Registry +tz1iVCqjMRG7MfBoFnVjbLQyeLAeJQfLxXze,Bakery IL Payouts +tz1cYufsxHXJcvANhvS55h3aY32a9BAFB494,Bakery-IL +tz1Pv7ha8wzAyRHkR9RahWapdopK1tt6nVz1,Baking co-operative +tz1RV1MBbZMR68tacosb7Mwj6LkbPSUS1er1,Baking Tacos +tz1S8MNvuFEUsWgjHvi3AxibRBf388NhT1q2,Binance +tz1eogbwM5NdoojKEvvSnjYGcJapZgADRK3m,Binance Delegator +tz1XXayQohB8XRXN7kMoHbf2NFwNiH3oMRQQ,Bit Cat +tz1KtGwriE7VuLwT3LwuvU9Nv4wAxP7XZ57d,Bitfinex +tz1LmaFsWRkjr7QMCx5PtV6xTUz3AmEpKQiF,Blockpower +tz1d2EwsrmoCPmhNUjt7unCQ8a9RwVsVXEmd,BTFD +tz3bEQoFCZEEfZMskefZ8q8e4eiHH1pssRax,Ceibo XTZ +tz1Z7eWGw18LqUgRqmDqNZFQx7f8GEHXRfT8,Coinbase 1 +tz1bDhCGNZLQw1QXgf6MCzo6EtAVSGkqEB11,Coinbase 2 +tz1irJKkXS2DBWkU1NnmFQx1c1L7pbGg4yhk,Coinbase Custody +tz1NcoDFXMAfB26mpBhVrdSHmppyTeccT6Fi,Coinbase Pro +tz1dgoZ2xHX6Aou5KbFNFX5XiGJ6UpkqCyQb,CoinEconomy +tz1dbfppLAAxXZNtf2SDps7rch3qfUznKSoK,Coinhouse +tz1SYq214SCBy9naR6cvycQsYcUGpBqQAE8d,Coinone +tz1eEnQhbwf6trb8Q8mPb2RaPkNk2rN7BKi8,Cryptium Labs +tz1cNARmnRRrvZgspPr2rSTUWq5xtGTuKuHY,Cryptium Labs Payout +tz1Tnjaxk6tbAeC2TmMApPh8UsrEVQvhHvx5,Crypto Delegate +tz1MyXTZmeMCM4yFnrER9LNYDZ9t2rHYDvcH,Crypto Delegate Payouts +tz1NqKH9D5HdfvoEQssMS5Up13CVypVxLA1k,Cypher-Baker +tz1Lc87p6zRaDwzJs9kHvdpm7XzeWE8QTwVB,Dash Master +KT1A13N5XXJLgD9rQ6BF6xg71yhNhQhrCxia,Dashmaster Payouts +tz1ZRWFLgT9sz8iFi1VYWPfRYeUvUSFAaDao,Delegated Capital +tz1LsuPvP2u63nouVGAPi13AYqqB2VQjhoSF,Drunk Tortuga Bakery +tz1W1f1JrE7VsqgpUpj1iiDobqP5TixgZhDk,Elite Tezos +tz1PK8bNrfPbc73BtHH1A63jfU5UtbRFkvU4,EON +tz1MXFrtZoaXckE41bjUCSjAjAap3AFDSr3N,Everstake +tz1WcCGBxxPq61ddkp8X7vYMrWsBFMCxe5RD,FightCovid +tz1Scdr2HsZiQjc7bHMeBbmDRXYVvdhjJbBh,Figment +tz1ZQppA6UerMz5CJtGvZmmB6z8L9syq7ixu,First Block +KT1GJBSMRGApA2DnSivs73Fhau6dbfP1Sntd,Flippin Tacos Payouts 1 +tz1TacosCtikEQCJWYDZZ9MuA42sVpbbKZFF,Flippin Tacos Payouts 2 +tz1TzaNn7wSQSP5gYPXCnNzBCpyMiidCq1PX,Flippin' tacos +tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9,Foundation Baker 1 +tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5,Foundation Baker 2 +tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8,Foundation Baker 3 +tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk,Foundation Baker 4 +tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV,Foundation Baker 5 +tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r,Foundation Baker 6 +tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K,Foundation Baker 7 +tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN,Foundation Baker 8 +tz1P6kxtzRrJ8KfXE4kKqR1RBiSpEgimxN5o,Fractalide +tz1QLXqnfN51dkjeghXvKHkJfhvGiM5gK4tc,Fresh Tezos +tz1djRgXXWWJiY1rpMECCxr5d9ZBqWewuiU1,Galleon Support +tz1NpWrAyDL9k2Lmnyxcgr9xuJakbBxdq7FB,Gate.io +tz1WCd2jm4uSt4vntk4vSuUWoZQGhLcDuR9q,Happy Tezos +tz1acsihTQWHEnxxNz7EEsBDLMTztoZQE9SW,Happy Tezos Payout +tz1KzSC1J9aBxKp7u8TUnpN8L7S65PBRkgdF,HashQuark +tz1SohptP53wDPZhzTWzDUFAUcWF6DMBpaJV,Hayek Lab +tz1gg5bjopPcr9agjamyu9BbXKLibNc2rbAq,hodl.farm +tz1hodLFy8YDHtf1qJ3XH1gCyeXfwWEitkj9,hodl.farm Payouts +tz1Z3KCf8CLGAYfvVWPEr562jDDyWkwNF7sT,Hot Stake +tz1VHFxUuBhwopxC9YC9gm5s2MHBHLyCtvN1,Hyperblocks +tz1dp3tScJZy8Go3byPdioSujgDi5CdemiuZ,iBakeTezos +tz1ZePm7HyANf3P9hZfhmm8ShbrbCP1nYwT5,iBakeTezos Payouts 1 +tz1VQq41XksF8oJ4ntxbh99g5wrqcreVf7e1,iBakeTezos Payouts 2 +tz1Ryat7ZuRapjGUgsPym9DuMGrTYYyDBJoq,Illuminarean +tz1bRaSjKZrSrSeQHBDiCqjKXqtZYZM1t8FW,Imma Baker +tz1awXW7wuXy21c66vBudMXQVAPgRnqqwgTH,Infinity Stones +tz1ZTG13gkvouxSANka3HG3uys8C5gu3DPXZ,Just a Baker +tz1gfArv665EUkSg2ojMBzcbfwuPxAvqPvjo,Kraken +tz1bDXD6nNSrebqmAnnKKwnX1QdePSMCj4MX,Kraken 1 +tz1YpWsMwc4gSyjtxEF3JbmN6YrGiDidaSmg,Kraken 2 +tz1aqcYgG6NuViML5vdWhohHJBYxcDVLNUsE,LetzBake! +tz1MTAmuFmKu3MoxnkVsMj7xvSScj6d83hu5,LetzBake! Payouts +tz1VmiY38m3y95HqQLjMwqnMS7sdMfGomzKi,Lucid Mining +KT1THshzbGq3NJfL8w9j3ppbUSCfy5tQvWch,Lucid Mining Payouts 1 +tz1LsnEuv4fLsgxU9HJK21v8zHwcuBjb99Cd,Lucid Mining Payouts 2 +tz1MkjfzHNF3kk7PMUDVsaTrF8iRD6fpHftR,LuxBaker +tz1aEsrxrhTYNcg897HjSWnJ5yjR95pxVD1d,Mama Baker +tz1PWCDnz783NNGGQjEFFsHtrcK5yBW4E2rm,Melange +tz1cb8xcmJWcdVU7cNAd93MfEReorvP52P8x,Mint Capital +tz1NEKxGEHsFufk87CVZcrqWu8o22qh46GK6,Money Every 3 Days +tz1egbN6RK2bM5vt4aAZw6r9j4nL8z49bPdS,Money Every 3 Days +tz1c7pVR4w3KSQarJdBeh4NS2WxMUFBy1rHQ,MyBakerSpace +tz1d6Fx42mYgVFnHUW8T8A7WBfJ6nD9pVok8,MyTezosBaking +tz1Y2ufHak4AcisVazSS1ymGgv4HAmFGagh5,MyTezosBaking Payouts +tz1YdCPrYbksK7HCoYKDyzgfXwY16Fy9rrGa,Norn Delegate +tz1KtvGSYU5hdKD288a1koTBURWYuADJGrLE,Ø Crypto Pool +tz1UuQ4HWDu3ALNRgAq94dX9MhqhQhnuY3gC,OKEx +tz1X1fpAZtwQk94QXUgZwfgsvkQgyc2KHp9d,ownBLOCK +tz1hTFcQk2KJRPzZyHkCwbj7E1zY1xBkiHsk,ownBLOCK +tz1P2Po7YM526ughEsRbY4oR9zaUPDZjxFrb,P2P Validator +tz1Ldzz6k1BHdhuKvAtMRX7h5kJSMHESMHLC,PayTezos +tz1c9HEyZdAWXZCLE7h3KcZCyykjZr7VY3QM,PayTezos Payouts +KT1JjzGWesNFaHv9iyBwLBzLyTpTZ3HeBf7o,PayTezos Payouts KT +tz1Yju7jmmsaUiG9qQLoYv35v5pHgnWoLWbt,Polychain Labs +tz2TSvNTh2epDMhZHrw73nV9piBX7kLZ9K9m,Polychain Labs 2 +tz3TSvNTh2epDMhZHrw73nV9piBX7kLZ9K9m,Polychain Labs II +tz1Vyuu4EJ5Nym4JcrfRLnp3hpaq1DSEp1Ke,POS Bakerz +tz1V4qCyvPKZ5UeqdH14HN42rxvNPQfc9UZg,Shake 'n Bake +tz1ZuxNqUk7odhMC4Bfx2NXXeej9ReXKewa8,SNZ Holding +tz1MDKr36woXfVtrtXfV1ppPJERxPcm2wU6V,Stack Tezos +tz1V2FYediKBAEaTpXXJBSjuQpjkyCzrTSiE,Stake House +tz2FCNBrERXtaTtNX6iimR1UJ5JSDxvdHM93,Stake.fish +tz1MowoYvqgxhKToQMAmGgTtjfZcRfS733JY,StakeBake +tz1RCFbB9GpALpsZtu6J58sb74dm8qe6XBzv,Staked +tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194,StakeNow +tz1Ywgcavxq9D6hL32Q2AQWHAux9MrWqGoZC,StakeNow Payouts +KT1EctCuorV2NfVb1XTQgvzJ88MQtWP8cMMv,Staker +tz1go7f6mEQfT2xX2LuHAqgnRGN6c2zHPf5c,Stakery +tz1gs78wrxS34XhGPECmN5fABKpEuhUGwAXS,Stakery Payouts +tz1WpeqFaBG9Jm73Dmgqamy8eF8NWLz9JCoY,Staking Facilities +tz1STeamwbp68THcny9zk3LsbG3H36DMvbRK,Staking Team +tz1e2meErj7eEfXwqr7bDK6N1YatmLaugfMp,STB +tz1dNVDWPf3Q59SdJqnjdnu277iyvReiRS9M,Steak.and.Bake +tz1fvBC1cHJzPtUW3ZKd2GZDj7P4XVuLk9vW,Steaker +tz1LH4L6XYT2JNPhvWYC4Zq3XEiGgEwzNRvo,Stir Delegate +tz1Lhf4J9Qxoe3DZ2nfe8FGDnvVj7oKjnMY6,Tez Baker +KT1TUi7KGFcTLku6niB2iGqzkv49ZLYxLRy1,Tez Baker Payouts 1 +KT1Q6YAVkqcEmXAM7SUcQS9YSbLwbqMaVe3Z,Tez Baker Payouts 2 +KT1KcLH9Qz6QJucvajEdXQvcSHgxfwuwDjNe,Tez Baker Payouts 3 +KT1V86WP8CuxAtQ4xkpZ5fVtXv5XzK2F81mp,Tez Baker Payouts 4 +tz1NXPdPC2DfyA5ohitnS6ryqrDKbFJeu5Nn,Tez Boss +tz1hgjLqCZWCtarByUMYd81F4FKRY5t1V978,Tez Dispensery +tz1bakeKFwqmtLBzghw8CFnqFvRxLj849Vfg,Tez Milk +tz1UUgPwikRHW1mEyVZfGYy6QaxrY6Y7WaG5,Tez Patisserie +tz1abTjX2tjtMdaq5VCzkDtBnMSCFPW2oRPa,TEZ Rocket +tz1isXamBXpTUgbByQ6gXgZQg4GWNW7r6rKE,Tez Whale +tz1WHALEgzq8cUUkg1vBKrpQrg9mysEX9NYR,Tez Whale Payouts 1 +tz1WHALEP56kjCTuUFLWjcBkfFpyevC4xo3t,Tez Whale Payouts 2 +tz1LBEKXaxQbd5Gtzbc1ATCwc3pppu81aWGc,Tez-Baking +KT1BvVxWM6cjFuJNet4R9m64VDCN2iMvjuGE,Tez-Baking Token +tz1YhNsiRRU8aHNGg7NK3uuP6UDAyacJernB,TezBaguette +tz3LBtbMMvvguWQupgEmtfjtXy77cHgdr5TE,Tezcentral +tz1Yc6ATtfUJyDjHwJ8WoVL22sJueDenueke,TezDele Baker A +tz1SdwBHocSrcuMFNLPUg4LPRfx9eaqjVUEL,TezDele Baker B +tz1LnWa4AiFCjMezTgSNa65QFoo7DQGEWgEU,Tezeract +tz1VceyYUpq1gk5dtp6jXQRtCtY8hm5DKt72,Tezetetic +tz1MR42UxXgFmxsP4iVEF8e8a7q17TPLicpt,Tezetetic Payout +tz1W5VkdB5s7ENMESVBtwyt9kyvLqPcUczRT,Tezgate +tz1g4o4a2wxpzJ7EgG3onFM5TLaPyiRFjFhL,Tezgate Payouts +tz1MQJPGNMijnXnVoBENFz9rUhaPt3S7rWoz,Tezmania +tz1TaLYBeGZD3yKVHQGBM857CcNnFFNceLYh,Tezocracy +KT1CthjeJp2pLbSdo6xZR1VfgUD44QYDNmSZ,Tezocracy Payouts 1 +tz1hfXvfcbbfcA77U8RXn7xLbRn3NHU4ksU1,Tezocracy Payouts 2 +tz1RSWMYKGAykpizFteowByYMueCYv9TMn1L,Tezos Alliance +tz1PriNQyDC7d5ccPAD96ugujYy5YbdGLdQ5,Tezos Baker JP +tz1fZ767VDbqx4DeKiFswPSHh513f51mKEUZ,Tezos Bakery +tz1PYLN9TsKZHfn2GtrXnxkeGvahmYdBTG5v,Tezos Bakes 2 +tz1b2mCFQm8isfJRGUctgyxHhHcmTNTRPdps,Tezos Barcelona +tz1Z7bxn1M4ewEWUekPaZfiwtKQRFroezUjY,Tezos Bay +tz1Y42aKCk53vMbaJNpf1hBg1rznGdBxHJ5C,Tezos Berlin +tz1Z2jXfEXL7dXhs6bsLmyLFLfmAkXBzA9WE,Tezos Boutique +tz1aKxnrzx5PXZJe7unufEswVRCMU9yafmfb,Tezos Canada +tz3PdGc7U5tiyqPgTSgqCDct94qd6ovQwP6u,Tezos Capital +tz1TDSmoZXwVevLTEvKCTHWpomG76oC9S2fJ,Tezos Capital Legacy +tz1hThMBD8jQjFt78heuCnKxJnJtQo9Ao25X,Tezos Chef +tz1aCJuxfZPsXZ3PmfA6MvjsRTpGVVDMmb8X,Tezos Chef Payouts +tz1WJpSCYdQE9merfftanA8eFsqUPTpwAt8a,Tezos China +tz1YTyvABUyhE7JHpxMVBVqjZnZM4ofMrWKE,Tezos Delegate EU +tz1U638Z3xWRiXDDKx2S125MCCaAeGDdA896,Tezos France +tz1TAzNBtnq6vF6kg34ApectxXoxZVTyapq3,Tezos Gold +tz1WnfXMPaNTBmH7DBPwqCWs9cPDJdkGBTZ8,Tezos HODL +tz1bLwpPfr3xqy1gWBF4sGvv8bLJyPHR11kx,Tezos Hot Bakery +tz1Kn5Dyir12f7NafEztgooypotBkE92mKDr,Tezos Hot Bakery Payout +tz1Vm5cfHncKGBo7YvZfHc4mmudY4qpWzvSB,Tezos Istanbul +tz1MBERDzXMvMQPCMWBCEgxyKrdU71HgXaM8,Tezos Istanbul Payouts 1 +tz1Uf4Raxz7JzNdpAyM3893qLtqTRwNrdKVK,Tezos Istanbul Payouts 2 +tz1b7YSEeNRqgmjuX4d4aiai2sQTF4A7WBZf,Tezos Japan +tz1ijdEfmUoQXJfSiKCLRioeCfSrKP3sVsff,Tezos Japan 2 +tz1Nn14BBsDULrPXtkM9UQeXaE4iqJhmqmK5,Tezos Kitchen +tz1PPUo28B8BroqmVCMMNDudG4ShA2bzicrU,Tezos Korea +tz1S1Aew75hMrPUymqenKfHo8FspppXKpW7h,Tezos Mars +tz1TgvirXEwVJPg1cCrxT9zFubdzw7Ng6Ke9,Tezos Moon +tz1b3SaPHFSw51r92ARcV5mGyYbSSsdFd5Gz,Tezos MX +tz1djEKk6j1FqigTgbRsunbnY9BB7qsn1aAQ,Tezos Ninja +tz1eFXLaCUcKbhXkLBGUGMGNY9tKHjWGG25V,Tezos Node Spain +tz1SkU5S3dDRi6f8LHCvYeK3H3Mjq4MS8uHW,Tezos OTC +tz1PeZx7FXy7QRuMREGXGxeipb24RsMMzUNe,Tezos Panda +KT1PnRgUjx7dD4xrnXnpJHz9MfnCWoeT6UBr,Tezos Panda Payouts +tz1cZfFQpcYhwDp7y1njZXDsZqCrn2NqmVof,Tezos Rio +tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY,Tezos Seoul +tz1RAzdkp1x5ZDqms4ZUrSdfJuUYGH9JTPK2,Tezos Spanish +tz1hAYfexyzPGG6RhZZMpDvAHifubsbb6kgn,Tezos Suisse +tz1bkKTY9Y3rTsHbpr2fbGUCRm736LLquQfM,Tezos Tacos +tz1VYQpZvjVhv1CdcENuCNWJQXu1TWBJ8KTD,Tezos Tokyo +tz1bHzftcTKZMTZgLLtnrXydCm6UEqf4ivca,Tezos Vote +tz1ggJg924JdWdww6oLGFksRgKctfrDMUNfE,Tezos Vote Payouts +tz1WctSwL49BZtdWAQ75NY6vr4xsAx6oKdah,Tezos Wake n'Bake +tz1MkaTVYdM9QvXxKvYEwG9BPN2XxriTs7Jh,Tezos.bi23 +tz1fb7c66UwePkkfDXz4ajFaBP9hVNLdS7JJ,Tezos.nu +tz1gfipKzYrNRT14oSNQJMCdRRsUtcbZoKsd,Tezos.Rio_old +tz1YvG6poSGhPLTdzbzCmqShMndzoyhyVg7J,tezosbaking.club +tz1c3Wh8gNMMsYwZd67JndQpYxdaaPUV27E7,TezosBC +KT1E5rWDHRA679Q98Tks3DppCqvBLUzmnBEe,TezosBC Payouts +tz1QGZ3dD2YpRKZ4APeso6EXTeyCUUkw6MQC,TezosBC_old +tz1YKh8T79LAtWxX29N5VedCSmaZGw9LNVxQ,TezosBr +tz1TzHDLTNK1J1fCotGBLuE43DoogC5pHGyT,TezosHODL Payouts +tz1V3yg82mcrPJbegqVCPn6bC8w1CSTRp3f8,TezoShop +tz1hxtCEcD7idQJEDiJEq37vBkoRcwF6KC2X,TezoShop Payout +tz1b9MYGrbN1NAxphLEsPPNT9JC7aNFc5nA4,TezosRUs +tz1R6Ej25VSerE3MkSoEEeBjKHCDTFbpKuSX,TezosSEAsia +tz1Lh9jeLSWDHYy8AshvG2dpNQseDaHg7cms,TezosSpanish +tz1LLNkQK4UQV6QcFShiXJ2vT2ELw449MzAA,TezosTeam +tz1YrmJw6Lje27gWqZ94gU9mNavEjkHu1xGc,TezosTeam Payouts +tz1UJvHTgpVzcKWhTazGxVcn5wsHru5Gietg,Tezry +tz1Vtimi84kLh9RANfRVX2JvYtP4NPCT1aFm,TezTech Labs +tz1Zcxkfa5jKrRbBThG765GP29bUCU3C4ok5,Tezz City +tz1fUjvVhJrHLZCbhPNvDRckxApqbkievJHN,Tezzieland +tz1WntXgznbivRjyhE7Y5jEzoebAhMPB2iJa,Tezzieland Payouts +tz3adcvQaKXTCg12zbninqo3q8ptKKtDFTLv,Tezzigator +tz1iZEKy4LaAjnTmn2RuGDf2iqdAQKnRi8kY,Tezzigator Legacy +tz1dhwJY5B1jKdkF7zcyu1v6EBEQjpGmAp1E,Tezzz +KT1U1JZaXoG4u1EPnhHL4R4otzkWc1L34q3c,Trianon KYC Registrar +KT1RM4Nzwe9kgyiF3WpqPdUezJ2SS7ixUdHX,Trianon STO Issuer +KT1XHAmdRKugP1Q38CxDmpcRSxq143KpEiYx,Trianon STO Token +tz1Zhv3RkfU2pHrmaiDyxp7kFZpZrUCu1CiF,TZ Bake +tz1XQ7SRj4QQWjaeebNd8dFwuTrCot3GGDRF,Tz Baker +tz1cX93Q3KsiTADpCC4f12TBvAmS5tw7CW19,Tz Bakery +tz1bkg7rynMXVcjomoe3diB4URfv8GU2GAcw,Tz Bank +tz1PesW5khQNhy4revu2ETvMtWPtuVyH2XkZ,Tz Dutch +tz1iJ4qgGTzyhaYEzd1RnC6duEkLBd1nzexh,Tz Envoy +tz1Vd1rXpV8hTHbFXCXN3c3qzCsgcU5BZw1e,Tz Node +tz1Yh1nLXfwqVpP8btykuRguu61n3veVmADa,TzBakeOven +KT1LH2o12xVRwTpJMZ6QJG74Fox8gE9QieFd,TZBTC +KT1NCuMkcRrGDSiB5BX4xyBGEUtyyF7YDDYz,TZBTC 2 +tz1Pwgj6j55akKCyvTwwr9X4np1RskSXpQY4,Validators.com +tz1aiYKXmSRckyJ9EybKmpVry373rfyngJU8,View Nodes +tz1iMAHAVpkCVegF9FLGWUpQQeiAHh4ffdLQ,Wetez Wallet +tz1LesY3S4wfe15SNm1W3qJmQzWxLqVjTruH,Xtez +tz1TcH4Nb3aHNDJ7CGZhU7jgAK1BkSP4Lxds,XTZ Antipodes +tz1P9GwUWeAyPoT3ZwBc6gWCTP9exEPaVU3w,XTZ Bakery I +tz1TnYeuR4HhUTapQiXvzcjmcpvZYwR1B5S3,XTZ Bakery II +tz1Suih9uWEnubDeqCTCEfueSCGWYohjyaA5,XTZ Black +tz1Xek93iSXXckyQ6aYLVS5Rr2tge2en7ZxS,XTZ Delegate +tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj,XTZ Land +tz1KfEsrtDaA1sX7vdM4qmEPWuSytuqCDp5j,XTZ Master +tz1XTZM55hF7g98CzY88MXWhmK8QioGXHtuY,XTZ Master Payout +tz1XTZMBAARXu3zmFno6SMT88MvqSvbD5qrh,XTZ Master Payouts +tz1ZKi4VrDMEQpypn2NTn9pPbZL3xLRkETLA,XTZ-Baker +tz1Q8QkSBS63ZQnH3fBTiAMPes9R666Rn6Sc,YieldWallet +tz1eZwq8b5cvE2bPKokatLkVMzkxz24z3Don,You Loaf We Bake +tz1L3vFD8mFzBaS8yLHFsd7qDJY1t276Dh8i,Zednode \ No newline at end of file