Lorre lightweight indexing

conseil-common/src/main/scala/tech/cryptonomic/conseil/common/tezos/TezosOptics.scala

@@ -409,12 +409,121 @@ object TezosOptics {
       * @return a Map holding for each group both external and internal operations' results
       */
     def extractOperationsAlongWithInternalResults(
-        block: Block
+        block: Block,
+        knownAddresses: Option[List[Tables.KnownAddressesRow]]
     ): Map[OperationsGroup, (List[Operation], List[InternalOperationResults.InternalOperationResult])] =
       block.operationGroups.map { group =>
         val updatedGroup = addIndexesToOperationGroup(group)
 
-        val internal = updatedGroup.contents.flatMap { op =>
+        val filteredOperations = updatedGroup.contents.filter { op =>
+          knownAddresses match {
+            case Some(ka) =>
+              val addresses = ka.map(_.address)
+              op match {
+                case Endorsement(level, metadata, blockOrder) => false
+                case EndorsementWithSlot(endorsement, metadata, blockOrder) => false
+                case Preendorsement(level, metadata, blockOrder) => false
+                case SeedNonceRevelation(level, nonce, metadata, blockOrder) => false
+                case ActivateAccount(pkh, secret, metadata, blockOrder) => addresses.contains(pkh.value)
+                case Reveal(counter, fee, gas_limit, storage_limit, public_key, source, metadata, blockOrder) =>
+                  addresses.contains(public_key.value) || addresses.contains(source.value)
+                case Transaction(
+                      counter,
+                      amount,
+                      fee,
+                      gas_limit,
+                      storage_limit,
+                      source,
+                      destination,
+                      parameters,
+                      parameters_micheline,
+                      metadata,
+                      blockOrder
+                    ) =>
+                  addresses.contains(source.value) || addresses.contains(destination.id)
+                case Origination(
+                      counter,
+                      fee,
+                      source,
+                      balance,
+                      gas_limit,
+                      storage_limit,
+                      manager_pubkey,
+                      delegatable,
+                      delegate,
+                      spendable,
+                      script,
+                      metadata,
+                      blockOrder
+                    ) =>
+                  addresses.contains(source.value) || addresses.contains(
+                    delegate.getOrElse(PublicKeyHash("")).value
+                  ) || addresses.contains(manager_pubkey.getOrElse(PublicKeyHash("")).value)
+                case Delegation(counter, source, fee, gas_limit, storage_limit, delegate, metadata, blockOrder) =>
+                  addresses.contains(source.value) || addresses.contains(delegate.getOrElse(PublicKeyHash("")).value)
+                case DoubleEndorsementEvidence(blockOrder) => false
+                case DoublePreendorsementEvidence(blockOrder) => false
+                case DoubleBakingEvidence(blockOrder) => false
+                case Proposals(source, period, proposals, blockOrder) =>
+                  addresses.contains(source.getOrElse(ContractId("")).id)
+                case Ballot(ballot, proposal, source, period, blockOrder) =>
+                  addresses.contains(source.getOrElse(ContractId("")).id)
+                case RegisterGlobalConstant(
+                      source,
+                      fee,
+                      counter,
+                      gas_limit,
+                      storage_limit,
+                      value,
+                      blockOrder,
+                      metadata
+                    ) =>
+                  addresses.contains(source.getOrElse(ContractId("")).id)
+                case SetDepositsLimit(source, fee, counter, gas_limit, storage_limit, limit, blockOrder, metadata) =>
+                  addresses.contains(source.getOrElse(ContractId("")).id)
+                case TxRollupCommit(counter, fee, source, gas_limit, storage_limit, rollup, metadata, blockOrder) =>
+                  addresses.contains(source.value)
+                case TxRollupFinalizeCommitment(
+                      counter,
+                      fee,
+                      source,
+                      gas_limit,
+                      storage_limit,
+                      rollup,
+                      metadata,
+                      blockOrder
+                    ) =>
+                  addresses.contains(source.value)
+                case TxRollupOrigination(counter, fee, source, gas_limit, storage_limit, metadata, blockOrder) =>
+                  addresses.contains(source.value)
+                case TxRollupSubmitBatch(
+                      counter,
+                      fee,
+                      source,
+                      gas_limit,
+                      storage_limit,
+                      rollup,
+                      metadata,
+                      blockOrder
+                    ) =>
+                  addresses.contains(source.value)
+                case TxRollupDispatchTickets(
+                      counter,
+                      fee,
+                      source,
+                      gas_limit,
+                      storage_limit,
+                      tx_rollup,
+                      metadata,
+                      blockOrder
+                    ) =>
+                  addresses.contains(source.value)
+                case DefaultOperation(kind, blockOrder) => true
+              }
+            case None => true
+          }
+        }
+        val internal = filteredOperations.flatMap { op =>
           op match {
             case r: Reveal => r.metadata.internal_operation_results.toList.flatten
             case t: Transaction => t.metadata.internal_operation_results.toList.flatten
@@ -423,7 +532,7 @@ object TezosOptics {
             case _ => List.empty
           }
         }
-        updatedGroup -> (updatedGroup.contents, internal)
+        updatedGroup -> (filteredOperations, internal)
       }.toMap
 
     /** Extracts all operations, primary and internal, and pre-order traverse

conseil-lorre/src/main/resources/application.conf

@@ -49,6 +49,8 @@ lorre {
     rights-processing-is-on: ${?CONSEIL_LORRE_RIGHTS_PROCESSING_ENABLED}
     baker-features-are-on: false
     baker-features-are-on: ${?CONSEIL_LORRE_BAKER_FEATURES_ENABLED}
+    lightweight-indexing: false
+    lightweight-indexing: ${?CONSEIL_LORRE_LIGHTWEIGHT_INDEXING_ENABLED}
   }
 
   # TODO Should we wrap configuration from below into 'Tezos' secion? The same in the *Configuration class?

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala

@@ -110,7 +110,8 @@ final case class Features(
     metadataFetchingIsOn: Boolean,
     registeredTokensIsOn: Boolean,
     rightsProcessingIsOn: Boolean,
-    bakerFeaturesAreOn: Boolean
+    bakerFeaturesAreOn: Boolean,
+    lightweightIndexing: Boolean
 )
 
 final case class TokenContracts(

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseConversions.scala

@@ -829,34 +829,37 @@ private[tezos] object TezosDatabaseConversions {
     * To correctly create the relation on the db, we must first store the operations, get
     * each generated id, and pass it to the associated balance-updates
     */
-  implicit val blockToOperationTablesData = new Conversion[List, Block, OperationTablesData] {
-    import OperationBalances._
-    import SymbolSourceLabels.Show._
-    import tech.cryptonomic.conseil.common.util.Conversion.Syntax._
-
-    override def convert(from: Block) =
-      TezosOptics.Blocks
-        .extractOperationsAlongWithInternalResults(from)
-        .flatMap { case (group, (operations, internalResults)) =>
-          val mainOperationData = operations.map(op =>
-            (from, group.hash, op).convertTo[Tables.OperationsRow] ->
-              BlockTagged
-                .fromBlockData(from.data, op)
+  implicit val blockToOperationTablesData =
+    new Conversion[List, (Block, Option[List[Tables.KnownAddressesRow]]), OperationTablesData] {
+      import OperationBalances._
+      import SymbolSourceLabels.Show._
+      import tech.cryptonomic.conseil.common.util.Conversion.Syntax._
+
+      override def convert(from: (Block, Option[List[Tables.KnownAddressesRow]])) = {
+        val (block, knownAddresses) = from
+        TezosOptics.Blocks
+          .extractOperationsAlongWithInternalResults(block, knownAddresses)
+          .flatMap { case (group, (operations, internalResults)) =>
+            val mainOperationData = operations.map(op =>
+              (block, group.hash, op).convertTo[Tables.OperationsRow] ->
+                BlockTagged
+                  .fromBlockData(block.data, op)
+                  .convertToA[List, Tables.BalanceUpdatesRow]
+            )
+            val internalOperationData = internalResults.map { case oop =>
+              val op = oop.convertTo[Operation]
+              (block, group.hash, op)
+                .convertTo[Tables.OperationsRow]
+                .copy(internal = true, nonce = Some(oop.nonce.toString)) -> BlockTagged
+                .fromBlockData(block.data, op)
                 .convertToA[List, Tables.BalanceUpdatesRow]
-          )
-          val internalOperationData = internalResults.map { case oop =>
-            val op = oop.convertTo[Operation]
-            (from, group.hash, op)
-              .convertTo[Tables.OperationsRow]
-              .copy(internal = true, nonce = Some(oop.nonce.toString)) -> BlockTagged
-              .fromBlockData(from.data, op)
-              .convertToA[List, Tables.BalanceUpdatesRow]
+            }
+            mainOperationData ++ internalOperationData
           }
-          mainOperationData ++ internalOperationData
-        }
-        .toList
+          .toList
+      }
 
-  }
+    }
 
   implicit val blockAccountsAssociationToCheckpointRow =
     new Conversion[

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala

@@ -13,7 +13,6 @@ import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query => _}
 import tech.cryptonomic.conseil.common.sql.CustomProfileExtension
 import tech.cryptonomic.conseil.common.tezos.{Fork, Tables}
 import tech.cryptonomic.conseil.common.tezos.Tables.{
-  AccountsRow,
   GovernanceRow,
   OperationsRow,
   OriginatedAccountMapsRow,
@@ -37,6 +36,8 @@ import java.util.UUID
 import slick.dbio.DBIOAction
 import tech.cryptonomic.conseil.indexer.tezos.RegisteredTokensFetcher.RegisteredToken
 
+import scala.io.Source
+
 /**
   * Functions for writing Tezos data to a database.
   */
@@ -101,7 +102,8 @@ object TezosDatabaseOperations extends ConseilLogSupport {
     */
   def writeBlocks(
       blocks: List[Block],
-      tokenContracts: TokenContracts
+      tokenContracts: TokenContracts,
+      knownAddresses: Option[List[Tables.KnownAddressesRow]]
   )(implicit ec: ExecutionContext, tnsContracts: TNSContract): DBIO[Unit] = {
     // Kleisli is a Function with effects, Kleisli[F, A, B] ~= A => F[B]
     import TezosDatabaseConversions.OperationTablesData
@@ -118,6 +120,11 @@ object TezosDatabaseOperations extends ConseilLogSupport {
     val saveBlocksAction = Tables.Blocks ++= blocks.map(_.convertTo[BlocksRow])
     val saveBlocksBalanceUpdatesAction = Tables.BalanceUpdates ++= blocks.flatMap { block =>
       block.data.convertToA[List, BalanceUpdatesRow]
+    }.filter { updatesRow =>
+      knownAddresses match {
+        case Some(value) => value.map(_.address).contains(updatesRow.accountId)
+        case None => true
+      }
     }
 
     val saveGroupsAction = Tables.OperationGroups ++= blocks.flatMap(_.convertToA[List, OperationGroupsRow])
@@ -147,7 +154,9 @@ object TezosDatabaseOperations extends ConseilLogSupport {
       saveBlocksAction,
       saveBlocksBalanceUpdatesAction,
       saveGroupsAction,
-      saveOperationsAndBalances.traverse(blocks.flatMap(_.convertToA[List, OperationTablesData])),
+      saveOperationsAndBalances.traverse(
+        blocks.flatMap(block => (block -> knownAddresses).convertToA[List, OperationTablesData])
+      ),
       saveBigMaps(blocks)(ec, tokenContracts, tnsContracts)
     )
 
@@ -396,7 +405,8 @@ object TezosDatabaseOperations extends ConseilLogSupport {
     */
   def writeBlocksAndCheckpointAccounts(
       blocks: List[Block],
-      accountUpdates: List[BlockTagged[List[AccountId]]]
+      accountUpdates: List[BlockTagged[List[AccountId]]],
+      knownAddresses: Option[List[Tables.KnownAddressesRow]]
   )(implicit ec: ExecutionContext, tnsContracts: TNSContract): DBIO[Option[Int]] = {
     logger.info("Writing blocks and account checkpoints to the DB...")
     //sequence both operations in a single transaction
@@ -407,7 +417,9 @@ object TezosDatabaseOperations extends ConseilLogSupport {
             ContractId(address) -> interfaces
         }.toList
       )
-      (writeBlocks(blocks, tokens) andThen writeAccountsCheckpoint(accountUpdates.map(_.asTuple))).transactionally
+      (writeBlocks(blocks, tokens, knownAddresses) andThen writeAccountsCheckpoint(
+        accountUpdates.map(_.asTuple)
+      )).transactionally
     }
   }
 
@@ -834,7 +846,6 @@ object TezosDatabaseOperations extends ConseilLogSupport {
     import io.circe.parser.decode
     import RegisteredTokensFetcher.decoder
     import java.io.{BufferedReader, InputStreamReader}
-    import scala.io.Source
 
     val file = getClass.getResource(s"/tezos/registered_tokens/$network.json")
     val content = Source.fromFile(file.toURI).getLines.mkString